WIP new testing infrastructure

infrastructure/Dockerfile

@@ -10,4 +10,4 @@ COPY package.json gatsby/infrastructure/
 COPY yarn.lock gatsby/infrastructure/
 RUN cd gatsby/infrastructure && yarn
 COPY build-site.js gatsby/infrastructure/
-CMD ["node", "gatsby/infrastructure/build-site.js"]
+ENTRYPOINT ["node", "gatsby/infrastructure/build-site.js"]

infrastructure/build-site.js

@@ -1,23 +1,23 @@
-const shell = require("shelljs")
-const { spawn } = require("child_process")
-const path = require("path")
-const GraphQLClient = require("graphql-request").GraphQLClient
+const shell = require(`shelljs`)
+const { spawn } = require(`child_process`)
+const path = require(`path`)
+const GraphQLClient = require(`graphql-request`).GraphQLClient
 const queue = require(`async/queue`)
-const s3 = require("s3")
+const s3 = require(`s3`)
 
-const commitId = process.env.CODEBUILD_SOURCE_VERSION
+const commitId = process.env.COMMIT.slice(0, 7)
 
 const s3Client = s3.createClient({
   s3Options: {
     accessKeyId: process.env.accessKeyId,
     secretAccessKey: process.env.secretAccessKey,
-    region: "us-east-1",
-    endpoint: "s3.us-east-1.amazonaws.com",
+    region: `us-east-1`,
+    endpoint: `s3.us-east-1.amazonaws.com`,
   },
 })
 
 const client = new GraphQLClient(
-  "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr",
+  `https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr`,
   {
     headers: {
       Authorization: `Bearer ${process.env.GRAPHCOOL_TOKEN}`,
@@ -58,8 +58,8 @@ function createLogLine(logLine, buildId, stderr = false) {
   )
 }
 
-const getCommitObjectByHash = commitId => {
-  return client
+const getCommitObjectByHash = commitId =>
+  client
     .request(
       `
       {
@@ -72,7 +72,6 @@ const getCommitObjectByHash = commitId => {
     `
     )
     .then(result => result.allCommits[0].id)
-}
 
 console.log(process.env)
 
@@ -82,7 +81,7 @@ var q = queue(function({ logLine, buildId, stderr }, callback) {
 
 const Main = async () => {
   if (!process.env.PATH_TO_SITE) {
-    console.log("Missing required environment variable PATH_TO_SITE")
+    console.log(`Missing required environment variable PATH_TO_SITE`)
     process.exit(1)
   }
 
@@ -123,23 +122,23 @@ const Main = async () => {
   child.on(`error`, err => console.log(`err:`, err))
   child.stdout.pipe(process.stdout)
   child.stderr.pipe(process.stderr)
-  child.stdout.on("data", data => {
+  child.stdout.on(`data`, data => {
     // Create new logline
-    q.push({ logLine: data.toString("utf8"), buildId })
+    q.push({ logLine: data.toString(`utf8`), buildId })
   })
-  child.stderr.on("data", data => {
+  child.stderr.on(`data`, data => {
     // Create new logline
-    q.push({ logLine: data.toString("utf8"), buildId, stderr: true })
+    q.push({ logLine: data.toString(`utf8`), buildId, stderr: true })
   })
 
   // On end
-  child.on("exit", (code, signal) => {
+  child.on(`exit`, (code, signal) => {
     console.log(
-      "child process exited with " + `code ${code} and signal ${signal}`
+      `child process exited with ` + `code ${code} and signal ${signal}`
     )
     // Gatsby build failed
     if (code !== 0) {
-      updateBuild(buildId, "FAILURE")
+      updateBuild(buildId, `FAILURE`)
       process.exit(code)
     }
     const publicDir = `${pathToSite}/public`
@@ -158,13 +157,13 @@ const Main = async () => {
     )
     upload.on(`error`, error => {
       console.error(error)
-      updateBuild(buildId, "FAILURE").then(() => {
+      updateBuild(buildId, `FAILURE`).then(() => {
         process.exit(code)
       })
     })
     // 2. Write final status of build and exit.
     upload.on(`end`, () => {
-      updateBuild(buildId, "SUCCESS").then(() => process.exit())
+      updateBuild(buildId, `SUCCESS`).then(() => process.exit())
     })
   })
 }

infrastructure/functions/onGithubWebhook/create-site-build.js

@@ -0,0 +1,40 @@
+var aws4 = require(`hyper-aws4`)
+var fetch = require(`node-fetch`)
+var Hyper = require(`hyper-api`)
+
+let api = new Hyper({
+  credential: {
+    accessKey: process.env.hyperAccessKey,
+    secretKey: process.env.hyperSecretKey,
+  },
+})
+
+module.exports = ({ pathToSite, commit }) => {
+  const config = {
+    Env: [
+      `PATH_TO_SITE=${pathToSite}`,
+      `GRAPHCOOL_TOKEN=${process.env.GRAPHCOOL_TOKEN}`,
+      `CODEBUILD_SOURCE_VERSION=${commit}`,
+      `COMMIT=${commit}`,
+      `accessKeyId=${process.env.accessKeyId}`,
+      `secretAccessKey=${process.env.secretAccessKey}`,
+    ],
+    Image: `gatsbyjs/gatsby-dev-builds`,
+    Labels: {
+      sh_hyper_instancetype: `m2`,
+    },
+  }
+
+  console.log(`site build job config`, config)
+  return api
+    .post(`/containers/create`, config)
+    .then(c => {
+      console.log(c)
+      return api
+        .post(`/containers/${c.Id}/start`)
+        .then(c => console.log(c))
+        .catch(e => console.log(e))
+      // TODO attach then once it's stopped, delete the job.
+    })
+    .catch(e => console.log(e))
+}

infrastructure/functions/onGithubWebhook/handler.js

@@ -1,20 +1,22 @@
 // handler.js
 
 "use strict"
-const axios = require("axios")
-const Libhoney = require("libhoney").default
-const flatten = require("flat")
-const GraphQLClient = require("graphql-request").GraphQLClient
+const axios = require(`axios`)
+const Libhoney = require(`libhoney`).default
+const flatten = require(`flat`)
+const GraphQLClient = require(`graphql-request`).GraphQLClient
+
+const createSiteBuild = require(`./create-site-build`)
 
 const hny = new Libhoney({
   writeKey: process.env.HONEYCOMB_KEY,
-  dataset: "gatsbyjs-os.lambda.github-webhook",
+  dataset: `gatsbyjs-os.lambda.github-webhook`,
 })
 
 console.log(`env vars`, process.env)
 
 const client = new GraphQLClient(
-  "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr",
+  `https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr`,
   {
     headers: {
       Authorization: `Bearer ${process.env.GRAPHCOOL_TOKEN}`,
@@ -59,15 +61,39 @@ const createBranchIfDoesNotExist = branch =>
 
 const createCommit = (commit, branchId) => {
   hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit))
-  return client.request(`
+
+  return client
+    .request(
+      `
     mutation {
-      createCommit(authorName: "${commit.author.name}",authorUsername: "${commit
-    .author.username}", authorEmail: "${commit.author
-    .email}", hash: "${commit.tree_id}", message: "${commit.message}", branchIds: ["${branchId}"]) {
+      createCommit(authorName: "${commit.author.name}",authorUsername: "${
+        commit.author.username
+      }", authorEmail: "${commit.author.email}", hash: "${
+        commit.tree_id
+      }", message: "${commit.message}", branchIds: ["${branchId}"]) {
         id
       }
     }
-  `)
+  `
+    )
+    .then(() =>
+      // TODO create build jobs for www, examples/image-processing, examples/using-glamor, examples/using-excel
+      Promise.all([
+        createSiteBuild({ pathToSite: `www`, commit: commit.tree_id }),
+        createSiteBuild({
+          pathToSite: `examples/image-processing`,
+          commit: commit.tree_id,
+        }),
+        createSiteBuild({
+          pathToSite: `examples/using-glamor`,
+          commit: commit.tree_id,
+        }),
+        createSiteBuild({
+          pathToSite: `examples/using-excel`,
+          commit: commit.tree_id,
+        }),
+      ])
+    )
 }
 
 module.exports.event = function(event, context, callback) {

infrastructure/functions/onGithubWebhook/package.json

@@ -12,6 +12,9 @@
     "axios": "^0.17.0",
     "flat": "^4.0.0",
     "graphql-request": "^1.4.0",
-    "libhoney": "^1.0.0-beta.10"
+    "hyper-api": "^1.6.0",
+    "hyper-aws4": "^1.1.3",
+    "libhoney": "^1.0.0-beta.10",
+    "node-fetch": "^1.7.3"
   }
 }

infrastructure/functions/onGithubWebhook/yarn.lock

@@ -13,6 +13,10 @@ axios@^0.17.0:
     follow-redirects "^1.2.3"
     is-buffer "^1.1.5"
 
+bluebird@^3.4.6:
+  version "3.5.1"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9"
+
 combined-stream@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.5.tgz#938370a57b4a51dea2c77c15d5c5fdf895164009"
@@ -92,6 +96,20 @@ graphql-request@^1.4.0:
   dependencies:
     cross-fetch "0.0.8"
 
+hyper-api@^1.6.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/hyper-api/-/hyper-api-1.6.0.tgz#2f0d2ed3c3c7009e127a28a4c991043af2efefc7"
+  dependencies:
+    bluebird "^3.4.6"
+    hyper-aws4 "^1.1.1"
+    node-fetch "^1.6.0"
+
+hyper-aws4@^1.1.1, hyper-aws4@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/hyper-aws4/-/hyper-aws4-1.1.3.tgz#c1882e746af5445d065a941df22ed165e332bdd7"
+  dependencies:
+    lodash "^4.13.1"
+
 iconv-lite@~0.4.13:
   version "0.4.19"
   resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b"
@@ -119,6 +137,10 @@ libhoney@^1.0.0-beta.10:
     superagent "^2.3.0"
     urljoin "^0.1.5"
 
+lodash@^4.13.1:
+  version "4.17.4"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
+
 methods@^1.1.1:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
@@ -141,7 +163,7 @@ ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
 
-node-fetch@1.7.3:
+node-fetch@1.7.3, node-fetch@^1.6.0, node-fetch@^1.7.3:
   version "1.7.3"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef"
   dependencies:

infrastructure/package.json

@@ -10,7 +10,11 @@
   "license": "ISC",
   "dependencies": {
     "async": "^2.6.0",
+    "aws-sdk": "^2.173.0",
     "graphql-request": "^1.4.0",
+    "hyper-api": "^1.6.0",
+    "hyper-aws4": "^1.1.3",
+    "node-fetch": "^1.7.3",
     "s3": "^4.4.0",
     "shelljs": "^0.7.8"
   }