diff --git a/infrastructure/Dockerfile b/infrastructure/Dockerfile index 8268e14ad2faf..64ff7b5f7b3bc 100644 --- a/infrastructure/Dockerfile +++ b/infrastructure/Dockerfile @@ -10,4 +10,4 @@ COPY package.json gatsby/infrastructure/ COPY yarn.lock gatsby/infrastructure/ RUN cd gatsby/infrastructure && yarn COPY build-site.js gatsby/infrastructure/ -CMD ["node", "gatsby/infrastructure/build-site.js"] +ENTRYPOINT ["node", "gatsby/infrastructure/build-site.js"] diff --git a/infrastructure/blue b/infrastructure/blue deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/infrastructure/build-site.js b/infrastructure/build-site.js index f0b6ad14968e1..e22256764f5fa 100644 --- a/infrastructure/build-site.js +++ b/infrastructure/build-site.js @@ -1,23 +1,23 @@ -const shell = require("shelljs") -const { spawn } = require("child_process") -const path = require("path") -const GraphQLClient = require("graphql-request").GraphQLClient +const shell = require(`shelljs`) +const { spawn } = require(`child_process`) +const path = require(`path`) +const GraphQLClient = require(`graphql-request`).GraphQLClient const queue = require(`async/queue`) -const s3 = require("s3") +const s3 = require(`s3`) -const commitId = process.env.CODEBUILD_SOURCE_VERSION +const commitId = process.env.COMMIT.slice(0, 7) const s3Client = s3.createClient({ s3Options: { accessKeyId: process.env.accessKeyId, secretAccessKey: process.env.secretAccessKey, - region: "us-east-1", - endpoint: "s3.us-east-1.amazonaws.com", + region: `us-east-1`, + endpoint: `s3.us-east-1.amazonaws.com`, }, }) const client = new GraphQLClient( - "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr", + `https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr`, { headers: { Authorization: `Bearer ${process.env.GRAPHCOOL_TOKEN}`, @@ -58,8 +58,8 @@ function createLogLine(logLine, buildId, stderr = false) { ) } -const getCommitObjectByHash = commitId => { - return client +const getCommitObjectByHash = commitId => + client .request( ` { @@ -72,7 +72,6 @@ const getCommitObjectByHash = commitId => { ` ) .then(result => result.allCommits[0].id) -} console.log(process.env) @@ -82,7 +81,7 @@ var q = queue(function({ logLine, buildId, stderr }, callback) { const Main = async () => { if (!process.env.PATH_TO_SITE) { - console.log("Missing required environment variable PATH_TO_SITE") + console.log(`Missing required environment variable PATH_TO_SITE`) process.exit(1) } @@ -123,23 +122,23 @@ const Main = async () => { child.on(`error`, err => console.log(`err:`, err)) child.stdout.pipe(process.stdout) child.stderr.pipe(process.stderr) - child.stdout.on("data", data => { + child.stdout.on(`data`, data => { // Create new logline - q.push({ logLine: data.toString("utf8"), buildId }) + q.push({ logLine: data.toString(`utf8`), buildId }) }) - child.stderr.on("data", data => { + child.stderr.on(`data`, data => { // Create new logline - q.push({ logLine: data.toString("utf8"), buildId, stderr: true }) + q.push({ logLine: data.toString(`utf8`), buildId, stderr: true }) }) // On end - child.on("exit", (code, signal) => { + child.on(`exit`, (code, signal) => { console.log( - "child process exited with " + `code ${code} and signal ${signal}` + `child process exited with ` + `code ${code} and signal ${signal}` ) // Gatsby build failed if (code !== 0) { - updateBuild(buildId, "FAILURE") + updateBuild(buildId, `FAILURE`) process.exit(code) } const publicDir = `${pathToSite}/public` @@ -158,13 +157,13 @@ const Main = async () => { ) upload.on(`error`, error => { console.error(error) - updateBuild(buildId, "FAILURE").then(() => { + updateBuild(buildId, `FAILURE`).then(() => { process.exit(code) }) }) // 2. Write final status of build and exit. upload.on(`end`, () => { - updateBuild(buildId, "SUCCESS").then(() => process.exit()) + updateBuild(buildId, `SUCCESS`).then(() => process.exit()) }) }) } diff --git a/infrastructure/functions/onGithubWebhook/create-site-build.js b/infrastructure/functions/onGithubWebhook/create-site-build.js new file mode 100644 index 0000000000000..3e001c6b46553 --- /dev/null +++ b/infrastructure/functions/onGithubWebhook/create-site-build.js @@ -0,0 +1,40 @@ +var aws4 = require(`hyper-aws4`) +var fetch = require(`node-fetch`) +var Hyper = require(`hyper-api`) + +let api = new Hyper({ + credential: { + accessKey: process.env.hyperAccessKey, + secretKey: process.env.hyperSecretKey, + }, +}) + +module.exports = ({ pathToSite, commit }) => { + const config = { + Env: [ + `PATH_TO_SITE=${pathToSite}`, + `GRAPHCOOL_TOKEN=${process.env.GRAPHCOOL_TOKEN}`, + `CODEBUILD_SOURCE_VERSION=${commit}`, + `COMMIT=${commit}`, + `accessKeyId=${process.env.accessKeyId}`, + `secretAccessKey=${process.env.secretAccessKey}`, + ], + Image: `gatsbyjs/gatsby-dev-builds`, + Labels: { + sh_hyper_instancetype: `m2`, + }, + } + + console.log(`site build job config`, config) + return api + .post(`/containers/create`, config) + .then(c => { + console.log(c) + return api + .post(`/containers/${c.Id}/start`) + .then(c => console.log(c)) + .catch(e => console.log(e)) + // TODO attach then once it's stopped, delete the job. + }) + .catch(e => console.log(e)) +} diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 38d7ca2764441..cf4145c1e2ce6 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -1,20 +1,22 @@ // handler.js "use strict" -const axios = require("axios") -const Libhoney = require("libhoney").default -const flatten = require("flat") -const GraphQLClient = require("graphql-request").GraphQLClient +const axios = require(`axios`) +const Libhoney = require(`libhoney`).default +const flatten = require(`flat`) +const GraphQLClient = require(`graphql-request`).GraphQLClient + +const createSiteBuild = require(`./create-site-build`) const hny = new Libhoney({ writeKey: process.env.HONEYCOMB_KEY, - dataset: "gatsbyjs-os.lambda.github-webhook", + dataset: `gatsbyjs-os.lambda.github-webhook`, }) console.log(`env vars`, process.env) const client = new GraphQLClient( - "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr", + `https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr`, { headers: { Authorization: `Bearer ${process.env.GRAPHCOOL_TOKEN}`, @@ -59,15 +61,39 @@ const createBranchIfDoesNotExist = branch => const createCommit = (commit, branchId) => { hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit)) - return client.request(` + + return client + .request( + ` mutation { - createCommit(authorName: "${commit.author.name}",authorUsername: "${commit - .author.username}", authorEmail: "${commit.author - .email}", hash: "${commit.tree_id}", message: "${commit.message}", branchIds: ["${branchId}"]) { + createCommit(authorName: "${commit.author.name}",authorUsername: "${ + commit.author.username + }", authorEmail: "${commit.author.email}", hash: "${ + commit.tree_id + }", message: "${commit.message}", branchIds: ["${branchId}"]) { id } } - `) + ` + ) + .then(() => + // TODO create build jobs for www, examples/image-processing, examples/using-glamor, examples/using-excel + Promise.all([ + createSiteBuild({ pathToSite: `www`, commit: commit.tree_id }), + createSiteBuild({ + pathToSite: `examples/image-processing`, + commit: commit.tree_id, + }), + createSiteBuild({ + pathToSite: `examples/using-glamor`, + commit: commit.tree_id, + }), + createSiteBuild({ + pathToSite: `examples/using-excel`, + commit: commit.tree_id, + }), + ]) + ) } module.exports.event = function(event, context, callback) { diff --git a/infrastructure/functions/onGithubWebhook/package.json b/infrastructure/functions/onGithubWebhook/package.json index 26a2dd04a4dde..a4d356c54cd53 100644 --- a/infrastructure/functions/onGithubWebhook/package.json +++ b/infrastructure/functions/onGithubWebhook/package.json @@ -12,6 +12,9 @@ "axios": "^0.17.0", "flat": "^4.0.0", "graphql-request": "^1.4.0", - "libhoney": "^1.0.0-beta.10" + "hyper-api": "^1.6.0", + "hyper-aws4": "^1.1.3", + "libhoney": "^1.0.0-beta.10", + "node-fetch": "^1.7.3" } } diff --git a/infrastructure/functions/onGithubWebhook/yarn.lock b/infrastructure/functions/onGithubWebhook/yarn.lock index 86241b9a11626..eb7a92328f133 100644 --- a/infrastructure/functions/onGithubWebhook/yarn.lock +++ b/infrastructure/functions/onGithubWebhook/yarn.lock @@ -13,6 +13,10 @@ axios@^0.17.0: follow-redirects "^1.2.3" is-buffer "^1.1.5" +bluebird@^3.4.6: + version "3.5.1" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9" + combined-stream@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.5.tgz#938370a57b4a51dea2c77c15d5c5fdf895164009" @@ -92,6 +96,20 @@ graphql-request@^1.4.0: dependencies: cross-fetch "0.0.8" +hyper-api@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/hyper-api/-/hyper-api-1.6.0.tgz#2f0d2ed3c3c7009e127a28a4c991043af2efefc7" + dependencies: + bluebird "^3.4.6" + hyper-aws4 "^1.1.1" + node-fetch "^1.6.0" + +hyper-aws4@^1.1.1, hyper-aws4@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/hyper-aws4/-/hyper-aws4-1.1.3.tgz#c1882e746af5445d065a941df22ed165e332bdd7" + dependencies: + lodash "^4.13.1" + iconv-lite@~0.4.13: version "0.4.19" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" @@ -119,6 +137,10 @@ libhoney@^1.0.0-beta.10: superagent "^2.3.0" urljoin "^0.1.5" +lodash@^4.13.1: + version "4.17.4" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" + methods@^1.1.1: version "1.1.2" resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" @@ -141,7 +163,7 @@ ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" -node-fetch@1.7.3: +node-fetch@1.7.3, node-fetch@^1.6.0, node-fetch@^1.7.3: version "1.7.3" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" dependencies: diff --git a/infrastructure/package.json b/infrastructure/package.json index b6ea36a688ca9..c2d125970ba94 100644 --- a/infrastructure/package.json +++ b/infrastructure/package.json @@ -10,7 +10,11 @@ "license": "ISC", "dependencies": { "async": "^2.6.0", + "aws-sdk": "^2.173.0", "graphql-request": "^1.4.0", + "hyper-api": "^1.6.0", + "hyper-aws4": "^1.1.3", + "node-fetch": "^1.7.3", "s3": "^4.4.0", "shelljs": "^0.7.8" } diff --git a/infrastructure/yarn.lock b/infrastructure/yarn.lock index 928e6ae368cf4..fc28fabe11096 100644 --- a/infrastructure/yarn.lock +++ b/infrastructure/yarn.lock @@ -8,6 +8,21 @@ async@^2.6.0: dependencies: lodash "^4.14.0" +aws-sdk@^2.173.0: + version "2.173.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.173.0.tgz#2b29486f16e1d04159d0513aa47f5dd70370116d" + dependencies: + buffer "4.9.1" + crypto-browserify "1.0.9" + events "^1.1.1" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.1.0" + xml2js "0.4.17" + xmlbuilder "4.2.1" + aws-sdk@~2.0.31: version "2.0.31" resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.0.31.tgz#e72cf1fdc69015bd9fd2bdf3d3b88c16507d268e" @@ -19,6 +34,14 @@ balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" +base64-js@^1.0.2: + version "1.2.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.1.tgz#a91947da1f4a516ea38e5b4ec0ec3773675e0886" + +bluebird@^3.4.6: + version "3.5.1" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9" + brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -26,6 +49,14 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +buffer@4.9.1: + version "4.9.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + isarray "^1.0.0" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -37,12 +68,20 @@ cross-fetch@0.0.8: node-fetch "1.7.3" whatwg-fetch "2.0.3" +crypto-browserify@1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-1.0.9.tgz#cc5449685dfb85eb11c9828acc7cb87ab5bbfcc0" + encoding@^0.1.11: version "0.1.12" resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.12.tgz#538b66f3ee62cd1ab51ec323829d1f9480c74beb" dependencies: iconv-lite "~0.4.13" +events@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + fd-slicer@~1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.0.1.tgz#8b5bcbd9ec327c5041bf9ab023fd6750f1177e65" @@ -80,10 +119,28 @@ graphql-request@^1.4.0: dependencies: cross-fetch "0.0.8" +hyper-api@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/hyper-api/-/hyper-api-1.6.0.tgz#2f0d2ed3c3c7009e127a28a4c991043af2efefc7" + dependencies: + bluebird "^3.4.6" + hyper-aws4 "^1.1.1" + node-fetch "^1.6.0" + +hyper-aws4@^1.1.1, hyper-aws4@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/hyper-aws4/-/hyper-aws4-1.1.3.tgz#c1882e746af5445d065a941df22ed165e332bdd7" + dependencies: + lodash "^4.13.1" + iconv-lite@~0.4.13: version "0.4.19" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" +ieee754@^1.1.4: + version "1.1.8" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" + inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -103,7 +160,15 @@ is-stream@^1.0.1: version "1.1.0" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" -lodash@^4.14.0: +isarray@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + +lodash@^4.0.0, lodash@^4.13.1, lodash@^4.14.0: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" @@ -131,7 +196,7 @@ natives@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/natives/-/natives-1.1.0.tgz#e9ff841418a6b2ec7a495e939984f78f163e6e31" -node-fetch@1.7.3: +node-fetch@1.7.3, node-fetch@^1.6.0, node-fetch@^1.7.3: version "1.7.3" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" dependencies: @@ -156,6 +221,14 @@ pend@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" +punycode@1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" + +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + rechoir@^0.6.2: version "0.6.2" resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" @@ -190,6 +263,14 @@ sax@0.4.2: version "0.4.2" resolved "https://registry.yarnpkg.com/sax/-/sax-0.4.2.tgz#39f3b601733d6bec97105b242a2a40fd6978ac3c" +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + +sax@>=0.6.0: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + shelljs@^0.7.8: version "0.7.8" resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.7.8.tgz#decbcf874b0d1e5fb72e14b164a9683048e9acb3" @@ -202,6 +283,17 @@ streamsink@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/streamsink/-/streamsink-1.2.0.tgz#efafee9f1e22d3591ed7de3dcaa95c3f5e79f73c" +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + dependencies: + punycode "1.3.2" + querystring "0.2.0" + +uuid@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" + whatwg-fetch@2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-2.0.3.tgz#9c84ec2dcf68187ff00bc64e1274b442176e1c84" @@ -216,6 +308,19 @@ xml2js@0.2.6: dependencies: sax "0.4.2" +xml2js@0.4.17: + version "0.4.17" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.17.tgz#17be93eaae3f3b779359c795b419705a8817e868" + dependencies: + sax ">=0.6.0" + xmlbuilder "^4.1.0" + xmlbuilder@0.4.2: version "0.4.2" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-0.4.2.tgz#1776d65f3fdbad470a08d8604cdeb1c4e540ff83" + +xmlbuilder@4.2.1, xmlbuilder@^4.1.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-4.2.1.tgz#aa58a3041a066f90eaa16c2f5389ff19f3f461a5" + dependencies: + lodash "^4.0.0"