From 5d179f5d04468f91705a555f3b7c9b6bb56b7f90 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 6 Nov 2017 17:02:09 -0800 Subject: [PATCH 01/32] Add lambda function for receiving github webhooks --- .gitignore | 1 + .../functions/onGithubWebhook/handler.js | 26 +++ .../functions/onGithubWebhook/package.json | 16 ++ .../functions/onGithubWebhook/serverless.yml | 106 +++++++++++ .../functions/onGithubWebhook/yarn.lock | 170 ++++++++++++++++++ 5 files changed, 319 insertions(+) create mode 100644 infrastructure/functions/onGithubWebhook/handler.js create mode 100644 infrastructure/functions/onGithubWebhook/package.json create mode 100644 infrastructure/functions/onGithubWebhook/serverless.yml create mode 100644 infrastructure/functions/onGithubWebhook/yarn.lock diff --git a/.gitignore b/.gitignore index 46c283f992d49..f5bd238b99fba 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ node_modules/ .idea/ .vscode/ +.serverless/ diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js new file mode 100644 index 0000000000000..f1eaf7ae6fba4 --- /dev/null +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -0,0 +1,26 @@ +// handler.js + +"use strict" +const axios = require("axios") +const Libhoney = require("libhoney").default +const flatten = require("flat") + +const hny = new Libhoney({ + writeKey: process.env.HONEYCOMB_KEY, + dataset: "gatsbyjs-os.lambda.github-webhook", +}) + +module.exports.event = function(event, context, callback) { + if (event && event.body) { + event.body = JSON.parse(event.body) + } + console.log(event) // Contains incoming request data (e.g., query params, headers and more) + hny.sendNow(flatten(event)) + + const response = { + statusCode: 200, + body: ``, + } + + callback(null, response) +} diff --git a/infrastructure/functions/onGithubWebhook/package.json b/infrastructure/functions/onGithubWebhook/package.json new file mode 100644 index 0000000000000..1a8a0a8e46407 --- /dev/null +++ b/infrastructure/functions/onGithubWebhook/package.json @@ -0,0 +1,16 @@ +{ + "name": "ongithubwebhook", + "version": "1.0.0", + "description": "", + "main": "handler.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "axios": "^0.17.0", + "flat": "^4.0.0", + "libhoney": "^1.0.0-beta.10" + } +} diff --git a/infrastructure/functions/onGithubWebhook/serverless.yml b/infrastructure/functions/onGithubWebhook/serverless.yml new file mode 100644 index 0000000000000..d031731be221b --- /dev/null +++ b/infrastructure/functions/onGithubWebhook/serverless.yml @@ -0,0 +1,106 @@ +# Welcome to Serverless! +# +# This file is the main config file for your service. +# It's very minimal at this point and uses default values. +# You can always add more config options for more control. +# We've included some commented out config examples here. +# Just uncomment any of them to get that config option. +# +# For full config options, check the docs: +# docs.serverless.com +# +# Happy Coding! + +service: onGithubWebhook + +# You can pin your service to only deploy with a specific Serverless version +# Check out our docs for more details +# frameworkVersion: "=X.X.X" + +provider: + name: aws + runtime: nodejs6.10 + +# you can overwrite defaults here +# stage: dev +# region: us-east-1 + +# you can add statements to the Lambda function's IAM Role here +# iamRoleStatements: +# - Effect: "Allow" +# Action: +# - "s3:ListBucket" +# Resource: { "Fn::Join" : ["", ["arn:aws:s3:::", { "Ref" : "ServerlessDeploymentBucket" } ] ] } +# - Effect: "Allow" +# Action: +# - "s3:PutObject" +# Resource: +# Fn::Join: +# - "" +# - - "arn:aws:s3:::" +# - "Ref" : "ServerlessDeploymentBucket" +# - "/*" + +# you can define service wide environment variables here +# environment: +# variable1: value1 + +# you can add packaging information here +#package: +# include: +# - include-me.js +# - include-me-dir/** +# exclude: +# - exclude-me.js +# - exclude-me-dir/** + +functions: + event: + handler: handler.event + events: + - http: POST event + +# The following are a few example events you can configure +# NOTE: Please make sure to change your handler code to work with those events +# Check the event documentation for details +# events: +# - http: +# path: users/create +# method: get +# - s3: ${env:BUCKET} +# - schedule: rate(10 minutes) +# - sns: greeter-topic +# - stream: arn:aws:dynamodb:region:XXXXXX:table/foo/stream/1970-01-01T00:00:00.000 +# - alexaSkill +# - alexaSmartHome: amzn1.ask.skill.xx-xx-xx-xx +# - iot: +# sql: "SELECT * FROM 'some_topic'" +# - cloudwatchEvent: +# event: +# source: +# - "aws.ec2" +# detail-type: +# - "EC2 Instance State-change Notification" +# detail: +# state: +# - pending +# - cloudwatchLog: '/aws/lambda/hello' +# - cognitoUserPool: +# pool: MyUserPool +# trigger: PreSignUp + +# Define function environment variables here +# environment: +# variable2: value2 + +# you can add CloudFormation resource templates here +#resources: +# Resources: +# NewResource: +# Type: AWS::S3::Bucket +# Properties: +# BucketName: my-new-bucket +# Outputs: +# NewOutput: +# Description: "Description for the output" +# Value: "Some output value" diff --git a/infrastructure/functions/onGithubWebhook/yarn.lock b/infrastructure/functions/onGithubWebhook/yarn.lock new file mode 100644 index 0000000000000..9311745875a87 --- /dev/null +++ b/infrastructure/functions/onGithubWebhook/yarn.lock @@ -0,0 +1,170 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +async@^1.5.2: + version "1.5.2" + resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" + +axios@^0.17.0: + version "0.17.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.17.0.tgz#7da747916db803f761651d6091d708789b953c6a" + dependencies: + follow-redirects "^1.2.3" + is-buffer "^1.1.5" + +combined-stream@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.5.tgz#938370a57b4a51dea2c77c15d5c5fdf895164009" + dependencies: + delayed-stream "~1.0.0" + +component-emitter@^1.2.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6" + +cookiejar@^2.0.6: + version "2.1.1" + resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.1.tgz#41ad57b1b555951ec171412a81942b1e8200d34a" + +core-util-is@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + +debug@^2.2.0, debug@^2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + dependencies: + ms "2.0.0" + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + +extend@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" + +extend@~2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/extend/-/extend-2.0.1.tgz#1ee8010689e7395ff9448241c98652bc759a8260" + +flat@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/flat/-/flat-4.0.0.tgz#3abc7f3b588e64ce77dc42fd59aa35806622fea8" + dependencies: + is-buffer "~1.1.5" + +follow-redirects@^1.2.3: + version "1.2.5" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.2.5.tgz#ffd3e14cbdd5eaa72f61b6368c1f68516c2a26cc" + dependencies: + debug "^2.6.9" + +form-data@1.0.0-rc4: + version "1.0.0-rc4" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-1.0.0-rc4.tgz#05ac6bc22227b43e4461f488161554699d4f8b5e" + dependencies: + async "^1.5.2" + combined-stream "^1.0.5" + mime-types "^2.1.10" + +formidable@^1.0.17: + version "1.1.1" + resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.1.1.tgz#96b8886f7c3c3508b932d6bd70c4d3a88f35f1a9" + +inherits@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + +is-buffer@^1.1.5, is-buffer@~1.1.5: + version "1.1.6" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" + +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + +libhoney@^1.0.0-beta.10: + version "1.0.0-beta.10" + resolved "https://registry.yarnpkg.com/libhoney/-/libhoney-1.0.0-beta.10.tgz#5ee682f38fa7ef8103810237d7f94345c6fb5134" + dependencies: + superagent "^2.3.0" + urljoin "^0.1.5" + +methods@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" + +mime-db@~1.30.0: + version "1.30.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" + +mime-types@^2.1.10: + version "2.1.17" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" + dependencies: + mime-db "~1.30.0" + +mime@^1.3.4: + version "1.4.1" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.4.1.tgz#121f9ebc49e3766f311a76e1fa1c8003c4b03aa6" + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + +process-nextick-args@~1.0.6: + version "1.0.7" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" + +qs@^6.1.0: + version "6.5.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" + +readable-stream@^2.0.5: + version "2.3.3" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~1.0.6" + safe-buffer "~5.1.1" + string_decoder "~1.0.3" + util-deprecate "~1.0.1" + +safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" + +string_decoder@~1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab" + dependencies: + safe-buffer "~5.1.0" + +superagent@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/superagent/-/superagent-2.3.0.tgz#703529a0714e57e123959ddefbce193b2e50d115" + dependencies: + component-emitter "^1.2.0" + cookiejar "^2.0.6" + debug "^2.2.0" + extend "^3.0.0" + form-data "1.0.0-rc4" + formidable "^1.0.17" + methods "^1.1.1" + mime "^1.3.4" + qs "^6.1.0" + readable-stream "^2.0.5" + +urljoin@^0.1.5: + version "0.1.5" + resolved "https://registry.yarnpkg.com/urljoin/-/urljoin-0.1.5.tgz#b25d2c6112c55ac9d50096a49a0f1fb7f4f53921" + dependencies: + extend "~2.0.0" + +util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" From 36b73df27d4cc98ecda4072a623042f16a267ea7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 9 Nov 2017 17:24:43 -0800 Subject: [PATCH 02/32] Record in graph.cool when new branches/commits are pushed --- .../functions/onGithubWebhook/handler.js | 74 ++++++++++++++++++- .../functions/onGithubWebhook/package.json | 1 + .../functions/onGithubWebhook/yarn.lock | 38 ++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index f1eaf7ae6fba4..8c379566a0450 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -4,12 +4,66 @@ const axios = require("axios") const Libhoney = require("libhoney").default const flatten = require("flat") +const GraphQLClient = require("graphql-request").GraphQLClient const hny = new Libhoney({ writeKey: process.env.HONEYCOMB_KEY, dataset: "gatsbyjs-os.lambda.github-webhook", }) +const client = new GraphQLClient( + "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr", + { + headers: { + Authorization: process.env.GRAPHCOOL_TOKEN, + }, + } +) + +function createBranch(branch) { + hny.sendNow({ createBranch: branch }) + return client.request(` + { + createBranch(name: "${branch}", commits: []) { + id + } + } + `) +} + +function getBranch(branch) { + return client.request(` + { + allBranches(filter: {name: "${branch}"}) { + id + name + } + } + `) +} + +const createBranchIfDoesNotExist = branch => + getBranch(branch).then(result => { + if (result.allBranches.length === 0) { + return createBranch(branch).then( + newBranch => newBranch.data.createBranch.id + ) + } else { + return result.allBranches[0].id + } + }) + +const createCommit = (commit, branchId) => { + hny.sendNow({ createCommit: true, branchId, ...commit }) + return client.request(` + { + createCommit(author: "${commit.author}", hash: "${commit.sha}", message: "${commit.message}", branchIds: [${branchId}]) { + id + } + } + `) +} + module.exports.event = function(event, context, callback) { if (event && event.body) { event.body = JSON.parse(event.body) @@ -22,5 +76,23 @@ module.exports.event = function(event, context, callback) { body: ``, } - callback(null, response) + new Promise(resolve => { + // Commit push + if ( + event.headers[`X-GitHub-Event`] === `push` && + event.body.ref.split(`/`)[1] === `heads` + ) { + const branchName = event.body.ref.split(`/`)[2] + + // Check if the branch exists + createBranchIfDoesNotExist(branchName).then(branchId => { + // Create commits + resolve( + Promise.all( + event.body.commits.map(commit => createCommit(commit, branchId)) + ) + ) + }) + } + }).then(() => callback(null, response)) } diff --git a/infrastructure/functions/onGithubWebhook/package.json b/infrastructure/functions/onGithubWebhook/package.json index 1a8a0a8e46407..26a2dd04a4dde 100644 --- a/infrastructure/functions/onGithubWebhook/package.json +++ b/infrastructure/functions/onGithubWebhook/package.json @@ -11,6 +11,7 @@ "dependencies": { "axios": "^0.17.0", "flat": "^4.0.0", + "graphql-request": "^1.4.0", "libhoney": "^1.0.0-beta.10" } } diff --git a/infrastructure/functions/onGithubWebhook/yarn.lock b/infrastructure/functions/onGithubWebhook/yarn.lock index 9311745875a87..86241b9a11626 100644 --- a/infrastructure/functions/onGithubWebhook/yarn.lock +++ b/infrastructure/functions/onGithubWebhook/yarn.lock @@ -31,6 +31,13 @@ core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" +cross-fetch@0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-0.0.8.tgz#01ed94dc407df2c00f1807fde700a7cfa48a205c" + dependencies: + node-fetch "1.7.3" + whatwg-fetch "2.0.3" + debug@^2.2.0, debug@^2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -41,6 +48,12 @@ delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" +encoding@^0.1.11: + version "0.1.12" + resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.12.tgz#538b66f3ee62cd1ab51ec323829d1f9480c74beb" + dependencies: + iconv-lite "~0.4.13" + extend@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" @@ -73,6 +86,16 @@ formidable@^1.0.17: version "1.1.1" resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.1.1.tgz#96b8886f7c3c3508b932d6bd70c4d3a88f35f1a9" +graphql-request@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/graphql-request/-/graphql-request-1.4.0.tgz#f5b067c83070296d93fb45760e83dfad0d9f537a" + dependencies: + cross-fetch "0.0.8" + +iconv-lite@~0.4.13: + version "0.4.19" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" + inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -81,6 +104,10 @@ is-buffer@^1.1.5, is-buffer@~1.1.5: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" +is-stream@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" + isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" @@ -114,6 +141,13 @@ ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" +node-fetch@1.7.3: + version "1.7.3" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" + dependencies: + encoding "^0.1.11" + is-stream "^1.0.1" + process-nextick-args@~1.0.6: version "1.0.7" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" @@ -168,3 +202,7 @@ urljoin@^0.1.5: util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + +whatwg-fetch@2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-2.0.3.tgz#9c84ec2dcf68187ff00bc64e1274b442176e1c84" From 098bf6022dab563f8637833a97242fb09d4efc9b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 9 Nov 2017 17:28:59 -0800 Subject: [PATCH 03/32] Use node 6 syntax :-(... --- infrastructure/functions/onGithubWebhook/handler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 8c379566a0450..0a1b7bee8eb6c 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -54,7 +54,7 @@ const createBranchIfDoesNotExist = branch => }) const createCommit = (commit, branchId) => { - hny.sendNow({ createCommit: true, branchId, ...commit }) + hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit)) return client.request(` { createCommit(author: "${commit.author}", hash: "${commit.sha}", message: "${commit.message}", branchIds: [${branchId}]) { From 2da4798ffb0168dea9f47d88de74729c74012f0f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 9 Nov 2017 17:35:37 -0800 Subject: [PATCH 04/32] Add graphql operator --- infrastructure/functions/onGithubWebhook/handler.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 0a1b7bee8eb6c..554b582668e90 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -23,7 +23,7 @@ const client = new GraphQLClient( function createBranch(branch) { hny.sendNow({ createBranch: branch }) return client.request(` - { + mutation { createBranch(name: "${branch}", commits: []) { id } @@ -33,7 +33,7 @@ function createBranch(branch) { function getBranch(branch) { return client.request(` - { + query { allBranches(filter: {name: "${branch}"}) { id name @@ -56,7 +56,7 @@ const createBranchIfDoesNotExist = branch => const createCommit = (commit, branchId) => { hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit)) return client.request(` - { + mutation { createCommit(author: "${commit.author}", hash: "${commit.sha}", message: "${commit.message}", branchIds: [${branchId}]) { id } From 184076a6a15a34e97ad67eb521661324855ff7fc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 9 Nov 2017 17:45:08 -0800 Subject: [PATCH 05/32] Try try again --- infrastructure/functions/onGithubWebhook/handler.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 554b582668e90..26339ef1b014f 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -11,6 +11,8 @@ const hny = new Libhoney({ dataset: "gatsbyjs-os.lambda.github-webhook", }) +console.log(`env vars`, process.env) + const client = new GraphQLClient( "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr", { From a5d074b1af45d5dd97010d9088de348f863c3f67 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 9 Nov 2017 17:51:36 -0800 Subject: [PATCH 06/32] missing the Bearer part of authorization --- infrastructure/functions/onGithubWebhook/handler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 26339ef1b014f..9d40e67a79655 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -17,7 +17,7 @@ const client = new GraphQLClient( "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr", { headers: { - Authorization: process.env.GRAPHCOOL_TOKEN, + Authorization: `Bearer ${process.env.GRAPHCOOL_TOKEN}`, }, } ) From f1ff0574ba8919b86a4fa071212ad0ac152d728c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 10 Nov 2017 13:33:39 -0800 Subject: [PATCH 07/32] Moer fixes --- infrastructure/functions/onGithubWebhook/handler.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 9d40e67a79655..26446719495a7 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -47,10 +47,12 @@ function getBranch(branch) { const createBranchIfDoesNotExist = branch => getBranch(branch).then(result => { if (result.allBranches.length === 0) { - return createBranch(branch).then( - newBranch => newBranch.data.createBranch.id - ) + return createBranch(branch).then(newBranch => { + console.log(`newBranch`, newBranch) + return newBranch.createBranch.id + }) } else { + console.log(`allBranches`, result) return result.allBranches[0].id } }) @@ -59,7 +61,7 @@ const createCommit = (commit, branchId) => { hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit)) return client.request(` mutation { - createCommit(author: "${commit.author}", hash: "${commit.sha}", message: "${commit.message}", branchIds: [${branchId}]) { + createCommit(author: "${commit.author}", hash: "${commit.sha}", message: "${commit.message}", branchIds: ["${branchId}"]) { id } } From c2140079e5b48a2d4b3d730b34ebe19bbfd68df9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 10 Nov 2017 13:36:50 -0800 Subject: [PATCH 08/32] =?UTF-8?q?Stringify=20author=20object=20=E2=80=94?= =?UTF-8?q?=20probably=20should=20change=20schema=20but=20this=20is=20a=20?= =?UTF-8?q?prototype=20so=20:shrug:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infrastructure/functions/onGithubWebhook/handler.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 26446719495a7..9137d00c734da 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -61,7 +61,9 @@ const createCommit = (commit, branchId) => { hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit)) return client.request(` mutation { - createCommit(author: "${commit.author}", hash: "${commit.sha}", message: "${commit.message}", branchIds: ["${branchId}"]) { + createCommit(author: "${JSON.stringify( + commit.author + )}", hash: "${commit.sha}", message: "${commit.message}", branchIds: ["${branchId}"]) { id } } From fde02ac8e0fc48e768ef6d578b2b4bf3a1934e27 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 10 Nov 2017 13:43:39 -0800 Subject: [PATCH 09/32] Ok, did need to break out data as JSON.stringify doesn't escape and don't want to double-stringify --- infrastructure/functions/onGithubWebhook/handler.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 9137d00c734da..9370bb71d17fd 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -61,9 +61,9 @@ const createCommit = (commit, branchId) => { hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit)) return client.request(` mutation { - createCommit(author: "${JSON.stringify( - commit.author - )}", hash: "${commit.sha}", message: "${commit.message}", branchIds: ["${branchId}"]) { + createCommit(authorName: "${commit.author.name}",authorUsername: "${commit + .author.username}", authorEmail: "${commit.author + .email}", hash: "${commit.sha}", message: "${commit.message}", branchIds: ["${branchId}"]) { id } } From 645c1896d2ef281ed72fb06f6f42d224cfc0f280 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 10 Nov 2017 18:34:00 -0800 Subject: [PATCH 10/32] test commit --- infrastructure/blue | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 infrastructure/blue diff --git a/infrastructure/blue b/infrastructure/blue new file mode 100644 index 0000000000000..e69de29bb2d1d From 2a775523750e47d5aef2ad4bbe8d2bb22ea9cc2f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 14 Nov 2017 10:55:57 -0800 Subject: [PATCH 11/32] Figuring out still Github's naming... --- infrastructure/functions/onGithubWebhook/handler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 9370bb71d17fd..38d7ca2764441 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -63,7 +63,7 @@ const createCommit = (commit, branchId) => { mutation { createCommit(authorName: "${commit.author.name}",authorUsername: "${commit .author.username}", authorEmail: "${commit.author - .email}", hash: "${commit.sha}", message: "${commit.message}", branchIds: ["${branchId}"]) { + .email}", hash: "${commit.tree_id}", message: "${commit.message}", branchIds: ["${branchId}"]) { id } } From 1124a063ec2b2d32d826ce451d01db865cf02a12 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 14 Nov 2017 17:22:37 -0800 Subject: [PATCH 12/32] Add Dockerfile + site build script --- infrastructure/Dockerfile | 13 +++ infrastructure/build-site.js | 172 +++++++++++++++++++++++++++ infrastructure/package.json | 17 +++ infrastructure/yarn.lock | 221 +++++++++++++++++++++++++++++++++++ 4 files changed, 423 insertions(+) create mode 100644 infrastructure/Dockerfile create mode 100644 infrastructure/build-site.js create mode 100644 infrastructure/package.json create mode 100644 infrastructure/yarn.lock diff --git a/infrastructure/Dockerfile b/infrastructure/Dockerfile new file mode 100644 index 0000000000000..8268e14ad2faf --- /dev/null +++ b/infrastructure/Dockerfile @@ -0,0 +1,13 @@ +FROM node:8 +RUN npm install -g yarn@1.3.2 +RUN yarn global add gatsby-cli gatsby-dev-cli +RUN mkdir ~/.ssh && echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config +RUN git clone https://github.com/gatsbyjs/gatsby.git +RUN wget https://github.com/github/hub/releases/download/v2.3.0-pre10/hub-linux-amd64-2.3.0-pre10.tgz --directory-prefix /usr/local/bin +RUN cd gatsby && gatsby-dev --set-path-to-repo . && yarn bootstrap +RUN mkdir -p gatsby/infrastructure +COPY package.json gatsby/infrastructure/ +COPY yarn.lock gatsby/infrastructure/ +RUN cd gatsby/infrastructure && yarn +COPY build-site.js gatsby/infrastructure/ +CMD ["node", "gatsby/infrastructure/build-site.js"] diff --git a/infrastructure/build-site.js b/infrastructure/build-site.js new file mode 100644 index 0000000000000..f0b6ad14968e1 --- /dev/null +++ b/infrastructure/build-site.js @@ -0,0 +1,172 @@ +const shell = require("shelljs") +const { spawn } = require("child_process") +const path = require("path") +const GraphQLClient = require("graphql-request").GraphQLClient +const queue = require(`async/queue`) +const s3 = require("s3") + +const commitId = process.env.CODEBUILD_SOURCE_VERSION + +const s3Client = s3.createClient({ + s3Options: { + accessKeyId: process.env.accessKeyId, + secretAccessKey: process.env.secretAccessKey, + region: "us-east-1", + endpoint: "s3.us-east-1.amazonaws.com", + }, +}) + +const client = new GraphQLClient( + "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr", + { + headers: { + Authorization: `Bearer ${process.env.GRAPHCOOL_TOKEN}`, + }, + } +) + +function createBuild(sha, commitObjId, url) { + return client.request(` + mutation { + createBuild(gitSha: "${sha}", result: BUILDING, url: "${url}", commitId: "${commitObjId}", logsIds: []) { + id + } + } + `) +} + +function updateBuild(buildId, status) { + return client.request(` + mutation { + updateBuild(id: "${buildId}", result: ${status}) { + id + } + } + `) +} + +function createLogLine(logLine, buildId, stderr = false) { + return client.request( + ` + mutation createLogLine($text: String!, $buildId: ID!, $stderr: Boolean) { + createLogLine(text: $text, buildId: $buildId, stderr: $stderr) { + id + } + } + `, + { text: logLine, buildId, stderr } + ) +} + +const getCommitObjectByHash = commitId => { + return client + .request( + ` + { + allCommits( + filter: { hash: "46f5ed9d8d05a9608bc9c98fcb5867a892149e94" } + ) { + id + } + } + ` + ) + .then(result => result.allCommits[0].id) +} + +console.log(process.env) + +var q = queue(function({ logLine, buildId, stderr }, callback) { + createLogLine(logLine, buildId, stderr).then(callback) +}, 1) + +const Main = async () => { + if (!process.env.PATH_TO_SITE) { + console.log("Missing required environment variable PATH_TO_SITE") + process.exit(1) + } + + console.log(process.cwd()) + + // Navigate to Gatsby directory + shell.cd(`/gatsby/`) + + // Ensure we have the latest code + shell.exec(`git fetch`) + + // Check if the commit exists (later we'll grab pull requests as well + // but we're just building branches on gatsbyjs/gatsby while testing) + const commitExists = shell.exec(`git cat-file -e ${commitId}`) + console.log(`commitExists`, commitExists) + if (commitExists.code !== 0) { + process.exit(commitExists.code) + } + + // It does! So let's checkout our commit and initialize the environment. + shell.exec(`git checkout -b newbranch ${commitId}`) + shell.exec(`yarn bootstrap`) + + // Now go to the site and install + const pathToSite = path.join(`/gatsby/`, process.env.PATH_TO_SITE) + shell.cd(pathToSite) + shell.exec(`yarn`) + shell.exec(`gatsby-dev --scan-once --quiet`) + + // Create the build + const commitObjId = await getCommitObjectByHash(commitId) + const url = `${process.env.PATH_TO_SITE}---${commitId}.gatsbydev.com` + const result = await createBuild(commitId, commitObjId, url) + const buildId = result.createBuild.id + + // Start building! + const child = spawn(`gatsby`, [`build`]) + child.on(`error`, err => console.log(`err:`, err)) + child.stdout.pipe(process.stdout) + child.stderr.pipe(process.stderr) + child.stdout.on("data", data => { + // Create new logline + q.push({ logLine: data.toString("utf8"), buildId }) + }) + child.stderr.on("data", data => { + // Create new logline + q.push({ logLine: data.toString("utf8"), buildId, stderr: true }) + }) + + // On end + child.on("exit", (code, signal) => { + console.log( + "child process exited with " + `code ${code} and signal ${signal}` + ) + // Gatsby build failed + if (code !== 0) { + updateBuild(buildId, "FAILURE") + process.exit(code) + } + const publicDir = `${pathToSite}/public` + console.log(`uploading files from ${publicDir}`) + + // 1. Push built files to s3 bucket + const upload = s3Client.uploadDir({ + localDir: publicDir, + s3Params: { + Prefix: url, + Bucket: `gatsby-js-builds`, + }, + }) + upload.on(`fileUploadEnd`, () => + console.log(`${upload.progressAmount / upload.progressTotal}`) + ) + upload.on(`error`, error => { + console.error(error) + updateBuild(buildId, "FAILURE").then(() => { + process.exit(code) + }) + }) + // 2. Write final status of build and exit. + upload.on(`end`, () => { + updateBuild(buildId, "SUCCESS").then(() => process.exit()) + }) + }) +} + +Main() diff --git a/infrastructure/package.json b/infrastructure/package.json new file mode 100644 index 0000000000000..b6ea36a688ca9 --- /dev/null +++ b/infrastructure/package.json @@ -0,0 +1,17 @@ +{ + "name": "infrastructure", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "async": "^2.6.0", + "graphql-request": "^1.4.0", + "s3": "^4.4.0", + "shelljs": "^0.7.8" + } +} diff --git a/infrastructure/yarn.lock b/infrastructure/yarn.lock new file mode 100644 index 0000000000000..928e6ae368cf4 --- /dev/null +++ b/infrastructure/yarn.lock @@ -0,0 +1,221 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +async@^2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/async/-/async-2.6.0.tgz#61a29abb6fcc026fea77e56d1c6ec53a795951f4" + dependencies: + lodash "^4.14.0" + +aws-sdk@~2.0.31: + version "2.0.31" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.0.31.tgz#e72cf1fdc69015bd9fd2bdf3d3b88c16507d268e" + dependencies: + xml2js "0.2.6" + xmlbuilder "0.4.2" + +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" + +brace-expansion@^1.1.7: + version "1.1.8" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + +cross-fetch@0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-0.0.8.tgz#01ed94dc407df2c00f1807fde700a7cfa48a205c" + dependencies: + node-fetch "1.7.3" + whatwg-fetch "2.0.3" + +encoding@^0.1.11: + version "0.1.12" + resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.12.tgz#538b66f3ee62cd1ab51ec323829d1f9480c74beb" + dependencies: + iconv-lite "~0.4.13" + +fd-slicer@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.0.1.tgz#8b5bcbd9ec327c5041bf9ab023fd6750f1177e65" + dependencies: + pend "~1.2.0" + +findit2@~2.2.3: + version "2.2.3" + resolved "https://registry.yarnpkg.com/findit2/-/findit2-2.2.3.tgz#58a466697df8a6205cdfdbf395536b8bd777a5f6" + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + +glob@^7.0.0: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + +graceful-fs@~3.0.5: + version "3.0.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-3.0.11.tgz#7613c778a1afea62f25c630a086d7f3acbbdd818" + dependencies: + natives "^1.1.0" + +graphql-request@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/graphql-request/-/graphql-request-1.4.0.tgz#f5b067c83070296d93fb45760e83dfad0d9f537a" + dependencies: + cross-fetch "0.0.8" + +iconv-lite@~0.4.13: + version "0.4.19" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" + +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + +interpret@^1.0.0: + version "1.0.4" + resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.0.4.tgz#820cdd588b868ffb191a809506d6c9c8f212b1b0" + +is-stream@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" + +lodash@^4.14.0: + version "4.17.4" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" + +mime@~1.2.11: + version "1.2.11" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" + +minimatch@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + dependencies: + brace-expansion "^1.1.7" + +minimist@0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" + +mkdirp@~0.5.0: + version "0.5.1" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" + dependencies: + minimist "0.0.8" + +natives@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/natives/-/natives-1.1.0.tgz#e9ff841418a6b2ec7a495e939984f78f163e6e31" + +node-fetch@1.7.3: + version "1.7.3" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" + dependencies: + encoding "^0.1.11" + is-stream "^1.0.1" + +once@^1.3.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + dependencies: + wrappy "1" + +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + +path-parse@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1" + +pend@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" + +rechoir@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" + dependencies: + resolve "^1.1.6" + +resolve@^1.1.6: + version "1.5.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36" + dependencies: + path-parse "^1.0.5" + +rimraf@~2.2.8: + version "2.2.8" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.2.8.tgz#e439be2aaee327321952730f99a8929e4fc50582" + +s3@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/s3/-/s3-4.4.0.tgz#56a4f775515a7b6b9c8e5c6b1ab51f9037669f1f" + dependencies: + aws-sdk "~2.0.31" + fd-slicer "~1.0.0" + findit2 "~2.2.3" + graceful-fs "~3.0.5" + mime "~1.2.11" + mkdirp "~0.5.0" + pend "~1.2.0" + rimraf "~2.2.8" + streamsink "~1.2.0" + +sax@0.4.2: + version "0.4.2" + resolved "https://registry.yarnpkg.com/sax/-/sax-0.4.2.tgz#39f3b601733d6bec97105b242a2a40fd6978ac3c" + +shelljs@^0.7.8: + version "0.7.8" + resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.7.8.tgz#decbcf874b0d1e5fb72e14b164a9683048e9acb3" + dependencies: + glob "^7.0.0" + interpret "^1.0.0" + rechoir "^0.6.2" + +streamsink@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/streamsink/-/streamsink-1.2.0.tgz#efafee9f1e22d3591ed7de3dcaa95c3f5e79f73c" + +whatwg-fetch@2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-2.0.3.tgz#9c84ec2dcf68187ff00bc64e1274b442176e1c84" + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + +xml2js@0.2.6: + version "0.2.6" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.2.6.tgz#d209c4e4dda1fc9c452141ef41c077f5adfdf6c4" + dependencies: + sax "0.4.2" + +xmlbuilder@0.4.2: + version "0.4.2" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-0.4.2.tgz#1776d65f3fdbad470a08d8604cdeb1c4e540ff83" From abf8568c48366f0f0404efe05811b8491ec82557 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 20 Dec 2017 17:39:48 -0800 Subject: [PATCH 13/32] Add stab at script for creating CodeBuild projects --- infrastructure/Dockerfile | 1 + infrastructure/blue | 0 infrastructure/build-site.js | 41 +++++---- infrastructure/buildspec.yml | 7 ++ .../codebuild-project-template.json | 24 +++++ infrastructure/create-codebuild-project.js | 47 ++++++++++ infrastructure/package.json | 1 + infrastructure/yarn.lock | 89 ++++++++++++++++++- 8 files changed, 188 insertions(+), 22 deletions(-) delete mode 100644 infrastructure/blue create mode 100644 infrastructure/buildspec.yml create mode 100644 infrastructure/codebuild-project-template.json create mode 100644 infrastructure/create-codebuild-project.js diff --git a/infrastructure/Dockerfile b/infrastructure/Dockerfile index 8268e14ad2faf..5cf335a10d776 100644 --- a/infrastructure/Dockerfile +++ b/infrastructure/Dockerfile @@ -10,4 +10,5 @@ COPY package.json gatsby/infrastructure/ COPY yarn.lock gatsby/infrastructure/ RUN cd gatsby/infrastructure && yarn COPY build-site.js gatsby/infrastructure/ +COPY buildspec.yml gatsby/infrastructure/ CMD ["node", "gatsby/infrastructure/build-site.js"] diff --git a/infrastructure/blue b/infrastructure/blue deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/infrastructure/build-site.js b/infrastructure/build-site.js index f0b6ad14968e1..8cb7ac87ec895 100644 --- a/infrastructure/build-site.js +++ b/infrastructure/build-site.js @@ -1,9 +1,9 @@ -const shell = require("shelljs") -const { spawn } = require("child_process") -const path = require("path") -const GraphQLClient = require("graphql-request").GraphQLClient +const shell = require(`shelljs`) +const { spawn } = require(`child_process`) +const path = require(`path`) +const GraphQLClient = require(`graphql-request`).GraphQLClient const queue = require(`async/queue`) -const s3 = require("s3") +const s3 = require(`s3`) const commitId = process.env.CODEBUILD_SOURCE_VERSION @@ -11,13 +11,13 @@ const s3Client = s3.createClient({ s3Options: { accessKeyId: process.env.accessKeyId, secretAccessKey: process.env.secretAccessKey, - region: "us-east-1", - endpoint: "s3.us-east-1.amazonaws.com", + region: `us-east-1`, + endpoint: `s3.us-east-1.amazonaws.com`, }, }) const client = new GraphQLClient( - "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr", + `https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr`, { headers: { Authorization: `Bearer ${process.env.GRAPHCOOL_TOKEN}`, @@ -58,8 +58,8 @@ function createLogLine(logLine, buildId, stderr = false) { ) } -const getCommitObjectByHash = commitId => { - return client +const getCommitObjectByHash = commitId => + client .request( ` { @@ -72,7 +72,6 @@ const getCommitObjectByHash = commitId => { ` ) .then(result => result.allCommits[0].id) -} console.log(process.env) @@ -82,7 +81,7 @@ var q = queue(function({ logLine, buildId, stderr }, callback) { const Main = async () => { if (!process.env.PATH_TO_SITE) { - console.log("Missing required environment variable PATH_TO_SITE") + console.log(`Missing required environment variable PATH_TO_SITE`) process.exit(1) } @@ -123,23 +122,23 @@ const Main = async () => { child.on(`error`, err => console.log(`err:`, err)) child.stdout.pipe(process.stdout) child.stderr.pipe(process.stderr) - child.stdout.on("data", data => { + child.stdout.on(`data`, data => { // Create new logline - q.push({ logLine: data.toString("utf8"), buildId }) + q.push({ logLine: data.toString(`utf8`), buildId }) }) - child.stderr.on("data", data => { + child.stderr.on(`data`, data => { // Create new logline - q.push({ logLine: data.toString("utf8"), buildId, stderr: true }) + q.push({ logLine: data.toString(`utf8`), buildId, stderr: true }) }) // On end - child.on("exit", (code, signal) => { + child.on(`exit`, (code, signal) => { console.log( - "child process exited with " + `code ${code} and signal ${signal}` + `child process exited with ` + `code ${code} and signal ${signal}` ) // Gatsby build failed if (code !== 0) { - updateBuild(buildId, "FAILURE") + updateBuild(buildId, `FAILURE`) process.exit(code) } const publicDir = `${pathToSite}/public` @@ -158,13 +157,13 @@ const Main = async () => { ) upload.on(`error`, error => { console.error(error) - updateBuild(buildId, "FAILURE").then(() => { + updateBuild(buildId, `FAILURE`).then(() => { process.exit(code) }) }) // 2. Write final status of build and exit. upload.on(`end`, () => { - updateBuild(buildId, "SUCCESS").then(() => process.exit()) + updateBuild(buildId, `SUCCESS`).then(() => process.exit()) }) }) } diff --git a/infrastructure/buildspec.yml b/infrastructure/buildspec.yml new file mode 100644 index 0000000000000..e5b0300fa3109 --- /dev/null +++ b/infrastructure/buildspec.yml @@ -0,0 +1,7 @@ +version: 0.2 + +phases: + build: + commands: + - env + - node infrastructure/build-site.js diff --git a/infrastructure/codebuild-project-template.json b/infrastructure/codebuild-project-template.json new file mode 100644 index 0000000000000..3c17468f721a0 --- /dev/null +++ b/infrastructure/codebuild-project-template.json @@ -0,0 +1,24 @@ +{ + "name": "gatsby-js-commit-build", + "description": "Build example sites on commits to Gatsby", + "source": { + "type": "GITHUB", + "location": "https://github.com/gatsbyjs/gatsby.git", + "buildspec": "", + "auth": { + "type": "OAUTH", + "resource": "" + } + }, + "artifacts": { + "type": "NO_ARTIFACTS" + }, + "environment": { + "type": "LINUX_CONTAINER", + "image": "gatsbyjs/gatsby-dev-builds", + "computeType": "BUILD_GENERAL1_SMALL", + "privilegedMode": true + }, + "serviceRole": "arn:aws:iam::886436322012:role/CodeBuildServiceRoleGatsbyJSDev", + "timeoutInMinutes": 15 +} diff --git a/infrastructure/create-codebuild-project.js b/infrastructure/create-codebuild-project.js new file mode 100644 index 0000000000000..80485c0486f56 --- /dev/null +++ b/infrastructure/create-codebuild-project.js @@ -0,0 +1,47 @@ +const AWS = require(`aws-sdk`) +AWS.config.region = `us-east-1` +console.log(AWS.config) + +const codebuild = new AWS.CodeBuild({ apiVersion: `2016-10-06` }) + +const key = `TESTING_CREATING_PROJECT3` + +const params = { + artifacts: { + type: `NO_ARTIFACTS`, + }, + environment: { + type: `LINUX_CONTAINER`, + image: `gatsbyjs/gatsby-dev-builds`, + computeType: `BUILD_GENERAL1_SMALL`, + privilegedMode: true, + }, + name: key, + source: { + type: `GITHUB`, + auth: { + type: `OAUTH`, + }, + buildspec: `infrastructure/buildspec.yml`, + location: `https://github.com/gatsbyjs/gatsby.git`, + }, + badgeEnabled: false, + serviceRole: `arn:aws:iam::886436322012:role/CodeBuildServiceRoleGatsbyJSDev`, + timeoutInMinutes: 15, +} + +codebuild.createProject(params, function(err, data) { + if (err) + console.log(err, err.stack) // an error occurred + else console.log(data) // successful response + + console.log(`WEBHOOK`) + var params = { + projectName: key /* required */, + } + codebuild.createWebhook(params, function(err, data) { + if (err) + console.log(err, err.stack) // an error occurred + else console.log(data) // successful response + }) +}) diff --git a/infrastructure/package.json b/infrastructure/package.json index b6ea36a688ca9..e0d6bd1c4973f 100644 --- a/infrastructure/package.json +++ b/infrastructure/package.json @@ -10,6 +10,7 @@ "license": "ISC", "dependencies": { "async": "^2.6.0", + "aws-sdk": "^2.173.0", "graphql-request": "^1.4.0", "s3": "^4.4.0", "shelljs": "^0.7.8" diff --git a/infrastructure/yarn.lock b/infrastructure/yarn.lock index 928e6ae368cf4..f29fcd85510da 100644 --- a/infrastructure/yarn.lock +++ b/infrastructure/yarn.lock @@ -8,6 +8,21 @@ async@^2.6.0: dependencies: lodash "^4.14.0" +aws-sdk@^2.173.0: + version "2.173.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.173.0.tgz#2b29486f16e1d04159d0513aa47f5dd70370116d" + dependencies: + buffer "4.9.1" + crypto-browserify "1.0.9" + events "^1.1.1" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.1.0" + xml2js "0.4.17" + xmlbuilder "4.2.1" + aws-sdk@~2.0.31: version "2.0.31" resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.0.31.tgz#e72cf1fdc69015bd9fd2bdf3d3b88c16507d268e" @@ -19,6 +34,10 @@ balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" +base64-js@^1.0.2: + version "1.2.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.1.tgz#a91947da1f4a516ea38e5b4ec0ec3773675e0886" + brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -26,6 +45,14 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +buffer@4.9.1: + version "4.9.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + isarray "^1.0.0" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -37,12 +64,20 @@ cross-fetch@0.0.8: node-fetch "1.7.3" whatwg-fetch "2.0.3" +crypto-browserify@1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-1.0.9.tgz#cc5449685dfb85eb11c9828acc7cb87ab5bbfcc0" + encoding@^0.1.11: version "0.1.12" resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.12.tgz#538b66f3ee62cd1ab51ec323829d1f9480c74beb" dependencies: iconv-lite "~0.4.13" +events@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + fd-slicer@~1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.0.1.tgz#8b5bcbd9ec327c5041bf9ab023fd6750f1177e65" @@ -84,6 +119,10 @@ iconv-lite@~0.4.13: version "0.4.19" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" +ieee754@^1.1.4: + version "1.1.8" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" + inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -103,7 +142,15 @@ is-stream@^1.0.1: version "1.1.0" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" -lodash@^4.14.0: +isarray@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + +lodash@^4.0.0, lodash@^4.14.0: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" @@ -156,6 +203,14 @@ pend@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" +punycode@1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" + +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + rechoir@^0.6.2: version "0.6.2" resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" @@ -190,6 +245,14 @@ sax@0.4.2: version "0.4.2" resolved "https://registry.yarnpkg.com/sax/-/sax-0.4.2.tgz#39f3b601733d6bec97105b242a2a40fd6978ac3c" +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + +sax@>=0.6.0: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + shelljs@^0.7.8: version "0.7.8" resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.7.8.tgz#decbcf874b0d1e5fb72e14b164a9683048e9acb3" @@ -202,6 +265,17 @@ streamsink@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/streamsink/-/streamsink-1.2.0.tgz#efafee9f1e22d3591ed7de3dcaa95c3f5e79f73c" +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + dependencies: + punycode "1.3.2" + querystring "0.2.0" + +uuid@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" + whatwg-fetch@2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-2.0.3.tgz#9c84ec2dcf68187ff00bc64e1274b442176e1c84" @@ -216,6 +290,19 @@ xml2js@0.2.6: dependencies: sax "0.4.2" +xml2js@0.4.17: + version "0.4.17" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.17.tgz#17be93eaae3f3b779359c795b419705a8817e868" + dependencies: + sax ">=0.6.0" + xmlbuilder "^4.1.0" + xmlbuilder@0.4.2: version "0.4.2" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-0.4.2.tgz#1776d65f3fdbad470a08d8604cdeb1c4e540ff83" + +xmlbuilder@4.2.1, xmlbuilder@^4.1.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-4.2.1.tgz#aa58a3041a066f90eaa16c2f5389ff19f3f461a5" + dependencies: + lodash "^4.0.0" From 0ca767b381e98c67079658d71f7cda4470aa90b8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 20 Dec 2017 17:45:02 -0800 Subject: [PATCH 14/32] debugging --- infrastructure/buildspec.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrastructure/buildspec.yml b/infrastructure/buildspec.yml index e5b0300fa3109..41d8c423bb4a0 100644 --- a/infrastructure/buildspec.yml +++ b/infrastructure/buildspec.yml @@ -4,4 +4,6 @@ phases: build: commands: - env + - pwd + - ls * - node infrastructure/build-site.js From a2f57bcd2c14bfb2ad499ca7e005db2303f1e938 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 20 Dec 2017 17:49:34 -0800 Subject: [PATCH 15/32] moer debugging --- infrastructure/buildspec.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/infrastructure/buildspec.yml b/infrastructure/buildspec.yml index 41d8c423bb4a0..6b59fbaadfe93 100644 --- a/infrastructure/buildspec.yml +++ b/infrastructure/buildspec.yml @@ -6,4 +6,7 @@ phases: - env - pwd - ls * + - ls ../../* + - ls ../../../* + - ls ../../../../* - node infrastructure/build-site.js From 003532aed5628f9ce72a4b7e658b072f4ec9cd9d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 20 Dec 2017 17:53:08 -0800 Subject: [PATCH 16/32] MOST GO DEEPER --- infrastructure/buildspec.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/buildspec.yml b/infrastructure/buildspec.yml index 6b59fbaadfe93..7e0039b255575 100644 --- a/infrastructure/buildspec.yml +++ b/infrastructure/buildspec.yml @@ -9,4 +9,5 @@ phases: - ls ../../* - ls ../../../* - ls ../../../../* + - ls ../../../../../* - node infrastructure/build-site.js From 65b97ee99d039eaca16811a6a15dcb364c4d29ff Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 20 Dec 2017 17:56:53 -0800 Subject: [PATCH 17/32] MUST GO DEEPER --- infrastructure/buildspec.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/infrastructure/buildspec.yml b/infrastructure/buildspec.yml index 7e0039b255575..c866745d5ed50 100644 --- a/infrastructure/buildspec.yml +++ b/infrastructure/buildspec.yml @@ -10,4 +10,9 @@ phases: - ls ../../../* - ls ../../../../* - ls ../../../../../* + - ls ../../../../../../* + - ls ../../../../../../../* + - ls ../../../../../../../../* + - ls ../../../../../../../../../* + - ls ../../../../../../../../../../* - node infrastructure/build-site.js From 57b7e56f941ecbe71fb8ba9f12b699f5614974de Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 20 Dec 2017 18:23:52 -0800 Subject: [PATCH 18/32] Switch to ENTRYPOINT to force using our command --- infrastructure/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/Dockerfile b/infrastructure/Dockerfile index 5cf335a10d776..ca8d373766806 100644 --- a/infrastructure/Dockerfile +++ b/infrastructure/Dockerfile @@ -11,4 +11,4 @@ COPY yarn.lock gatsby/infrastructure/ RUN cd gatsby/infrastructure && yarn COPY build-site.js gatsby/infrastructure/ COPY buildspec.yml gatsby/infrastructure/ -CMD ["node", "gatsby/infrastructure/build-site.js"] +ENTRYPOINT ["node", "gatsby/infrastructure/build-site.js"] From a60f1a98d4a43a230d1063521b68e038846bd2d5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 20 Dec 2017 18:38:39 -0800 Subject: [PATCH 19/32] Use the Docker image built from this branch --- infrastructure/create-codebuild-project.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/create-codebuild-project.js b/infrastructure/create-codebuild-project.js index 80485c0486f56..57f1c1aaa4acb 100644 --- a/infrastructure/create-codebuild-project.js +++ b/infrastructure/create-codebuild-project.js @@ -12,7 +12,7 @@ const params = { }, environment: { type: `LINUX_CONTAINER`, - image: `gatsbyjs/gatsby-dev-builds`, + image: `gatsbyjs/gatsby-dev-builds#infrastructure`, computeType: `BUILD_GENERAL1_SMALL`, privilegedMode: true, }, From bff046826f1bef5a9e8d062373c774089f6fbff0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 21 Dec 2017 09:47:21 -0800 Subject: [PATCH 20/32] Small fixes --- infrastructure/create-codebuild-project.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/create-codebuild-project.js b/infrastructure/create-codebuild-project.js index 57f1c1aaa4acb..bfdf263691b67 100644 --- a/infrastructure/create-codebuild-project.js +++ b/infrastructure/create-codebuild-project.js @@ -4,7 +4,7 @@ console.log(AWS.config) const codebuild = new AWS.CodeBuild({ apiVersion: `2016-10-06` }) -const key = `TESTING_CREATING_PROJECT3` +const key = `TESTING_CREATING_PROJECT5` const params = { artifacts: { @@ -12,7 +12,7 @@ const params = { }, environment: { type: `LINUX_CONTAINER`, - image: `gatsbyjs/gatsby-dev-builds#infrastructure`, + image: `gatsbyjs/gatsby-dev-builds:infrastructure`, computeType: `BUILD_GENERAL1_SMALL`, privilegedMode: true, }, From f674ad288518217c912b77ded56df6357453fab1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 21 Dec 2017 10:59:07 -0800 Subject: [PATCH 21/32] test --- infrastructure/create-codebuild-project.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/create-codebuild-project.js b/infrastructure/create-codebuild-project.js index bfdf263691b67..fb2006fa0c490 100644 --- a/infrastructure/create-codebuild-project.js +++ b/infrastructure/create-codebuild-project.js @@ -4,7 +4,7 @@ console.log(AWS.config) const codebuild = new AWS.CodeBuild({ apiVersion: `2016-10-06` }) -const key = `TESTING_CREATING_PROJECT5` +const key = `TESTING_CREATING_PROJECT6` const params = { artifacts: { @@ -22,7 +22,7 @@ const params = { auth: { type: `OAUTH`, }, - buildspec: `infrastructure/buildspec.yml`, + buildspec: ``, location: `https://github.com/gatsbyjs/gatsby.git`, }, badgeEnabled: false, From acdeca981e94527c679bb1050214bedaaa5b003a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 08:15:11 -0800 Subject: [PATCH 22/32] Add script for creating build jobs on hyper.sh + remove codebuild code --- infrastructure/buildspec.yml | 18 ------- .../codebuild-project-template.json | 24 ---------- infrastructure/create-codebuild-project.js | 47 ------------------- infrastructure/create-job.js | 37 +++++++++++++++ infrastructure/package.json | 3 ++ infrastructure/yarn.lock | 22 ++++++++- 6 files changed, 60 insertions(+), 91 deletions(-) delete mode 100644 infrastructure/buildspec.yml delete mode 100644 infrastructure/codebuild-project-template.json delete mode 100644 infrastructure/create-codebuild-project.js create mode 100644 infrastructure/create-job.js diff --git a/infrastructure/buildspec.yml b/infrastructure/buildspec.yml deleted file mode 100644 index c866745d5ed50..0000000000000 --- a/infrastructure/buildspec.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: 0.2 - -phases: - build: - commands: - - env - - pwd - - ls * - - ls ../../* - - ls ../../../* - - ls ../../../../* - - ls ../../../../../* - - ls ../../../../../../* - - ls ../../../../../../../* - - ls ../../../../../../../../* - - ls ../../../../../../../../../* - - ls ../../../../../../../../../../* - - node infrastructure/build-site.js diff --git a/infrastructure/codebuild-project-template.json b/infrastructure/codebuild-project-template.json deleted file mode 100644 index 3c17468f721a0..0000000000000 --- a/infrastructure/codebuild-project-template.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "gatsby-js-commit-build", - "description": "Build example sites on commits to Gatsby", - "source": { - "type": "GITHUB", - "location": "https://github.com/gatsbyjs/gatsby.git", - "buildspec": "", - "auth": { - "type": "OAUTH", - "resource": "" - } - }, - "artifacts": { - "type": "NO_ARTIFACTS" - }, - "environment": { - "type": "LINUX_CONTAINER", - "image": "gatsbyjs/gatsby-dev-builds", - "computeType": "BUILD_GENERAL1_SMALL", - "privilegedMode": true - }, - "serviceRole": "arn:aws:iam::886436322012:role/CodeBuildServiceRoleGatsbyJSDev", - "timeoutInMinutes": 15 -} diff --git a/infrastructure/create-codebuild-project.js b/infrastructure/create-codebuild-project.js deleted file mode 100644 index fb2006fa0c490..0000000000000 --- a/infrastructure/create-codebuild-project.js +++ /dev/null @@ -1,47 +0,0 @@ -const AWS = require(`aws-sdk`) -AWS.config.region = `us-east-1` -console.log(AWS.config) - -const codebuild = new AWS.CodeBuild({ apiVersion: `2016-10-06` }) - -const key = `TESTING_CREATING_PROJECT6` - -const params = { - artifacts: { - type: `NO_ARTIFACTS`, - }, - environment: { - type: `LINUX_CONTAINER`, - image: `gatsbyjs/gatsby-dev-builds:infrastructure`, - computeType: `BUILD_GENERAL1_SMALL`, - privilegedMode: true, - }, - name: key, - source: { - type: `GITHUB`, - auth: { - type: `OAUTH`, - }, - buildspec: ``, - location: `https://github.com/gatsbyjs/gatsby.git`, - }, - badgeEnabled: false, - serviceRole: `arn:aws:iam::886436322012:role/CodeBuildServiceRoleGatsbyJSDev`, - timeoutInMinutes: 15, -} - -codebuild.createProject(params, function(err, data) { - if (err) - console.log(err, err.stack) // an error occurred - else console.log(data) // successful response - - console.log(`WEBHOOK`) - var params = { - projectName: key /* required */, - } - codebuild.createWebhook(params, function(err, data) { - if (err) - console.log(err, err.stack) // an error occurred - else console.log(data) // successful response - }) -}) diff --git a/infrastructure/create-job.js b/infrastructure/create-job.js new file mode 100644 index 0000000000000..66cd86d245264 --- /dev/null +++ b/infrastructure/create-job.js @@ -0,0 +1,37 @@ +var aws4 = require("hyper-aws4") +var fetch = require("node-fetch") +var Hyper = require("hyper-api") + +let api = new Hyper({ + credential: { + accessKey: process.env.hyperAccessKey, + secretKey: process.env.hyperSecretKey, + }, +}) + +const config = { + Env: [ + `PATH_TO_SITE=${process.env.PATH_TO_SITE}`, + `GRAPHCOOL_TOKEN=${process.env.GRAPHCOOL_TOKEN}`, + `CODEBUILD_SOURCE_VERSION=${process.env.COMMIT}`, + `accessKeyId=${process.env.accessKeyId}`, + `secretAccessKey=${process.env.secretAccessKey}`, + ], + Image: "gatsbyjs/gatsby-dev-builds", + Labels: { + sh_hyper_instancetype: "m2", + }, +} + +api + .post("/containers/create", config) + .then(c => { + console.log(c) + setTimeout(() => { + api + .post(`/containers/${c.Id}/start`) + .then(c => console.log(c)) + .catch(e => console.log(e)) + }, 2000) + }) + .catch(e => console.log(e)) diff --git a/infrastructure/package.json b/infrastructure/package.json index e0d6bd1c4973f..c2d125970ba94 100644 --- a/infrastructure/package.json +++ b/infrastructure/package.json @@ -12,6 +12,9 @@ "async": "^2.6.0", "aws-sdk": "^2.173.0", "graphql-request": "^1.4.0", + "hyper-api": "^1.6.0", + "hyper-aws4": "^1.1.3", + "node-fetch": "^1.7.3", "s3": "^4.4.0", "shelljs": "^0.7.8" } diff --git a/infrastructure/yarn.lock b/infrastructure/yarn.lock index f29fcd85510da..fc28fabe11096 100644 --- a/infrastructure/yarn.lock +++ b/infrastructure/yarn.lock @@ -38,6 +38,10 @@ base64-js@^1.0.2: version "1.2.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.1.tgz#a91947da1f4a516ea38e5b4ec0ec3773675e0886" +bluebird@^3.4.6: + version "3.5.1" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9" + brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -115,6 +119,20 @@ graphql-request@^1.4.0: dependencies: cross-fetch "0.0.8" +hyper-api@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/hyper-api/-/hyper-api-1.6.0.tgz#2f0d2ed3c3c7009e127a28a4c991043af2efefc7" + dependencies: + bluebird "^3.4.6" + hyper-aws4 "^1.1.1" + node-fetch "^1.6.0" + +hyper-aws4@^1.1.1, hyper-aws4@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/hyper-aws4/-/hyper-aws4-1.1.3.tgz#c1882e746af5445d065a941df22ed165e332bdd7" + dependencies: + lodash "^4.13.1" + iconv-lite@~0.4.13: version "0.4.19" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" @@ -150,7 +168,7 @@ jmespath@0.15.0: version "0.15.0" resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" -lodash@^4.0.0, lodash@^4.14.0: +lodash@^4.0.0, lodash@^4.13.1, lodash@^4.14.0: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" @@ -178,7 +196,7 @@ natives@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/natives/-/natives-1.1.0.tgz#e9ff841418a6b2ec7a495e939984f78f163e6e31" -node-fetch@1.7.3: +node-fetch@1.7.3, node-fetch@^1.6.0, node-fetch@^1.7.3: version "1.7.3" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" dependencies: From e8882a46cc212ba21c1ad9b6e52f478ead355901 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 11:04:02 -0800 Subject: [PATCH 23/32] Shorten commit hash so don't exceed 63 character subdomain limit --- infrastructure/build-site.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/build-site.js b/infrastructure/build-site.js index 8cb7ac87ec895..ea0353249542e 100644 --- a/infrastructure/build-site.js +++ b/infrastructure/build-site.js @@ -5,7 +5,7 @@ const GraphQLClient = require(`graphql-request`).GraphQLClient const queue = require(`async/queue`) const s3 = require(`s3`) -const commitId = process.env.CODEBUILD_SOURCE_VERSION +const commitId = process.env.CODEBUILD_SOURCE_VERSION.slice(0, 7) const s3Client = s3.createClient({ s3Options: { From 255c391dd9a361b4fadaea4d91fb4aaef8f02a06 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 11:20:00 -0800 Subject: [PATCH 24/32] Use COMMIT instead of old CODEBUILD_SOURCE_VERSION env var --- infrastructure/build-site.js | 2 +- infrastructure/create-job.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/infrastructure/build-site.js b/infrastructure/build-site.js index ea0353249542e..e22256764f5fa 100644 --- a/infrastructure/build-site.js +++ b/infrastructure/build-site.js @@ -5,7 +5,7 @@ const GraphQLClient = require(`graphql-request`).GraphQLClient const queue = require(`async/queue`) const s3 = require(`s3`) -const commitId = process.env.CODEBUILD_SOURCE_VERSION.slice(0, 7) +const commitId = process.env.COMMIT.slice(0, 7) const s3Client = s3.createClient({ s3Options: { diff --git a/infrastructure/create-job.js b/infrastructure/create-job.js index 66cd86d245264..276361c9498e7 100644 --- a/infrastructure/create-job.js +++ b/infrastructure/create-job.js @@ -14,6 +14,7 @@ const config = { `PATH_TO_SITE=${process.env.PATH_TO_SITE}`, `GRAPHCOOL_TOKEN=${process.env.GRAPHCOOL_TOKEN}`, `CODEBUILD_SOURCE_VERSION=${process.env.COMMIT}`, + `COMMIT=${process.env.COMMIT}`, `accessKeyId=${process.env.accessKeyId}`, `secretAccessKey=${process.env.secretAccessKey}`, ], From 5df5cbf259735f9a92ec4107ee10e807889b3a54 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 11:49:10 -0800 Subject: [PATCH 25/32] Remove copying buildspec.yml from Dockerfile --- infrastructure/Dockerfile | 1 - infrastructure/create-job.js | 1 + infrastructure/functions/onGithubWebhook/handler.js | 9 ++++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/infrastructure/Dockerfile b/infrastructure/Dockerfile index ca8d373766806..64ff7b5f7b3bc 100644 --- a/infrastructure/Dockerfile +++ b/infrastructure/Dockerfile @@ -10,5 +10,4 @@ COPY package.json gatsby/infrastructure/ COPY yarn.lock gatsby/infrastructure/ RUN cd gatsby/infrastructure && yarn COPY build-site.js gatsby/infrastructure/ -COPY buildspec.yml gatsby/infrastructure/ ENTRYPOINT ["node", "gatsby/infrastructure/build-site.js"] diff --git a/infrastructure/create-job.js b/infrastructure/create-job.js index 276361c9498e7..b64e1d8095be3 100644 --- a/infrastructure/create-job.js +++ b/infrastructure/create-job.js @@ -34,5 +34,6 @@ api .then(c => console.log(c)) .catch(e => console.log(e)) }, 2000) + // TODO attach then once it's stopped, delete the job. }) .catch(e => console.log(e)) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 38d7ca2764441..7955843f9eae6 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -59,11 +59,14 @@ const createBranchIfDoesNotExist = branch => const createCommit = (commit, branchId) => { hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit)) + // TODO create build jobs for www, examples/image-processing, examples/using-glamor, examples/using-excel return client.request(` mutation { - createCommit(authorName: "${commit.author.name}",authorUsername: "${commit - .author.username}", authorEmail: "${commit.author - .email}", hash: "${commit.tree_id}", message: "${commit.message}", branchIds: ["${branchId}"]) { + createCommit(authorName: "${commit.author.name}",authorUsername: "${ + commit.author.username + }", authorEmail: "${commit.author.email}", hash: "${ + commit.tree_id + }", message: "${commit.message}", branchIds: ["${branchId}"]) { id } } From 29a76e58d16e589ac68c423a37e60f30d8e58c0d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 13:38:46 -0800 Subject: [PATCH 26/32] Test triggering site builds from lambda --- infrastructure/create-job.js | 56 ++++++++++--------- .../functions/onGithubWebhook/handler.js | 8 +++ 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/infrastructure/create-job.js b/infrastructure/create-job.js index b64e1d8095be3..e3fc37c4a9306 100644 --- a/infrastructure/create-job.js +++ b/infrastructure/create-job.js @@ -9,31 +9,33 @@ let api = new Hyper({ }, }) -const config = { - Env: [ - `PATH_TO_SITE=${process.env.PATH_TO_SITE}`, - `GRAPHCOOL_TOKEN=${process.env.GRAPHCOOL_TOKEN}`, - `CODEBUILD_SOURCE_VERSION=${process.env.COMMIT}`, - `COMMIT=${process.env.COMMIT}`, - `accessKeyId=${process.env.accessKeyId}`, - `secretAccessKey=${process.env.secretAccessKey}`, - ], - Image: "gatsbyjs/gatsby-dev-builds", - Labels: { - sh_hyper_instancetype: "m2", - }, -} +module.exports = ({ pathToSite, commit }) => { + const config = { + Env: [ + `PATH_TO_SITE=${pathToSite}`, + `GRAPHCOOL_TOKEN=${process.env.GRAPHCOOL_TOKEN}`, + `CODEBUILD_SOURCE_VERSION=${commit}`, + `COMMIT=${commit}`, + `accessKeyId=${process.env.accessKeyId}`, + `secretAccessKey=${process.env.secretAccessKey}`, + ], + Image: "gatsbyjs/gatsby-dev-builds", + Labels: { + sh_hyper_instancetype: "m2", + }, + } -api - .post("/containers/create", config) - .then(c => { - console.log(c) - setTimeout(() => { - api - .post(`/containers/${c.Id}/start`) - .then(c => console.log(c)) - .catch(e => console.log(e)) - }, 2000) - // TODO attach then once it's stopped, delete the job. - }) - .catch(e => console.log(e)) + api + .post("/containers/create", config) + .then(c => { + console.log(c) + setTimeout(() => { + api + .post(`/containers/${c.Id}/start`) + .then(c => console.log(c)) + .catch(e => console.log(e)) + }, 2000) + // TODO attach then once it's stopped, delete the job. + }) + .catch(e => console.log(e)) +} diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 7955843f9eae6..3862ada2255c7 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -6,6 +6,8 @@ const Libhoney = require("libhoney").default const flatten = require("flat") const GraphQLClient = require("graphql-request").GraphQLClient +const createBuildJob = require(`../../create-job`) + const hny = new Libhoney({ writeKey: process.env.HONEYCOMB_KEY, dataset: "gatsbyjs-os.lambda.github-webhook", @@ -59,7 +61,13 @@ const createBranchIfDoesNotExist = branch => const createCommit = (commit, branchId) => { hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit)) + // TODO create build jobs for www, examples/image-processing, examples/using-glamor, examples/using-excel + createBuildJob({ pathToSite: `www`, commit }) + createBuildJob({ pathToSite: `examples/image-processing`, commit }) + createBuildJob({ pathToSite: `examples/using-glamor`, commit }) + createBuildJob({ pathToSite: `examples/using-excel`, commit }) + return client.request(` mutation { createCommit(authorName: "${commit.author.name}",authorUsername: "${ From 881dfc2f1ec44f7f34abd0ac4d5ce4aa9044edf2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 13:43:09 -0800 Subject: [PATCH 27/32] Rename and pull module into lambda function --- .../onGithubWebhook/create-site-build.js} | 0 .../functions/onGithubWebhook/handler.js | 10 ++++---- .../functions/onGithubWebhook/package.json | 5 +++- .../functions/onGithubWebhook/yarn.lock | 24 ++++++++++++++++++- 4 files changed, 32 insertions(+), 7 deletions(-) rename infrastructure/{create-job.js => functions/onGithubWebhook/create-site-build.js} (100%) diff --git a/infrastructure/create-job.js b/infrastructure/functions/onGithubWebhook/create-site-build.js similarity index 100% rename from infrastructure/create-job.js rename to infrastructure/functions/onGithubWebhook/create-site-build.js diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 3862ada2255c7..620f20998e887 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -6,7 +6,7 @@ const Libhoney = require("libhoney").default const flatten = require("flat") const GraphQLClient = require("graphql-request").GraphQLClient -const createBuildJob = require(`../../create-job`) +const createSiteBuild = require(`./create-site-build`) const hny = new Libhoney({ writeKey: process.env.HONEYCOMB_KEY, @@ -63,10 +63,10 @@ const createCommit = (commit, branchId) => { hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit)) // TODO create build jobs for www, examples/image-processing, examples/using-glamor, examples/using-excel - createBuildJob({ pathToSite: `www`, commit }) - createBuildJob({ pathToSite: `examples/image-processing`, commit }) - createBuildJob({ pathToSite: `examples/using-glamor`, commit }) - createBuildJob({ pathToSite: `examples/using-excel`, commit }) + createSiteBuild({ pathToSite: `www`, commit }) + createSiteBuild({ pathToSite: `examples/image-processing`, commit }) + createSiteBuild({ pathToSite: `examples/using-glamor`, commit }) + createSiteBuild({ pathToSite: `examples/using-excel`, commit }) return client.request(` mutation { diff --git a/infrastructure/functions/onGithubWebhook/package.json b/infrastructure/functions/onGithubWebhook/package.json index 26a2dd04a4dde..a4d356c54cd53 100644 --- a/infrastructure/functions/onGithubWebhook/package.json +++ b/infrastructure/functions/onGithubWebhook/package.json @@ -12,6 +12,9 @@ "axios": "^0.17.0", "flat": "^4.0.0", "graphql-request": "^1.4.0", - "libhoney": "^1.0.0-beta.10" + "hyper-api": "^1.6.0", + "hyper-aws4": "^1.1.3", + "libhoney": "^1.0.0-beta.10", + "node-fetch": "^1.7.3" } } diff --git a/infrastructure/functions/onGithubWebhook/yarn.lock b/infrastructure/functions/onGithubWebhook/yarn.lock index 86241b9a11626..eb7a92328f133 100644 --- a/infrastructure/functions/onGithubWebhook/yarn.lock +++ b/infrastructure/functions/onGithubWebhook/yarn.lock @@ -13,6 +13,10 @@ axios@^0.17.0: follow-redirects "^1.2.3" is-buffer "^1.1.5" +bluebird@^3.4.6: + version "3.5.1" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9" + combined-stream@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.5.tgz#938370a57b4a51dea2c77c15d5c5fdf895164009" @@ -92,6 +96,20 @@ graphql-request@^1.4.0: dependencies: cross-fetch "0.0.8" +hyper-api@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/hyper-api/-/hyper-api-1.6.0.tgz#2f0d2ed3c3c7009e127a28a4c991043af2efefc7" + dependencies: + bluebird "^3.4.6" + hyper-aws4 "^1.1.1" + node-fetch "^1.6.0" + +hyper-aws4@^1.1.1, hyper-aws4@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/hyper-aws4/-/hyper-aws4-1.1.3.tgz#c1882e746af5445d065a941df22ed165e332bdd7" + dependencies: + lodash "^4.13.1" + iconv-lite@~0.4.13: version "0.4.19" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b" @@ -119,6 +137,10 @@ libhoney@^1.0.0-beta.10: superagent "^2.3.0" urljoin "^0.1.5" +lodash@^4.13.1: + version "4.17.4" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" + methods@^1.1.1: version "1.1.2" resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" @@ -141,7 +163,7 @@ ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" -node-fetch@1.7.3: +node-fetch@1.7.3, node-fetch@^1.6.0, node-fetch@^1.7.3: version "1.7.3" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" dependencies: From 854d8ab4a1548e2885885c0a559aebf7fe87fdb9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 13:50:39 -0800 Subject: [PATCH 28/32] format --- .../functions/onGithubWebhook/create-site-build.js | 12 ++++++------ infrastructure/functions/onGithubWebhook/handler.js | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/infrastructure/functions/onGithubWebhook/create-site-build.js b/infrastructure/functions/onGithubWebhook/create-site-build.js index e3fc37c4a9306..1f197cba42276 100644 --- a/infrastructure/functions/onGithubWebhook/create-site-build.js +++ b/infrastructure/functions/onGithubWebhook/create-site-build.js @@ -1,6 +1,6 @@ -var aws4 = require("hyper-aws4") -var fetch = require("node-fetch") -var Hyper = require("hyper-api") +var aws4 = require(`hyper-aws4`) +var fetch = require(`node-fetch`) +var Hyper = require(`hyper-api`) let api = new Hyper({ credential: { @@ -19,14 +19,14 @@ module.exports = ({ pathToSite, commit }) => { `accessKeyId=${process.env.accessKeyId}`, `secretAccessKey=${process.env.secretAccessKey}`, ], - Image: "gatsbyjs/gatsby-dev-builds", + Image: `gatsbyjs/gatsby-dev-builds`, Labels: { - sh_hyper_instancetype: "m2", + sh_hyper_instancetype: `m2`, }, } api - .post("/containers/create", config) + .post(`/containers/create`, config) .then(c => { console.log(c) setTimeout(() => { diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 620f20998e887..29602cd1980fd 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -1,22 +1,22 @@ // handler.js "use strict" -const axios = require("axios") -const Libhoney = require("libhoney").default -const flatten = require("flat") -const GraphQLClient = require("graphql-request").GraphQLClient +const axios = require(`axios`) +const Libhoney = require(`libhoney`).default +const flatten = require(`flat`) +const GraphQLClient = require(`graphql-request`).GraphQLClient const createSiteBuild = require(`./create-site-build`) const hny = new Libhoney({ writeKey: process.env.HONEYCOMB_KEY, - dataset: "gatsbyjs-os.lambda.github-webhook", + dataset: `gatsbyjs-os.lambda.github-webhook`, }) console.log(`env vars`, process.env) const client = new GraphQLClient( - "https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr", + `https://api.graph.cool/simple/v1/cj8xuo77f0a3a0164y7jketkr`, { headers: { Authorization: `Bearer ${process.env.GRAPHCOOL_TOKEN}`, From 959f89b80c302c9fb487aa8271714c344d4c42ef Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 13:57:33 -0800 Subject: [PATCH 29/32] Wait until site build jobs are created/started before quitting lambda --- .../onGithubWebhook/create-site-build.js | 12 ++++---- .../functions/onGithubWebhook/handler.js | 30 +++++++++++-------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/infrastructure/functions/onGithubWebhook/create-site-build.js b/infrastructure/functions/onGithubWebhook/create-site-build.js index 1f197cba42276..9cd6a5f272de8 100644 --- a/infrastructure/functions/onGithubWebhook/create-site-build.js +++ b/infrastructure/functions/onGithubWebhook/create-site-build.js @@ -25,16 +25,14 @@ module.exports = ({ pathToSite, commit }) => { }, } - api + return api .post(`/containers/create`, config) .then(c => { console.log(c) - setTimeout(() => { - api - .post(`/containers/${c.Id}/start`) - .then(c => console.log(c)) - .catch(e => console.log(e)) - }, 2000) + return api + .post(`/containers/${c.Id}/start`) + .then(c => console.log(c)) + .catch(e => console.log(e)) // TODO attach then once it's stopped, delete the job. }) .catch(e => console.log(e)) diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 29602cd1980fd..55529fc56ce9d 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -62,23 +62,29 @@ const createBranchIfDoesNotExist = branch => const createCommit = (commit, branchId) => { hny.sendNow(Object.assign({ createCommit: true, branchId: branchId }, commit)) - // TODO create build jobs for www, examples/image-processing, examples/using-glamor, examples/using-excel - createSiteBuild({ pathToSite: `www`, commit }) - createSiteBuild({ pathToSite: `examples/image-processing`, commit }) - createSiteBuild({ pathToSite: `examples/using-glamor`, commit }) - createSiteBuild({ pathToSite: `examples/using-excel`, commit }) - - return client.request(` + return client + .request( + ` mutation { createCommit(authorName: "${commit.author.name}",authorUsername: "${ - commit.author.username - }", authorEmail: "${commit.author.email}", hash: "${ - commit.tree_id - }", message: "${commit.message}", branchIds: ["${branchId}"]) { + commit.author.username + }", authorEmail: "${commit.author.email}", hash: "${ + commit.tree_id + }", message: "${commit.message}", branchIds: ["${branchId}"]) { id } } - `) + ` + ) + .then(() => + // TODO create build jobs for www, examples/image-processing, examples/using-glamor, examples/using-excel + Promise.all([ + createSiteBuild({ pathToSite: `www`, commit }), + createSiteBuild({ pathToSite: `examples/image-processing`, commit }), + createSiteBuild({ pathToSite: `examples/using-glamor`, commit }), + createSiteBuild({ pathToSite: `examples/using-excel`, commit }), + ]) + ) } module.exports.event = function(event, context, callback) { From 8112e1575c7ca43f052383709f0e261fcf2dc300 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 14:29:06 -0800 Subject: [PATCH 30/32] Fix hopefully trigger site builds --- .../onGithubWebhook/create-site-build.js | 5 +++-- .../functions/onGithubWebhook/handler.js | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/infrastructure/functions/onGithubWebhook/create-site-build.js b/infrastructure/functions/onGithubWebhook/create-site-build.js index 9cd6a5f272de8..b7959f4dfc598 100644 --- a/infrastructure/functions/onGithubWebhook/create-site-build.js +++ b/infrastructure/functions/onGithubWebhook/create-site-build.js @@ -16,8 +16,8 @@ module.exports = ({ pathToSite, commit }) => { `GRAPHCOOL_TOKEN=${process.env.GRAPHCOOL_TOKEN}`, `CODEBUILD_SOURCE_VERSION=${commit}`, `COMMIT=${commit}`, - `accessKeyId=${process.env.accessKeyId}`, - `secretAccessKey=${process.env.secretAccessKey}`, + `accessKeyId=${process.env.AWS_ACCESS_KEY_ID}`, + `secretAccessKey=${process.env.AWS_SECRET_ACCESS_KEY}`, ], Image: `gatsbyjs/gatsby-dev-builds`, Labels: { @@ -25,6 +25,7 @@ module.exports = ({ pathToSite, commit }) => { }, } + console.log(config) return api .post(`/containers/create`, config) .then(c => { diff --git a/infrastructure/functions/onGithubWebhook/handler.js b/infrastructure/functions/onGithubWebhook/handler.js index 55529fc56ce9d..cf4145c1e2ce6 100644 --- a/infrastructure/functions/onGithubWebhook/handler.js +++ b/infrastructure/functions/onGithubWebhook/handler.js @@ -79,10 +79,19 @@ const createCommit = (commit, branchId) => { .then(() => // TODO create build jobs for www, examples/image-processing, examples/using-glamor, examples/using-excel Promise.all([ - createSiteBuild({ pathToSite: `www`, commit }), - createSiteBuild({ pathToSite: `examples/image-processing`, commit }), - createSiteBuild({ pathToSite: `examples/using-glamor`, commit }), - createSiteBuild({ pathToSite: `examples/using-excel`, commit }), + createSiteBuild({ pathToSite: `www`, commit: commit.tree_id }), + createSiteBuild({ + pathToSite: `examples/image-processing`, + commit: commit.tree_id, + }), + createSiteBuild({ + pathToSite: `examples/using-glamor`, + commit: commit.tree_id, + }), + createSiteBuild({ + pathToSite: `examples/using-excel`, + commit: commit.tree_id, + }), ]) ) } From bc6b84d448cee6ef248bf440bc48015dd00a954d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 14:40:17 -0800 Subject: [PATCH 31/32] Use different keys --- infrastructure/functions/onGithubWebhook/create-site-build.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/functions/onGithubWebhook/create-site-build.js b/infrastructure/functions/onGithubWebhook/create-site-build.js index b7959f4dfc598..a5e28752ce213 100644 --- a/infrastructure/functions/onGithubWebhook/create-site-build.js +++ b/infrastructure/functions/onGithubWebhook/create-site-build.js @@ -16,8 +16,8 @@ module.exports = ({ pathToSite, commit }) => { `GRAPHCOOL_TOKEN=${process.env.GRAPHCOOL_TOKEN}`, `CODEBUILD_SOURCE_VERSION=${commit}`, `COMMIT=${commit}`, - `accessKeyId=${process.env.AWS_ACCESS_KEY_ID}`, - `secretAccessKey=${process.env.AWS_SECRET_ACCESS_KEY}`, + `accessKeyId=${process.env.accessKeyId}`, + `secretAccessKey=${process.env.secretAccessKey}`, ], Image: `gatsbyjs/gatsby-dev-builds`, Labels: { From 5e2482c8c40149669a9a03cea4c6b933b80fe282 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 22 Dec 2017 14:44:15 -0800 Subject: [PATCH 32/32] Fairly meaningless commit to test things --- infrastructure/functions/onGithubWebhook/create-site-build.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/functions/onGithubWebhook/create-site-build.js b/infrastructure/functions/onGithubWebhook/create-site-build.js index a5e28752ce213..3e001c6b46553 100644 --- a/infrastructure/functions/onGithubWebhook/create-site-build.js +++ b/infrastructure/functions/onGithubWebhook/create-site-build.js @@ -25,7 +25,7 @@ module.exports = ({ pathToSite, commit }) => { }, } - console.log(config) + console.log(`site build job config`, config) return api .post(`/containers/create`, config) .then(c => {