Base setup for scaling and cleanup lambdas

.ci/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:12
+
+WORKDIR /lambda
+
+COPY . /lambda
+
+RUN apt-get update \
+    && apt-get install -y zip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN yarn install \
+    && yarn run dist
+

.ci/build.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+lambaSrcDirs=("modules/runner-binaries-syncer/lambdas/runner-binaries-syncer" "modules/runners/lambdas/scale-runners" "modules/webhook/lambdas/webhook")
+repoRoot=$(dirname "${BASH_SOURCE[0]}")/..
+
+for lambdaDir in ${lambaSrcDirs[@]}; do
+    cd $repoRoot/${lambdaDir}
+    docker build -t lambda -f ../../../../.ci/Dockerfile .
+    docker create --name lambda lambda
+    zipName=$(basename "$PWD")
+    docker cp lambda:/lambda/${zipName}.zip ${zipName}.zip
+    docker rm lambda
+done

.github/workflows/lambda-scale-runners.yml

@@ -0,0 +1,26 @@
+name: Lambda Scale Runners
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - .github/workflows/lambda-scale-runners.yml
+      - "modules/runners/lambdas/scale-runners/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: node:12
+    defaults:
+      run:
+        working-directory: modules/runners/lambdas/scale-runners
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: yarn install
+      - name: Run tests
+        run: yarn test
+      - name: Build distribution
+        run: yarn build

.github/workflows/terraform.yml

@@ -19,9 +19,10 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v2
       - name: "Fake zip files" # Validate will fail if it cannot find the zip files
-        run:
-          touch modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip
+        run: |
           touch modules/webhook/lambdas/webhook/webhook.zip
+          touch modules/runners/lambdas/scale-runners/scale-runners.zip
+          touch modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip
       - name: "Terraform Format"
         uses: hashicorp/terraform-github-actions@master
         with:

.gitignore

@@ -16,3 +16,4 @@ example/*.secrets*.tfvars
 *.zip
 *.gz
 *.tgz
+*.env

examples/default/main.tf

@@ -8,20 +8,28 @@ resource "random_password" "random" {
   length = 32
 }
 
-
 module "runners" {
   source = "../../"
 
   aws_region = local.aws_region
   vpc_id     = module.vpc.vpc_id
+  subnet_ids = module.vpc.private_subnets
 
   environment = local.environment
   tags = {
     Project = "ProjectX"
   }
 
-  github_app_webhook_secret = random_password.random.result
+  github_app = {
+    key_base64     = var.github_app_key_base64
+    id             = var.github_app_id
+    client_id      = var.github_app_client_id
+    client_secret  = var.github_app_client_secret
+    webhook_secret = random_password.random.result
+  }
 
+  enable_organization_runners = false
+  runner_extra_labels         = "default,example"
 }
 
 

examples/default/outputs.tf

@@ -1,17 +1,12 @@
-output "action_runners" {
+output "runners" {
   value = {
     runners = module.runners.runners
   }
 }
 
-
-output "lambda_binaries_syncer_name" {
-  value = module.runners.binaries_syncer.lambda.id
-}
-
-
-output "github_app_webhook_secret" {
-  value = random_password.random.result
+output "webhook" {
+  value = {
+    secret  = random_password.random.result
+    gateway = module.runners.webhook.gateway
+  }
 }
-
-

examples/default/providers.tf

@@ -1,5 +1,5 @@
 provider "aws" {
   region  = local.aws_region
-  version = "2.59"
+  version = "2.61"
 }
 

examples/default/variables.tf

@@ -0,0 +1,9 @@
+
+variable "github_app_key_base64" {}
+
+variable "github_app_id" {}
+
+variable "github_app_client_id" {}
+
+variable "github_app_client_secret" {}
+

main.tf

@@ -15,6 +15,7 @@ resource "random_string" "random" {
 resource "aws_sqs_queue" "queued_builds" {
   name                        = "${var.environment}-queued-builds.fifo"
   delay_seconds               = 30
+  visibility_timeout_seconds  = 60
   fifo_queue                  = true
   receive_wait_time_seconds   = 10
   content_based_deduplication = true
@@ -30,19 +31,27 @@ module "webhook" {
   tags        = local.tags
 
   sqs_build_queue           = aws_sqs_queue.queued_builds
-  github_app_webhook_secret = var.github_app_webhook_secret
+  github_app_webhook_secret = var.github_app.webhook_secret
 }
 
 module "runners" {
   source = "./modules/runners"
 
   aws_region  = var.aws_region
   vpc_id      = var.vpc_id
+  subnet_ids  = var.subnet_ids
   environment = var.environment
   tags        = local.tags
 
   s3_bucket_runner_binaries   = module.runner_binaries.bucket
   s3_location_runner_binaries = local.s3_action_runner_url
+
+  sqs                             = aws_sqs_queue.queued_builds
+  github_app                      = var.github_app
+  enable_organization_runners     = var.enable_organization_runners
+  scale_down_schedule_expression  = var.scale_down_schedule_expression
+  minimum_running_time_in_minutes = var.minimum_running_time_in_minutes
+  runner_extra_labels             = var.runner_extra_labels
 }
 
 module "runner_binaries" {

modules/runner-binaries-syncer/runner-binaries-syncer.tf

@@ -73,7 +73,7 @@ resource "aws_cloudwatch_event_target" "syncer" {
   arn  = aws_lambda_function.syncer.arn
 }
 
-resource "aws_lambda_permission" "allow_cloudwatch_to_call_check_foo" {
+resource "aws_lambda_permission" "syncer" {
   statement_id  = "AllowExecutionFromCloudWatch"
   action        = "lambda:InvokeFunction"
   function_name = aws_lambda_function.syncer.function_name

modules/runners/lambdas/.gitignore

@@ -0,0 +1,15 @@
+# dependencies
+node_modules/
+
+# production
+dist/
+build/
+
+# misc
+.DS_Store
+.env*
+*.zip
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

modules/runners/lambdas/scale-runners/.nvmrc

@@ -0,0 +1 @@
+v12.16.1

modules/runners/lambdas/scale-runners/.prettierrc

@@ -0,0 +1,5 @@
+{
+  "printWidth": 120,
+  "singleQuote": true,
+  "trailingComma": "all"
+}

modules/runners/lambdas/scale-runners/jest.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+};

modules/runners/lambdas/scale-runners/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "github-runner-lambda-scale-runners",
+  "version": "1.0.0",
+  "main": "lambda.ts",
+  "license": "MIT",
+  "scripts": {
+    "start": "ts-node-dev src/local.ts",
+    "test": "NODE_ENV=test jest",
+    "watch": "ts-node-dev --respawn --exit-child src/local.ts",
+    "build": "ncc build src/lambda.ts -o dist",
+    "dist": "yarn build && cd dist && zip ../scale-runners.zip index.js"
+  },
+  "devDependencies": {
+    "@types/aws-lambda": "^8.10.51",
+    "@types/express": "^4.17.3",
+    "@types/jest": "^25.2.1",
+    "@types/node": "^13.13.4",
+    "@zeit/ncc": "^0.22.1",
+    "aws-sdk": "^2.671.0",
+    "jest": "^25.4.0",
+    "ts-jest": "^25.4.0",
+    "ts-node-dev": "^1.0.0-pre.44",
+    "typescript": "^3.8.3"
+  },
+  "dependencies": {
+    "@octokit/auth-app": "^2.4.5",
+    "@octokit/rest": "^17.6.0",
+    "moment": "^2.25.3",
+    "yn": "^4.0.0"
+  }
+}

modules/runners/lambdas/scale-runners/src/lambda.ts

@@ -0,0 +1,26 @@
+import { scaleUp } from './scale-runners/scale-up';
+import { scaleDown } from './scale-runners/scale-down';
+import { SQSEvent } from 'aws-lambda';
+
+module.exports.scaleUp = async (event: SQSEvent, context: any, callback: any) => {
+  console.log(event);
+  try {
+    for (const e of event.Records) {
+      await scaleUp(e.eventSource, JSON.parse(e.body));
+    }
+    return callback(null);
+  } catch (e) {
+    console.error(e);
+    return callback('Failed handling SQS event');
+  }
+};
+
+module.exports.scaleDown = async (event: any, context: any, callback: any) => {
+  try {
+    scaleDown();
+    return callback(null);
+  } catch (e) {
+    console.error(e);
+    return callback('Failed');
+  }
+};