From 5e17d6f0a083a110411c4415df4b25d79291c3d8 Mon Sep 17 00:00:00 2001 From: Niek Palm Date: Fri, 21 Oct 2022 09:17:03 +0200 Subject: [PATCH] feat: Add multi-runner capability (#2472) * feat: Remove support check_run (#2521) * chore: Remove support check_run * format, lint * feat: Remove old scale down mechanism (< 0.19.0) (#2519) fix: Remove old cleanup mechanism (< 0.19.0) * feat: added changes for multi runner. * fix: region. * fix: more fixes. * tuple to list. * fixes. * fixes. * fixes. * fixes. * fixes. * fixes. * fix: formatting. * fix: formatting. * fix: formatting. * fix: moved some blocks outside runner config. * fix: few more updates * fix: liniting. * fix: updated example output * changed runner group name. * fix: updated the tests. * fix: addressed review comments. * fix: linting issues. * fix: formatting. * fix: updated tf version. * fix: Remove removed prerelease option * Add ubuntu runner to example * refactor: use each instead of count * fix: few small issues. * refactor: syncer to count for multi runner * fix: comments. * fix: added Readme. * fix: errors. * move variable to runner config * fix: updated the readme. * Add todos * feat: added windows runner configuration, completed todos and added the weight for runner config matchers. * chore: Update docs * fix: reverted tf versions. * fix: addressed comments. * fix: missed. * fix: formatting. * Update terraform versions in CI * Update terraform versions in CI * Update docs * fix: coverage. * Update docs * improve test coverage webhook * Apply suggestions from code review * fix: formatting. * fix: fixed merge issues. * fix: syntax. Co-authored-by: Niek Palm Co-authored-by: Niek Palm Co-authored-by: navdeepg2021 --- .github/workflows/terraform.yml | 35 +- README.md | 27 +- examples/ephemeral/main.tf | 3 - examples/multi-runner/.terraform.lock.hcl | 60 +++ examples/multi-runner/README.md | 48 ++ .../multi-runner/lambdas-download/main.tf | 25 + examples/multi-runner/main.tf | 165 ++++++ examples/multi-runner/outputs.tf | 8 + examples/multi-runner/providers.tf | 3 + examples/multi-runner/templates/user-data.sh | 84 +++ examples/multi-runner/variables.tf | 9 + examples/multi-runner/versions.tf | 15 + examples/multi-runner/vpc.tf | 21 + main.tf | 32 +- modules/download-lambda/README.md | 18 - modules/multi-runner/README.md | 81 +++ modules/multi-runner/main.tf | 23 + modules/multi-runner/outputs.tf | 36 ++ modules/multi-runner/queues.tf | 72 +++ modules/multi-runner/runner-binaries.tf | 36 ++ modules/multi-runner/runners.tf | 94 ++++ modules/multi-runner/ssm.tf | 8 + modules/multi-runner/variables.tf | 491 ++++++++++++++++++ modules/multi-runner/webhook.tf | 27 + modules/runner-binaries-syncer/README.md | 20 +- .../lambdas/runner-binaries-syncer/yarn.lock | 85 ++- modules/runner-binaries-syncer/variables.tf | 5 - modules/runners/README.md | 20 +- modules/runners/lambdas/runners/yarn.lock | 8 +- modules/webhook/README.md | 20 +- .../webhook/lambdas/webhook/jest.config.js | 10 +- .../lambdas/webhook/src/sqs/index.test.ts | 59 ++- .../webhook/lambdas/webhook/src/sqs/index.ts | 21 +- .../lambdas/webhook/src/ssm/index.test.ts | 42 ++ .../webhook/src/webhook/handler.test.ts | 228 ++++++-- .../lambdas/webhook/src/webhook/handler.ts | 83 ++- .../multi_runner_configurations.json | 30 ++ .../policies/lambda-publish-sqs-policy.json | 2 +- modules/webhook/variables.tf | 46 +- modules/webhook/webhook.tf | 20 +- policies/lambda-publish-sqs-policy.json | 2 +- variables.tf | 6 - 42 files changed, 1842 insertions(+), 286 deletions(-) create mode 100644 examples/multi-runner/.terraform.lock.hcl create mode 100644 examples/multi-runner/README.md create mode 100644 examples/multi-runner/lambdas-download/main.tf create mode 100644 examples/multi-runner/main.tf create mode 100644 examples/multi-runner/outputs.tf create mode 100644 examples/multi-runner/providers.tf create mode 100644 examples/multi-runner/templates/user-data.sh create mode 100644 examples/multi-runner/variables.tf create mode 100644 examples/multi-runner/versions.tf create mode 100644 examples/multi-runner/vpc.tf create mode 100644 modules/multi-runner/README.md create mode 100644 modules/multi-runner/main.tf create mode 100644 modules/multi-runner/outputs.tf create mode 100644 modules/multi-runner/queues.tf create mode 100644 modules/multi-runner/runner-binaries.tf create mode 100644 modules/multi-runner/runners.tf create mode 100644 modules/multi-runner/ssm.tf create mode 100644 modules/multi-runner/variables.tf create mode 100644 modules/multi-runner/webhook.tf create mode 100644 modules/webhook/lambdas/webhook/test/resources/multi_runner_configurations.json diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index fe428a9f1b..70d005c291 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -15,7 +15,7 @@ jobs: name: Verify module strategy: matrix: - terraform: [1.1.3, "latest"] + terraform: [1.3.2, "latest"] runs-on: ubuntu-latest container: image: hashicorp/terraform:${{ matrix.terraform }} @@ -29,7 +29,7 @@ jobs: touch modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip - name: terraform init run: terraform init -get -backend=false -input=false - - if: contains(matrix.terraform, '1.1.') + - if: contains(matrix.terraform, '1.3.') name: check terraform formatting run: terraform fmt -recursive -check=true -write=false - if: contains(matrix.terraform, 'latest') # check formatting for the latest release but avoid failing the build @@ -44,7 +44,7 @@ jobs: strategy: fail-fast: false matrix: - terraform: [1.0.11, 1.1.3, "latest"] + terraform: [1.0.11, 1.1.9, 1.2.9, "latest"] example: ["default", "ubuntu", "prebuilt", "arm64", "ephemeral", "windows"] defaults: @@ -57,7 +57,7 @@ jobs: - uses: actions/checkout@v3 - name: terraform init run: terraform init -get -backend=false -input=false - - if: contains(matrix.terraform, '1.1.') + - if: contains(matrix.terraform, '1.3.') name: check terraform formatting run: terraform fmt -recursive -check=true -write=false - if: contains(matrix.terraform, 'latest') # check formatting for the latest release but avoid failing the build @@ -66,3 +66,30 @@ jobs: continue-on-error: true - name: validate terraform011 run: terraform validate + + + verify_multi_runner_example: + name: Verify Multi-Runner examples + strategy: + fail-fast: false + matrix: + terraform: [1.3.2, "latest"] + defaults: + run: + working-directory: examples/multi-runner + runs-on: ubuntu-latest + container: + image: hashicorp/terraform:${{ matrix.terraform }} + steps: + - uses: actions/checkout@v3 + - name: terraform init + run: terraform init -get -backend=false -input=false + - if: contains(matrix.terraform, '1.3.') + name: check terraform formatting + run: terraform fmt -recursive -check=true -write=false + - if: contains(matrix.terraform, 'latest') # check formatting for the latest release but avoid failing the build + name: check terraform formatting + run: terraform fmt -recursive -check=true -write=false + continue-on-error: true + - name: validate terraform + run: terraform validate diff --git a/README.md b/README.md index c31d943e28..144147a2a3 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ This [Terraform](https://www.terraform.io/) module creates the required infrastr - [Motivation](#motivation) - [Overview](#overview) - [Major configuration options.](#major-configuration-options) - - [ARM64 support via Graviton/Graviton2 instance-types](#arm64-support-via-gravitongraviton2-instance-types) - [Usages](#usages) - [Setup GitHub App (part 1)](#setup-github-app-part-1) - [Setup terraform module](#setup-terraform-module) @@ -25,7 +24,6 @@ This [Terraform](https://www.terraform.io/) module creates the required infrastr - [Experimental - Optional queue to publish GitHub workflow job events](#experimental---optional-queue-to-publish-github-workflow-job-events) - [Examples](#examples) - [Sub modules](#sub-modules) - - [ARM64 configuration for submodules](#arm64-configuration-for-submodules) - [Debugging](#debugging) - [Security Consideration](#security-consideration) - [Requirements](#requirements) @@ -81,16 +79,13 @@ Besides these permissions, the lambdas also need permission to CloudWatch (for l To be able to support a number of use-cases the module has quite a lot of configuration options. We try to choose reasonable defaults. The several examples also show for the main cases how to configure the runners. - Org vs Repo level. You can configure the module to connect the runners in GitHub on an org level and share the runners in your org. Or set the runners on repo level and the module will install the runner to the repo. There can be multiple repos but runners are not shared between repos. -- Checkrun vs Workflow job event. You can configure the webhook in GitHub to send checkrun or workflow job events to the webhook. Workflow job events are introduced by GitHub in September 2021 and are designed to support scalable runners. We advise when possible using the workflow job event, you can set `runner_enable_workflow_job_labels_check = true` to let the webhook only accept jobs based on the labels configured. The webhook will check the custom labels provided via the variable `runner_extra_labels` and the GitHub managed labels, "self-hosted", OS and architecture. The OS and architecture are derived from the settings. By default the check is disabled. +- Multi-Runner module. This modules allows to create multiple runner configurations with a single webhook and single GitHub App to simply deployment of different types of runners. Refer to the [ReadMe](.modules/../modules/multi-runner/README.md) for more information to understand the functionality. +- Workflow job event. You can configure the webhook in GitHub to send workflow job events to the webhook. Workflow job events are introduced by GitHub in September 2021 and are designed to support scalable runners. We advise when possible using the workflow job event. - Linux vs Windows. you can configure the OS types linux and win. Linux will be used by default. - Re-use vs Ephemeral. By default runners are re-used for till detected idle. Once idle they will be removed from the pool. To improve security we are introducing ephemeral runners. Those runners are only used for one job. Ephemeral runners are only working in combination with the workflow job event. We also suggest using a pre-build AMI to improve the start time of jobs. - GitHub Cloud vs GitHub Enterprise Server (GHES). The runner support GitHub Cloud as well GitHub Enterprise Server. For GHES we rely on our community to test and support. We have no possibility to test ourselves on GHES. - Spot vs on-demand. The runners use either the EC2 spot or on-demand life cycle. Runners will be created via the AWS [CreateFleet API](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html). The module (scale up lambda) will request via the CreateFleet API to create instances in one of the subnets and of the specified instance types. - - -#### ARM64 support via Graviton/Graviton2 instance-types - -When using the default example or top-level module, specifying `instance_types` that match a Graviton/Graviton 2 (ARM64) architecture (e.g. a1, t4g or any 6th-gen `g` or `gd` type), you must also specify `runner_architecture = "arm64"` and the sub-modules will be automatically configured to provision with ARM64 AMIs and leverage GitHub's ARM64 action runner. See below for more details. +- ARM64 support via Graviton/Graviton2 instance-types. When using the default example or top-level module, specifying `instance_types` that match a Graviton/Graviton 2 (ARM64) architecture (e.g. a1, t4g or any 6th-gen `g` or `gd` type), you must also specify `runner_architecture = "arm64"` and the sub-modules will be automatically configured to provision with ARM64 AMIs and leverage GitHub's ARM64 action runner. See below for more details. ## Usages @@ -334,11 +329,13 @@ Examples are located in the [examples](./examples) directory. The following exam - _[Default](examples/default/README.md)_: The default example of the module - _[ARM64](examples/arm64/README.md)_: Example usage with ARM64 architecture -- _[Ubuntu](examples/ubuntu/README.md)_: Example usage of creating a runner using Ubuntu AMIs. -- _[Windows](examples/windows/README.md)_: Example usage of creating a runner using Windows as the OS. - _[Ephemeral](examples/ephemeral/README.md)_: Example usages of ephemeral runners based on the default example. -- _[Prebuilt Images](examples/prebuilt/README.md)_: Example usages of deploying runners with a custom prebuilt image. +- _[Multi Runner](examples/multi-runner/README.md)_ : Example usage of creating a multi runner which creates multiple runners/ configurations with a single deployment - _[Permissions boundary](examples/permissions-boundary/README.md)_: Example usages of permissions boundaries. +- _[Prebuilt Images](examples/prebuilt/README.md)_: Example usages of deploying runners with a custom prebuilt image. +- _[Ubuntu](examples/ubuntu/README.md)_: Example usage of creating a runner using Ubuntu AMIs. +- _[Windows](examples/windows/README.md)_: Example usage of creating a runner using Windows as the OS. + ## Sub modules @@ -349,15 +346,14 @@ The following submodules are the core of the module and are mandatory: - _[runner-binaries-syncer](./modules/runner-binaries-syncer/README.md)_ - Syncs the action runner distribution. - _[runners](./modules/runners/README.md)_ - Scales the action runners up and down - _[webhook](./modules/webhook/README.md)_ - Handles GitHub webhooks +- _[multi-runner](./modules/multi-runner/README.md) - Creates multiple runner configurations in a single deployment The following sub modules are optional and are provided as example or utility: - _[download-lambda](./modules/download-lambda/README.md)_ - Utility module to download lambda artifacts from GitHub Release - _[setup-iam-permissions](./modules/setup-iam-permissions/README.md)_ - Example module to setup permission boundaries -### ARM64 configuration for submodules - -When using the top level module configure `runner_architecture = "arm64"` and ensure the list of `instance_types` matches. When not using the top-level, ensure these properties are set on the submodules. +ARM64 configuration for submodules. When using the top level module configure `runner_architecture = "arm64"` and ensure the list of `instance_types` matches. When not using the top-level, ensure these properties are set on the submodules. ## Debugging @@ -484,8 +480,7 @@ We welcome any improvement to the standard module to make the default as secure | [runner\_boot\_time\_in\_minutes](#input\_runner\_boot\_time\_in\_minutes) | The minimum time for an EC2 runner to boot and register as a runner. | `number` | `5` | no | | [runner\_ec2\_tags](#input\_runner\_ec2\_tags) | Map of tags that will be added to the launch template instance tag specifications. | `map(string)` | `{}` | no | | [runner\_egress\_rules](#input\_runner\_egress\_rules) | List of egress rules for the GitHub runner instances. | 
list(object({
    cidr_blocks      = list(string)
    ipv6_cidr_blocks = list(string)
    prefix_list_ids  = list(string)
    from_port        = number
    protocol         = string
    security_groups  = list(string)
    self             = bool
    to_port          = number
    description      = string
  }))
| 
[
  {
    "cidr_blocks": [
      "0.0.0.0/0"
    ],
    "description": null,
    "from_port": 0,
    "ipv6_cidr_blocks": [
      "::/0"
    ],
    "prefix_list_ids": null,
    "protocol": "-1",
    "security_groups": null,
    "self": null,
    "to_port": 0
  }
]
| no | -| [runner\_enable\_workflow\_job\_labels\_check](#input\_runner\_enable\_workflow\_job\_labels\_check) | If set to true all labels in the workflow job even are matched against the custom labels and GitHub labels (os, architecture and `self-hosted`). When the labels are not matching the event is dropped at the webhook. | `bool` | `false` | no | -| [runner\_enable\_workflow\_job\_labels\_check\_all](#input\_runner\_enable\_workflow\_job\_labels\_check\_all) | If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ label matches it will trigger the webhook. `runner_enable_workflow_job_labels_check` must be true for this to take effect. | `bool` | `true` | no | +| [runner\_enable\_workflow\_job\_labels\_check\_all](#input\_runner\_enable\_workflow\_job\_labels\_check\_all) | If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ label matches it will trigger the webhook. | `bool` | `true` | no | | [runner\_extra\_labels](#input\_runner\_extra\_labels) | Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `enable_workflow_job_labels_check`. GitHub read-only labels should not be provided. | `string` | `""` | no | | [runner\_group\_name](#input\_runner\_group\_name) | Name of the runner group. | `string` | `"Default"` | no | | [runner\_iam\_role\_managed\_policy\_arns](#input\_runner\_iam\_role\_managed\_policy\_arns) | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no | diff --git a/examples/ephemeral/main.tf b/examples/ephemeral/main.tf index 5bd0e97c82..5e9c52c615 100644 --- a/examples/ephemeral/main.tf +++ b/examples/ephemeral/main.tf @@ -43,9 +43,6 @@ module "runners" { enable_organization_runners = true runner_extra_labels = "default,example" - # enable workflow labels check - # runner_enable_workflow_job_labels_check = true - # enable access to the runners via SSM enable_ssm_on_runners = true diff --git a/examples/multi-runner/.terraform.lock.hcl b/examples/multi-runner/.terraform.lock.hcl new file mode 100644 index 0000000000..7561c244d0 --- /dev/null +++ b/examples/multi-runner/.terraform.lock.hcl @@ -0,0 +1,60 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.34.0" + constraints = ">= 3.63.0, ~> 4.0" + hashes = [ + "h1:SDqaa/BVMQMzQ1bWQfrcsC4jfaywFeUq03jsojDNnyY=", + "zh:2bdc9b908008c1e874d8ba7e2cfabd856cafb63c52fef51a1fdeef2f5584bffd", + "zh:43c5364e3161be3856e56494cbb8b21d513fc05875f1b40e66e583602154dd0a", + "zh:44e763adae92489f223f65866c1f8b5342e7e85b95daa8d1f483a2afb47f7db3", + "zh:62bfabb3a1a31814cb3fadc356539d8253b95abacfffd90984affb54c9a53a86", + "zh:6495ce67897d2d5648d466c09e8588e837c2878322988738a95c06926044b05d", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b1546b2ac61d7cc27a8eba160ae1b6ac581d2c4af824a400d6511e4998da398a", + "zh:c8c362c5527f0533bde581e41cdb2bdf42aea557762f326dbfa122fdf001fb10", + "zh:c8cc28fb41f73ca09f590bace2204ea325f6116be0bbce6abfebd393d028f12c", + "zh:db0601c9bd12ca028d60ac87e85320285ebc64857715f7908dd6a283e5f44d45", + "zh:e64d2193236d05348ba2e8b99650d9274e5af80be39b3ee28bbe442b0684d8a3", + "zh:ff6228f3751e1f0ee7dc086d09e32d39ca6556f0b5267f36aae67881d29ace94", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.2.3" + hashes = [ + "h1:FvRIEgCmAezgZUqb2F+PZ9WnSSnR5zbEM2ZI+GLmbMk=", + "zh:04f0978bb3e052707b8e82e46780c371ac1c66b689b4a23bbc2f58865ab7d5c0", + "zh:6484f1b3e9e3771eb7cc8e8bab8b35f939a55d550b3f4fb2ab141a24269ee6aa", + "zh:78a56d59a013cb0f7eb1c92815d6eb5cf07f8b5f0ae20b96d049e73db915b238", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8aa9950f4c4db37239bcb62e19910c49e47043f6c8587e5b0396619923657797", + "zh:996beea85f9084a725ff0e6473a4594deb5266727c5f56e9c1c7c62ded6addbb", + "zh:9a7ef7a21f48fabfd145b2e2a4240ca57517ad155017e86a30860d7c0c109de3", + "zh:a63e70ac052aa25120113bcddd50c1f3cfe61f681a93a50cea5595a4b2cc3e1c", + "zh:a6e8d46f94108e049ad85dbed60354236dc0b9b5ec8eabe01c4580280a43d3b8", + "zh:bb112ce7efbfcfa0e65ed97fa245ef348e0fd5bfa5a7e4ab2091a9bd469f0a9e", + "zh:d7bec0da5c094c6955efed100f3fe22fca8866859f87c025be1760feb174d6d9", + "zh:fb9f271b72094d07cef8154cd3d50e9aa818a0ea39130bc193132ad7b23076fd", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.4.3" + hashes = [ + "h1:saZR+mhthL0OZl4SyHXZraxyaBNVMxiZzks78nWcZ2o=", + "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", + "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", + "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3", + "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5", + "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda", + "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6", + "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1", + "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d", + "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8", + "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", + ] +} diff --git a/examples/multi-runner/README.md b/examples/multi-runner/README.md new file mode 100644 index 0000000000..f0811c2510 --- /dev/null +++ b/examples/multi-runner/README.md @@ -0,0 +1,48 @@ +# Action runners deployment of Multiple-Runner-Configurations-Together example + +This module shows how to create GitHub action runners with multiple runner configuration together in one deployment. This example has the configurations for the following runner types with the relevant labels supported by them as matchers: + +- Linux ARM64 `["self-hosted", "linux", "arm64", "amazon"]` +- Linux Ubuntu `["self-hosted", "linux", "x64", "ubuntu"]` +- Linux X64 `["self-hosted", "linux", "x64", "amazon"]` +- Windows X64 `["self-hosted", "windows", "x64", "servercore-2022"]` + +The module will decide the runner for the workflow job based on the match in the labels defined in the workflow job and runner configuration. Also the runner configuration allows the match to be exact or non-exact match. We recommend to use only exact matches. + +For exact match, all the labels defined in the workflow should be present in the runner configuration matchers and for non-exact match, some of the labels in the workflow, when present in runner configuration, shall be enough for the runner configuration to be used for the job. First the exact matchers are applied, next the non exact ones. + +## Webhook + +For the list of provided runner configurations, there will be a single webhook and only a single Github App to receive the notifications for all types of workflow triggers. + +## Lambda distribution + +Per combination of OS and architecture a lambda distribution syncer will be created. For this example there will be three instances (windows X64, linux X64, linux ARM). + +## Usages + +Steps for the full setup, such as creating a GitHub app can be found in the root module's [README](../../README.md). First download the Lambda releases from GitHub. Alternatively you can build the lambdas locally with Node or Docker, there is a simple build script in `/.ci/build.sh`. In the `main.tf` you can simply remove the location of the lambda zip files, the default location will work in this case. + +> Ensure you have set the version in `lambdas-download/main.tf` for running the example. The version needs to be set to a GitHub release version, see https://github.com/philips-labs/terraform-aws-github-runner/releases + +```bash +cd lambdas-download +terraform init +terraform apply +cd .. +``` + +Before running Terraform, ensure the GitHub app is configured. See the [configuration details](../../README.md#usages) for more details. + +```bash +terraform init +terraform apply +``` + +You can receive the webhook details by running: + +```bash +terraform output -raw webhook_secret +``` + +Be-aware some shells will print some end of line character `%`. diff --git a/examples/multi-runner/lambdas-download/main.tf b/examples/multi-runner/lambdas-download/main.tf new file mode 100644 index 0000000000..87f31bd8a9 --- /dev/null +++ b/examples/multi-runner/lambdas-download/main.tf @@ -0,0 +1,25 @@ +locals { + version = "" +} + +module "lambdas" { + source = "../../../modules/download-lambda" + lambdas = [ + { + name = "webhook" + tag = local.version + }, + { + name = "runners" + tag = local.version + }, + { + name = "runner-binaries-syncer" + tag = local.version + } + ] +} + +output "files" { + value = module.lambdas.files +} diff --git a/examples/multi-runner/main.tf b/examples/multi-runner/main.tf new file mode 100644 index 0000000000..461ee3696d --- /dev/null +++ b/examples/multi-runner/main.tf @@ -0,0 +1,165 @@ +locals { + environment = var.environment != null ? var.environment : "multi-runner" + aws_region = "eu-west-1" +} + +resource "random_id" "random" { + byte_length = 20 +} +data "aws_caller_identity" "current" {} + + +################################################################################ +### Hybrid account +################################################################################ + +module "multi-runner" { + source = "../../modules/multi-runner" + multi_runner_config = { + "linux-arm64" = { + matcherConfig : { + labelMatchers = ["self-hosted", "linux", "arm64", "amazon"] + exactMatch = true + } + fifo = true + delay_webhook_event = 0 + redrive_build_queue = { + enabled = false + maxReceiveCount = null + } + runner_config = { + runner_os = "linux" + runner_architecture = "arm64" + runner_extra_labels = "arm" + enable_ssm_on_runners = true + instance_types = ["t4g.large", "c6g.large"] + runners_maximum_count = 1 + scale_down_schedule_expression = "cron(* * * * ? *)" + } + }, + "linux-ubuntu" = { + matcherConfig : { + labelMatchers = ["self-hosted", "linux", "x64", "ubuntu"] + exactMatch = true + } + fifo = true + delay_webhook_event = 0 + redrive_build_queue = { + enabled = false + maxReceiveCount = null + } + runner_config = { + runner_os = "linux" + runner_architecture = "x64" + runner_extra_labels = "ubuntu" + enable_ssm_on_runners = true + instance_types = ["m5ad.large", "m5a.large"] + runners_maximum_count = 1 + scale_down_schedule_expression = "cron(* * * * ? *)" + runner_run_as = "ubuntu" + userdata_template = "./templates/user-data.sh" + ami_owners = ["099720109477"] # Canonical's Amazon account ID + + ami_filter = { + name = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + block_device_mappings = [{ + # Set the block device name for Ubuntu root device + device_name = "/dev/sda1" + delete_on_termination = true + volume_type = "gp3" + volume_size = 30 + encrypted = true + iops = null + throughput = null + kms_key_id = null + snapshot_id = null + }] + runner_log_files = [ + { + log_group_name = "syslog" + prefix_log_group = true + file_path = "/var/log/syslog" + log_stream_name = "{instance_id}" + }, + { + log_group_name = "user_data" + prefix_log_group = true + file_path = "/var/log/user-data.log" + log_stream_name = "{instance_id}/user_data" + }, + { + log_group_name = "runner" + prefix_log_group = true + file_path = "/opt/actions-runner/_diag/Runner_**.log", + log_stream_name = "{instance_id}/runner" + } + ] + } + }, + "windows-x64" = { + matcherConfig : { + labelMatchers = ["self-hosted", "windows", "x64", "servercore-2022"] + exactMatch = true + } + fifo = true + delay_webhook_event = 5 + runner_config = { + runner_os = "windows" + runner_architecture = "x64" + enable_ssm_on_runners = true + instance_types = ["m5.large", "c5.large"] + runner_extra_labels = "servercore-2022" + runners_maximum_count = 1 + scale_down_schedule_expression = "cron(* * * * ? *)" + runner_boot_time_in_minutes = 20 + ami_filter = { + name = ["Windows_Server-2022-English-Core-ContainersLatest-*"] + } + } + }, + "linux-x64" = { + matcherConfig : { + labelMatchers = ["self-hosted", "linux", "x64", "amazon"] + exactMatch = false + } + fifo = true + delay_webhook_event = 0 + runner_config = { + runner_os = "linux" + runner_architecture = "x64" + create_service_linked_role_spot = true + enable_ssm_on_runners = true + instance_types = ["m5ad.large", "m5a.large"] + runner_extra_labels = "amazon" + runners_maximum_count = 1 + enable_ephemeral_runners = true + scale_down_schedule_expression = "cron(* * * * ? *)" + } + } + } + aws_region = local.aws_region + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + runners_scale_up_lambda_timeout = 60 + runners_scale_down_lambda_timeout = 60 + prefix = local.environment + tags = { + Project = "ProjectX" + } + github_app = { + key_base64 = var.github_app_key_base64 + id = var.github_app_id + webhook_secret = random_id.random.hex + } + + # Assuming local build lambda's to use pre build ones, uncomment the lines below and download the + # lambda zip files lambda_download + # webhook_lambda_zip = "lambdas-download/webhook.zip" + # runner_binaries_syncer_lambda_zip = "lambdas-download/runner-binaries-syncer.zip" + # runners_lambda_zip = "lambdas-download/runners.zip" + + # override delay of events in seconds + + # log_level = "debug" +} diff --git a/examples/multi-runner/outputs.tf b/examples/multi-runner/outputs.tf new file mode 100644 index 0000000000..8ecbfc1001 --- /dev/null +++ b/examples/multi-runner/outputs.tf @@ -0,0 +1,8 @@ +output "webhook_endpoint" { + value = module.multi-runner.webhook.endpoint +} + +output "webhook_secret" { + sensitive = true + value = random_id.random.hex +} diff --git a/examples/multi-runner/providers.tf b/examples/multi-runner/providers.tf new file mode 100644 index 0000000000..b6c81d5415 --- /dev/null +++ b/examples/multi-runner/providers.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = local.aws_region +} diff --git a/examples/multi-runner/templates/user-data.sh b/examples/multi-runner/templates/user-data.sh new file mode 100644 index 0000000000..752a0de0e3 --- /dev/null +++ b/examples/multi-runner/templates/user-data.sh @@ -0,0 +1,84 @@ +#!/bin/bash +exec > >(tee /var/log/user-data.log | logger -t user-data -s 2>/dev/console) 2>&1 + + +# AWS suggest to create a log for debug purpose based on https://aws.amazon.com/premiumsupport/knowledge-center/ec2-linux-log-user-data/ +# As side effect all command, set +x disable debugging explicitly. +# +# An alternative for masking tokens could be: exec > >(sed 's/--token\ [^ ]* /--token\ *** /g' > /var/log/user-data.log) 2>&1 +set +x + +%{ if enable_debug_logging } +set -x +%{ endif } + +${pre_install} + +# Install AWS CLI +apt-get update +DEBIAN_FRONTEND=noninteractive apt-get install -y \ + awscli \ + build-essential \ + curl \ + git \ + iptables \ + jq \ + uidmap \ + unzip \ + wget + +user_name=ubuntu +user_id=$(id -ru $user_name) + +# install and configure cloudwatch logging agent +wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb +dpkg -i -E ./amazon-cloudwatch-agent.deb +amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c ssm:${ssm_key_cloudwatch_agent_config} + +# configure systemd for running service in users accounts +cat >/etc/systemd/user@UID.service <<-EOF + +[Unit] +Description=User Manager for UID %i +After=user-runtime-dir@%i.service +Wants=user-runtime-dir@%i.service + +[Service] +LimitNOFILE=infinity +LimitNPROC=infinity +User=%i +PAMName=systemd-user +Type=notify + +[Install] +WantedBy=default.target + +EOF + +echo export XDG_RUNTIME_DIR=/run/user/$user_id >>/home/$user_name/.bashrc + +systemctl daemon-reload +systemctl enable user@UID.service +systemctl start user@UID.service + +curl -fsSL https://get.docker.com/rootless >>/opt/rootless.sh && chmod 755 /opt/rootless.sh +su -l $user_name -c /opt/rootless.sh +echo export DOCKER_HOST=unix:///run/user/$user_id/docker.sock >>/home/$user_name/.bashrc +echo export PATH=/home/$user_name/bin:$PATH >>/home/$user_name/.bashrc + +# Run docker service by default +loginctl enable-linger $user_name +su -l $user_name -c "systemctl --user enable docker" + +${install_runner} + +# config runner for rootless docker +cd /opt/actions-runner/ +echo DOCKER_HOST=unix:///run/user/$user_id/docker.sock >>.env +echo PATH=/home/$user_name/bin:$PATH >>.env + +${post_install} + +cd /opt/actions-runner + +${start_runner} diff --git a/examples/multi-runner/variables.tf b/examples/multi-runner/variables.tf new file mode 100644 index 0000000000..0efdc263a2 --- /dev/null +++ b/examples/multi-runner/variables.tf @@ -0,0 +1,9 @@ + +variable "github_app_key_base64" {} + +variable "github_app_id" {} + +variable "environment" { + type = string + default = null +} diff --git a/examples/multi-runner/versions.tf b/examples/multi-runner/versions.tf new file mode 100644 index 0000000000..a1455f3305 --- /dev/null +++ b/examples/multi-runner/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + local = { + source = "hashicorp/local" + } + random = { + source = "hashicorp/random" + } + } + required_version = ">= 1.3" +} diff --git a/examples/multi-runner/vpc.tf b/examples/multi-runner/vpc.tf new file mode 100644 index 0000000000..6b19a06b3f --- /dev/null +++ b/examples/multi-runner/vpc.tf @@ -0,0 +1,21 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "3.11.2" + + name = "vpc-${local.environment}" + cidr = "10.0.0.0/16" + + azs = ["${local.aws_region}a", "${local.aws_region}b", "${local.aws_region}c"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] + + enable_dns_hostnames = true + enable_nat_gateway = true + map_public_ip_on_launch = false + single_nat_gateway = true + + tags = { + Environment = local.environment + } + +} diff --git a/main.tf b/main.tf index cb6d3c9133..7f5766a132 100644 --- a/main.tf +++ b/main.tf @@ -9,6 +9,7 @@ locals { } default_runner_labels = "self-hosted,${var.runner_os},${var.runner_architecture}" + runner_labels = var.runner_extra_labels != "" ? "${local.default_runner_labels},${var.runner_extra_labels}" : local.default_runner_labels } resource "random_string" "random" { @@ -122,12 +123,21 @@ module "ssm" { module "webhook" { source = "./modules/webhook" - aws_region = var.aws_region - prefix = var.prefix - tags = local.tags - kms_key_arn = var.kms_key_arn - sqs_build_queue = aws_sqs_queue.queued_builds - sqs_build_queue_fifo = var.fifo_build_queue + prefix = var.prefix + tags = local.tags + kms_key_arn = var.kms_key_arn + + runner_config = { + "${aws_sqs_queue.queued_builds.id}" = { + id : aws_sqs_queue.queued_builds.id + arn : aws_sqs_queue.queued_builds.arn + fifo : var.fifo_build_queue + matcherConfig : { + labelMatchers : split(",", local.runner_labels) + exactMatch : var.runner_enable_workflow_job_labels_check_all + } + } + } sqs_workflow_job_queue = length(aws_sqs_queue.webhook_events_workflow_job_queue) > 0 ? aws_sqs_queue.webhook_events_workflow_job_queue[0] : null github_app_webhook_secret_arn = module.ssm.parameters.github_app_webhook_secret.arn @@ -142,11 +152,6 @@ module "webhook" { logging_retention_in_days = var.logging_retention_in_days logging_kms_key_id = var.logging_kms_key_id - # labels - enable_workflow_job_labels_check = var.runner_enable_workflow_job_labels_check - workflow_job_labels_check_all = var.runner_enable_workflow_job_labels_check_all - runner_labels = var.runner_extra_labels != "" ? "${local.default_runner_labels},${var.runner_extra_labels}" : local.default_runner_labels - role_path = var.role_path role_permissions_boundary = var.role_permissions_boundary repository_white_list = var.repository_white_list @@ -258,9 +263,8 @@ module "runner_binaries" { source = "./modules/runner-binaries-syncer" - aws_region = var.aws_region - prefix = var.prefix - tags = local.tags + prefix = var.prefix + tags = local.tags distribution_bucket_name = "${var.prefix}-dist-${random_string.random.result}" s3_logging_bucket = var.runner_binaries_s3_logging_bucket diff --git a/modules/download-lambda/README.md b/modules/download-lambda/README.md index 42800ad53d..b0875b2007 100644 --- a/modules/download-lambda/README.md +++ b/modules/download-lambda/README.md @@ -60,21 +60,3 @@ No modules. |------|-------------| | [files](#output\_files) | n/a | - -## Philips Forest - -This module is part of the Philips Forest. - -```plain - ___ _ - / __\__ _ __ ___ ___| |_ - / _\/ _ \| '__/ _ \/ __| __| - / / | (_) | | | __/\__ \ |_ - \/ \___/|_| \___||___/\__| - - Infrastructure -``` - -Talk to the forestkeepers in the `forest`-channel on Slack. - -[![Slack](https://philips-software-slackin.now.sh/badge.svg)](https://philips-software-slackin.now.sh) diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md new file mode 100644 index 0000000000..e17ce15bd4 --- /dev/null +++ b/modules/multi-runner/README.md @@ -0,0 +1,81 @@ +# Module - Multi runner + +> This module replaces the top-level module to make it easy to create with one deployment multiple type of runners. + +This module create many runners with a single GitHub app. The module utiliazed the internal modules and deploys parts of the stack for each runner defined. + +The module takes a configuration as input containing a matcher for the labels. The [webhook](../webhook/README.md) lambda is using the configuration to delegate events based on the labels in the workflow job and sent them to a dedicated queue based on the configuration. Events on each queue are processed by a dedicated lambda per configuration to scale runners. + +For each configuration: + +- When enabled the [distritbution sycner](../runner-binaries-syncer/README.md) is deployed for each unique combination of OS and architecture. +- For each configuration a queue is created and [runner module](../runners/README.md) is deployed + + +## Matching + +Matching of the configuration is done based on the labels specified in labelMatchers configuration. The webhook is processing the workflow_job event and match the labels against the labels specified in labelMatchers configuration in the order of configuration with exact-match true first, followed by all exact matches false. + + +## The catch + +Controlling which event is taken up by which runner is not to this module. It is completely done by GitHub. This means when potentially different runners can run the same job there is nothing that can be done to guarantee a certain runner will take up the job. + +An example, given you have two runners one with the labels. `self-hosted, linux, x64, large` and one with the labels `self-hosted, linux, x64, small`. Once you define a subset of the labels in the worklfow, for example `self-hosted, linux, x64`. Both runners can take the job potentially. You can define to scale one of the runners for the event, but still there is no guarantee that the scaled runner take the job. The workflow with subset of labels (`self-hosted, linux, x64`) can take up runner with specific labels (`self-hosted, linux, x64, large`) and leave the workflow with labels (`self-hosted, linux, x64, large`) be without the runner. +The only mitigation that is available right now is to use a small pool of runners. Pool instances can also exists for a short amount of time and only created once in x time based on a cron expressions. + + +## Usages + +A complate example is available in the examples, see the [multi-runner example](../../examples/multi-runner/) for actual implementation. + + +```hcl + +module "multi-runner" { + prefix = "multi-runner" + + github_app = { + # app details + } + + multi_runner_config = { + "linux-arm" = { + matcherConfig : { + labelMatchers = ["self-hosted", "linux", "arm64", "arm"] + exactMatch = true + } + runner_config = { + runner_os = "linux" + runner_architecture = "arm64" + runner_extra_labels = "arm" + enable_ssm_on_runners = true + instance_types = ["t4g.large", "c6g.large"] + ... + } + ... + }, + "linux-x64" = { + matcherConfig : { + labelMatchers = ["self-hosted", "linux", "x64"] + exactMatch = false + } + runner_config = { + runner_os = "linux" + runner_architecture = "x64" + instance_types = ["m5ad.large", "m5a.large"] + enable_ephemeral_runners = true + ... + } + delay_webhook_event = 0 + ... + } + } + +} + +``` + + + + diff --git a/modules/multi-runner/main.tf b/modules/multi-runner/main.tf new file mode 100644 index 0000000000..5ce4a7c2c4 --- /dev/null +++ b/modules/multi-runner/main.tf @@ -0,0 +1,23 @@ +locals { + tags = merge(var.tags, { + "ghr:environment" = var.prefix + }) + + github_app_parameters = { + id = module.ssm.parameters.github_app_id + key_base64 = module.ssm.parameters.github_app_key_base64 + } + + default_runner_labels = "self-hosted" + + runner_config = { for k, v in var.multi_runner_config : k => merge({ id = aws_sqs_queue.queued_builds[k].id, arn = aws_sqs_queue.queued_builds[k].arn }, v) } + + tmp_distinct_list_unique_os_and_arch = distinct([for i, config in local.runner_config : { "os_type" : config.runner_config.runner_os, "architecture" : config.runner_config.runner_architecture } if config.runner_config.enable_runner_binaries_syncer]) + unique_os_and_arch = { for i, v in local.tmp_distinct_list_unique_os_and_arch : "${v.os_type}_${v.architecture}" => v } +} + +resource "random_string" "random" { + length = 24 + special = false + upper = false +} diff --git a/modules/multi-runner/outputs.tf b/modules/multi-runner/outputs.tf new file mode 100644 index 0000000000..f051ff6d38 --- /dev/null +++ b/modules/multi-runner/outputs.tf @@ -0,0 +1,36 @@ +output "runners" { + value = [for runner in module.runners : { + launch_template_name = runner.launch_template.name + launch_template_id = runner.launch_template.id + launch_template_version = runner.launch_template.latest_version + launch_template_ami_id = runner.launch_template.image_id + lambda_up = runner.lambda_scale_up + lambda_down = runner.lambda_scale_down + role_runner = runner.role_runner + role_scale_up = runner.role_scale_up + role_scale_down = runner.role_scale_down + role_pool = runner.role_pool + }] +} + +output "binaries_syncer" { + value = [for runner_binary in module.runner_binaries : { + lambda = runner_binary.lambda + lambda_role = runner_binary.lambda_role + location = "s3://runner_binary.bucket.id}/runner_binary.bucket.key" + bucket = runner_binary.bucket + }] +} + +output "webhook" { + value = { + gateway = module.webhook.gateway + lambda = module.webhook.lambda + lambda_role = module.webhook.role + endpoint = "${module.webhook.gateway.api_endpoint}/${module.webhook.endpoint_relative_path}" + } +} + +output "ssm_parameters" { + value = module.ssm.parameters +} diff --git a/modules/multi-runner/queues.tf b/modules/multi-runner/queues.tf new file mode 100644 index 0000000000..7684f91f65 --- /dev/null +++ b/modules/multi-runner/queues.tf @@ -0,0 +1,72 @@ + +data "aws_iam_policy_document" "deny_unsecure_transport" { + statement { + sid = "DenyUnsecureTransport" + + effect = "Deny" + + principals { + type = "AWS" + identifiers = ["*"] + } + + actions = [ + "sqs:*" + ] + + resources = [ + "*" + ] + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} + + +resource "aws_sqs_queue" "queued_builds" { + for_each = var.multi_runner_config + name = "${var.prefix}-${each.key}-queued-builds${each.value.fifo ? ".fifo" : ""}" + delay_seconds = each.value.runner_config.delay_webhook_event + visibility_timeout_seconds = var.runners_scale_up_lambda_timeout + message_retention_seconds = each.value.runner_config.job_queue_retention_in_seconds + fifo_queue = each.value.fifo + receive_wait_time_seconds = 0 + content_based_deduplication = each.value.fifo + redrive_policy = each.value.redrive_build_queue.enabled ? jsonencode({ + deadLetterTargetArn = aws_sqs_queue.queued_builds_dlq[each.key].arn, + maxReceiveCount = each.value.redrive_build_queue.maxReceiveCount + }) : null + + sqs_managed_sse_enabled = var.queue_encryption.sqs_managed_sse_enabled + kms_master_key_id = var.queue_encryption.kms_master_key_id + kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds + + tags = var.tags +} + +resource "aws_sqs_queue_policy" "build_queue_policy" { + for_each = var.multi_runner_config + queue_url = aws_sqs_queue.queued_builds[each.key].id + policy = data.aws_iam_policy_document.deny_unsecure_transport.json +} + +resource "aws_sqs_queue" "queued_builds_dlq" { + for_each = var.multi_runner_config + name = "${var.prefix}-${each.key}-queued-builds_dead_letter" + + sqs_managed_sse_enabled = var.queue_encryption.sqs_managed_sse_enabled + kms_master_key_id = var.queue_encryption.kms_master_key_id + kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds + + tags = var.tags +} + +resource "aws_sqs_queue_policy" "build_queue_dlq_policy" { + for_each = var.multi_runner_config + queue_url = aws_sqs_queue.queued_builds_dlq[each.key].id + policy = data.aws_iam_policy_document.deny_unsecure_transport.json +} diff --git a/modules/multi-runner/runner-binaries.tf b/modules/multi-runner/runner-binaries.tf new file mode 100644 index 0000000000..54e832ba12 --- /dev/null +++ b/modules/multi-runner/runner-binaries.tf @@ -0,0 +1,36 @@ +module "runner_binaries" { + source = "../runner-binaries-syncer" + for_each = local.unique_os_and_arch + prefix = "${var.prefix}-${each.value.os_type}-${each.value.architecture}" + tags = local.tags + + distribution_bucket_name = "${var.prefix}-${each.value.os_type}-${each.value.architecture}-dist-${random_string.random.result}" + + runner_os = each.value.os_type + runner_architecture = each.value.architecture + + lambda_s3_bucket = var.lambda_s3_bucket + syncer_lambda_s3_key = var.syncer_lambda_s3_key + syncer_lambda_s3_object_version = var.syncer_lambda_s3_object_version + lambda_runtime = var.lambda_runtime + lambda_architecture = var.lambda_architecture + lambda_zip = var.runner_binaries_syncer_lambda_zip + lambda_timeout = var.runner_binaries_syncer_lambda_timeout + logging_retention_in_days = var.logging_retention_in_days + logging_kms_key_id = var.logging_kms_key_id + + server_side_encryption_configuration = var.runner_binaries_s3_sse_configuration + + role_path = var.role_path + role_permissions_boundary = var.role_permissions_boundary + + log_type = var.log_type + log_level = var.log_level + + lambda_principals = var.lambda_principals +} +locals { + runner_binaries_by_os_and_arch_map = { + for k, v in module.runner_binaries : k => { arn = v.bucket.arn, id = v.bucket.id, key = v.runner_distribution_object_key } + } +} diff --git a/modules/multi-runner/runners.tf b/modules/multi-runner/runners.tf new file mode 100644 index 0000000000..bfbd308fa8 --- /dev/null +++ b/modules/multi-runner/runners.tf @@ -0,0 +1,94 @@ +module "runners" { + source = "../runners" + for_each = local.runner_config + aws_region = var.aws_region + aws_partition = var.aws_partition + vpc_id = var.vpc_id + subnet_ids = var.subnet_ids + prefix = "${var.prefix}-${each.key}" + tags = merge(local.tags, { + "ghr:environment" = "${var.prefix}-${each.key}" + }) + + s3_runner_binaries = each.value.runner_config.enable_runner_binaries_syncer ? local.runner_binaries_by_os_and_arch_map["${each.value.runner_config.runner_os}_${each.value.runner_config.runner_architecture}"] : {} + + runner_os = each.value.runner_config.runner_os + instance_types = each.value.runner_config.instance_types + instance_target_capacity_type = each.value.runner_config.instance_target_capacity_type + instance_allocation_strategy = each.value.runner_config.instance_allocation_strategy + instance_max_spot_price = each.value.runner_config.instance_max_spot_price + block_device_mappings = each.value.runner_config.block_device_mappings + + runner_architecture = each.value.runner_config.runner_architecture + ami_filter = each.value.runner_config.ami_filter + ami_owners = each.value.runner_config.ami_owners + + sqs_build_queue = { "arn" : each.value.arn } + github_app_parameters = local.github_app_parameters + enable_organization_runners = each.value.runner_config.enable_organization_runners + enable_ephemeral_runners = each.value.runner_config.enable_ephemeral_runners + enable_job_queued_check = each.value.runner_config.enable_job_queued_check + disable_runner_autoupdate = each.value.runner_config.disable_runner_autoupdate + enable_managed_runner_security_group = var.enable_managed_runner_security_group + enable_runner_detailed_monitoring = each.value.runner_config.enable_runner_detailed_monitoring + scale_down_schedule_expression = each.value.runner_config.scale_down_schedule_expression + minimum_running_time_in_minutes = each.value.runner_config.minimum_running_time_in_minutes + runner_boot_time_in_minutes = each.value.runner_config.runner_boot_time_in_minutes + runner_extra_labels = each.value.runner_config.runner_extra_labels + runner_as_root = each.value.runner_config.runner_as_root + runner_run_as = each.value.runner_config.runner_run_as + runners_maximum_count = each.value.runner_config.runners_maximum_count + idle_config = each.value.runner_config.idle_config + enable_ssm_on_runners = each.value.runner_config.enable_ssm_on_runners + egress_rules = var.runner_egress_rules + runner_additional_security_group_ids = var.runner_additional_security_group_ids + metadata_options = each.value.runner_config.runner_metadata_options + + enable_runner_binaries_syncer = each.value.runner_config.enable_runner_binaries_syncer + lambda_s3_bucket = var.lambda_s3_bucket + runners_lambda_s3_key = var.runners_lambda_s3_key + runners_lambda_s3_object_version = var.runners_lambda_s3_object_version + lambda_runtime = var.lambda_runtime + lambda_architecture = var.lambda_architecture + lambda_zip = var.runners_lambda_zip + lambda_timeout_scale_up = var.runners_scale_up_lambda_timeout + lambda_timeout_scale_down = var.runners_scale_down_lambda_timeout + lambda_subnet_ids = var.lambda_subnet_ids + lambda_security_group_ids = var.lambda_security_group_ids + logging_retention_in_days = var.logging_retention_in_days + logging_kms_key_id = var.logging_kms_key_id + enable_cloudwatch_agent = each.value.runner_config.enable_cloudwatch_agent + cloudwatch_config = var.cloudwatch_config + runner_log_files = each.value.runner_config.runner_log_files + runner_group_name = each.value.runner_config.runner_group_name + + scale_up_reserved_concurrent_executions = each.value.runner_config.scale_up_reserved_concurrent_executions + + instance_profile_path = var.instance_profile_path + role_path = var.role_path + role_permissions_boundary = var.role_permissions_boundary + + enabled_userdata = each.value.runner_config.enabled_userdata + userdata_template = each.value.runner_config.userdata_template + userdata_pre_install = each.value.runner_config.userdata_pre_install + userdata_post_install = each.value.runner_config.userdata_post_install + key_name = var.key_name + runner_ec2_tags = each.value.runner_config.runner_ec2_tags + + create_service_linked_role_spot = each.value.runner_config.create_service_linked_role_spot + + runner_iam_role_managed_policy_arns = each.value.runner_config.runner_iam_role_managed_policy_arns + + ghes_url = var.ghes_url + ghes_ssl_verify = var.ghes_ssl_verify + + kms_key_arn = var.kms_key_arn + + log_type = var.log_type + log_level = var.log_level + + pool_config = each.value.runner_config.pool_config + pool_lambda_timeout = var.pool_lambda_timeout + pool_runner_owner = each.value.runner_config.pool_runner_owner + pool_lambda_reserved_concurrent_executions = var.pool_lambda_reserved_concurrent_executions +} diff --git a/modules/multi-runner/ssm.tf b/modules/multi-runner/ssm.tf new file mode 100644 index 0000000000..9d26d55bc6 --- /dev/null +++ b/modules/multi-runner/ssm.tf @@ -0,0 +1,8 @@ +module "ssm" { + source = "../ssm" + + kms_key_arn = var.kms_key_arn + prefix = var.prefix + github_app = var.github_app + tags = local.tags +} diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf new file mode 100644 index 0000000000..abba020f5a --- /dev/null +++ b/modules/multi-runner/variables.tf @@ -0,0 +1,491 @@ +variable "github_app" { + description = "GitHub app parameters, see your github app. Ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`)." + type = object({ + key_base64 = string + id = string + webhook_secret = string + }) +} + +variable "prefix" { + description = "The prefix used for naming resources" + type = string + default = "github-actions" +} + +variable "kms_key_arn" { + description = "Optional CMK Key ARN to be used for Parameter Store." + type = string + default = null +} + +variable "tags" { + description = "Map of tags that will be added to created resources. By default resources will be tagged with name and environment." + type = map(string) + default = {} +} + +variable "runner_extra_labels" { + description = "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `enable_workflow_job_labels_check`. GitHub read-only labels should not be provided." + type = string + default = "" +} + +variable "multi_runner_config" { + type = map(object({ + runner_config = object({ + runner_os = string + runner_architecture = string + runner_metadata_options = optional(map(any), { + http_endpoint = "enabled" + http_tokens = "optional" + http_put_response_hop_limit = 1 + }) + ami_filter = optional(map(list(string)), null) + ami_owners = optional(list(string), ["amazon"]) + create_service_linked_role_spot = optional(bool, false) + delay_webhook_event = optional(number, 30) + disable_runner_autoupdate = optional(bool, false) + enable_ephemeral_runners = optional(bool, false) + enable_job_queued_check = optional(bool, null) + enable_organization_runners = optional(bool, false) + enable_runner_binaries_syncer = optional(bool, true) + enable_ssm_on_runners = optional(bool, false) + enabled_userdata = optional(bool, true) + instance_allocation_strategy = optional(string, "lowest-price") + instance_max_spot_price = optional(string, null) + instance_target_capacity_type = optional(string, "spot") + instance_types = list(string) + job_queue_retention_in_seconds = optional(number, 86400) + minimum_running_time_in_minutes = optional(number, null) + pool_runner_owner = optional(string, null) + runner_as_root = optional(bool, false) + runner_boot_time_in_minutes = optional(number, 5) + runner_extra_labels = string + runner_group_name = optional(string, "Default") + runner_run_as = optional(string, "ec2-user") + runners_maximum_count = number + scale_down_schedule_expression = optional(string, "cron(*/5 * * * ? *)") + scale_up_reserved_concurrent_executions = optional(number, 1) + userdata_template = optional(string, null) + enable_runner_detailed_monitoring = optional(bool, false) + enable_cloudwatch_agent = optional(bool, true) + userdata_pre_install = optional(string, "") + userdata_post_install = optional(string, "") + runner_ec2_tags = optional(map(string), {}) + runner_iam_role_managed_policy_arns = optional(list(string), []) + idle_config = optional(list(object({ + cron = string + timeZone = string + idleCount = number + })), []) + runner_log_files = optional(list(object({ + log_group_name = string + prefix_log_group = bool + file_path = string + log_stream_name = string + })), null) + block_device_mappings = optional(list(object({ + delete_on_termination = bool + device_name = string + encrypted = bool + iops = number + kms_key_id = string + snapshot_id = string + throughput = number + volume_size = number + volume_type = string + })), [{ + delete_on_termination = true + device_name = "/dev/xvda" + encrypted = true + iops = null + kms_key_id = null + snapshot_id = null + throughput = null + volume_size = 30 + volume_type = "gp3" + }]) + pool_config = optional(list(object({ + schedule_expression = string + size = number + })), []) + }) + + matcherConfig = object({ + labelMatchers = list(string) + exactMatch = optional(bool, false) + }) + fifo = optional(bool, false) + redrive_build_queue = optional(object({ + enabled = bool + maxReceiveCount = number + }), { + enabled = false + maxReceiveCount = null + }) + })) + description = < This module is treated as internal module, breaking changes will not trigger a major release bump. + This module creates a lambda that will sync GitHub action binary to a S3 bucket, the lambda will be triggered via a CloudWatch event. The distribution is cached to avoid the latency of downloading the distribution during the setup. After deployment the lambda will be triggered via an S3 object created at deployment time. ## Usages @@ -120,21 +122,3 @@ No modules. | [lambda\_role](#output\_lambda\_role) | n/a | | [runner\_distribution\_object\_key](#output\_runner\_distribution\_object\_key) | n/a | - -## Philips Forest - -This module is part of the Philips Forest. - -```plain - ___ _ - / __\__ _ __ ___ ___| |_ - / _\/ _ \| '__/ _ \/ __| __| - / / | (_) | | | __/\__ \ |_ - \/ \___/|_| \___||___/\__| - - Infrastructure -``` - -Talk to the forestkeepers in the `forest`-channel on Slack. - -[![Slack](https://philips-software-slackin.now.sh/badge.svg)](https://philips-software-slackin.now.sh) diff --git a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock index 4bd0472061..fc40073d30 100644 --- a/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock +++ b/modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/yarn.lock @@ -117,6 +117,15 @@ "@jridgewell/gen-mapping" "^0.3.2" jsesc "^2.5.1" +"@babel/generator@^7.19.3", "@babel/generator@^7.19.4": + version "7.19.5" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.19.5.tgz#da3f4b301c8086717eee9cab14da91b1fa5dcca7" + integrity sha512-DxbNz9Lz4aMZ99qPpO1raTbcrI1ZeYh+9NR9qhfkQIbFtVEqotHojEBxHzmxhVONkGt6VyrqVQcgpefMy9pqcg== + dependencies: + "@babel/types" "^7.19.4" + "@jridgewell/gen-mapping" "^0.3.2" + jsesc "^2.5.1" + "@babel/helper-compilation-targets@^7.17.7": version "7.18.9" resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.18.9.tgz#69e64f57b524cde3e5ff6cc5a9f4a387ee5563bf" @@ -174,6 +183,14 @@ "@babel/template" "^7.18.10" "@babel/types" "^7.19.0" +"@babel/helper-function-name@^7.19.0": + version "7.19.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.19.0.tgz#941574ed5390682e872e52d3f38ce9d1bef4648c" + integrity sha512-WAwHBINyrpqywkUH0nTnNgI5ina5TFn85HKS0pbPDfxFfhyR/aNQEn4hGi1P1JyT//I0t4OgXUlofzWILRvS5w== + dependencies: + "@babel/template" "^7.18.10" + "@babel/types" "^7.19.0" + "@babel/helper-get-function-arity@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.16.7.tgz#ea08ac753117a669f1508ba06ebcc49156387419" @@ -528,6 +545,22 @@ debug "^4.1.0" globals "^11.1.0" +"@babel/traverse@^7.19.0", "@babel/traverse@^7.19.3", "@babel/traverse@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.19.4.tgz#f117820e18b1e59448a6c1fa9d0ff08f7ac459a8" + integrity sha512-w3K1i+V5u2aJUOXBFFC5pveFLmtq1s3qcdDNC2qRI6WPBQIDaKFqXxDEqDO/h1dQ3HjsZoZMyIy6jGLq0xtw+g== + dependencies: + "@babel/code-frame" "^7.18.6" + "@babel/generator" "^7.19.4" + "@babel/helper-environment-visitor" "^7.18.9" + "@babel/helper-function-name" "^7.19.0" + "@babel/helper-hoist-variables" "^7.18.6" + "@babel/helper-split-export-declaration" "^7.18.6" + "@babel/parser" "^7.19.4" + "@babel/types" "^7.19.4" + debug "^4.1.0" + globals "^11.1.0" + "@babel/types@7.17.0": version "7.17.0" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.17.0.tgz#a826e368bccb6b3d84acd76acad5c0d87342390b" @@ -554,6 +587,15 @@ "@babel/helper-validator-identifier" "^7.19.1" to-fast-properties "^2.0.0" +"@babel/types@^7.18.10", "@babel/types@^7.19.0", "@babel/types@^7.19.3", "@babel/types@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.19.4.tgz#0dd5c91c573a202d600490a35b33246fed8a41c7" + integrity sha512-M5LK7nAeS6+9j7hAq+b3fQs+pNfUtTGq+yFFfHnauFA8zQtLRfmuipmsKDKKLuyG+wC8ABW43A153YNawNTEtw== + dependencies: + "@babel/helper-string-parser" "^7.19.4" + "@babel/helper-validator-identifier" "^7.19.1" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -866,9 +908,11 @@ "@jridgewell/sourcemap-codec" "^1.4.10" "@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.15": + version "0.3.17" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985" integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g== + dependencies: "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" @@ -1412,6 +1456,7 @@ axios@^1.1.3: form-data "^4.0.0" proxy-from-env "^1.1.0" + babel-jest@^29.2.0: version "29.2.0" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.2.0.tgz#088624f037da90e69a06073305276cbd111d68a8" @@ -1421,6 +1466,7 @@ babel-jest@^29.2.0: "@types/babel__core" "^7.1.14" babel-plugin-istanbul "^6.1.1" babel-preset-jest "^29.2.0" + chalk "^4.0.0" graceful-fs "^4.2.9" slash "^3.0.0" @@ -1436,10 +1482,12 @@ babel-plugin-istanbul@^6.1.1: istanbul-lib-instrument "^5.0.4" test-exclude "^6.0.0" + babel-plugin-jest-hoist@^29.2.0: version "29.2.0" resolved "https://registry.yarnpkg.com/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.2.0.tgz#23ee99c37390a98cfddf3ef4a78674180d823094" integrity sha512-TnspP2WNiR3GLfCsUNHqeXw0RoQ2f9U5hQ5L3XFpwuO8htQmSrhh8qsB6vi5Yi8+kuynN1yjDjQsPfkebmB6ZA== + dependencies: "@babel/template" "^7.3.3" "@babel/types" "^7.3.3" @@ -1588,6 +1636,7 @@ caniuse-lite@^1.0.30001400: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001419.tgz#3542722d57d567c8210d5e4d0f9f17336b776457" integrity sha512-aFO1r+g6R7TW+PNQxKzjITwLOyDhVRLjW0LcwS/HCZGUUKTGNp9+IwLC4xyDSZBygVL/mxaFR3HIV6wEKQuSzw== + chalk@^2.0.0: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -1766,6 +1815,11 @@ diff-sequences@^29.2.0: resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.2.0.tgz#4c55b5b40706c7b5d2c5c75999a50c56d214e8f6" integrity sha512-413SY5JpYeSBZxmenGEmCVQ8mCgtFJF0w9PROdaS6z987XC2Pd2GOKqOITLtMftmyFZqgtCOb/QA7/Z3ZXfzIw== +diff-sequences@^29.0.0: + version "29.0.0" + resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.0.0.tgz#bae49972ef3933556bcb0800b72e8579d19d9e4f" + integrity sha512-7Qe/zd1wxSDL4D/X/FPjOMB+ZMDt71W94KYaq05I2l0oQqgXgs7s4ftYYmV38gBSrPz2vcygxfs1xn0FT+rKNA== + diff@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" @@ -1802,6 +1856,7 @@ electron-to-chromium@^1.4.251: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.282.tgz#02af3fd6051e97ac3388a4b11d455bc1ca49838f" integrity sha512-Dki0WhHNh/br/Xi1vAkueU5mtIc9XLHcMKB6tNfQKk+kPG0TEUjRh5QEMAUbRp30/rYNMFD1zKKvbVzwq/4wmg== + emittery@^0.10.2: version "0.10.2" resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.10.2.tgz#902eec8aedb8c41938c46e9385e9db7e03182933" @@ -2048,6 +2103,7 @@ expect@^29.2.0: jest-message-util "^29.2.0" jest-util "^29.2.0" + fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -2939,7 +2995,19 @@ jest-util@^29.0.0, jest-util@^29.2.0, jest-util@^29.2.1: resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.2.1.tgz#f26872ba0dc8cbefaba32c34f98935f6cf5fc747" integrity sha512-P5VWDj25r7kj7kl4pN2rG/RN2c1TLfYYYZYULnS/35nFDjBai+hBeo3MDrYZS7p6IoY3YHZnt2vq4L6mKnLk0g== dependencies: - "@jest/types" "^29.2.1" + "@jest/types" "^29.2.0" + "@types/node" "*" + chalk "^4.0.0" + ci-info "^3.2.0" + graceful-fs "^4.2.9" + picomatch "^2.2.3" + +jest-util@^29.0.0, jest-util@^29.1.2: + version "29.1.2" + resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.1.2.tgz#ac5798e93cb6a6703084e194cfa0898d66126df1" + integrity sha512-vPCk9F353i0Ymx3WQq3+a4lZ07NXu9Ca8wya6o4Fe4/aO1e1awMMprZ3woPFpKwghEOW+UXgd15vVotuNN9ONQ== + dependencies: + "@jest/types" "^29.1.2" "@types/node" "*" chalk "^4.0.0" ci-info "^3.2.0" @@ -3385,6 +3453,15 @@ pretty-format@^29.2.0: ansi-styles "^5.0.0" react-is "^18.0.0" +pretty-format@^29.1.2: + version "29.1.2" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.1.2.tgz#b1f6b75be7d699be1a051f5da36e8ae9e76a8e6a" + integrity sha512-CGJ6VVGXVRP2o2Dorl4mAwwvDWT25luIsYhkyVQW32E4nL+TgW939J7LlKT/npq5Cpq6j3s+sy+13yk7xYpBmg== + dependencies: + "@jest/schemas" "^29.0.0" + ansi-styles "^5.0.0" + react-is "^18.0.0" + progress@^2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" @@ -3433,6 +3510,11 @@ react-is@^18.0.0: resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b" integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w== +react-is@^18.0.0: + version "18.2.0" + resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b" + integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w== + readdirp@~3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" @@ -3728,6 +3810,7 @@ supports-color@^8.0.0: dependencies: has-flag "^4.0.0" + table@^6.0.9: version "6.7.5" resolved "https://registry.yarnpkg.com/table/-/table-6.7.5.tgz#f04478c351ef3d8c7904f0e8be90a1b62417d238" diff --git a/modules/runner-binaries-syncer/variables.tf b/modules/runner-binaries-syncer/variables.tf index 0c59811e75..5eb41cd960 100644 --- a/modules/runner-binaries-syncer/variables.tf +++ b/modules/runner-binaries-syncer/variables.tf @@ -1,8 +1,3 @@ -variable "aws_region" { - description = "AWS region." - type = string -} - variable "tags" { description = "Map of tags that will be added to created resources. By default resources will be tagged with name and environment." type = map(string) diff --git a/modules/runners/README.md b/modules/runners/README.md index 8f3b51b1f3..802223b7c5 100644 --- a/modules/runners/README.md +++ b/modules/runners/README.md @@ -1,5 +1,7 @@ # Module - Scale runners +> This module is treated as internal module, breaking changes will not trigger a major release bump. + This module creates resources required to run the GitHub action runner on AWS EC2 spot instances. The life cycle of the runners on AWS is managed by two lambda functions. One function will handle scaling up, the other scaling down. ## Overview @@ -205,21 +207,3 @@ yarn run dist | [role\_scale\_down](#output\_role\_scale\_down) | n/a | | [role\_scale\_up](#output\_role\_scale\_up) | n/a | - -## Philips Forest - -This module is part of the Philips Forest. - -```plain - ___ _ - / __\__ _ __ ___ ___| |_ - / _\/ _ \| '__/ _ \/ __| __| - / / | (_) | | | __/\__ \ |_ - \/ \___/|_| \___||___/\__| - - Infrastructure -``` - -Talk to the forestkeepers in the `forest`-channel on Slack. - -[![Slack](https://philips-software-slackin.now.sh/badge.svg)](https://philips-software-slackin.now.sh) diff --git a/modules/runners/lambdas/runners/yarn.lock b/modules/runners/lambdas/runners/yarn.lock index b6546449b1..a1bcdd5212 100644 --- a/modules/runners/lambdas/runners/yarn.lock +++ b/modules/runners/lambdas/runners/yarn.lock @@ -3637,10 +3637,10 @@ jest-util@^29.0.0, jest-util@^29.2.0: graceful-fs "^4.2.9" picomatch "^2.2.3" -jest-util@^29.2.1: - version "29.2.1" - resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.2.1.tgz#f26872ba0dc8cbefaba32c34f98935f6cf5fc747" - integrity sha512-P5VWDj25r7kj7kl4pN2rG/RN2c1TLfYYYZYULnS/35nFDjBai+hBeo3MDrYZS7p6IoY3YHZnt2vq4L6mKnLk0g== +jest-util@^29.0.0, jest-util@^29.1.2: + version "29.1.2" + resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.1.2.tgz#ac5798e93cb6a6703084e194cfa0898d66126df1" + integrity sha512-vPCk9F353i0Ymx3WQq3+a4lZ07NXu9Ca8wya6o4Fe4/aO1e1awMMprZ3woPFpKwghEOW+UXgd15vVotuNN9ONQ== dependencies: "@jest/types" "^29.2.1" "@types/node" "*" diff --git a/modules/webhook/README.md b/modules/webhook/README.md index 56896a2744..522a7098a1 100644 --- a/modules/webhook/README.md +++ b/modules/webhook/README.md @@ -1,5 +1,7 @@ # Module - GitHub App web hook +> This module is treated as internal module, breaking changes will not trigger a major release bump. + This module creates an API gateway endpoint and lambda function to handle GitHub App webhook events. ## Usages @@ -109,21 +111,3 @@ No modules. | [lambda](#output\_lambda) | n/a | | [role](#output\_role) | n/a | - -## Philips Forest - -This module is part of the Philips Forest. - -```plain - ___ _ - / __\__ _ __ ___ ___| |_ - / _\/ _ \| '__/ _ \/ __| __| - / / | (_) | | | __/\__ \ |_ - \/ \___/|_| \___||___/\__| - - Infrastructure -``` - -Talk to the forestkeepers in the `forest`-channel on Slack. - -[![Slack](https://philips-software-slackin.now.sh/badge.svg)](https://philips-software-slackin.now.sh) diff --git a/modules/webhook/lambdas/webhook/jest.config.js b/modules/webhook/lambdas/webhook/jest.config.js index 02a6524ce9..4d0bf9e89c 100644 --- a/modules/webhook/lambdas/webhook/jest.config.js +++ b/modules/webhook/lambdas/webhook/jest.config.js @@ -2,13 +2,13 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', collectCoverage: true, - collectCoverageFrom: ['src/**/*.{ts,js,jsx}', '!src/**/*local*.ts'], + collectCoverageFrom: ['src/**/*.{ts,js,jsx}', '!src/**/*local*.ts', '!src/**/*.d.ts'], coverageThreshold: { global: { - branches: 85, - functions: 85, - lines: 85, - statements: 85 + branches: 87, + functions: 99, + lines: 99, + statements: 99 } } }; diff --git a/modules/webhook/lambdas/webhook/src/sqs/index.test.ts b/modules/webhook/lambdas/webhook/src/sqs/index.test.ts index 32761e4526..47b91e5bac 100644 --- a/modules/webhook/lambdas/webhook/src/sqs/index.test.ts +++ b/modules/webhook/lambdas/webhook/src/sqs/index.test.ts @@ -15,39 +15,37 @@ jest.mock('aws-sdk', () => ({ })); describe('Test sending message to SQS.', () => { - const message: ActionRequestMessage = { + const queueUrl = 'https://sqs.eu-west-1.amazonaws.com/123456789/queued-builds'; + const message = { eventType: 'type', id: 0, installationId: 0, repositoryName: 'test', repositoryOwner: 'owner', + queueId: queueUrl, + queueFifo: false, }; + const sqsMessage: SQS.Types.SendMessageRequest = { - QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/123456789/queued-builds', + QueueUrl: queueUrl, MessageBody: JSON.stringify(message), }; afterEach(() => { jest.clearAllMocks(); }); - it('no fifo queue, based on defaults', async () => { - // Arrange - process.env.SQS_URL_WEBHOOK = sqsMessage.QueueUrl; - - // Act - const result = await sendActionRequest(message); - - // Assert - expect(mockSQS.sendMessage).toBeCalledWith(sqsMessage); - expect(result).resolves; - }); it('no fifo queue', async () => { // Arrange - process.env.SQS_URL_WEBHOOK = sqsMessage.QueueUrl; - process.env.SQS_IS_FIFO = 'false'; - + const no_fifo_message: ActionRequestMessage = { + ...message, + queueFifo: false, + }; + const sqsMessage: SQS.Types.SendMessageRequest = { + QueueUrl: queueUrl, + MessageBody: JSON.stringify(no_fifo_message), + }; // Act - const result = await sendActionRequest(message); + const result = await sendActionRequest(no_fifo_message); // Assert expect(mockSQS.sendMessage).toBeCalledWith(sqsMessage); @@ -56,11 +54,16 @@ describe('Test sending message to SQS.', () => { it('use a fifo queue', async () => { // Arrange - process.env.SQS_URL_WEBHOOK = sqsMessage.QueueUrl; - process.env.SQS_IS_FIFO = 'true'; - + const fifo_message: ActionRequestMessage = { + ...message, + queueFifo: true, + }; + const sqsMessage: SQS.Types.SendMessageRequest = { + QueueUrl: queueUrl, + MessageBody: JSON.stringify(fifo_message), + }; // Act - const result = await sendActionRequest(message); + const result = await sendActionRequest(fifo_message); // Assert expect(mockSQS.sendMessage).toBeCalledWith({ ...sqsMessage, MessageGroupId: String(message.id) }); @@ -98,4 +101,18 @@ describe('Test sending message to SQS.', () => { // Assert expect(mockSQS.sendMessage).not.toBeCalledWith(sqsMessage); }); + it('Catch the exception when even copy queue throws exception', async () => { + // Arrange + process.env.SQS_WORKFLOW_JOB_QUEUE = sqsMessage.QueueUrl; + const mockSQS = { + sendMessage: jest.fn(() => { + throw new Error(); + }), + }; + jest.mock('aws-sdk', () => ({ + SQS: jest.fn().mockImplementation(() => mockSQS), + })); + await expect(mockSQS.sendMessage).toThrowError(); + await expect(sendWebhookEventToWorkflowJobQueue(message)).resolves.not.toThrowError(); + }); }); diff --git a/modules/webhook/lambdas/webhook/src/sqs/index.ts b/modules/webhook/lambdas/webhook/src/sqs/index.ts index a8c3dd08a2..024b04062b 100644 --- a/modules/webhook/lambdas/webhook/src/sqs/index.ts +++ b/modules/webhook/lambdas/webhook/src/sqs/index.ts @@ -1,5 +1,6 @@ import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { SQS } from 'aws-sdk'; +import { bool } from 'aws-sdk/clients/signer'; import { LogFields, logger } from '../webhook/logger'; @@ -9,6 +10,19 @@ export interface ActionRequestMessage { repositoryName: string; repositoryOwner: string; installationId: number; + queueId: string; + queueFifo: bool; +} + +export interface MatcherConfig { + labelMatchers: string[]; + exactMatch: bool; +} +export interface QueueConfig { + matcherConfig: MatcherConfig; + id: string; + arn: string; + fifo: bool; } export interface GithubWorkflowEvent { workflowJobEvent: WorkflowJobEvent; @@ -17,16 +31,13 @@ export interface GithubWorkflowEvent { export const sendActionRequest = async (message: ActionRequestMessage): Promise => { const sqs = new SQS({ region: process.env.AWS_REGION }); - const useFifoQueueEnv = process.env.SQS_IS_FIFO || 'false'; - const useFifoQueue = JSON.parse(useFifoQueueEnv) as boolean; - const sqsMessage: SQS.Types.SendMessageRequest = { - QueueUrl: String(process.env.SQS_URL_WEBHOOK), + QueueUrl: message.queueId, MessageBody: JSON.stringify(message), }; logger.debug(`sending message to SQS: ${JSON.stringify(sqsMessage)}`, LogFields.print()); - if (useFifoQueue) { + if (message.queueFifo) { sqsMessage.MessageGroupId = String(message.id); } diff --git a/modules/webhook/lambdas/webhook/src/ssm/index.test.ts b/modules/webhook/lambdas/webhook/src/ssm/index.test.ts index 0f002dbacf..3795dc2f58 100644 --- a/modules/webhook/lambdas/webhook/src/ssm/index.test.ts +++ b/modules/webhook/lambdas/webhook/src/ssm/index.test.ts @@ -11,6 +11,7 @@ const ENVIRONMENT = 'dev'; beforeEach(() => { jest.resetModules(); jest.clearAllMocks(); + jest.resetAllMocks(); process.env = { ...cleanEnv }; nock.disableNetConnect(); }); @@ -39,4 +40,45 @@ describe('Test getParameterValue', () => { // Assert expect(result).toBe(parameterValue); }); + + test('Gets parameters and returns value undefined', async () => { + // Arrange + const parameterValue = undefined; + const parameterName = 'testParam'; + const output: GetParameterCommandOutput = { + Parameter: { + Name: parameterName, + Type: 'SecureString', + Value: parameterValue, + }, + $metadata: { + httpStatusCode: 200, + }, + }; + SSM.prototype.getParameter = jest.fn().mockResolvedValue(output); + + // Act + const result = await getParameterValue(ENVIRONMENT, parameterName); + + // Assert + expect(result).toBe(undefined); + }); + + test('Gets parameters and returns undefined', async () => { + // Arrange + const parameterName = 'testParam'; + const output: GetParameterCommandOutput = { + $metadata: { + httpStatusCode: 200, + }, + }; + + SSM.prototype.getParameter = jest.fn().mockResolvedValue(output); + + // Act + const result = await getParameterValue(ENVIRONMENT, parameterName); + + // Assert + expect(result).toBe(undefined); + }); }); diff --git a/modules/webhook/lambdas/webhook/src/webhook/handler.test.ts b/modules/webhook/lambdas/webhook/src/webhook/handler.test.ts index f518f18f69..e79a7068e8 100644 --- a/modules/webhook/lambdas/webhook/src/webhook/handler.test.ts +++ b/modules/webhook/lambdas/webhook/src/webhook/handler.test.ts @@ -4,7 +4,8 @@ import nock from 'nock'; import checkrun_event from '../../test/resources/github_check_run_event.json'; import workflowjob_event from '../../test/resources/github_workflowjob_event.json'; -import { sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '../sqs'; +import queuesConfig from '../../test/resources/multi_runner_configurations.json'; +import { sendActionRequest } from '../sqs'; import { getParameterValue } from '../ssm'; import { handle } from './handler'; @@ -18,6 +19,17 @@ const webhooks = new Webhooks({ secret: secret, }); +const mockSQS = { + sendMessage: jest.fn(() => { + { + return { promise: jest.fn() }; + } + }), +}; +jest.mock('aws-sdk', () => ({ + SQS: jest.fn().mockImplementation(() => mockSQS), +})); + describe('handler', () => { let originalError: Console['error']; @@ -49,7 +61,7 @@ describe('handler', () => { describe('Test for workflowjob event: ', () => { beforeEach(() => { - process.env.DISABLE_CHECK_WORKFLOW_JOB_LABELS = 'false'; + process.env.RUNNER_CONFIG = JSON.stringify(queuesConfig); }); it('handles workflow job events', async () => { const event = JSON.stringify(workflowjob_event); @@ -132,9 +144,22 @@ describe('handler', () => { }); it('Check runner labels accept test job', async () => { - process.env.RUNNER_LABELS = '["self-hosted", "test"]'; - process.env.ENABLE_WORKFLOW_JOB_LABELS_CHECK = 'true'; - process.env.WORKFLOW_JOB_LABELS_CHECK_ALL = 'true'; + process.env.RUNNER_CONFIG = JSON.stringify([ + { + ...queuesConfig[0], + matcherConfig: { + labelMatchers: ['self-hosted', 'test'], + exactMatch: true, + }, + }, + { + ...queuesConfig[1], + matcherConfig: { + labelMatchers: ['self-hosted', 'test1'], + exactMatch: true, + }, + }, + ]); const event = JSON.stringify({ ...workflowjob_event, workflow_job: { @@ -151,9 +176,22 @@ describe('handler', () => { }); it('Check runner labels accept job with mixed order.', async () => { - process.env.RUNNER_LABELS = '["linux", "TEST", "self-hosted"]'; - process.env.ENABLE_WORKFLOW_JOB_LABELS_CHECK = 'true'; - process.env.WORKFLOW_JOB_LABELS_CHECK_ALL = 'true'; + process.env.RUNNER_CONFIG = JSON.stringify([ + { + ...queuesConfig[0], + matcherConfig: { + labelMatchers: ['linux', 'TEST', 'self-hosted'], + exactMatch: true, + }, + }, + { + ...queuesConfig[1], + matcherConfig: { + labelMatchers: ['self-hosted', 'test1'], + exactMatch: true, + }, + }, + ]); const event = JSON.stringify({ ...workflowjob_event, workflow_job: { @@ -170,9 +208,22 @@ describe('handler', () => { }); it('Check webhook accept jobs where not all labels are provided in job.', async () => { - process.env.RUNNER_LABELS = '["self-hosted", "test", "test2"]'; - process.env.ENABLE_WORKFLOW_JOB_LABELS_CHECK = 'true'; - process.env.WORKFLOW_JOB_LABELS_CHECK_ALL = 'true'; + process.env.RUNNER_CONFIG = JSON.stringify([ + { + ...queuesConfig[0], + matcherConfig: { + labelMatchers: ['self-hosted', 'test', 'test2'], + exactMatch: true, + }, + }, + { + ...queuesConfig[1], + matcherConfig: { + labelMatchers: ['self-hosted', 'test1'], + exactMatch: true, + }, + }, + ]); const event = JSON.stringify({ ...workflowjob_event, workflow_job: { @@ -189,9 +240,22 @@ describe('handler', () => { }); it('Check webhook does not accept jobs where not all labels are supported by the runner.', async () => { - process.env.RUNNER_LABELS = '["self-hosted", "x64", "linux", "test"]'; - process.env.ENABLE_WORKFLOW_JOB_LABELS_CHECK = 'true'; - process.env.WORKFLOW_JOB_LABELS_CHECK_ALL = 'true'; + process.env.RUNNER_CONFIG = JSON.stringify([ + { + ...queuesConfig[0], + matcherConfig: { + labelMatchers: ['self-hosted', 'x64', 'linux', 'test'], + exactMatch: true, + }, + }, + { + ...queuesConfig[1], + matcherConfig: { + labelMatchers: ['self-hosted', 'x64', 'linux', 'test1'], + exactMatch: true, + }, + }, + ]); const event = JSON.stringify({ ...workflowjob_event, workflow_job: { @@ -208,9 +272,22 @@ describe('handler', () => { }); it('Check webhook will accept jobs with a single acceptable label.', async () => { - process.env.RUNNER_LABELS = '["self-hosted", "x64", "linux", "test"]'; - process.env.ENABLE_WORKFLOW_JOB_LABELS_CHECK = 'true'; - process.env.WORKFLOW_JOB_LABELS_CHECK_ALL = 'false'; + process.env.RUNNER_CONFIG = JSON.stringify([ + { + ...queuesConfig[0], + matcherConfig: { + labelMatchers: ['self-hosted', 'test', 'test2'], + exactMatch: true, + }, + }, + { + ...queuesConfig[1], + matcherConfig: { + labelMatchers: ['self-hosted', 'x64'], + exactMatch: false, + }, + }, + ]); const event = JSON.stringify({ ...workflowjob_event, workflow_job: { @@ -227,9 +304,22 @@ describe('handler', () => { }); it('Check webhook will not accept jobs without correct label when job label check all is false.', async () => { - process.env.RUNNER_LABELS = '["self-hosted", "x64", "linux", "test"]'; - process.env.ENABLE_WORKFLOW_JOB_LABELS_CHECK = 'true'; - process.env.WORKFLOW_JOB_LABELS_CHECK_ALL = 'false'; + process.env.RUNNER_CONFIG = JSON.stringify([ + { + ...queuesConfig[0], + matcherConfig: { + labelMatchers: ['self-hosted', 'x64', 'linux', 'test'], + exactMatch: false, + }, + }, + { + ...queuesConfig[1], + matcherConfig: { + labelMatchers: ['self-hosted', 'x64', 'linux', 'test1'], + exactMatch: false, + }, + }, + ]); const event = JSON.stringify({ ...workflowjob_event, workflow_job: { @@ -244,6 +334,88 @@ describe('handler', () => { expect(resp.statusCode).toBe(202); expect(sendActionRequest).not.toBeCalled; }); + it('Check webhook will accept jobs for specific labels if workflow labels are specific', async () => { + process.env.RUNNER_CONFIG = JSON.stringify([ + { + ...queuesConfig[0], + matcherConfig: { + labelMatchers: ['self-hosted'], + exactMatch: false, + }, + id: 'ubuntu-queue-id', + }, + { + ...queuesConfig[1], + matcherConfig: { + labelMatchers: ['self-hosted'], + exactMatch: false, + }, + id: 'default-queue-id', + }, + ]); + const event = JSON.stringify({ + ...workflowjob_event, + workflow_job: { + ...workflowjob_event.workflow_job, + labels: ['self-hosted', 'ubuntu', 'x64', 'linux'], + }, + }); + const resp = await handle( + { 'X-Hub-Signature': await webhooks.sign(event), 'X-GitHub-Event': 'workflow_job' }, + event, + ); + expect(resp.statusCode).toBe(201); + expect(sendActionRequest).toBeCalledWith({ + id: workflowjob_event.workflow_job.id, + repositoryName: workflowjob_event.repository.name, + repositoryOwner: workflowjob_event.repository.owner.login, + eventType: 'workflow_job', + installationId: 0, + queueId: 'ubuntu-queue-id', + queueFifo: false, + }); + }); + it('Check webhook will accept jobs for latest labels if workflow labels are not specific', async () => { + process.env.RUNNER_CONFIG = JSON.stringify([ + { + ...queuesConfig[0], + matcherConfig: { + labelMatchers: ['self-hosted'], + exactMatch: false, + }, + id: 'ubuntu-queue-id', + }, + { + ...queuesConfig[1], + matcherConfig: { + labelMatchers: ['self-hosted'], + exactMatch: false, + }, + id: 'default-queue-id', + }, + ]); + const event = JSON.stringify({ + ...workflowjob_event, + workflow_job: { + ...workflowjob_event.workflow_job, + labels: ['self-hosted', 'linux', 'x64'], + }, + }); + const resp = await handle( + { 'X-Hub-Signature': await webhooks.sign(event), 'X-GitHub-Event': 'workflow_job' }, + event, + ); + expect(resp.statusCode).toBe(201); + expect(sendActionRequest).toBeCalledWith({ + id: workflowjob_event.workflow_job.id, + repositoryName: workflowjob_event.repository.name, + repositoryOwner: workflowjob_event.repository.owner.login, + eventType: 'workflow_job', + installationId: 0, + queueId: 'ubuntu-queue-id', + queueFifo: false, + }); + }); }); describe('Test for check_run is ignored.', () => { @@ -257,20 +429,4 @@ describe('handler', () => { expect(sendActionRequest).toBeCalledTimes(0); }); }); - - describe('Test for webhook events to be sent to workflow job queue: ', () => { - beforeEach(() => { - process.env.SQS_WORKFLOW_JOB_QUEUE = - 'https://sqs.eu-west-1.amazonaws.com/123456789/webhook_events_workflow_job_queue'; - }); - it('sends webhook events to workflow job queue', async () => { - const event = JSON.stringify(workflowjob_event); - const resp = await handle( - { 'X-Hub-Signature': await webhooks.sign(event), 'X-GitHub-Event': 'workflow_job' }, - event, - ); - expect(resp.statusCode).toBe(201); - expect(sendWebhookEventToWorkflowJobQueue).toBeCalled(); - }); - }); }); diff --git a/modules/webhook/lambdas/webhook/src/webhook/handler.ts b/modules/webhook/lambdas/webhook/src/webhook/handler.ts index 6ab87d2971..257e68850a 100644 --- a/modules/webhook/lambdas/webhook/src/webhook/handler.ts +++ b/modules/webhook/lambdas/webhook/src/webhook/handler.ts @@ -3,7 +3,7 @@ import { CheckRunEvent, WorkflowJobEvent } from '@octokit/webhooks-types'; import { IncomingHttpHeaders } from 'http'; import { Response } from '../lambda'; -import { sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '../sqs'; +import { QueueConfig, sendActionRequest, sendWebhookEventToWorkflowJobQueue } from '../sqs'; import { getParameterValue } from '../ssm'; import { LogFields, logger as rootLogger } from './logger'; @@ -11,8 +11,7 @@ const supportedEvents = ['workflow_job']; const logger = rootLogger.getChildLogger(); export async function handle(headers: IncomingHttpHeaders, body: string): Promise { - const { environment, repositoryWhiteList, enableWorkflowLabelCheck, workflowLabelCheckAll, runnerLabels } = - readEnvironmentVariables(); + const { environment, repositoryWhiteList, queuesConfig } = readEnvironmentVariables(); // ensure header keys lower case since github headers can contain capitals. for (const key in headers) { @@ -52,7 +51,6 @@ export async function handle(headers: IncomingHttpHeaders, body: string): Promis */ LogFields.fields.completed_at = payload[githubEvent]?.completed_at; LogFields.fields.conclusion = payload[githubEvent]?.conclusion; - if (isRepoNotAllowed(payload.repository.full_name, repositoryWhiteList)) { logger.error(`Received event from unauthorized repository ${payload.repository.full_name}`, LogFields.print()); return { @@ -61,23 +59,11 @@ export async function handle(headers: IncomingHttpHeaders, body: string): Promis } logger.info(`Processing Github event`, LogFields.print()); + logger.debug(`Queue configuration: ${queuesConfig}`, LogFields.print()); - if (githubEvent == 'workflow_job') { - const workflowEventPayload = payload as WorkflowJobEvent; - response = await handleWorkflowJob( - workflowEventPayload, - githubEvent, - enableWorkflowLabelCheck, - workflowLabelCheckAll, - runnerLabels, - ); - } else { - response = { - statusCode: 202, - body: `Received event '${githubEvent}' ignored.`, - }; - } - + const workflowJobEvent = payload as WorkflowJobEvent; + response = await handleWorkflowJob(workflowJobEvent, githubEvent, queuesConfig); + await sendWorkflowJobEvents(githubEvent, workflowJobEvent); return response; } async function sendWorkflowJobEvents(githubEvent: string, workflowEventPayload: WorkflowJobEvent) { @@ -88,15 +74,11 @@ async function sendWorkflowJobEvents(githubEvent: string, workflowEventPayload: function readEnvironmentVariables() { const environment = process.env.ENVIRONMENT; - const enableWorkflowLabelCheckEnv = process.env.ENABLE_WORKFLOW_JOB_LABELS_CHECK || 'false'; - const enableWorkflowLabelCheck = JSON.parse(enableWorkflowLabelCheckEnv) as boolean; - const workflowLabelCheckAllEnv = process.env.WORKFLOW_JOB_LABELS_CHECK_ALL || 'false'; - const workflowLabelCheckAll = JSON.parse(workflowLabelCheckAllEnv) as boolean; const repositoryWhiteListEnv = process.env.REPOSITORY_WHITE_LIST || '[]'; const repositoryWhiteList = JSON.parse(repositoryWhiteListEnv) as Array; - const runnerLabelsEnv = (process.env.RUNNER_LABELS || '[]').toLowerCase(); - const runnerLabels = JSON.parse(runnerLabelsEnv) as Array; - return { environment, repositoryWhiteList, enableWorkflowLabelCheck, workflowLabelCheckAll, runnerLabels }; + const queuesConfigEnv = process.env.RUNNER_CONFIG || '[]'; + const queuesConfig = JSON.parse(queuesConfigEnv) as Array; + return { environment, repositoryWhiteList, queuesConfig }; } async function verifySignature( @@ -134,11 +116,32 @@ async function verifySignature( async function handleWorkflowJob( body: WorkflowJobEvent, githubEvent: string, - enableWorkflowLabelCheck: boolean, - workflowLabelCheckAll: boolean, - runnerLabels: string[], + queuesConfig: Array, ): Promise { - if (enableWorkflowLabelCheck && !canRunJob(body, runnerLabels, workflowLabelCheckAll)) { + const installationId = getInstallationId(body); + if (body.action === 'queued') { + // sort the queuesConfig by order of matcher config exact match, with all true matches lined up ahead. + queuesConfig.sort((a, b) => { + return a.matcherConfig.exactMatch === b.matcherConfig.exactMatch ? 0 : a.matcherConfig.exactMatch ? -1 : 1; + }); + for (const queue of queuesConfig) { + if (canRunJob(body.workflow_job.labels, queue.matcherConfig.labelMatchers, queue.matcherConfig.exactMatch)) { + await sendActionRequest({ + id: body.workflow_job.id, + repositoryName: body.repository.name, + repositoryOwner: body.repository.owner.login, + eventType: githubEvent, + installationId: installationId, + queueId: queue.id, + queueFifo: queue.fifo, + }); + logger.info( + `Successfully queued job for ${body.repository.full_name} to the queue ${queue.id}`, + LogFields.print(), + ); + return { statusCode: 201 }; + } + } logger.warn( `Received event contains runner labels '${body.workflow_job.labels}' that are not accepted.`, LogFields.print(), @@ -148,18 +151,6 @@ async function handleWorkflowJob( body: `Received event contains runner labels '${body.workflow_job.labels}' that are not accepted.`, }; } - - const installationId = getInstallationId(body); - if (body.action === 'queued') { - await sendActionRequest({ - id: body.workflow_job.id, - repositoryName: body.repository.name, - repositoryOwner: body.repository.owner.login, - eventType: githubEvent, - installationId: installationId, - }); - logger.info(`Successfully queued job for ${body.repository.full_name}`, LogFields.print()); - } return { statusCode: 201 }; } @@ -175,8 +166,10 @@ function isRepoNotAllowed(repoFullName: string, repositoryWhiteList: string[]): return repositoryWhiteList.length > 0 && !repositoryWhiteList.includes(repoFullName); } -function canRunJob(job: WorkflowJobEvent, runnerLabels: string[], workflowLabelCheckAll: boolean): boolean { - const workflowJobLabels = job.workflow_job.labels; +function canRunJob(workflowJobLabels: string[], runnerLabels: string[], workflowLabelCheckAll: boolean): boolean { + runnerLabels = runnerLabels.map((element) => { + return element.toLowerCase(); + }); const match = workflowLabelCheckAll ? workflowJobLabels.every((l) => runnerLabels.includes(l.toLowerCase())) : workflowJobLabels.some((l) => runnerLabels.includes(l.toLowerCase())); diff --git a/modules/webhook/lambdas/webhook/test/resources/multi_runner_configurations.json b/modules/webhook/lambdas/webhook/test/resources/multi_runner_configurations.json new file mode 100644 index 0000000000..de0ed6945c --- /dev/null +++ b/modules/webhook/lambdas/webhook/test/resources/multi_runner_configurations.json @@ -0,0 +1,30 @@ +[ + { + "id": "ubuntu-queue-id", + "arn": "queueARN", + "fifo": false, + "matcherConfig": { + "labelMatchers": [ + "self-hosted", + "linux", + "x64", + "ubuntu" + ], + "exactMatch": true + } + }, + { + "id": "latest-queue-id", + "arn": "queueARN", + "fifo": false, + "matcherConfig": { + "labelMatchers": [ + "self-hosted", + "linux", + "x64", + "latest" + ], + "exactMatch": false + } + } +] diff --git a/modules/webhook/policies/lambda-publish-sqs-policy.json b/modules/webhook/policies/lambda-publish-sqs-policy.json index 84de5ee3f9..031560874b 100644 --- a/modules/webhook/policies/lambda-publish-sqs-policy.json +++ b/modules/webhook/policies/lambda-publish-sqs-policy.json @@ -4,7 +4,7 @@ { "Effect": "Allow", "Action": ["sqs:SendMessage", "sqs:GetQueueAttributes"], - "Resource": "${sqs_resource_arn}" + "Resource": ${sqs_resource_arns} } ] } diff --git a/modules/webhook/variables.tf b/modules/webhook/variables.tf index de26bb8b5a..59f2b05e2e 100644 --- a/modules/webhook/variables.tf +++ b/modules/webhook/variables.tf @@ -1,8 +1,3 @@ -variable "aws_region" { - description = "AWS region." - type = string -} - variable "environment" { description = "A name that identifies the environment, used as prefix and for tagging." type = string @@ -30,12 +25,17 @@ variable "tags" { default = {} } -variable "sqs_build_queue" { - description = "SQS queue to publish accepted build events." - type = object({ - id = string - arn = string - }) +variable "runner_config" { + description = "SQS queue to publish accepted build events based on the runner type." + type = map(object({ + arn = string + id = string + fifo = bool + matcherConfig = object({ + labelMatchers = list(string) + exactMatch = bool + }) + })) } variable "sqs_workflow_job_queue" { description = "SQS queue to monitor github events." @@ -117,24 +117,6 @@ variable "kms_key_arn" { default = null } -variable "runner_labels" { - description = "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `enable_workflow_job_labels_check`. GitHub read-only labels should not be provided." - type = string - default = "" -} - -variable "enable_workflow_job_labels_check" { - description = "If set to true all labels in the workflow job even are matched against the custom labels and GitHub labels (os, architecture and `self-hosted`). When the labels are not matching the event is dropped at the webhook." - type = bool - default = false -} - -variable "workflow_job_labels_check_all" { - description = "If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ label matches it will trigger the webhook. `enable_workflow_job_labels_check` must be true for this to take effect." - type = bool - default = true -} - variable "log_type" { description = "Logging format for lambda logging. Valid values are 'json', 'pretty', 'hidden'. " type = string @@ -173,12 +155,6 @@ variable "disable_check_wokflow_job_labels" { default = false } -variable "sqs_build_queue_fifo" { - description = "Enable a FIFO queue to remain the order of events received by the webhook. Suggest to set to true for repo level runners." - type = bool - default = false -} - variable "lambda_runtime" { description = "AWS Lambda runtime." type = string diff --git a/modules/webhook/webhook.tf b/modules/webhook/webhook.tf index 08bb943290..a98771ca7f 100644 --- a/modules/webhook/webhook.tf +++ b/modules/webhook/webhook.tf @@ -13,16 +13,12 @@ resource "aws_lambda_function" "webhook" { environment { variables = { - ENABLE_WORKFLOW_JOB_LABELS_CHECK = var.enable_workflow_job_labels_check - WORKFLOW_JOB_LABELS_CHECK_ALL = var.workflow_job_labels_check_all - ENVIRONMENT = var.prefix - LOG_LEVEL = var.log_level - LOG_TYPE = var.log_type - REPOSITORY_WHITE_LIST = jsonencode(var.repository_white_list) - RUNNER_LABELS = jsonencode(split(",", lower(var.runner_labels))) - SQS_URL_WEBHOOK = var.sqs_build_queue.id - SQS_IS_FIFO = var.sqs_build_queue_fifo - SQS_WORKFLOW_JOB_QUEUE = try(var.sqs_workflow_job_queue, null) != null ? var.sqs_workflow_job_queue.id : "" + ENVIRONMENT = var.prefix + LOG_LEVEL = var.log_level + LOG_TYPE = var.log_type + REPOSITORY_WHITE_LIST = jsonencode(var.repository_white_list) + RUNNER_CONFIG = jsonencode([for k, v in var.runner_config : v]) + SQS_WORKFLOW_JOB_QUEUE = try(var.sqs_workflow_job_queue, null) != null ? var.sqs_workflow_job_queue.id : "" } } @@ -76,7 +72,7 @@ resource "aws_iam_role_policy" "webhook_sqs" { role = aws_iam_role.webhook_lambda.name policy = templatefile("${path.module}/policies/lambda-publish-sqs-policy.json", { - sqs_resource_arn = var.sqs_build_queue.arn + sqs_resource_arns = jsonencode([for k, v in var.runner_config : v.arn]) }) } resource "aws_iam_role_policy" "webhook_workflow_job_sqs" { @@ -85,7 +81,7 @@ resource "aws_iam_role_policy" "webhook_workflow_job_sqs" { role = aws_iam_role.webhook_lambda.name policy = templatefile("${path.module}/policies/lambda-publish-sqs-policy.json", { - sqs_resource_arn = var.sqs_workflow_job_queue.arn + sqs_resource_arns = jsonencode([var.sqs_workflow_job_queue.arn]) }) } diff --git a/policies/lambda-publish-sqs-policy.json b/policies/lambda-publish-sqs-policy.json index 84de5ee3f9..eec8baa7eb 100644 --- a/policies/lambda-publish-sqs-policy.json +++ b/policies/lambda-publish-sqs-policy.json @@ -4,7 +4,7 @@ { "Effect": "Allow", "Action": ["sqs:SendMessage", "sqs:GetQueueAttributes"], - "Resource": "${sqs_resource_arn}" + "Resource": ${sqs_resource_arn} } ] } diff --git a/variables.tf b/variables.tf index 4e5db061cf..cd62c30618 100644 --- a/variables.tf +++ b/variables.tf @@ -549,12 +549,6 @@ variable "log_level" { } } -variable "runner_enable_workflow_job_labels_check" { - description = "If set to true all labels in the workflow job even are matched against the custom labels and GitHub labels (os, architecture and `self-hosted`). When the labels are not matching the event is dropped at the webhook." - type = bool - default = false -} - variable "runner_enable_workflow_job_labels_check_all" { description = "If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ label matches it will trigger the webhook. `runner_enable_workflow_job_labels_check` must be true for this to take effect." type = bool