feat: Replace environment variable by prefix (philips-labs#1858)

We're looking at using this module to deploy multiple sets of runner types (x86_64 and arm architectures) but within the same conceptual "environment".

We use the "Environment" tag throughout our tooling, but the constraints of using the "environment" variable for resource naming mean that we need to essentially supply different environment names (eg "env-amd64" and "env-arm64"), even though they are not in different environments.

We also use the dot character (".") in our environment names, which isn't allowed in some resource names (eg SQS queue name).

This PR replaces the "environmet" variable by "prefix" to prefix resources crated by the module. The prefix is also used to set the tag: "ghr:environment" with the value of prefix for lambda's to orchestrate the instnaces.

You can still set the tag "environment" to all resources via the AWS provider.


Co-authored-by: Niek Palm <niek.palm@philips.com>

examples/arm64/main.tf

@@ -19,7 +19,7 @@ module "runners" {
   vpc_id                          = module.vpc.vpc_id
   subnet_ids                      = module.vpc.private_subnets
 
-  environment = local.environment
+  prefix = local.environment
   tags = {
     Project = "ProjectX"
   }

examples/default/main.tf

@@ -19,7 +19,7 @@ module "runners" {
   vpc_id                          = module.vpc.vpc_id
   subnet_ids                      = module.vpc.private_subnets
 
-  environment = local.environment
+  prefix = local.environment
   tags = {
     Project = "ProjectX"
   }

examples/ephemeral/main.tf

@@ -16,7 +16,7 @@ module "runners" {
   vpc_id                          = module.vpc.vpc_id
   subnet_ids                      = module.vpc.private_subnets
 
-  environment = local.environment
+  prefix = local.environment
   tags = {
     Project = "ProjectX"
   }

examples/permissions-boundary/main.tf

@@ -35,7 +35,7 @@ module "runners" {
   subnet_ids  = module.vpc.private_subnets
   kms_key_arn = aws_kms_key.github.key_id
 
-  environment = local.environment
+  prefix = local.environment
   tags = {
     Project = "ProjectX"
   }

examples/prebuilt/main.tf

@@ -15,7 +15,7 @@ module "runners" {
   vpc_id                          = module.vpc.vpc_id
   subnet_ids                      = module.vpc.private_subnets
 
-  environment = local.environment
+  prefix = local.environment
 
   github_app = {
     key_base64     = var.github_app_key_base64

examples/ubuntu/main.tf

@@ -16,7 +16,7 @@ module "runners" {
   vpc_id     = module.vpc.vpc_id
   subnet_ids = module.vpc.private_subnets
 
-  environment = local.environment
+  prefix = local.environment
   tags = {
     Project = "ProjectX"
   }

examples/windows/main.tf

@@ -10,10 +10,10 @@ resource "random_id" "random" {
 module "runners" {
   source = "../../"
 
-  aws_region  = local.aws_region
-  vpc_id      = module.vpc.vpc_id
-  subnet_ids  = module.vpc.private_subnets
-  environment = local.environment
+  aws_region = local.aws_region
+  vpc_id     = module.vpc.vpc_id
+  subnet_ids = module.vpc.private_subnets
+  prefix     = local.environment
 
   github_app = {
     key_base64     = var.github_app_key_base64

main.tf

@@ -1,7 +1,6 @@
 locals {
   tags = merge(var.tags, {
-    Environment       = var.environment,
-    "ghr:environment" = format("%s", var.environment)
+    "ghr:environment" = var.prefix
   })
 
   s3_action_runner_url = "s3://${module.runner_binaries.bucket.id}/${module.runner_binaries.runner_distribution_object_key}"
@@ -50,7 +49,7 @@ resource "aws_sqs_queue_policy" "build_queue_policy" {
 }
 
 resource "aws_sqs_queue" "queued_builds" {
-  name                        = "${var.environment}-queued-builds${var.fifo_build_queue ? ".fifo" : ""}"
+  name                        = "${var.prefix}-queued-builds${var.fifo_build_queue ? ".fifo" : ""}"
   delay_seconds               = var.delay_webhook_event
   visibility_timeout_seconds  = var.runners_scale_up_lambda_timeout
   message_retention_seconds   = var.job_queue_retention_in_seconds
@@ -74,7 +73,7 @@ resource "aws_sqs_queue_policy" "build_queue_dlq_policy" {
 
 resource "aws_sqs_queue" "queued_builds_dlq" {
   count = var.redrive_build_queue.enabled ? 1 : 0
-  name  = "${var.environment}-queued-builds_dead_letter"
+  name  = "${var.prefix}-queued-builds_dead_letter"
 
   tags = var.tags
 }
@@ -83,7 +82,7 @@ module "ssm" {
   source = "./modules/ssm"
 
   kms_key_arn = var.kms_key_arn
-  environment = var.environment
+  prefix      = var.prefix
   github_app  = var.github_app
   tags        = local.tags
 }
@@ -92,7 +91,7 @@ module "webhook" {
   source = "./modules/webhook"
 
   aws_region  = var.aws_region
-  environment = var.environment
+  prefix      = var.prefix
   tags        = local.tags
   kms_key_arn = var.kms_key_arn
 
@@ -127,7 +126,7 @@ module "runners" {
   aws_partition = var.aws_partition
   vpc_id        = var.vpc_id
   subnet_ids    = var.subnet_ids
-  environment   = var.environment
+  prefix        = var.prefix
   tags          = local.tags
 
   s3_bucket_runner_binaries   = module.runner_binaries.bucket
@@ -214,11 +213,11 @@ module "runners" {
 module "runner_binaries" {
   source = "./modules/runner-binaries-syncer"
 
-  aws_region  = var.aws_region
-  environment = var.environment
-  tags        = local.tags
+  aws_region = var.aws_region
+  prefix     = var.prefix
+  tags       = local.tags
 
-  distribution_bucket_name = "${var.environment}-dist-${random_string.random.result}"
+  distribution_bucket_name = "${var.prefix}-dist-${random_string.random.result}"
 
   runner_os                        = var.runner_os
   runner_architecture              = var.runner_architecture
@@ -244,10 +243,10 @@ module "runner_binaries" {
 }
 
 resource "aws_resourcegroups_group" "resourcegroups_group" {
-  name = "${var.environment}-group"
+  name = "${var.prefix}-group"
   resource_query {
     query = templatefile("${path.module}/templates/resource-group.json", {
-      environment = var.environment
+      environment = var.prefix
     })
   }
 }

modules/runner-binaries-syncer/runner-binaries-syncer.tf

@@ -1,6 +1,6 @@
 locals {
   lambda_zip = var.lambda_zip == null ? "${path.module}/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip" : var.lambda_zip
-  role_path  = var.role_path == null ? "/${var.environment}/" : var.role_path
+  role_path  = var.role_path == null ? "/${var.prefix}/" : var.role_path
   gh_binary_os_label = {
     windows = "win",
     linux   = "linux"
@@ -13,7 +13,7 @@ resource "aws_lambda_function" "syncer" {
   s3_object_version = var.syncer_lambda_s3_object_version != null ? var.syncer_lambda_s3_object_version : null
   filename          = var.lambda_s3_bucket == null ? local.lambda_zip : null
   source_code_hash  = var.lambda_s3_bucket == null ? filebase64sha256(local.lambda_zip) : null
-  function_name     = "${var.environment}-syncer"
+  function_name     = "${var.prefix}-syncer"
   role              = aws_iam_role.syncer_lambda.arn
   handler           = "index.handler"
   runtime           = "nodejs14.x"
@@ -63,7 +63,7 @@ resource "aws_cloudwatch_log_group" "syncer" {
 }
 
 resource "aws_iam_role" "syncer_lambda" {
-  name                 = "${var.environment}-action-syncer-lambda-role"
+  name                 = "${var.prefix}-action-syncer-lambda-role"
   assume_role_policy   = data.aws_iam_policy_document.lambda_assume_role_policy.json
   path                 = local.role_path
   permissions_boundary = var.role_permissions_boundary
@@ -92,7 +92,7 @@ data "aws_iam_policy_document" "lambda_assume_role_policy" {
 }
 
 resource "aws_iam_role_policy" "lambda_logging" {
-  name = "${var.environment}-lambda-logging-policy-syncer"
+  name = "${var.prefix}-lambda-logging-policy-syncer"
   role = aws_iam_role.syncer_lambda.id
 
   policy = templatefile("${path.module}/policies/lambda-cloudwatch.json", {
@@ -101,7 +101,7 @@ resource "aws_iam_role_policy" "lambda_logging" {
 }
 
 resource "aws_iam_role_policy" "syncer" {
-  name = "${var.environment}-lambda-syncer-s3-policy"
+  name = "${var.prefix}-lambda-syncer-s3-policy"
   role = aws_iam_role.syncer_lambda.id
 
   policy = templatefile("${path.module}/policies/lambda-syncer.json", {
@@ -110,7 +110,7 @@ resource "aws_iam_role_policy" "syncer" {
 }
 
 resource "aws_cloudwatch_event_rule" "syncer" {
-  name                = "${var.environment}-syncer-rule"
+  name                = "${var.prefix}-syncer-rule"
   schedule_expression = var.lambda_schedule_expression
   tags                = var.tags
 }

modules/runner-binaries-syncer/variables.tf

@@ -12,6 +12,18 @@ variable "tags" {
 variable "environment" {
   description = "A name that identifies the environment, used as prefix and for tagging."
   type        = string
+  default     = null
+
+  validation {
+    condition     = var.environment == null
+    error_message = "The \"environment\" variable is no longer used. To migrate, set the \"prefix\" variable to the original value of \"environment\" and optionally, add \"Environment\" to the \"tags\" variable map with the same value."
+  }
+}
+
+variable "prefix" {
+  description = "The prefix used for naming resources"
+  type        = string
+  default     = "github-actions"
 }
 
 variable "distribution_bucket_name" {

modules/runners/README.md

@@ -130,7 +130,7 @@ yarn run dist
 | <a name="input_enable_runner_detailed_monitoring"></a> [enable\_runner\_detailed\_monitoring](#input\_enable\_runner\_detailed\_monitoring) | Enable detailed monitoring for runners | `bool` | `false` | no |
 | <a name="input_enable_ssm_on_runners"></a> [enable\_ssm\_on\_runners](#input\_enable\_ssm\_on\_runners) | Enable to allow access to the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances. | `bool` | n/a | yes |
 | <a name="input_enabled_userdata"></a> [enabled\_userdata](#input\_enabled\_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI | `bool` | `true` | no |
-| <a name="input_environment"></a> [environment](#input\_environment) | A name that identifies the environment, used as prefix and for tagging. | `string` | n/a | yes |
+| <a name="input_prefix"></a> [prefix](#input\_prefix) | The prefix used for naming resources | `string` | `github-actions` | no |
 | <a name="input_ghes_ssl_verify"></a> [ghes\_ssl\_verify](#input\_ghes\_ssl\_verify) | GitHub Enterprise SSL verification. Set to 'false' when custom certificate (chains) is used for GitHub Enterprise Server (insecure). | `bool` | `true` | no |
 | <a name="input_ghes_url"></a> [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. DO NOT SET IF USING PUBLIC GITHUB | `string` | `null` | no |
 | <a name="input_github_app_parameters"></a> [github\_app\_parameters](#input\_github\_app\_parameters) | Parameter Store for GitHub App Parameters. | <pre>object({<br>    key_base64 = map(string)<br>    id         = map(string)<br>  })</pre> | n/a | yes |
@@ -171,7 +171,7 @@ yarn run dist
 | <a name="input_runner_extra_labels"></a> [runner\_extra\_labels](#input\_runner\_extra\_labels) | Extra labels for the runners (GitHub). Separate each label by a comma | `string` | `""` | no |
 | <a name="input_runner_group_name"></a> [runner\_group\_name](#input\_runner\_group\_name) | Name of the runner group. | `string` | `"Default"` | no |
 | <a name="input_runner_iam_role_managed_policy_arns"></a> [runner\_iam\_role\_managed\_policy\_arns](#input\_runner\_iam\_role\_managed\_policy\_arns) | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no |
-| <a name="input_runner_log_files"></a> [runner\_log\_files](#input\_runner\_log\_files) | (optional) List of logfiles to send to CloudWatch, will only be used if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: If true, the log group name will be prefixed with `/github-self-hosted-runners/<var.environment>`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | <pre>list(object({<br>    log_group_name   = string<br>    prefix_log_group = bool<br>    file_path        = string<br>    log_stream_name  = string<br>  }))</pre> | `null` | no |
+| <a name="input_runner_log_files"></a> [runner\_log\_files](#input\_runner\_log\_files) | (optional) List of logfiles to send to CloudWatch, will only be used if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: If true, the log group name will be prefixed with `/github-self-hosted-runners/<var.prefix>`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | <pre>list(object({<br>    log_group_name   = string<br>    prefix_log_group = bool<br>    file_path        = string<br>    log_stream_name  = string<br>  }))</pre> | `null` | no |
 | <a name="input_runner_os"></a> [runner\_os](#input\_runner\_os) | The EC2 Operating System type to use for action runner instances (linux,windows). | `string` | `"linux"` | no |
 | <a name="input_runner_run_as"></a> [runner\_run\_as](#input\_runner\_run\_as) | Run the GitHub actions agent as user. | `string` | `"ec2-user"` | no |
 | <a name="input_runners_lambda_s3_key"></a> [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no |
@@ -183,7 +183,7 @@ yarn run dist
 | <a name="input_scale_up_reserved_concurrent_executions"></a> [scale\_up\_reserved\_concurrent\_executions](#input\_scale\_up\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no |
 | <a name="input_sqs_build_queue"></a> [sqs\_build\_queue](#input\_sqs\_build\_queue) | SQS queue to consume accepted build events. | <pre>object({<br>    arn = string<br>  })</pre> | n/a | yes |
 | <a name="input_subnet_ids"></a> [subnet\_ids](#input\_subnet\_ids) | List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. | `list(string)` | n/a | yes |
-| <a name="input_tags"></a> [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name. | `map(string)` | `{}` | no |
 | <a name="input_userdata_post_install"></a> [userdata\_post\_install](#input\_userdata\_post\_install) | User-data script snippet to insert after GitHub action runner install | `string` | `""` | no |
 | <a name="input_userdata_pre_install"></a> [userdata\_pre\_install](#input\_userdata\_pre\_install) | User-data script snippet to insert before GitHub action runner install | `string` | `""` | no |
 | <a name="input_userdata_template"></a> [userdata\_template](#input\_userdata\_template) | Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored. | `string` | `null` | no |

modules/runners/lambdas/runners/src/aws/runners.test.ts

@@ -101,7 +101,7 @@ describe('list instances', () => {
       Filters: [
         { Name: 'tag:Application', Values: ['github-action-runner'] },
         { Name: 'instance-state-name', Values: ['running', 'pending'] },
-        { Name: 'tag:Environment', Values: [ENVIRONMENT] },
+        { Name: 'tag:ghr:environment', Values: [ENVIRONMENT] },
       ],
     });
   });

modules/runners/lambdas/runners/src/aws/runners.ts

@@ -54,7 +54,7 @@ export async function listEC2Runners(filters: ListRunnerFilters | undefined = un
 
   if (filters) {
     if (filters.environment !== undefined) {
-      ec2Filters.push({ Name: 'tag:Environment', Values: [filters.environment] });
+      ec2Filters.push({ Name: 'tag:ghr:environment', Values: [filters.environment] });
     }
     if (filters.runnerType && filters.runnerOwner) {
       ec2Filters.push({ Name: `tag:Type`, Values: [filters.runnerType] });

modules/runners/logging.tf

@@ -30,7 +30,7 @@ locals {
     ]
   )
   logfiles = var.enable_cloudwatch_agent ? [for l in local.runner_log_files : {
-    "log_group_name" : l.prefix_log_group ? "/github-self-hosted-runners/${var.environment}/${l.log_group_name}" : "/${l.log_group_name}"
+    "log_group_name" : l.prefix_log_group ? "/github-self-hosted-runners/${var.prefix}/${l.log_group_name}" : "/${l.log_group_name}"
     "log_stream_name" : l.log_stream_name
     "file_path" : l.file_path
   }] : []
@@ -42,7 +42,7 @@ locals {
 
 resource "aws_ssm_parameter" "cloudwatch_agent_config_runner" {
   count = var.enable_cloudwatch_agent ? 1 : 0
-  name  = "${var.environment}-cloudwatch_agent_config_runner"
+  name  = "${var.prefix}-cloudwatch_agent_config_runner"
   type  = "String"
   value = var.cloudwatch_config != null ? var.cloudwatch_config : templatefile("${path.module}/templates/cloudwatch_config.json", {
     logfiles = jsonencode(local.logfiles)

modules/runners/main.tf

@@ -1,15 +1,15 @@
 locals {
   tags = merge(
     {
-      "Name" = format("%s-action-runner", var.environment)
+      "Name" = format("%s-action-runner", var.prefix)
     },
     var.tags,
   )
 
   name_sg               = var.overrides["name_sg"] == "" ? local.tags["Name"] : var.overrides["name_sg"]
   name_runner           = var.overrides["name_runner"] == "" ? local.tags["Name"] : var.overrides["name_runner"]
-  role_path             = var.role_path == null ? "/${var.environment}/" : var.role_path
-  instance_profile_path = var.instance_profile_path == null ? "/${var.environment}/" : var.instance_profile_path
+  role_path             = var.role_path == null ? "/${var.prefix}/" : var.role_path
+  instance_profile_path = var.instance_profile_path == null ? "/${var.prefix}/" : var.instance_profile_path
   lambda_zip            = var.lambda_zip == null ? "${path.module}/lambdas/runners/runners.zip" : var.lambda_zip
   userdata_template     = var.userdata_template == null ? local.default_userdata_template[var.runner_os] : var.userdata_template
   kms_key_arn           = var.kms_key_arn != null ? var.kms_key_arn : ""
@@ -54,7 +54,7 @@ data "aws_ami" "runner" {
 }
 
 resource "aws_launch_template" "runner" {
-  name = "${var.environment}-action-runner"
+  name = "${var.prefix}-action-runner"
 
   dynamic "block_device_mappings" {
     for_each = var.block_device_mappings != null ? var.block_device_mappings : []
@@ -131,7 +131,7 @@ resource "aws_launch_template" "runner" {
     ghes_url        = var.ghes_url
     ghes_ssl_verify = var.ghes_ssl_verify
     ## retain these for backwards compatibility
-    environment                     = var.environment
+    environment                     = var.prefix
     enable_cloudwatch_agent         = var.enable_cloudwatch_agent
     ssm_key_cloudwatch_agent_config = var.enable_cloudwatch_agent ? aws_ssm_parameter.cloudwatch_agent_config_runner[0].name : ""
   })) : ""
@@ -143,7 +143,7 @@ resource "aws_launch_template" "runner" {
 
 resource "aws_security_group" "runner_sg" {
   count       = var.enable_managed_runner_security_group ? 1 : 0
-  name_prefix = "${var.environment}-github-actions-runner-sg"
+  name_prefix = "${var.prefix}-github-actions-runner-sg"
   description = "Github Actions Runner security group"
 
   vpc_id = var.vpc_id

modules/runners/policies-runner.tf

@@ -1,15 +1,15 @@
 data "aws_caller_identity" "current" {}
 
 resource "aws_iam_role" "runner" {
-  name                 = "${var.environment}-runner-role"
+  name                 = "${var.prefix}-runner-role"
   assume_role_policy   = templatefile("${path.module}/policies/instance-role-trust-policy.json", {})
   path                 = local.role_path
   permissions_boundary = var.role_permissions_boundary
   tags                 = local.tags
 }
 
 resource "aws_iam_instance_profile" "runner" {
-  name = "${var.environment}-runner-profile"
+  name = "${var.prefix}-runner-profile"
   role = aws_iam_role.runner.name
   path = local.instance_profile_path
 }
@@ -26,8 +26,8 @@ resource "aws_iam_role_policy" "ssm_parameters" {
   role = aws_iam_role.runner.name
   policy = templatefile("${path.module}/policies/instance-ssm-parameters-policy.json",
     {
-      arn_ssm_parameters_prefix = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}-*"
-      arn_ssm_parameters_path   = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.environment}/*"
+      arn_ssm_parameters_prefix = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.prefix}-*"
+      arn_ssm_parameters_path   = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter/${var.prefix}/*"
     }
   )
 }

modules/runners/pool.tf

@@ -4,7 +4,7 @@ module "pool" {
   source = "./pool"
 
   config = {
-    environment = var.environment
+    prefix = var.prefix
     ghes = {
       ssl_verify = var.ghes_ssl_verify
       url        = var.ghes_url