diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 000000000..9c0423502 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,101 @@ +name: libp2p perf test + +# How to configure a repository for running this workflow: +# 1. Configure auth for the AWS provider as per https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration +# 2. Run 'terraform init' and 'terraform apply' in 'perf/terraform/configs/remote' to create the resources needed for this workflow +# 3. Go to https://console.aws.amazon.com/iamv2/home?#/users/details/perf?section=security_credentials +# 4. Click 'Create access key' to get the access key ID and secret access key +# 5. Go to https://github.com/libp2p/test-plans/settings/secrets/actions +# 6. Click 'New repository secret', set the name to 'PERF_AWS_SECRET_ACCESS_KEY', and paste the secret access key from step 5 +# 7. Go to https://github.com/libp2p/test-plans/settings/variables/actions +# 8. Click 'New repository variable', set the name to 'PERF_AWS_ACCESS_KEY_ID', and paste the access key ID from step 5 + +on: + workflow_dispatch: + inputs: + push: + description: 'Push the benchmark results to the repository' + required: false + default: 'true' + +jobs: + perf: + name: Perf + runs-on: ubuntu-latest + timeout-minutes: 40 + defaults: + run: + shell: bash + working-directory: perf + env: + AWS_ACCESS_KEY_ID: ${{ vars.PERF_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.PERF_AWS_SECRET_ACCESS_KEY }} + TF_IN_AUTOMATION: 1 + TF_INPUT: 0 + steps: + - name: Checkout test-plans + uses: actions/checkout@v3 + with: + repository: ${{ github.repository }} + ref: ${{ github.ref }} + - id: ssh + name: Generate SSH key + run: | + make ssh-keygen + echo "key<> $GITHUB_OUTPUT + while read -r line; do + echo "::add-mask::$line" + echo "$line" >> $GITHUB_OUTPUT + done < terraform/modules/short_lived/files/perf + echo "EOF" >> $GITHUB_OUTPUT + - name: Configure SSH + uses: webfactory/ssh-agent@d4b9b8ff72958532804b70bbe600ad43b36d5f2e # v0.8.0 + with: + ssh-private-key: ${{ steps.ssh.outputs.key }} + - name: Configure git + run: | + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com>" + git config --global user.name "${GITHUB_ACTOR}" + - name: Configure terraform + uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3 + - name: Init terraform + id: init + run: terraform init + working-directory: perf/terraform/configs/local + - name: Apply terraform + run: terraform apply -auto-approve + working-directory: perf/terraform/configs/local + - id: server + name: Retrieve server's IP + run: terraform output -raw server_ip + working-directory: perf/terraform/configs/local + - id: client + name: Retrieve client's IP + run: terraform output -raw client_ip + working-directory: perf/terraform/configs/local + - name: Download dependencies + run: npm ci + working-directory: perf/runner + - name: Run tests + env: + SERVER_IP: ${{ steps.server.outputs.stdout }} + CLIENT_IP: ${{ steps.client.outputs.stdout }} + run: npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP + working-directory: perf/runner + - name: Push + if: github.event.inputs.push == 'true' + run: | + git add benchmark-results.json + git commit -m "perf: update benchmark results" + git push + working-directory: perf/runner + - name: Archive + if: github.event.intputs.push == 'false' + uses: actions/upload-artifact@v2 + with: + name: benchmark-results + path: perf/runner/benchmark-results.json + - name: Destroy terraform + if: always() && steps.init.outputs.exitcode == 0 + run: terraform destroy -auto-approve + working-directory: perf/terraform/configs/local diff --git a/perf/Makefile b/perf/Makefile new file mode 100644 index 000000000..1d826b01c --- /dev/null +++ b/perf/Makefile @@ -0,0 +1,5 @@ +ssh-keygen: + ssh-keygen -t ed25519 -f ./terraform/modules/short_lived/files/perf -N '' + +ssh-add: + ssh-add ./terraform/modules/short_lived/files/perf diff --git a/perf/README.md b/perf/README.md index 5463f410b..36c17edf4 100644 --- a/perf/README.md +++ b/perf/README.md @@ -10,21 +10,27 @@ Benchmark results can be visualized with https://observablehq.com/@mxinden-works ## Provision infrastructure -1. `cd terraform` -2. Save your public SSH key as the file `./user.pub`. +### Bootstrap + +1. Save your public SSH key as the file `./terraform/modules/short_lived/files/perf.pub`; or generate a new key pair with `make ssh-keygen` and add it to your SSH agent with `make ssh-add`. +2. `cd terraform/configs/local` 3. `terraform init` 4. `terraform apply` +5. `CLIENT_IP=$(terraform output -raw client_ip)` +6. `SERVER_IP=$(terraform output -raw server_ip)` ## Build and run implementations +_WARNING_: Running the perf tests might take a while. + 1. `cd runner` 2. `npm ci` -3. `npm run start -- --client-public-ip $(terraform output -raw -state ../terraform/terraform.tfstate client_public_ip) --server-public-ip $(terraform output -raw -state ../terraform/terraform.tfstate server_public_ip)` +3. `npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP` ## Deprovision infrastructure -1. `cd terraform` -3. `terraform destroy` +1. `cd terraform/configs/local` +2. `terraform destroy` ## Adding a new implementation diff --git a/perf/terraform/.gitignore b/perf/terraform/.gitignore index bb09c5300..80795e780 100644 --- a/perf/terraform/.gitignore +++ b/perf/terraform/.gitignore @@ -24,7 +24,7 @@ override.tf.json *_override.tf.json # Include override files you do wish to add to version control using negated pattern -# !example_override.tf +!configs/remote/terraform_override.tf # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan # example: *tfplan* @@ -32,5 +32,3 @@ override.tf.json # Ignore CLI configuration files .terraformrc terraform.rc - -*.pub diff --git a/perf/terraform/.terraform.lock.hcl b/perf/terraform/.terraform.lock.hcl deleted file mode 100644 index e54d18d34..000000000 --- a/perf/terraform/.terraform.lock.hcl +++ /dev/null @@ -1,25 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "4.61.0" - constraints = "~> 4.0" - hashes = [ - "h1:mJSchOA6VkYwEsi+tuspadRmyyE+FGZGYJFUt5kHV+M=", - "zh:051e2588410b7448a5c4c30d668948dd6fdfa8037700bfc00fb228986ccbf3a5", - "zh:082fbcf9706b48d0880ba552a11c29527e228dadd6d83668d0789abda24e5922", - "zh:0e0e72f214fb24f4f9c601cab088a2d8e00ec3327c451bc753911951d773214a", - "zh:3af6d38ca733ca66cce15c6a5735ded7c18348ad26040ebd9a59778b2cd9cf6c", - "zh:404898bc2258bbb9527fa06c72cb927ca011fd9bc3f4b90931c0912652c3f9e9", - "zh:4f617653b0f17a7708bc896f029c4ab0b677a1a1c987bd77166acad1d82db469", - "zh:5dbe393355ac137aa3fd329e3d24871f27012d3ba93d714485b55820df240349", - "zh:6067c2127eb5c879227aca671f101de6dcba909d0d8d15d5711480351962a248", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:a939f94461f91aa3b7ec7096271e2714309bd917fe9a03e02f68afb556d65e0f", - "zh:b21227b9082e5fafe8b7c415dc6a99c0d82da05492457377a5fe7d4acaed80e2", - "zh:b8d9f09ed5fc8c654b768b7bee1237eaf1e2287c898249e740695055fb0fe072", - "zh:d360e1e185b148ff6b1d0ed4f7d574e08f2391697ab43df62085b04a1a5b1284", - "zh:da962da17ddda744911cb1e92b983fa3874d73a28f3ee72faa9ddb6680a63774", - "zh:e2f1c4f5ebeb4fd7ef690178168a4c529025b54a91bb7a087dcea48e0b82737a", - ] -} diff --git a/perf/terraform/configs/README.md b/perf/terraform/configs/README.md new file mode 100644 index 000000000..97ce004a4 --- /dev/null +++ b/perf/terraform/configs/README.md @@ -0,0 +1,13 @@ +# Configs + +The terraform configs defined in this directory are used to provision the infrastructure for the libp2p perf tests. + +The configs are named after the type of backend they use. The defaults for what parts of infrastructure they provision differ between the two. + +## local + +Terraform state in this configuration will be stored locally. The defaults are configured for a single performance benchmark run, i.e. `terraform apply` will bring up short-lived infrastructure only. It will skip long-lived infrastructure like the clean-up Lambda and the instance launch template. + +## remote + +Terraform state here will be stored remotely in an S3 bucket. `terraform apply` will only bring up the long-lived infrastructure needed to run the performance benchmarks It will skip short-lived infrastructure like launching EC2 instances. diff --git a/perf/terraform/configs/local/.terraform.lock.hcl b/perf/terraform/configs/local/.terraform.lock.hcl new file mode 100644 index 000000000..e490491e1 --- /dev/null +++ b/perf/terraform/configs/local/.terraform.lock.hcl @@ -0,0 +1,44 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.3.0" + hashes = [ + "h1:NaDbOqAcA9d8DiAS5/6+5smXwN3/+twJGb3QRiz6pNw=", + "zh:0869128d13abe12b297b0cd13b8767f10d6bf047f5afc4215615aabc39c2eb4f", + "zh:481ed837d63ba3aa45dd8736da83e911e3509dee0e7961bf5c00ed2644f807b3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9f08fe2977e2166849be24fb9f394e4d2697414d463f7996fd0d7beb4e19a29c", + "zh:9fe566deeafd460d27999ca0bbfd85426a5fcfcb40007b23884deb76da127b6f", + "zh:a1bd9a60925d9769e0da322e4523330ee86af9dc2e770cba1d0247a999ef29cb", + "zh:bb4094c8149f74308b22a87e1ac19bcccca76e8ef021b571074d9bccf1c0c6f0", + "zh:c8984c9def239041ce41ec8e19bbd76a49e74ed2024ff736dad60429dee89bcc", + "zh:ea4bb5ae73db1de3a586e62f39106f5e56770804a55aa5e6b4f642df973e0e75", + "zh:f44a9d596ecc3a8c5653f56ba0cd202ad93b49f76767f4608daf7260b813289e", + "zh:f5c5e6cc9f7f070020ab7d95fcc9ed8e20d5cf219978295a71236e22cbb6d508", + "zh:fd2273f51dcc8f43403bf1e425ba9db08a57c3ddcba5ad7a51742ccde21ca611", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.67.0" + constraints = "4.67.0" + hashes = [ + "h1:5Zfo3GfRSWBaXs4TGQNOflr1XaYj6pRnVJLX5VAjFX4=", + "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060", + "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6", + "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183", + "zh:4a002990b9f4d6d225d82cb2fb8805789ffef791999ee5d9cb1fef579aeff8f1", + "zh:559a2b5ace06b878c6de3ecf19b94fbae3512562f7a51e930674b16c2f606e29", + "zh:6a07da13b86b9753b95d4d8218f6dae874cf34699bca1470d6effbb4dee7f4b7", + "zh:768b3bfd126c3b77dc975c7c0e5db3207e4f9997cf41aa3385c63206242ba043", + "zh:7be5177e698d4b547083cc738b977742d70ed68487ce6f49ecd0c94dbf9d1362", + "zh:8b562a818915fb0d85959257095251a05c76f3467caa3ba95c583ba5fe043f9b", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c385d03a958b54e2afd5279cd8c7cbdd2d6ca5c7d6a333e61092331f38af7cf", + "zh:b3ca45f2821a89af417787df8289cb4314b273d29555ad3b2a5ab98bb4816b3b", + "zh:da3c317f1db2469615ab40aa6baba63b5643bae7110ff855277a1fb9d8eb4f2c", + "zh:dc6430622a8dc5cdab359a8704aec81d3825ea1d305bbb3bbd032b1c6adfae0c", + "zh:fac0d2ddeadf9ec53da87922f666e1e73a603a611c57bcbc4b86ac2821619b1d", + ] +} diff --git a/perf/terraform/configs/local/terraform.tf b/perf/terraform/configs/local/terraform.tf new file mode 100644 index 000000000..22c745f5a --- /dev/null +++ b/perf/terraform/configs/local/terraform.tf @@ -0,0 +1,121 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "4.67.0" + } + } +} + +locals { + tags = { + Project = "perf" + } +} + +provider "aws" { + alias = "us-west-2" + region = "us-west-2" + default_tags { + tags = local.tags + } +} + +provider "aws" { + alias = "us-east-1" + region = "us-east-1" + default_tags { + tags = local.tags + } +} + + +variable "ci_enabled" { + type = bool + description = "Whether or not to create resources required to automate the setup in CI (e.g. IAM user, cleanup Lambda)" + default = false +} + +variable "long_lived_enabled" { + type = bool + description = "Whether or not to create long lived resources (in CI, used across runs; e.g. VPCs)" + default = false +} + +variable "short_lived_enabled" { + type = bool + description = "Whether or not to create short lived resources (in CI, specific to each run; e.g. EC2 instances)" + default = true +} + +module "ci" { + count = var.ci_enabled ? 1 : 0 + + source = "../../modules/ci" + + regions = ["us-west-2", "us-east-1"] + tags = local.tags + + providers = { + aws = aws.us-west-2 + } +} + +module "long_lived_server" { + count = var.long_lived_enabled ? 1 : 0 + + source = "../../modules/long_lived" + + region = "us-west-2" + ami = "ami-0747e613a2a1ff483" + + providers = { + aws = aws.us-west-2 + } +} + +module "long_lived_client" { + count = var.long_lived_enabled ? 1 : 0 + + source = "../../modules/long_lived" + + region = "us-east-1" + ami = "ami-06e46074ae430fba6" + + providers = { + aws = aws.us-east-1 + } +} + +module "short_lived_server" { + count = var.short_lived_enabled ? 1 : 0 + + source = "../../modules/short_lived" + + providers = { + aws = aws.us-west-2 + } + + depends_on = [module.long_lived_server] +} + +module "short_lived_client" { + count = var.short_lived_enabled ? 1 : 0 + + source = "../../modules/short_lived" + + providers = { + aws = aws.us-east-1 + } + + depends_on = [module.long_lived_client] +} + +output "client_ip" { + value = var.short_lived_enabled ? module.short_lived_client[0].public_ip : null +} + +output "server_ip" { + value = var.short_lived_enabled ? module.short_lived_server[0].public_ip : null +} + diff --git a/perf/terraform/configs/remote/.terraform.lock.hcl b/perf/terraform/configs/remote/.terraform.lock.hcl new file mode 100644 index 000000000..e490491e1 --- /dev/null +++ b/perf/terraform/configs/remote/.terraform.lock.hcl @@ -0,0 +1,44 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.3.0" + hashes = [ + "h1:NaDbOqAcA9d8DiAS5/6+5smXwN3/+twJGb3QRiz6pNw=", + "zh:0869128d13abe12b297b0cd13b8767f10d6bf047f5afc4215615aabc39c2eb4f", + "zh:481ed837d63ba3aa45dd8736da83e911e3509dee0e7961bf5c00ed2644f807b3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9f08fe2977e2166849be24fb9f394e4d2697414d463f7996fd0d7beb4e19a29c", + "zh:9fe566deeafd460d27999ca0bbfd85426a5fcfcb40007b23884deb76da127b6f", + "zh:a1bd9a60925d9769e0da322e4523330ee86af9dc2e770cba1d0247a999ef29cb", + "zh:bb4094c8149f74308b22a87e1ac19bcccca76e8ef021b571074d9bccf1c0c6f0", + "zh:c8984c9def239041ce41ec8e19bbd76a49e74ed2024ff736dad60429dee89bcc", + "zh:ea4bb5ae73db1de3a586e62f39106f5e56770804a55aa5e6b4f642df973e0e75", + "zh:f44a9d596ecc3a8c5653f56ba0cd202ad93b49f76767f4608daf7260b813289e", + "zh:f5c5e6cc9f7f070020ab7d95fcc9ed8e20d5cf219978295a71236e22cbb6d508", + "zh:fd2273f51dcc8f43403bf1e425ba9db08a57c3ddcba5ad7a51742ccde21ca611", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.67.0" + constraints = "4.67.0" + hashes = [ + "h1:5Zfo3GfRSWBaXs4TGQNOflr1XaYj6pRnVJLX5VAjFX4=", + "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060", + "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6", + "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183", + "zh:4a002990b9f4d6d225d82cb2fb8805789ffef791999ee5d9cb1fef579aeff8f1", + "zh:559a2b5ace06b878c6de3ecf19b94fbae3512562f7a51e930674b16c2f606e29", + "zh:6a07da13b86b9753b95d4d8218f6dae874cf34699bca1470d6effbb4dee7f4b7", + "zh:768b3bfd126c3b77dc975c7c0e5db3207e4f9997cf41aa3385c63206242ba043", + "zh:7be5177e698d4b547083cc738b977742d70ed68487ce6f49ecd0c94dbf9d1362", + "zh:8b562a818915fb0d85959257095251a05c76f3467caa3ba95c583ba5fe043f9b", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c385d03a958b54e2afd5279cd8c7cbdd2d6ca5c7d6a333e61092331f38af7cf", + "zh:b3ca45f2821a89af417787df8289cb4314b273d29555ad3b2a5ab98bb4816b3b", + "zh:da3c317f1db2469615ab40aa6baba63b5643bae7110ff855277a1fb9d8eb4f2c", + "zh:dc6430622a8dc5cdab359a8704aec81d3825ea1d305bbb3bbd032b1c6adfae0c", + "zh:fac0d2ddeadf9ec53da87922f666e1e73a603a611c57bcbc4b86ac2821619b1d", + ] +} diff --git a/perf/terraform/configs/remote/terraform.tf b/perf/terraform/configs/remote/terraform.tf new file mode 120000 index 000000000..e51ea571a --- /dev/null +++ b/perf/terraform/configs/remote/terraform.tf @@ -0,0 +1 @@ +../local/terraform.tf \ No newline at end of file diff --git a/perf/terraform/configs/remote/terraform_override.tf b/perf/terraform/configs/remote/terraform_override.tf new file mode 100644 index 000000000..e12102fb0 --- /dev/null +++ b/perf/terraform/configs/remote/terraform_override.tf @@ -0,0 +1,19 @@ +terraform { + backend "s3" { + bucket = "libp2p-terraform-state" + key = "github.com/libp2p/test-plans/perf/terraform/configs/remote/terraform.tfstate" + region = "us-west-2" + } +} + +variable "ci_enabled" { + default = true +} + +variable "long_lived_enabled" { + default = true +} + +variable "short_lived_enabled" { + default = false +} diff --git a/perf/terraform/modules/ci/cleanup.tf b/perf/terraform/modules/ci/cleanup.tf new file mode 100644 index 000000000..27e7bcedc --- /dev/null +++ b/perf/terraform/modules/ci/cleanup.tf @@ -0,0 +1,107 @@ +data "archive_file" "cleanup" { + type = "zip" + source_file = "${path.module}/files/cleanup.py" + output_path = "${path.module}/files/cleanup.zip" +} + +resource "aws_lambda_function" "cleanup" { + filename = data.archive_file.cleanup.output_path + source_code_hash = data.archive_file.cleanup.output_base64sha256 + function_name = "perf-cleanup" + role = aws_iam_role.cleanup.arn + handler = "cleanup.lambda_handler" + runtime = "python3.9" + memory_size = 128 + timeout = 30 + + environment { + variables = { + REGIONS = jsonencode(var.regions) + TAGS = jsonencode(var.tags) + MAX_AGE_MINUTES = 50 + } + } +} + +resource "aws_cloudwatch_log_group" "cleanup" { + name = "/aws/lambda/${aws_lambda_function.cleanup.function_name}" + retention_in_days = 7 +} + +resource "aws_cloudwatch_event_rule" "cleanup" { + name = "perf-cleanup-rule" + schedule_expression = "cron(37 * * * ? *)" # 00:37, 01:37, 02:37, ..., 23:37 +} + +resource "aws_cloudwatch_event_target" "cleanup" { + rule = aws_cloudwatch_event_rule.cleanup.name + arn = aws_lambda_function.cleanup.arn +} + +resource "aws_lambda_permission" "cleanup" { + statement_id = "AllowExecutionFromCloudWatch" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.cleanup.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.cleanup.arn +} + +data "aws_iam_policy_document" "cleanup_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "cleanup" { + name = "perf-cleanup-lambda-role" + assume_role_policy = data.aws_iam_policy_document.cleanup_assume_role.json +} + +data "aws_iam_policy_document" "cleanup" { + statement { + actions = ["ec2:DescribeInstances", "ec2:DescribeTags", "ec2:DescribeKeyPairs"] + resources = ["*"] + effect = "Allow" + } + + statement { + actions = ["ec2:TerminateInstances", "ec2:DeleteKeyPair"] + resources = ["*"] + effect = "Allow" + + dynamic "condition" { + for_each = var.tags + + content { + test = "StringEquals" + variable = "ec2:ResourceTag/${condition.key}" + values = [condition.value] + } + } + } +} + +resource "aws_iam_role_policy" "cleanup" { + name = "perf-cleanup-lamda-policy" + role = aws_iam_role.cleanup.name + policy = data.aws_iam_policy_document.cleanup.json +} + +data "aws_iam_policy_document" "cleanup_logging" { + statement { + actions = ["logs:CreateLogStream", "logs:PutLogEvents"] + resources = ["${aws_cloudwatch_log_group.cleanup.arn}*"] + effect = "Allow" + } +} + +resource "aws_iam_role_policy" "cleanup_logging" { + name = "perf-lambda-logging" + role = aws_iam_role.cleanup.name + policy = data.aws_iam_policy_document.cleanup_logging.json +} diff --git a/perf/terraform/modules/ci/files/.gitignore b/perf/terraform/modules/ci/files/.gitignore new file mode 100644 index 000000000..47066beae --- /dev/null +++ b/perf/terraform/modules/ci/files/.gitignore @@ -0,0 +1,2 @@ +# generated ZIP for AWS Lambda +cleanup.zip diff --git a/perf/terraform/modules/ci/files/cleanup.json b/perf/terraform/modules/ci/files/cleanup.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/perf/terraform/modules/ci/files/cleanup.json @@ -0,0 +1 @@ +{} diff --git a/perf/terraform/modules/ci/files/cleanup.py b/perf/terraform/modules/ci/files/cleanup.py new file mode 100644 index 000000000..4cb2f483a --- /dev/null +++ b/perf/terraform/modules/ci/files/cleanup.py @@ -0,0 +1,45 @@ +import boto3 +import os +import json +import datetime + +regions = json.loads(os.environ['REGIONS']) # Assuming this is a JSON array +tags = json.loads(os.environ['TAGS']) # Assuming this is a JSON object +max_age_minutes = int(os.environ['MAX_AGE_MINUTES']) # Assuming this is an integer + +# TODO: Find and delete unused key pairs +def lambda_handler(event, context): + # iterate over all regions + for region in regions: + ec2 = boto3.client('ec2', region_name=region) + + now = datetime.datetime.now(datetime.timezone.utc) + + filters = [{'Name': 'instance-state-name', 'Values': ['running']}] + filters = filters + [{ + 'Name': 'tag:' + k, + 'Values': [v] + } for k, v in tags.items()] + + response = ec2.describe_instances(Filters=filters) + + instances = [] + + for reservation in response['Reservations']: + for instance in reservation['Instances']: + launch_time = instance['LaunchTime'] + instance_id = instance['InstanceId'] + + print( + f'Instance ID: {instance_id} has been running since {launch_time}.') + + if launch_time < now - datetime.timedelta(minutes=max_age_minutes): + print( + f'Instance ID: {instance_id} has been running for more than {max_age_minutes} minutes.') + instances.append(instance_id) + + if instances: + ec2.terminate_instances(InstanceIds=instances) + print(f'Terminating instances: {instances}') + + ec2.describe_ diff --git a/perf/terraform/modules/ci/files/cleanup.sh b/perf/terraform/modules/ci/files/cleanup.sh new file mode 100755 index 000000000..e540775c2 --- /dev/null +++ b/perf/terraform/modules/ci/files/cleanup.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# This script can be used to test the cleanup lambda. +# It requires the AWS CLI and SAM CLI to be installed. +# You can get SAM CLI at https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html + +sam local invoke Cleanup --template cleanup.yml --event cleanup.json diff --git a/perf/terraform/modules/ci/files/cleanup.yml b/perf/terraform/modules/ci/files/cleanup.yml new file mode 100644 index 000000000..21336ff6f --- /dev/null +++ b/perf/terraform/modules/ci/files/cleanup.yml @@ -0,0 +1,19 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: An AWS Lambda application. + +Resources: + Cleanup: + Type: AWS::Serverless::Function + Properties: + Handler: cleanup.lambda_handler + Runtime: python3.9 + CodeUri: . + Environment: + Variables: + REGIONS: '["us-west-2", "us-east-1"]' + TAGS: '{"Project":"perf", "Name":"node"}' + MAX_AGE_MINUTES: '30' + Policies: + - AmazonEC2FullAccess + Timeout: 30 diff --git a/perf/terraform/modules/ci/main.tf b/perf/terraform/modules/ci/main.tf new file mode 100644 index 000000000..6237e3306 --- /dev/null +++ b/perf/terraform/modules/ci/main.tf @@ -0,0 +1,38 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "4.67.0" + } + } +} + +variable "tags" { + type = map(string) + description = "Tags that the perf resources are tagged with" +} + +variable "regions" { + type = list(string) + description = "Regions that the perf resources are created in" +} + +resource "aws_iam_user" "perf" { + name = "perf" +} + +# TODO: Make the policy more restrictive; it needs to be able to create/destroy instances and key pairs +data "aws_iam_policy_document" "perf" { + statement { + actions = ["ec2:*"] + resources = ["*"] + effect = "Allow" + } +} + +resource "aws_iam_user_policy" "perf" { + name = "perf" + user = aws_iam_user.perf.name + + policy = data.aws_iam_policy_document.perf.json +} diff --git a/perf/terraform/user-data.sh b/perf/terraform/modules/long_lived/files/user-data.sh similarity index 100% rename from perf/terraform/user-data.sh rename to perf/terraform/modules/long_lived/files/user-data.sh diff --git a/perf/terraform/region/main.tf b/perf/terraform/modules/long_lived/main.tf similarity index 60% rename from perf/terraform/region/main.tf rename to perf/terraform/modules/long_lived/main.tf index 81a3db2bc..7fbc9588b 100644 --- a/perf/terraform/region/main.tf +++ b/perf/terraform/modules/long_lived/main.tf @@ -1,30 +1,26 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "4.67.0" + } + } +} + variable "region" { - description = "The AWS region to create resources in" + description = "The AWS region of the provider" } variable "ami" { description = "The Amazon Machine Image to use" } -variable "common_tags" { - type = map(string) - description = "Common tags to apply to all resources" -} - locals { availability_zone = "${var.region}a" } -provider "aws" { - region = var.region -} - resource "aws_vpc" "perf" { cidr_block = "10.0.0.0/16" - - tags = merge(var.common_tags, { - Name = "perf" - }) } resource "aws_subnet" "perf" { @@ -32,18 +28,10 @@ resource "aws_subnet" "perf" { cidr_block = "10.0.0.0/16" availability_zone = local.availability_zone map_public_ip_on_launch = true - - tags = merge(var.common_tags, { - Name = "perf" - }) } resource "aws_internet_gateway" "perf" { vpc_id = aws_vpc.perf.id - - tags = merge(var.common_tags, { - Name = "perf-igw" - }) } resource "aws_route_table" "perf" { @@ -53,10 +41,6 @@ resource "aws_route_table" "perf" { cidr_block = "0.0.0.0/0" gateway_id = aws_internet_gateway.perf.id } - - tags = merge(var.common_tags, { - Name = "perf-route-table" - }) } resource "aws_route_table_association" "perf" { @@ -105,39 +89,23 @@ resource "aws_security_group" "restricted_inbound" { protocol = "-1" cidr_blocks = ["0.0.0.0/0"] } - - tags = merge(var.common_tags, { - Name = "restricted_inbound_sg" - }) } -resource "aws_key_pair" "perf" { - key_name = "user-public-key" - public_key = file("./user.pub") -} - -resource "aws_instance" "node" { - ami = var.ami +resource "aws_launch_template" "perf" { + name = "perf-node" + image_id = var.ami instance_type = "m5n.8xlarge" - subnet_id = aws_subnet.perf.id - - key_name = aws_key_pair.perf.key_name - - vpc_security_group_ids = [aws_security_group.restricted_inbound.id] - # Debug via: # - /var/log/cloud-init.log and # - /var/log/cloud-init-output.log - user_data = file("./user-data.sh") - user_data_replace_on_change = true + user_data = filebase64("${path.module}/files/user-data.sh") - tags = merge(var.common_tags, { - Name = "node" - }) -} + instance_initiated_shutdown_behavior = "terminate" -output "node_public_ip" { - value = aws_instance.node.public_ip - description = "Public IP address of the node instance" + network_interfaces { + subnet_id = aws_subnet.perf.id + security_groups = [aws_security_group.restricted_inbound.id] + delete_on_termination = true + } } diff --git a/perf/terraform/modules/short_lived/files/.gitignore b/perf/terraform/modules/short_lived/files/.gitignore new file mode 100644 index 000000000..d53e7f9d3 --- /dev/null +++ b/perf/terraform/modules/short_lived/files/.gitignore @@ -0,0 +1,3 @@ +# generated SSH key +perf +perf.pub diff --git a/perf/terraform/modules/short_lived/main.tf b/perf/terraform/modules/short_lived/main.tf new file mode 100644 index 000000000..17782a51c --- /dev/null +++ b/perf/terraform/modules/short_lived/main.tf @@ -0,0 +1,29 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "4.67.0" + } + } +} + +resource "aws_key_pair" "perf" { + key_name_prefix = "perf-" + public_key = file("${path.module}/files/perf.pub") +} + +resource "aws_instance" "perf" { + tags = { + Name = "perf-node" + } + + launch_template { + name = "perf-node" + } + + key_name = aws_key_pair.perf.key_name +} + +output "public_ip" { + value = aws_instance.perf.public_ip +} diff --git a/perf/terraform/terraform.tf b/perf/terraform/terraform.tf deleted file mode 100644 index 79ad18050..000000000 --- a/perf/terraform/terraform.tf +++ /dev/null @@ -1,38 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 4.0" - } - } -} - -module "server_region" { - source = "./region" - region = "us-west-2" - ami = "ami-0747e613a2a1ff483" - - common_tags = { - Project = "perf" - } -} - -module "client_region" { - source = "./region" - region = "us-east-1" - ami = "ami-06e46074ae430fba6" - - common_tags = { - Project = "perf" - } -} - -output "server_public_ip" { - value = module.server_region.node_public_ip - description = "Public IP address of the server instance" -} - -output "client_public_ip" { - value = module.client_region.node_public_ip - description = "Public IP address of the client instance" -}