diff --git a/MAINTENANCE.md b/MAINTENANCE.md new file mode 100644 index 0000000..2b06aef --- /dev/null +++ b/MAINTENANCE.md @@ -0,0 +1,124 @@ +# Regular maintenance + +## Goals + +The goals of the regular maintenance process are to update dependencies used by +the CI infrastructure, to pull in bug fixes and improvements and to ensure the +deployed infrastructure doesn't fall too far behind which would result in costly +updates. + +## Overview + +These are the steps taken when performing the regular maintenance: + +1. Update binaries and dependencies used in the AMIs (Amazon Machine Images), + the disk images used to start the VMs that run the CI infrastructure. + [Packer](https://www.packer.io/) + is used to create these images, this step is referred to as the Packer update. +1. Update Node.js dependencies used by AWS Lambdas that perform cleanup tasks + like AMI de-registration. +1. Update the infrastructure using [Terraform](https://www.terraform.io/), so + that the VMs use the newly built images. + +## Example update + +See https://github.com/envoyproxy/ci-infra/pull/7 for an example of a PR that +performed this update. + +## Packer update + +All packer configuration files and scripts are in the [ami-build](ami-build/) +directory. + +### Update the AZP agent version + +Edit the [ami-build/agent-setup.sh](ami-build/agent-setup.sh) file and update +the `AGENT_VERSION` variable to the [latest released +version](https://github.com/microsoft/azure-pipelines-agent/releases) of the AZP +agent. + +### Update the Ubuntu OS version + +Packer is used to build two AMIs, one for x64 architecture (intel/amd) and one +for the arm64 architecture. The Packer configuration for these two AMIs is in +these files: + +- [ami-build/azp-x64.json](ami-build/azp-x64.json) +- [ami-build/azp-arm64.json](ami-build/azp-arm64.json) + +Refer to this +[howto](https://learn.hashicorp.com/tutorials/packer/aws-get-started-build-image?in=packer/aws-get-started) +for details on how to build AMIs with Packer. You can also review the +[documentation](https://www.packer.io/plugins/builders/amazon/ebs) for the +Amazon EBS Packer builder. + +Edit each of the Packer configuration files and update the `name` under the +`source_ami_filter` to the latest LTS (long-term support) version of the Ubuntu +server image. This +[tutorial](https://ubuntu.com/tutorials/search-and-launch-ubuntu-22-04-in-aws-using-cli#2-search-for-the-right-ami) +outlines how to list images available in AWS. + +### Update the bazel-remote version + +Edit the +[ami-build/scripts/install-bazel-remote.sh](ami-build/scripts/install-bazel-remote.sh) +file and modify the target of the `wget` command to the latest released +`bazel-remote` version from https://github.com/buchgr/bazel-remote/tags. + +### Build updated AMIs with Packer + +Once the updates are performed, build and push the new AMIs to AWS by running: + +- `packer build azp-x64.json`. +- `packer build azp-arm64.json`. + +Note that this step should be done shortly before updating the infrastructure +using Terraform, since the `azp-dereg-lambda` runs daily and removes all but +the latest AMI. If the infrastructure isn't updated to use the latest AMI, the +lambda may delete an AMI that is in use. + +## Node.js dependencies update + +The directories +[instances/azp-cleanup-snapshots](instances/azp-cleanup-snapshots) and +[instances/azp-dereg-lambda](instances/azp-dereg-lambda) contain two [AWS +Lambdas](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html) written in +Node.js. + +To update the dependencies, first make sure you have +[npm-check-updates](https://www.npmjs.com/package/npm-check-updates) installed. + +Then go to each of the two directories and run `ncu -u`. + +## Terraform update + +### Build Node.js zip files + +Go to directories +[instances/azp-cleanup-snapshots](instances/azp-cleanup-snapshots) and +[instances/azp-dereg-lambda](instances/azp-dereg-lambda) and run: + +- `npm run build` + +This will produce two zip files in the [instances](instances) directory that +will be used by Terraform. + +### Apply terraform configs + +You can refer to [this +documentation](https://learn.hashicorp.com/tutorials/terraform/aws-build?in=terraform/aws-get-started) +for details on how to manage AWS infrastructure using Terraform. + +First get the AZP token used between AZP and the CI Agent: + +- `export TF_VAR_azp_token=$(aws s3 cp s3://cncf-envoy-token/azp_token -)` + + +Then run the Terraform update step. This should only be done after the PR is +reviewed and approved. In short, execute: + +- `terraform init` - to initialize the local Terraform installation. +- `terraform fmt` - to format any Terraform configuration files that were + modified. +- `terraform apply` - to update the AWS infrastructure applying local changes + and switching to the new AMIs. diff --git a/README.md b/README.md index 4743018..dfb4fb5 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,8 @@ The general idea is: - A Lambda watches the EC2 CloudWatch instance termination event, and properly deregisters the agent from AZP. + +# Regulare maintenance + +The regular maintenance process is documented in +[MAINTENANCE.md](MAINTENANCE.md). diff --git a/ami-build/agent-setup.sh b/ami-build/agent-setup.sh index 5f7cba5..6de342b 100644 --- a/ami-build/agent-setup.sh +++ b/ami-build/agent-setup.sh @@ -33,7 +33,7 @@ sudo mkdir -p /srv/azure-pipelines sudo chown -R azure-pipelines:azure-pipelines /srv/azure-pipelines/ [[ "${ARCH}" == "amd64" ]] && ARCH=x64 -AGENT_VERSION=2.185.1 +AGENT_VERSION=2.211.0 AGENT_FILE=vsts-agent-linux-${ARCH}-${AGENT_VERSION} sudo -u azure-pipelines /bin/bash -c "wget -q -O - https://vstsagentpackage.azureedge.net/agent/${AGENT_VERSION}/${AGENT_FILE}.tar.gz | tar zx -C /srv/azure-pipelines" diff --git a/ami-build/azp-arm64.json b/ami-build/azp-arm64.json index 98a9d9c..183c369 100644 --- a/ami-build/azp-arm64.json +++ b/ami-build/azp-arm64.json @@ -7,7 +7,7 @@ "source_ami_filter": { "filters": { "virtualization-type": "hvm", - "name": "ubuntu/images/*ubuntu-bionic-18.04-arm64-server-*", + "name": "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-*", "root-device-type": "ebs" }, "owners": [ diff --git a/ami-build/azp-x64.json b/ami-build/azp-x64.json index f0c4a1a..63b211f 100644 --- a/ami-build/azp-x64.json +++ b/ami-build/azp-x64.json @@ -7,7 +7,7 @@ "source_ami_filter": { "filters": { "virtualization-type": "hvm", - "name": "ubuntu/images/*ubuntu-bionic-18.04-amd64-server-*", + "name": "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*", "root-device-type": "ebs" }, "owners": [ diff --git a/ami-build/scripts/install-bazel-remote.sh b/ami-build/scripts/install-bazel-remote.sh index 67dcd67..09b7a9e 100755 --- a/ami-build/scripts/install-bazel-remote.sh +++ b/ami-build/scripts/install-bazel-remote.sh @@ -2,17 +2,8 @@ set -eu -o pipefail -export GOARCH=$(dpkg --print-architecture) +export ARCH=$(dpkg --print-architecture) +[[ "${ARCH}" == "amd64" ]] && ARCH="x86_64" -BUILD_TMP="$(mktemp -d)" -trap "chmod +w -R ${BUILD_TMP} && rm -rf ${BUILD_TMP}" EXIT - -cd "${BUILD_TMP}" - -curl -fsSL https://golang.org/dl/go1.16.3.linux-${GOARCH}.tar.gz | tar zx -export GOPATH="${BUILD_TMP}/gopath" - -git clone https://github.com/buchgr/bazel-remote -cd bazel-remote -PATH="${BUILD_TMP}/go/bin:${PATH}" ./linux-build.sh -sudo cp ./bazel-remote /usr/local/bin/bazel-remote +sudo wget -O /usr/local/bin/bazel-remote https://github.com/buchgr/bazel-remote/releases/download/v2.3.9/bazel-remote-2.3.9-linux-${ARCH} +sudo chmod 0755 /usr/local/bin/bazel-remote diff --git a/instances/azp-build-asg/init.sh.tpl b/instances/azp-build-asg/init.sh.tpl index e770c43..77187d8 100644 --- a/instances/azp-build-asg/init.sh.tpl +++ b/instances/azp-build-asg/init.sh.tpl @@ -41,7 +41,7 @@ Type=simple Restart=always RestartSec=1 User=bazel-remote -ExecStart=/usr/local/bin/bazel-remote --experimental_remote_asset_api --s3.endpoint s3.$AWS_DEFAULT_REGION.amazonaws.com --s3.bucket ${bazel_cache_bucket} --s3.prefix ${cache_prefix} --s3.iam_role_endpoint http://169.254.169.254 --max_size 30 --dir /dev/shm/bazel-remote-cache +ExecStart=/usr/local/bin/bazel-remote --experimental_remote_asset_api --s3.endpoint s3.$AWS_DEFAULT_REGION.amazonaws.com --s3.bucket ${bazel_cache_bucket} --s3.prefix ${cache_prefix} --s3.iam_role_endpoint http://169.254.169.254 --s3.auth_method=iam_role --max_size 30 --dir /dev/shm/bazel-remote-cache [Install] WantedBy=multi-user.target diff --git a/instances/azp-cleanup-snapshots/package-lock.json b/instances/azp-cleanup-snapshots/package-lock.json index d751af7..38dd3db 100644 --- a/instances/azp-cleanup-snapshots/package-lock.json +++ b/instances/azp-cleanup-snapshots/package-lock.json @@ -5,24 +5,34 @@ "requires": true, "packages": { "": { + "name": "azp-cleanup-snapshots", "version": "1.0.0", "license": "MIT", "devDependencies": { - "prettier": "^2.0.5" + "prettier": "^2.7.1" } }, "node_modules/prettier": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.0.5.tgz", - "integrity": "sha512-7PtVymN48hGcO4fGjybyBSIWDsLU4H4XlvOHfq91pz9kkGlonzwTfYkaIEwiRg/dAJF9YlbsduBAgtYLi+8cFg==", - "dev": true + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", + "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", + "dev": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } } }, "dependencies": { "prettier": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.0.5.tgz", - "integrity": "sha512-7PtVymN48hGcO4fGjybyBSIWDsLU4H4XlvOHfq91pz9kkGlonzwTfYkaIEwiRg/dAJF9YlbsduBAgtYLi+8cFg==", + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", + "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", "dev": true } } diff --git a/instances/azp-cleanup-snapshots/package.json b/instances/azp-cleanup-snapshots/package.json index a3191dc..eb0c80d 100644 --- a/instances/azp-cleanup-snapshots/package.json +++ b/instances/azp-cleanup-snapshots/package.json @@ -10,6 +10,6 @@ }, "license": "MIT", "devDependencies": { - "prettier": "^2.0.5" + "prettier": "^2.7.1" } } diff --git a/instances/azp-dereg-lambda/package-lock.json b/instances/azp-dereg-lambda/package-lock.json index b497666..0b92ff9 100644 --- a/instances/azp-dereg-lambda/package-lock.json +++ b/instances/azp-dereg-lambda/package-lock.json @@ -5,27 +5,54 @@ "requires": true, "packages": { "": { + "name": "azp-dereg-lambda", "version": "1.0.0", "license": "MIT", "dependencies": { - "axios": "^0.21.1" + "axios": "^1.1.3" }, "devDependencies": { - "prettier": "^2.0.5" + "prettier": "^2.7.1" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "node_modules/axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.1.3.tgz", + "integrity": "sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", "dependencies": { - "follow-redirects": "^1.10.0" + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" } }, "node_modules/follow-redirects": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.0.tgz", - "integrity": "sha512-0vRwd7RKQBTt+mgu87mtYeofLFZpTas2S9zY+jIeuLJMNvudIgF52nr19q40HOwH5RrhWIPuj9puybzSJiRrVg==", + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", "funding": [ { "type": "individual", @@ -41,32 +68,126 @@ } } }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/prettier": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.0.5.tgz", - "integrity": "sha512-7PtVymN48hGcO4fGjybyBSIWDsLU4H4XlvOHfq91pz9kkGlonzwTfYkaIEwiRg/dAJF9YlbsduBAgtYLi+8cFg==", - "dev": true + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", + "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", + "dev": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" } }, "dependencies": { + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.1.3.tgz", + "integrity": "sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==", + "requires": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", "requires": { - "follow-redirects": "^1.10.0" + "delayed-stream": "~1.0.0" } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" + }, "follow-redirects": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.0.tgz", - "integrity": "sha512-0vRwd7RKQBTt+mgu87mtYeofLFZpTas2S9zY+jIeuLJMNvudIgF52nr19q40HOwH5RrhWIPuj9puybzSJiRrVg==" + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" + }, + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "requires": { + "mime-db": "1.52.0" + } }, "prettier": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.0.5.tgz", - "integrity": "sha512-7PtVymN48hGcO4fGjybyBSIWDsLU4H4XlvOHfq91pz9kkGlonzwTfYkaIEwiRg/dAJF9YlbsduBAgtYLi+8cFg==", + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", + "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", "dev": true + }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" } } } diff --git a/instances/azp-dereg-lambda/package.json b/instances/azp-dereg-lambda/package.json index 01c06cf..864cef3 100644 --- a/instances/azp-dereg-lambda/package.json +++ b/instances/azp-dereg-lambda/package.json @@ -10,9 +10,9 @@ }, "license": "MIT", "dependencies": { - "axios": "^0.21.1" + "axios": "^1.1.3" }, "devDependencies": { - "prettier": "^2.0.5" + "prettier": "^2.7.1" } }