From c2eb5463fd3ab0e3649fa3e8486a972ed471901b Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Mon, 1 Aug 2022 08:17:04 -0700 Subject: [PATCH 1/5] Use JQ args instead. --- .github/workflows/terminate-zombie-build-instances.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terminate-zombie-build-instances.yml b/.github/workflows/terminate-zombie-build-instances.yml index f45112a55cb6..fc88519f3c2f 100644 --- a/.github/workflows/terminate-zombie-build-instances.yml +++ b/.github/workflows/terminate-zombie-build-instances.yml @@ -30,7 +30,7 @@ jobs: # Since the AWS cli returns an ISO HH:MM timestamp, and Jq only accepts Z timestamps, we define a function toZ to convert this. export to_terminate=$(aws ec2 describe-instances --no-paginate --filters Name=instance-type,Values=c5.2xlarge Name=instance-state-name,Values=running \ --query 'Reservations[*].Instances[*].{Instance:InstanceId,LaunchTime:LaunchTime}' --output json \ - | jq 'def toZ(str): str | (split("+")[0] + "Z") | fromdate ; + | jq --arg TIME_LIMIT "$TIME_LIMIT" 'def toZ(str): str | (split("+")[0] + "Z") | fromdate ; flatten | map( { InstanceId: .Instance, LaunchTime: toZ(.LaunchTime) } ) | map( select ( .LaunchTime < (now - $TIME_LIMIT) ) )') echo "MARKED FOR TERMINATION: ${to_terminate}" From 78635b43af4f8e8a1e557357e616ca7ffd5a347b Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Mon, 1 Aug 2022 08:39:26 -0700 Subject: [PATCH 2/5] Directly reference env vars. --- .github/workflows/terminate-zombie-build-instances.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/terminate-zombie-build-instances.yml b/.github/workflows/terminate-zombie-build-instances.yml index fc88519f3c2f..ace2df00a382 100644 --- a/.github/workflows/terminate-zombie-build-instances.yml +++ b/.github/workflows/terminate-zombie-build-instances.yml @@ -30,8 +30,8 @@ jobs: # Since the AWS cli returns an ISO HH:MM timestamp, and Jq only accepts Z timestamps, we define a function toZ to convert this. export to_terminate=$(aws ec2 describe-instances --no-paginate --filters Name=instance-type,Values=c5.2xlarge Name=instance-state-name,Values=running \ --query 'Reservations[*].Instances[*].{Instance:InstanceId,LaunchTime:LaunchTime}' --output json \ - | jq --arg TIME_LIMIT "$TIME_LIMIT" 'def toZ(str): str | (split("+")[0] + "Z") | fromdate ; - flatten | map( { InstanceId: .Instance, LaunchTime: toZ(.LaunchTime) } ) | map( select ( .LaunchTime < (now - $TIME_LIMIT) ) )') + | jq 'def toZ(str): str | (split("+")[0] + "Z") | fromdate ; + flatten | map( { InstanceId: .Instance, LaunchTime: toZ(.LaunchTime) } ) | map( select ( .LaunchTime < (now - env.TIME_LIMIT) ) )') echo "MARKED FOR TERMINATION: ${to_terminate}" From 14ba4160a9984e2d8bd34b6228cdfc4ed02ebb11 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Mon, 1 Aug 2022 08:41:11 -0700 Subject: [PATCH 3/5] Export into env var. --- .github/workflows/terminate-zombie-build-instances.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terminate-zombie-build-instances.yml b/.github/workflows/terminate-zombie-build-instances.yml index ace2df00a382..c01c202f9ceb 100644 --- a/.github/workflows/terminate-zombie-build-instances.yml +++ b/.github/workflows/terminate-zombie-build-instances.yml @@ -22,7 +22,7 @@ jobs: run: | set -euxo pipefail - TIME_LIMIT=10800 # 3 hours + export TIME_LIMIT=10800 # 3 hours aws configure set default.region us-east-2 From 7dcdd90eceb21a99e8c07ebd422a5df2c708c373 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Mon, 1 Aug 2022 08:43:58 -0700 Subject: [PATCH 4/5] Make sure to convert to number. --- .github/workflows/terminate-zombie-build-instances.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terminate-zombie-build-instances.yml b/.github/workflows/terminate-zombie-build-instances.yml index c01c202f9ceb..99b5600243ba 100644 --- a/.github/workflows/terminate-zombie-build-instances.yml +++ b/.github/workflows/terminate-zombie-build-instances.yml @@ -31,7 +31,7 @@ jobs: export to_terminate=$(aws ec2 describe-instances --no-paginate --filters Name=instance-type,Values=c5.2xlarge Name=instance-state-name,Values=running \ --query 'Reservations[*].Instances[*].{Instance:InstanceId,LaunchTime:LaunchTime}' --output json \ | jq 'def toZ(str): str | (split("+")[0] + "Z") | fromdate ; - flatten | map( { InstanceId: .Instance, LaunchTime: toZ(.LaunchTime) } ) | map( select ( .LaunchTime < (now - env.TIME_LIMIT) ) )') + flatten | map( { InstanceId: .Instance, LaunchTime: toZ(.LaunchTime) } ) | map( select ( .LaunchTime < (now - (env.TIME_LIMIT|tonumber)) ) )') echo "MARKED FOR TERMINATION: ${to_terminate}" From b1b11f78289369bca7f5d371146b7be2a915f9af Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Mon, 1 Aug 2022 08:45:51 -0700 Subject: [PATCH 5/5] Increase to 4 hours. --- .github/workflows/terminate-zombie-build-instances.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/terminate-zombie-build-instances.yml b/.github/workflows/terminate-zombie-build-instances.yml index 99b5600243ba..8de735476c81 100644 --- a/.github/workflows/terminate-zombie-build-instances.yml +++ b/.github/workflows/terminate-zombie-build-instances.yml @@ -13,7 +13,7 @@ jobs: terminate: runs-on: ubuntu-latest steps: - - name: List and Terminate Instances Older Than 3 Hours + - name: List and Terminate Instances Older Than 4 Hours env: AWS_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} @@ -22,7 +22,7 @@ jobs: run: | set -euxo pipefail - export TIME_LIMIT=10800 # 3 hours + export TIME_LIMIT=14400 # 4 hours aws configure set default.region us-east-2