From 025be9195719279b51a85c14c6941cc18737fba4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 21 May 2024 12:46:19 +0000 Subject: [PATCH] chore(ci): spot capacity and reaping --- .github/ensure-builder/action.yml | 2 +- .github/ensure-tester/action.yml | 2 +- .github/spot-runner-action/dist/index.js | 2 +- .github/spot-runner-action/src/ec2.ts | 2 +- .github/workflows/ci.yml | 62 ++++++++---------------- scripts/run_on_builder | 2 +- scripts/run_on_tester | 2 +- 7 files changed, 27 insertions(+), 47 deletions(-) diff --git a/.github/ensure-builder/action.yml b/.github/ensure-builder/action.yml index 6cd8a433c31..ef482704f8d 100644 --- a/.github/ensure-builder/action.yml +++ b/.github/ensure-builder/action.yml @@ -24,7 +24,7 @@ runs: echo "spot_strategy=BestEffort" >> $GITHUB_OUTPUT echo "runner_label=$USERNAME-$runner_type" >> $GITHUB_OUTPUT if [[ $TYPE == builder-x86 ]]; then - # 128-core x86 instance types with least evictions + # 128-core x86 instance types, aws chooses for us based on capacity echo "instance_type=i4i.32xlarge m6a.32xlarge m6i.32xlarge m6id.32xlarge m6idn.32xlarge m6in.32xlarge m7a.32xlarge r6a.32xlarge r6i.32xlarge r6id.32xlarge r6in.32xlarge r7iz.32xlarge" >> $GITHUB_OUTPUT echo "ami_id=ami-04d8422a9ba4de80f" >> $GITHUB_OUTPUT echo "ebs_cache_size=256" >> $GITHUB_OUTPUT diff --git a/.github/ensure-tester/action.yml b/.github/ensure-tester/action.yml index 24d61cbbb65..a87c0bd9387 100644 --- a/.github/ensure-tester/action.yml +++ b/.github/ensure-tester/action.yml @@ -61,7 +61,7 @@ runs: ec2_subnet_id: subnet-4cfabd25 ec2_security_group_id: sg-0ccd4e5df0dcca0c9 ec2_key_name: "build-instance" - ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + ec2_instance_tags: '[]' # Set up a context for this run - name: Copy Repo to Tester diff --git a/.github/spot-runner-action/dist/index.js b/.github/spot-runner-action/dist/index.js index bca6f38f5f8..b7d177e1611 100644 --- a/.github/spot-runner-action/dist/index.js +++ b/.github/spot-runner-action/dist/index.js @@ -327,7 +327,7 @@ class Ec2Instance { LaunchTemplateConfigs: [fleetLaunchConfig], ClientToken: clientToken, SpotOptions: { - AllocationStrategy: "capacity-optimized", + AllocationStrategy: "price-capacity-optimized", }, TargetCapacitySpecification: { TotalTargetCapacity: 1, diff --git a/.github/spot-runner-action/src/ec2.ts b/.github/spot-runner-action/src/ec2.ts index 3879b1dea1d..695b7564464 100644 --- a/.github/spot-runner-action/src/ec2.ts +++ b/.github/spot-runner-action/src/ec2.ts @@ -246,7 +246,7 @@ export class Ec2Instance { LaunchTemplateConfigs: [fleetLaunchConfig], ClientToken: clientToken, SpotOptions: { - AllocationStrategy: "capacity-optimized", + AllocationStrategy: "price-capacity-optimized", }, TargetCapacitySpecification: { TotalTargetCapacity: 1, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 184848f54f4..bcca8dbff1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}" RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} - USERNAME: ${{ github.event.pull_request.user.login || github.actor }} + USERNAME: ubuntu-latest GITHUB_TOKEN: ${{ github.token }} GH_SELF_HOSTED_RUNNER_TOKEN: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -26,14 +26,6 @@ env: WAIT_FOR_RUNNERS: false jobs: - setup: - uses: ./.github/workflows/setup-runner.yml - with: - username: ${{ github.event.pull_request.user.login || github.actor }} - runner_type: builder-x86 - secrets: inherit - # job just to alert that you *might* hit missing CI files, but if not you can continue - ci-consistency: runs-on: ubuntu-latest # Only check PRs for consistency (not master) @@ -66,8 +58,7 @@ jobs: fi build: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} bench_list: ${{ steps.bench_list.outputs.list }} @@ -155,7 +146,7 @@ jobs: bench-summary: needs: bench-e2e - runs-on: ${{ inputs.username || github.actor }}-x86 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -182,8 +173,7 @@ jobs: run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment noir-format: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -202,8 +192,7 @@ jobs: # barretenberg (prover) native and AVM (public VM) tests # only ran on x86 for resource reasons (memory intensive) bb-native-tests: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -219,8 +208,7 @@ jobs: run: earthly-ci --no-output +test --hardware_concurrency=64 bb-js-test: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -233,8 +221,7 @@ jobs: run: earthly-ci --no-output ./+test noir-test: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -245,8 +232,7 @@ jobs: run: earthly-ci --no-output ./noir+test noir-examples: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -257,8 +243,7 @@ jobs: run: earthly-ci --no-output ./noir+examples noir-packages-test: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -269,8 +254,7 @@ jobs: run: earthly-ci --no-output ./noir+packages-test noir-projects: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -283,7 +267,7 @@ jobs: yarn-project-formatting: needs: noir-projects - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -298,7 +282,7 @@ jobs: yarn-project-test: # quiet machine needs: [noir-projects, build, bb-native-tests] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -312,7 +296,7 @@ jobs: prover-client-test: needs: noir-projects - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -325,7 +309,7 @@ jobs: build-acir-tests: needs: build - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -338,7 +322,7 @@ jobs: barretenberg-acir-tests-bb: needs: build-acir-tests - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -352,7 +336,7 @@ jobs: barretenberg-acir-tests-sol: needs: build-acir-tests - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -366,7 +350,7 @@ jobs: barretenberg-acir-tests-bb-js: needs: build-acir-tests - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -379,20 +363,19 @@ jobs: run: earthly-ci --no-output ./+barretenberg-acir-tests-bb.js l1-contracts-test: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } - uses: ./.github/ci-setup-action with: - concurrency_key: l1-contracts-${{ github.event.pull_request.user.login || github.actor }}-x86 + concurrency_key: l1-contracts-ubuntu-latest-x86 - name: "Test l1 contracts" run: earthly-ci --no-output ./l1-contracts+test docs-preview: needs: build - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 if: github.event.number steps: - uses: actions/checkout@v4 @@ -406,7 +389,6 @@ jobs: bb-bench: runs-on: ubuntu-latest - needs: setup steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -429,8 +411,7 @@ jobs: scripts/earthly-ci --no-output ./barretenberg/cpp/+bench-ultra-honk --bench_mode=cache protocol-circuits-gates-report: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ubuntu-latest-x86 permissions: pull-requests: write steps: @@ -465,7 +446,6 @@ jobs: merge-check: runs-on: ubuntu-latest needs: - - setup - e2e - bb-native-tests - bb-bench diff --git a/scripts/run_on_builder b/scripts/run_on_builder index e9522f973ed..a170a9727db 100755 --- a/scripts/run_on_builder +++ b/scripts/run_on_builder @@ -4,4 +4,4 @@ set -eu # Enter the repo root cd "$(dirname "$0")/.." -ssh -o ControlMaster=auto -o ControlPath=~/.ssh_mux_%h_%p_%r -o ControlPersist=10m -o TCPKeepAlive=no -o ServerAliveCountMax=20 -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -i "$BUILDER_SPOT_KEY" ubuntu@"$BUILDER_SPOT_IP" "$@" +ssh -o ControlMaster=auto -o ControlPath=~/.ssh_mux_%h_%p_%r -o ControlPersist=30s -o TCPKeepAlive=no -o ServerAliveCountMax=5 -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -i "$BUILDER_SPOT_KEY" ubuntu@"$BUILDER_SPOT_IP" "$@" diff --git a/scripts/run_on_tester b/scripts/run_on_tester index ae82e429f15..73c2ddbdd40 100755 --- a/scripts/run_on_tester +++ b/scripts/run_on_tester @@ -4,4 +4,4 @@ set -eu # Enter the repo root cd "$(dirname "$0")/.." -ssh -o ControlMaster=auto -o ControlPath=~/.ssh_mux_%h_%p_%r -o ControlPersist=10m -o TCPKeepAlive=no -o ServerAliveCountMax=20 -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -i "$SPOT_KEY" ubuntu@"$SPOT_IP" "$@" +ssh -o ControlMaster=auto -o ControlPath=~/.ssh_mux_%h_%p_%r -o ControlPersist=30s -o TCPKeepAlive=no -o ServerAliveCountMax=5 -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -i "$SPOT_KEY" ubuntu@"$SPOT_IP" "$@"