Benchmark: running individual benchmarks #29
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Benchmark: running individual benchmarks" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| benchmark_name: | |
| type: choice | |
| required: true | |
| description: The name of the benchmark to run | |
| options: | |
| - verify_fibair | |
| - fibonacci | |
| - revm_transfer | |
| - regex | |
| - base64_json | |
| - fib_e2e | |
| instance_type: | |
| type: string | |
| required: false | |
| description: The type of runner to start ({1,2,4,8,16,32,48,64}cpu-linux-{arm64,x64}) | |
| default: 64cpu-linux-arm64 | |
| memory_allocator: | |
| type: string | |
| required: false | |
| description: Memory allocator to use (mimalloc or jemalloc) | |
| default: mimalloc | |
| app_log_blowup: | |
| type: number | |
| required: false | |
| description: Application level log blowup | |
| default: 2 | |
| agg_log_blowup: | |
| type: number | |
| required: false | |
| description: Aggregation (leaf) level log blowup | |
| default: 2 | |
| root_log_blowup: | |
| type: number | |
| required: false | |
| description: Root level log blowup (only for e2e) | |
| default: 2 | |
| internal_log_blowup: | |
| type: number | |
| required: false | |
| description: Internal level log blowup (only for e2e) | |
| default: 2 | |
| max_segment_length: | |
| type: number | |
| required: false | |
| description: Max segment length for continuations, must be larger than 524288 | |
| default: 1048476 | |
| e2e_bench: | |
| type: boolean | |
| required: true | |
| description: Whether to run the e2e benchmark | |
| workflow_call: | |
| inputs: | |
| benchmark_name: | |
| type: string | |
| required: true | |
| description: The name of the benchmark to run | |
| benchmark_id: | |
| type: string | |
| required: true | |
| description: The id of the benchmark to run, must be unique within matrix | |
| instance_type: | |
| type: string | |
| required: false | |
| description: The type of runner to start ({1,2,4,8,16,32,48,64}cpu-linux-{arm64,x64}) | |
| default: 64cpu-linux-arm64 | |
| memory_allocator: | |
| type: string | |
| required: false | |
| description: Memory allocator to use (mimalloc or jemalloc) | |
| default: mimalloc | |
| app_log_blowup: | |
| type: number | |
| required: false | |
| description: Application level log blowup | |
| default: 2 | |
| agg_log_blowup: | |
| type: number | |
| required: false | |
| description: Aggregation (leaf) level log blowup | |
| default: 2 | |
| root_log_blowup: | |
| type: number | |
| required: false | |
| description: Root level log blowup (only for e2e) | |
| default: 2 | |
| internal_log_blowup: | |
| type: number | |
| required: false | |
| description: Internal level log blowup (only for e2e) | |
| default: 2 | |
| max_segment_length: | |
| type: number | |
| required: false | |
| description: Max segment length for continuations, must be larger than 524288 | |
| default: 1048476 | |
| e2e_bench: | |
| type: boolean | |
| required: true | |
| description: Whether to run the e2e benchmark | |
| env: | |
| S3_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/results | |
| S3_METRICS_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/metrics | |
| PUBLIC_S3_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/flamegraphs | |
| FEATURE_FLAGS: "bench-metrics,parallel,nightly-features" | |
| CMD_ARGS: "" | |
| INPUT_ARGS: "" | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| permissions: | |
| contents: write | |
| jobs: | |
| bench-new: | |
| name: Run benchmark on workflow ref/branch | |
| runs-on: | |
| - runs-on | |
| - runner=${{ inputs.instance_type }} | |
| - run-id=${{ github.run_id }} | |
| - family=m7 | |
| - tag=bench-${{ inputs.benchmark_name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} | |
| steps: | |
| ########################################################################## | |
| # Environment setup # | |
| ########################################################################## | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| - uses: dtolnay/rust-toolchain@nightly | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| cache-on-failure: true | |
| - name: Install architecture specific tools | |
| run: | | |
| source ci/scripts/utils.sh | |
| install_s5cmd | |
| - name: Display workflow inputs | |
| run: echo "${{ toJSON(inputs) }}" | |
| - name: Feature flags | |
| if: contains(github.event.pull_request.labels.*.name, 'run-benchmark') || (github.event_name == 'push' && github.ref == 'refs/heads/main') | |
| run: | | |
| echo "Adding aggregation feature flag" | |
| echo "FEATURE_FLAGS=${FEATURE_FLAGS},aggregation" >> $GITHUB_ENV | |
| - name: Setup e2e (halo2 and arguments) | |
| run: | | |
| E2E_BENCH=${{ inputs.e2e_bench }} | |
| echo "E2E_BENCH=${E2E_BENCH}" >> $GITHUB_ENV | |
| if [[ "${E2E_BENCH}" == "true" ]]; then | |
| ROOT_ARG="--root_log_blowup ${{ inputs.root_log_blowup }}" | |
| INTERNAL_ARG="--internal_log_blowup ${{ inputs.internal_log_blowup }}" | |
| echo "INPUT_ARGS=${ROOT_ARG} ${INTERNAL_ARG} ${INPUT_ARGS}" >> $GITHUB_ENV | |
| bash ./extensions/native/recursion/trusted_setup_s3.sh | |
| export PARAMS_DIR=$(pwd)/params | |
| fi | |
| - name: Set BIN_NAME and CMD_ARGS | |
| run: | | |
| CMD_ARGS="--features ${FEATURE_FLAGS}" | |
| echo "CMD_ARGS=${CMD_ARGS}" >> $GITHUB_ENV | |
| echo "BIN_NAME=${{ inputs.benchmark_name }}" >> $GITHUB_ENV | |
| - name: Set working directory | |
| id: set-working-dir | |
| run: | | |
| WORKING_DIR=$(jq -r --arg name "${{ inputs.benchmark_name }}" ' | |
| .benchmarks[] | | |
| select(.name == $name) | | |
| .working_directory | |
| ' ./ci/benchmark-config.json) | |
| RELATIVE_PATH=$(python3 -c "import os.path; print(os.path.relpath('.', '$WORKING_DIR'))") | |
| echo "working_dir=$WORKING_DIR" >> $GITHUB_OUTPUT | |
| echo "relative_path=$RELATIVE_PATH" >> $GITHUB_OUTPUT | |
| # Metric name is unique within a run (matrix) | |
| # When uploading to S3, use ${METRIC_NAME}-${current_sha}.[md/json] | |
| - name: Set metric name | |
| run: | | |
| METRIC_NAME=${{ inputs.benchmark_id || inputs.benchmark_name }} | |
| echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV | |
| METRIC_PATH=".bench_metrics/${METRIC_NAME}.json" | |
| echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV | |
| - name: Set input args | |
| run: | | |
| INSTANCE_TYPE="--instance_type ${{ inputs.instance_type }}" | |
| MEMORY_ALLOCATOR="--memory_allocator ${{ inputs.memory_allocator }}" | |
| APP_ARG="--app_log_blowup ${{ inputs.app_log_blowup }}" | |
| AGG_ARG="--agg_log_blowup ${{ inputs.agg_log_blowup }}" | |
| MAX_SEGMENT_LENGTH="--max_segment_length ${{ inputs.max_segment_length }}" | |
| OUTPUT_PATH="--output_path $METRIC_PATH" | |
| echo "INPUT_ARGS=${INSTANCE_TYPE} ${MEMORY_ALLOCATOR} ${APP_ARG} ${AGG_ARG} ${MAX_SEGMENT_LENGTH} ${OUTPUT_PATH} ${INPUT_ARGS}" >> $GITHUB_ENV | |
| ########################################################################## | |
| # Find working directory based on benchmark_name and run the benchmark # | |
| ########################################################################## | |
| - name: Run benchmark | |
| working-directory: ${{ steps.set-working-dir.outputs.working_dir }} | |
| run: | | |
| python3 ${{ steps.set-working-dir.outputs.relative_path }}/ci/scripts/bench.py $BIN_NAME $CMD_ARGS $INPUT_ARGS | |
| ########################################################################## | |
| # Generate result .md files and flamegraphs, store them in S3 # | |
| ########################################################################## | |
| - name: Upload metric json and compute diff with previous to generate markdown | |
| run: | | |
| current_sha=$(git rev-parse HEAD) | |
| echo "Current SHA: $current_sha" | |
| echo "current_sha=${current_sha}" >> $GITHUB_ENV | |
| s5cmd cp $METRIC_PATH ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}-${current_sha}.json | |
| source ci/scripts/utils.sh | |
| generate_markdown $METRIC_PATH $METRIC_NAME ${{ env.S3_METRICS_PATH }} "." | |
| # - name: Install inferno-flamegraph | |
| # run: cargo install inferno | |
| # - name: Generate flamegraphs | |
| # run: | | |
| # if [[ -f $METRIC_PATH ]]; then | |
| # python3 ci/scripts/metric_unify/flamegraph.py $METRIC_PATH | |
| # s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.PUBLIC_S3_PATH }}/${current_sha}/" | |
| # echo "UPLOAD_FLAMEGRAPHS=1" >> $GITHUB_ENV | |
| # fi | |
| - name: Add benchmark metadata and upload markdown | |
| id: add_metadata | |
| run: | | |
| source ci/scripts/utils.sh | |
| add_metadata results.md ${{ inputs.max_segment_length }} ${{ inputs.instance_type }} ${{ inputs.memory_allocator }} ${{ github.repository }} ${{ github.run_id }} | |
| s3_md_file="${METRIC_NAME}-${current_sha}.md" | |
| s5cmd cp results.md "${{ env.S3_PATH }}/${s3_md_file}" | |
| ########################################################################## | |
| # Update S3 with individual results upon a push event # | |
| ########################################################################## | |
| - name: Update latest main result in s3 | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| run: | | |
| s5cmd cp "${{ env.S3_PATH }}/${METRIC_NAME}-${{ env.current_sha }}.md" "${{ env.S3_PATH }}/main-${METRIC_NAME}.md" | |
| if [[ -f $METRIC_PATH ]]; then | |
| s5cmd cp $METRIC_PATH "${{ env.S3_METRICS_PATH }}/main-${METRIC_NAME}.json" | |
| fi | |
| ########################################################################## | |
| # Update benchmark-results with individual results # | |
| ########################################################################## | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: benchmark-results | |
| - name: Set up git | |
| run: | | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --global user.name "github-actions[bot]" | |
| - name: Set github pages path for dispatch | |
| run: | | |
| BENCHMARK_RESULTS_PATH="benchmarks-dispatch/${{ github.head_ref || github.ref }}" | |
| echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV | |
| - name: Set github pages path for PR | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| BENCHMARK_RESULTS_PATH="benchmarks-pr/${{ github.event.pull_request.number }}/individual" | |
| echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV | |
| - name: Set github pages path for push | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| run: | | |
| BENCHMARK_RESULTS_PATH="benchmarks/individual" | |
| echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV | |
| - name: Update PR github pages with new bench results | |
| if: github.event.pull_request.head.repo.fork == false # forks do not have write access | |
| run: | | |
| mkdir -p ${BENCHMARK_RESULTS_PATH} | |
| s3_md_file="${METRIC_NAME}-${current_sha}.md" | |
| s5cmd cp "${{ env.S3_PATH }}/${s3_md_file}" ${BENCHMARK_RESULTS_PATH}/${s3_md_file} | |
| git add ${BENCHMARK_RESULTS_PATH}/${s3_md_file} | |
| git commit --allow-empty -m "Update benchmark result at ${BENCHMARK_RESULTS_PATH}/${s3_md_file}" | |
| MAX_RETRIES=10 | |
| RETRY_DELAY=5 | |
| ATTEMPT=0 | |
| SUCCESS=false | |
| while [ $ATTEMPT -lt $MAX_RETRIES ]; do | |
| echo "Attempt $((ATTEMPT + 1)) to push of $MAX_RETRIES..." | |
| git fetch origin benchmark-results | |
| git merge origin/benchmark-results --no-edit | |
| if git push origin benchmark-results; then | |
| SUCCESS=true | |
| break | |
| else | |
| echo "Push failed. Retrying in $RETRY_DELAY seconds..." | |
| sleep $RETRY_DELAY | |
| ATTEMPT=$((ATTEMPT + 1)) | |
| fi | |
| done | |
| if [ "$SUCCESS" = false ]; then | |
| echo "PUSH_FAILED" | |
| exit 1 | |
| fi |