chore: use logs for benchmarking #1157
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Reports | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| jobs: | |
| build-nargo: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout Noir repo | |
| uses: actions/checkout@v4 | |
| - name: Setup toolchain | |
| uses: dtolnay/rust-toolchain@1.74.1 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: x86_64-unknown-linux-gnu | |
| cache-on-failure: true | |
| save-if: ${{ github.event_name != 'merge_group' }} | |
| - name: Build Nargo | |
| run: cargo build --package nargo_cli --release | |
| - name: Package artifacts | |
| run: | | |
| mkdir dist | |
| cp ./target/release/nargo ./dist/nargo | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: nargo | |
| path: ./dist/* | |
| retention-days: 3 | |
| compare_gates_reports: | |
| name: Circuit sizes | |
| needs: [build-nargo] | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install `bb` | |
| run: | | |
| ./scripts/install_bb.sh | |
| echo "$HOME/.bb/" >> $GITHUB_PATH | |
| - name: Download nargo binary | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nargo | |
| path: ./nargo | |
| - name: Set nargo on PATH | |
| run: | | |
| nargo_binary="${{ github.workspace }}/nargo/nargo" | |
| chmod +x $nargo_binary | |
| echo "$(dirname $nargo_binary)" >> $GITHUB_PATH | |
| export PATH="$PATH:$(dirname $nargo_binary)" | |
| nargo -V | |
| - name: Generate gates report | |
| working-directory: ./test_programs | |
| run: | | |
| ./rebuild.sh | |
| ./gates_report.sh | |
| mv gates_report.json ../gates_report.json | |
| - name: Compare gates reports | |
| id: gates_diff | |
| uses: noir-lang/noir-gates-diff@84ada11295b9a1e1da7325af4e45e2db9f775175 | |
| with: | |
| report: gates_report.json | |
| summaryQuantile: 0.9 # only display the 10% most significant circuit size diffs in the summary (defaults to 20%) | |
| - name: Add gates diff to sticky comment | |
| if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| # delete the comment in case changes no longer impact circuit sizes | |
| delete: ${{ !steps.gates_diff.outputs.markdown }} | |
| message: ${{ steps.gates_diff.outputs.markdown }} | |
| compare_brillig_bytecode_size_reports: | |
| name: Brillig bytecode sizes | |
| needs: [build-nargo] | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nargo binary | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nargo | |
| path: ./nargo | |
| - name: Set nargo on PATH | |
| run: | | |
| nargo_binary="${{ github.workspace }}/nargo/nargo" | |
| chmod +x $nargo_binary | |
| echo "$(dirname $nargo_binary)" >> $GITHUB_PATH | |
| export PATH="$PATH:$(dirname $nargo_binary)" | |
| nargo -V | |
| - name: Generate Brillig bytecode size report | |
| working-directory: ./test_programs | |
| run: | | |
| ./gates_report_brillig.sh | |
| mv gates_report_brillig.json ../gates_report_brillig.json | |
| - name: Compare Brillig bytecode size reports | |
| id: brillig_bytecode_diff | |
| uses: noir-lang/noir-gates-diff@84ada11295b9a1e1da7325af4e45e2db9f775175 | |
| with: | |
| report: gates_report_brillig.json | |
| header: | | |
| # Changes to Brillig bytecode sizes | |
| brillig_report: true | |
| summaryQuantile: 0.9 # only display the 10% most significant bytecode size diffs in the summary (defaults to 20%) | |
| - name: Add bytecode size diff to sticky comment | |
| if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| header: brillig | |
| # delete the comment in case changes no longer impact brillig bytecode sizes | |
| delete: ${{ !steps.brillig_bytecode_diff.outputs.markdown }} | |
| message: ${{ steps.brillig_bytecode_diff.outputs.markdown }} | |
| compare_brillig_execution_reports: | |
| name: Brillig execution trace sizes | |
| needs: [build-nargo] | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nargo binary | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nargo | |
| path: ./nargo | |
| - name: Set nargo on PATH | |
| run: | | |
| nargo_binary="${{ github.workspace }}/nargo/nargo" | |
| chmod +x $nargo_binary | |
| echo "$(dirname $nargo_binary)" >> $GITHUB_PATH | |
| export PATH="$PATH:$(dirname $nargo_binary)" | |
| nargo -V | |
| - name: Generate Brillig execution report | |
| working-directory: ./test_programs | |
| run: | | |
| ./gates_report_brillig_execution.sh | |
| mv gates_report_brillig_execution.json ../gates_report_brillig_execution.json | |
| - name: Compare Brillig execution reports | |
| id: brillig_execution_diff | |
| uses: noir-lang/noir-gates-diff@84ada11295b9a1e1da7325af4e45e2db9f775175 | |
| with: | |
| report: gates_report_brillig_execution.json | |
| header: | | |
| # Changes to number of Brillig opcodes executed | |
| brillig_report: true | |
| summaryQuantile: 0.9 # only display the 10% most significant bytecode size diffs in the summary (defaults to 20%) | |
| - name: Add bytecode size diff to sticky comment | |
| if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| header: brillig_execution | |
| # delete the comment in case changes no longer impact brillig bytecode sizes | |
| delete: ${{ !steps.brillig_execution_diff.outputs.markdown }} | |
| message: ${{ steps.brillig_execution_diff.outputs.markdown }} | |
| generate_memory_report: | |
| name: Peak memory usage | |
| needs: [build-nargo] | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nargo binary | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nargo | |
| path: ./nargo | |
| - name: Set nargo on PATH | |
| run: | | |
| nargo_binary="${{ github.workspace }}/nargo/nargo" | |
| chmod +x $nargo_binary | |
| echo "$(dirname $nargo_binary)" >> $GITHUB_PATH | |
| export PATH="$PATH:$(dirname $nargo_binary)" | |
| nargo -V | |
| - name: Generate Memory report | |
| working-directory: ./test_programs | |
| run: | | |
| ./memory_report.sh | |
| mv memory_report.json ../memory_report.json | |
| - name: Upload memory report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: in_progress_memory_report | |
| path: memory_report.json | |
| retention-days: 3 | |
| overwrite: true | |
| generate_compilation_and_execution_report: | |
| name: Compilation and execution time | |
| needs: [build-nargo] | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download nargo binary | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nargo | |
| path: ./nargo | |
| - name: Set nargo on PATH | |
| run: | | |
| nargo_binary="${{ github.workspace }}/nargo/nargo" | |
| chmod +x $nargo_binary | |
| echo "$(dirname $nargo_binary)" >> $GITHUB_PATH | |
| export PATH="$PATH:$(dirname $nargo_binary)" | |
| nargo -V | |
| - name: Generate Compilation report | |
| working-directory: ./test_programs | |
| run: | | |
| ./compilation_report.sh 0 1 | |
| mv compilation_report.json ../compilation_report.json | |
| - name: Generate Execution report | |
| working-directory: ./test_programs | |
| run: | | |
| ./execution_report.sh 0 1 | |
| mv execution_report.json ../execution_report.json | |
| - name: Upload compilation report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: in_progress_compilation_report | |
| path: compilation_report.json | |
| retention-days: 3 | |
| overwrite: true | |
| - name: Upload execution report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: in_progress_execution_report | |
| path: execution_report.json | |
| retention-days: 3 | |
| overwrite: true | |
| external_repo_compilation_and_execution_report: | |
| needs: [build-nargo] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts, is_library: true } | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root, take_average: true } | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner, take_average: true } | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail, take_average: true } | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset, take_average: true } | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private } | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public } | |
| name: External repo compilation and execution reports - ${{ matrix.project.repo }}/${{ matrix.project.path }} | |
| steps: | |
| - name: Download nargo binary | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nargo | |
| path: ./nargo | |
| - name: Set nargo on PATH | |
| run: | | |
| nargo_binary="${{ github.workspace }}/nargo/nargo" | |
| chmod +x $nargo_binary | |
| echo "$(dirname $nargo_binary)" >> $GITHUB_PATH | |
| export PATH="$PATH:$(dirname $nargo_binary)" | |
| nargo -V | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: scripts | |
| sparse-checkout: | | |
| test_programs/compilation_report.sh | |
| test_programs/execution_report.sh | |
| test_programs/parse_time.sh | |
| sparse-checkout-cone-mode: false | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ matrix.project.repo }} | |
| path: test-repo | |
| ref: ${{ matrix.project.ref }} | |
| - name: Generate compilation report without averages | |
| working-directory: ./test-repo/${{ matrix.project.path }} | |
| if: ${{ !matrix.project.take_average }} | |
| run: | | |
| mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh | |
| mv /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh | |
| ./compilation_report.sh 1 | |
| - name: Generate compilation report with averages | |
| working-directory: ./test-repo/${{ matrix.project.path }} | |
| if: ${{ matrix.project.take_average }} | |
| run: | | |
| mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh | |
| mv /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh | |
| ./compilation_report.sh 1 1 | |
| - name: Generate execution report without averages | |
| working-directory: ./test-repo/${{ matrix.project.path }} | |
| if: ${{ !matrix.project.is_library && !matrix.project.take_average }} | |
| run: | | |
| mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh | |
| ./execution_report.sh 1 | |
| - name: Generate execution report with averages | |
| working-directory: ./test-repo/${{ matrix.project.path }} | |
| if: ${{ !matrix.project.is_library && matrix.project.take_average }} | |
| run: | | |
| mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh | |
| ./execution_report.sh 1 1 | |
| - name: Move compilation report | |
| id: compilation_report | |
| shell: bash | |
| run: | | |
| PACKAGE_NAME=${{ matrix.project.path }} | |
| PACKAGE_NAME=$(basename $PACKAGE_NAME) | |
| mv ./test-repo/${{ matrix.project.path }}/compilation_report.json ./compilation_report_$PACKAGE_NAME.json | |
| echo "compilation_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT | |
| - name: Move execution report | |
| id: execution_report | |
| shell: bash | |
| if: ${{ !matrix.project.is_library }} | |
| run: | | |
| PACKAGE_NAME=${{ matrix.project.path }} | |
| PACKAGE_NAME=$(basename $PACKAGE_NAME) | |
| mv ./test-repo/${{ matrix.project.path }}/execution_report.json ./execution_report_$PACKAGE_NAME.json | |
| echo "execution_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT | |
| - name: Upload compilation report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }} | |
| path: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}.json | |
| retention-days: 3 | |
| overwrite: true | |
| - name: Upload execution report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: execution_report_${{ steps.execution_report.outputs.execution_report_name }} | |
| path: execution_report_${{ steps.execution_report.outputs.execution_report_name }}.json | |
| retention-days: 3 | |
| overwrite: true | |
| upload_compilation_report: | |
| name: Upload compilation report | |
| needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report] | |
| # We want this job to run even if one variation of the matrix in `external_repo_compilation_and_execution_report` fails | |
| if: always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download initial compilation report | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: in_progress_compilation_report | |
| - name: Download matrix compilation reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: compilation_report_* | |
| path: ./reports | |
| - name: Merge compilation reports using jq | |
| run: | | |
| mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh | |
| ./merge-bench-reports.sh compilation_report | |
| - name: Parse compilation report | |
| id: compilation_report | |
| uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4 | |
| with: | |
| report: compilation_report.json | |
| header: | | |
| # Compilation Report | |
| memory_report: false | |
| - name: Add memory report to sticky comment | |
| if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| header: compilation | |
| message: ${{ steps.compilation_report.outputs.markdown }} | |
| external_repo_memory_report: | |
| needs: [build-nargo] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # TODO: Bring this report back under a flag. The `noir-contracts` report takes just under 30 minutes. | |
| # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts } | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root } | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner } | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset } | |
| - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail } | |
| name: External repo memory report - ${{ matrix.project.repo }}/${{ matrix.project.path }} | |
| steps: | |
| - name: Download nargo binary | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: nargo | |
| path: ./nargo | |
| - name: Set nargo on PATH | |
| run: | | |
| nargo_binary="${{ github.workspace }}/nargo/nargo" | |
| chmod +x $nargo_binary | |
| echo "$(dirname $nargo_binary)" >> $GITHUB_PATH | |
| export PATH="$PATH:$(dirname $nargo_binary)" | |
| nargo -V | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: scripts | |
| sparse-checkout: | | |
| test_programs/memory_report.sh | |
| sparse-checkout-cone-mode: false | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ matrix.project.repo }} | |
| path: test-repo | |
| ref: ${{ matrix.project.ref }} | |
| - name: Generate compilation report | |
| working-directory: ./test-repo/${{ matrix.project.path }} | |
| run: | | |
| mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh | |
| chmod +x ./memory_report.sh | |
| ./memory_report.sh 1 | |
| - name: Move compilation report | |
| id: report | |
| shell: bash | |
| run: | | |
| PACKAGE_NAME=${{ matrix.project.path }} | |
| PACKAGE_NAME=$(basename $PACKAGE_NAME) | |
| mv ./test-repo/${{ matrix.project.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json | |
| echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT | |
| - name: Upload memory report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: memory_report_${{ steps.report.outputs.memory_report_name }} | |
| path: memory_report_${{ steps.report.outputs.memory_report_name }}.json | |
| retention-days: 3 | |
| overwrite: true | |
| upload_memory_report: | |
| name: Upload memory report | |
| needs: [generate_memory_report, external_repo_memory_report] | |
| # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails | |
| if: always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download initial memory report | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: in_progress_memory_report | |
| - name: Download matrix memory reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: memory_report_* | |
| path: ./reports | |
| - name: Merge memory reports using jq | |
| run: | | |
| mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh | |
| ./merge-bench-reports.sh memory_report | |
| - name: Parse memory report | |
| id: memory_report | |
| uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4 | |
| with: | |
| report: memory_report.json | |
| header: | | |
| # Memory Report | |
| memory_report: true | |
| - name: Add memory report to sticky comment | |
| if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| header: memory | |
| message: ${{ steps.memory_report.outputs.markdown }} | |
| upload_execution_report: | |
| name: Upload execution report | |
| needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report] | |
| # We want this job to run even if one variation of the matrix in `external_repo_compilation_and_execution_report` fails | |
| if: always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download initial execution report | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: in_progress_execution_report | |
| - name: Download matrix execution reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: execution_report_* | |
| path: ./reports | |
| - name: Merge execution reports using jq | |
| run: | | |
| mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh | |
| ./merge-bench-reports.sh execution_report | |
| - name: Parse execution report | |
| id: execution_report | |
| uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4 | |
| with: | |
| report: execution_report.json | |
| header: | | |
| # Execution Report | |
| execution_report: true | |
| - name: Add memory report to sticky comment | |
| if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| header: execution_time | |
| message: ${{ steps.execution_report.outputs.markdown }} | |