Benchmark.yml

name: Benchmark
on:
  workflow_dispatch:
    inputs:
      repository:
        type: string
        description: name of repository triggers this workflow, e.g. FluxML/NNlib.jl
      pr_id:
        type: string
        description: id of the pull request that triggers this workflow
      target_url:
        type: string
        description: url of target
      baseline_url:
        type: string
        description: url of baseline

jobs:
  Cache:
    runs-on: ubuntu-latest
    env:
      TARGET_URL: ${{ github.event.inputs.target_url || format('{0}#{1}', github.event.pull_request.head.repo.html_url, github.event.pull_request.head.sha) }}
      BASELINE_URL: ${{ github.event.inputs.baseline_url || format('{0}#{1}', github.event.pull_request.base.repo.html_url, github.event.pull_request.base.sha) }}
    steps:
      - uses: actions/checkout@v3
      -
        name: Print target&baseline repo url
        run: |
          echo "Target repo url: $TARGET_URL"
          echo "Baseline repo url: $BASELINE_URL"
      -
        name: Get current time
        run: echo "TIME=$(date +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV
      -
        id: restore-cache
        uses: actions/cache/restore@v3
        with:
          path: ~/.julia
          key: bm-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.TARGET_URL }}-${{ env.BASELINE_URL }}-${{ env.TIME }}
          restore-keys: bm-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.TARGET_URL }}-${{ env.BASELINE_URL }}-${{ env.TIME }}

      # cache-matched-key is '' when cache/restore cannot hit cache even used "restore-keys"
      - uses: julia-actions/setup-julia@v1
        if: ${{ steps.restore-cache.outputs.cache-matched-key == '' }}
      - uses: julia-actions/julia-buildpkg@v1
        if: ${{ steps.restore-cache.outputs.cache-matched-key == '' }}
      -
        name: Install dependencies
        if: ${{ steps.restore-cache.outputs.cache-matched-key == '' }}
        run: |
          julia --project=benchmark benchmark/runbenchmarks.jl --cache-setup \
            --arch=cpu \
            --target=$TARGET_URL --baseline=$BASELINE_URL
      -
        uses: actions/cache/save@v3
        if: ${{ steps.restore-cache.outputs.cache-matched-key == '' }}
        with:
          path: ~/.julia
          key: bm-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.TARGET_URL }}-${{ env.BASELINE_URL }}-${{ env.TIME }}


  RunBenchmark:
    needs: [Cache]
    runs-on: ubuntu-latest
    env:
      TARGET_URL: ${{ github.event.inputs.target_url || format('{0}#{1}', github.event.pull_request.head.repo.html_url, github.event.pull_request.head.sha) }}
      BASELINE_URL: ${{ github.event.inputs.baseline_url || format('{0}#{1}', github.event.pull_request.base.repo.html_url, github.event.pull_request.base.sha) }}
    strategy:
      matrix:
        bm: [
          # for test
          #"nnlib(attention,activations,gemm)", "nnlib(conv)",
          #"nnlib(pooling,softmax)", "nnlib(dropout,upsample)",
          "flux"
        ]
    steps:
      -
        name: Print target&baseline repo url
        run: |
          echo "Target repo url: $TARGET_URL"
          echo "Baseline repo url: $BASELINE_URL"
      -
        name: Checkout code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - run: |
          git fetch origin benchmark-tuning:benchmark-tuning
          git fetch origin benchmark-results:benchmark-results
      -
        name: Setup julia
        uses: julia-actions/setup-julia@v1
      -
        name: Restore cache
        uses: actions/cache@v3
        with:
          path: ~/.julia
          # This "key" is required, but the time of this job is definitely different from "Cache" job.
          # Therefore cache cannot be hit. So this is a fake key.
          key: bm-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.TARGET_URL }}-${{ env.BASELINE_URL }}-
          restore-keys: |
            bm-${{ github.run_id }}-${{ github.run_attempt }}-${{ env.TARGET_URL }}-${{ env.BASELINE_URL }}-
            bm-${{ github.run_id }}-${{ github.run_attempt }}-
      -
        name: Build pkg
        uses: julia-actions/julia-buildpkg@v1
      -
        id: benchmark
        name: Run benchmark
        run: |
          julia --project=benchmark benchmark/runbenchmarks.jl --pr \
            --arch=cpu \
            --target=$TARGET_URL --baseline=$BASELINE_URL \
            --enable="${{ matrix.bm }}" \
            --fetch-result
      -
        name: Print report
        run: |
          cat benchmark/report.md
      -
        name: Upload report
        uses: actions/upload-artifact@v3
        with:
          name: fluxml-benchmark-result-${{ matrix.bm }}
          path: |
            benchmark/result-baseline.json
            benchmark/result-target.json


  MergeReport:
    needs: [RunBenchmark]
    if: ${{ github.event_name == 'workflow_dispatch' }}
    runs-on: ubuntu-latest
    env:
      TARGET_URL: ${{ github.event.inputs.target_url || format('{0}#{1}', github.event.pull_request.head.repo.html_url, github.event.pull_request.head.sha) }}
      BASELINE_URL: ${{ github.event.inputs.baseline_url || format('{0}#{1}', github.event.pull_request.base.repo.html_url, github.event.pull_request.base.sha) }}
    steps:
      - uses: actions/checkout@v3
      -
        name: Download all reports
        uses: actions/download-artifact@v3
      - uses: julia-actions/setup-julia@v1
      - uses: julia-actions/julia-buildpkg@v1
      -
        name: Merge reports
        run: |
          julia --project=benchmark benchmark/runbenchmarks.jl --merge-reports \
            --arch=cpu \
            --target=$TARGET_URL --baseline=$BASELINE_URL \
            --push-result --push-password=${{ github.token }}
      -
        name: Upload final report
        uses: actions/upload-artifact@v3
        with:
          name: final-report
          path: benchmark/report.md


  Comment:
    needs: [MergeReport]
    if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.repository != '' && github.event.inputs.pr_id != '' }}
    runs-on: ubuntu-latest
    steps:
      -
        name: Download report
        uses: actions/download-artifact@v3
        with:
          name: final-report
      -
        name: Get app token
        id: get-app-token
        uses: tibdex/github-app-token@v1
        with: 
          app_id: ${{ secrets.BENCH_APP_ID }}
          installation_id: ${{ secrets.BENCH_INSTALL_ID }}
          private_key: ${{ secrets.BENCH_PRIVATE_KEY }}
      -
        name: Comment
        uses: peter-evans/create-or-update-comment@v2
        with:
          repository: ${{ github.event.inputs.repository }}
          issue-number: ${{ github.event.inputs.pr_id }}
          token: ${{ steps.get-app-token.outputs.token }}
          body-file: report.md


  DryPrint:
    needs: [MergeReport]
    if: ${{ github.event.inputs.repository == '' || github.event.inputs.pr_id == '' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: final-report
      - run: |
          cat report.md