Add a benchmarks bot.

.buildkite/pipeline.yml

@@ -1,73 +1,58 @@
 steps:
-  - label: "Julia 1.6"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: 1.6
-      - JuliaCI/julia-test#v1: ~
-      - JuliaCI/julia-coverage#v1:
-    agents:
-      queue: "juliagpu"
-      cuda: "*"
-      cap: "recent"
-    if: build.message !~ /\[skip tests\]/
-    timeout_in_minutes: 120
+  - group: ":julia: Julia"
+    key: "julia"
+    steps:
+      - label: "Julia {{matrix.julia}}"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "{{matrix.julia}}"
+          - JuliaCI/julia-test#v1: ~
+          - JuliaCI/julia-coverage#v1: ~
+        agents:
+          queue: "juliagpu"
+          cuda: "*"
+        if: build.message !~ /\[skip tests\]/
+        timeout_in_minutes: 120
+        matrix:
+          setup:
+            julia:
+              - "1.6"
+              - "1.7"
+              - "1.8"
+              - "1.9"
+          #     - "nightly"
+          # adjustments:
+          #   - with:
+          #       julia: "nightly"
+          #     soft_fail: true
 
-  - label: "Julia 1.7"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: 1.7
-      - JuliaCI/julia-test#v1: ~
-      - JuliaCI/julia-coverage#v1:
-    agents:
-      queue: "juliagpu"
-      cuda: "*"
-      cap: "recent"
-    if: build.message !~ /\[skip tests\]/
-    timeout_in_minutes: 120
+  - group: ":racehorse: Benchmarks"
+    steps:
+      - label: "Benchmarks on Julia {{matrix.julia}}"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: "{{matrix.julia}}"
+        command: |
+          julia --project -e '
+            println("--- :julia: Instantiating project")
+            using Pkg
+            Pkg.instantiate()
+            Pkg.activate("benchmarks")
+            Pkg.instantiate()
+            push!(LOAD_PATH, @__DIR__)
 
-  - label: "Julia 1.8"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: 1.8
-      - JuliaCI/julia-test#v1: ~
-      - JuliaCI/julia-coverage#v1:
-    agents:
-      queue: "juliagpu"
-      cuda: "*"
-      cap: "recent"
-    if: build.message !~ /\[skip tests\]/
-    timeout_in_minutes: 120
-    soft_fail:
-      - exit_status: 1
-
-  - label: "Julia 1.9"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: 1.9
-      - JuliaCI/julia-test#v1: ~
-      - JuliaCI/julia-coverage#v1:
-    agents:
-      queue: "juliagpu"
-      cuda: "*"
-      cap: "recent"
-    if: build.message !~ /\[skip tests\]/
-    timeout_in_minutes: 120
-    soft_fail:
-      - exit_status: 1
-
-  - label: "Julia nightly"
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "nightly"
-      - JuliaCI/julia-test#v1: ~
-    agents:
-      queue: "juliagpu"
-      cuda: "*"
-      cap: "recent"
-    if: build.message !~ /\[skip tests\]/
-    timeout_in_minutes: 120
-    soft_fail:
-      - exit_status: 1
+            println("+++ :julia: Benchmarking")
+            include("benchmarks/runbenchmarks.jl")'
+        agents:
+          queue: "juliagpu"
+          cuda: "*"
+        if: build.message !~ /\[skip benchmarks\]/
+        matrix:
+          setup:
+            julia:
+              - "1.9"
+        timeout_in_minutes: 30
 
 env:
   SECRET_CODECOV_TOKEN: "mqW1qFfd8v23i8ulnEaJEZUJj70lCAFZonhVkARc1z3VbB/Z0MDDB5BCzM3Rrgq3DQdNwc+ZSMluTNT0H7QIWuUN49eLqtLVl7gs0Io2MlMXnaD892I3n0+v0cV5oShBtY8iErgdmlCAw4EzfLv3fsXR0hPJyL4DUfyewhe6lDvWCYb8cZzoNfIQ8gmQN6oyJnbm8CmCKsrdzrDcC9NRhMI6jMYL8ZD9ry/m059S2hReyIdNtVikXPz9NBBmN+DAlGmp+MqQfcIlTeXIj0/U9OCi04GjbnT0IBFdjW4lTaNDIdLDnaRdumfO0C2g1lFJk73F+dsf/7goK6GpyyD/dw==;U2FsdGVkX192PcTTM+mOwv2BFs5wDMjTHE11fxBvTJ98468YZ6nlosYl9drk9bbZpYXR3cg4fEr/mK/dkDCJBA=="
+  SECRET_GITHUB_TOKEN: "lX9HgvfbQmLQyalEYzbp/mYpbyPPSzNkwjhI6VAyAFTMnKlX+FN8V2gWpKGVCsXuU5P1VgdTxuPcI/FiB2Rl0ng+i4hc/FY9vbl2xUl1tS6BJWtbHa8MZjpcRfGWsqvf5oFZxijiaYdAd4cDO5Ubh2zIYR9GNQTRqcX0yGTOlYpcnla77XfucrCuciD0mCchD4UXWMUjEoNOvd6PqIWlGazjym6Fllc90u6JNEa3m8/UQjjx6AwTW3+lhC0MyvuMDzFRc+KWvwNLc4bhNGoZUnoe+uoNVmILXow3FYbvqvr8aIKPQgbZBBCajPzrevCi1PvcKXsk+BhdeJEtOjEaDg==;U2FsdGVkX1860HAQDaVtnxbr2tOV98XJS345pfGCH7UoIAoBFhDYXolq6sgslgRSqOUcZlaU8Ifj9i3LPqxb7A=="

benchmarks/Project.toml

@@ -1,3 +1,5 @@
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
+GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"

benchmarks/runbenchmarks.jl

@@ -0,0 +1,182 @@
+using BenchmarkTools
+using GemmKernels, CUDA
+using Git: git
+import GitHub
+using Printf
+
+using StableRNGs
+rng = StableRNG(123)
+
+# to find untuned benchmarks
+BenchmarkTools.DEFAULT_PARAMETERS.evals = 0
+
+@info "Loading previous benchmark results"
+github_token = get(ENV, "GITHUB_TOKEN", nothing)
+benchmark_results = mktempdir()
+if github_token === nothing
+    run(`$(git()) clone -q https://github.com/JuliaGPU/GemmKernels.jl -b benchmark-results $benchmark_results`)
+else
+    run(`$(git()) clone -q https://$github_token:x-oauth-basic@github.com/JuliaGPU/GemmKernels.jl -b benchmark-results $benchmark_results`)
+end
+
+# load timings
+function load_results()
+    proc = open(`$(git()) -C $benchmark_results log --first-parent --pretty=format:"%H" origin/master`)
+    while !eof(proc)
+        commit = readline(proc)
+        results_file = joinpath(benchmark_results, "results-$commit.json")
+        if isfile(results_file)
+            timings = BenchmarkTools.load(results_file)[1]
+            close(proc)
+            return (; commit, timings)
+        end
+    end
+    return nothing
+end
+previous_results = load_results()
+if previous_results === nothing
+    @error "No previous benchmark results found"
+else
+    @info "Found previous timings for commit $(previous_results.commit)"
+end
+
+@info "Loading benchmarks"
+SUITE = BenchmarkGroup()
+include("wmma.jl")
+
+# load params
+paramsfile = joinpath(benchmark_results, "params.json")
+if !isfile(paramsfile)
+    @info "Tuning benchmarks"
+    tune!(SUITE)
+    BenchmarkTools.save(paramsfile, params(SUITE))
+else
+    loadparams!(SUITE, BenchmarkTools.load(paramsfile)[1], :evals, :samples)
+
+    # find untuned benchmarks for which we have the default evals==0
+    for (ids, benchmark) in BenchmarkTools.leaves(SUITE)
+        if params(benchmark).evals == 0
+            @info "Re-runing benchmarks"
+            tune!(SUITE)
+            BenchmarkTools.save(paramsfile, params(SUITE))
+            break
+        end
+    end
+end
+
+@info "Warming-up benchmarks"
+warmup(SUITE; verbose=false)
+
+@info "Running benchmarks"
+timings = run(SUITE; verbose=true)
+println(timings)
+
+# write results
+if get(ENV, "BUILDKITE_BRANCH", nothing) == "master"
+    commit = ENV["BUILDKITE_COMMIT"]
+    results_file = joinpath(benchmark_results, "results-$commit.json")
+    BenchmarkTools.save(results_file, timings)
+
+    # commit and push
+    run(`$(git()) -C $benchmark_results add $results_file`)
+    run(`$(git()) -C $benchmark_results commit -q -a -m "Results for $commit."`)
+    run(`$(git()) -C $benchmark_results push -q`)
+end
+
+# result rendering functions
+function markdown_escaped_code(str)
+    ticks = eachmatch(r"`+", str)
+    isempty(ticks) && return "`$str`"
+    ticks = maximum(x -> length(x.match), ticks) + 1
+    ticks = "`"^ticks
+    return string(ticks, startswith(str, '`') ? " " : "", str, endswith(str, '`') ? " " : "", ticks)
+end
+idrepr(id::Vector) = sprint(idrepr, id)
+function idrepr(io::IO, id::Vector)
+    print(io, "[")
+    first = true
+    for i in id
+        first ? (first = false) : print(io, ", ")
+        show(io, i)
+    end
+    print(io, "]")
+end
+idrepr_md(id::Vector) = markdown_escaped_code(idrepr(id))
+intpercent(p) = string(ceil(Int, p * 100), "%")
+function resultrow(ids, t::BenchmarkTools.TrialJudgement)
+    t_tol = intpercent(BenchmarkTools.params(t).time_tolerance)
+    m_tol = intpercent(BenchmarkTools.params(t).memory_tolerance)
+    t_ratio = @sprintf("%.2f", BenchmarkTools.time(BenchmarkTools.ratio(t)))
+    m_ratio =  @sprintf("%.2f", BenchmarkTools.memory(BenchmarkTools.ratio(t)))
+    t_mark = resultmark(BenchmarkTools.time(t))
+    m_mark = resultmark(BenchmarkTools.memory(t))
+    timestr = "$(t_ratio) ($(t_tol)) $(t_mark)"
+    memstr = "$(m_ratio) ($(m_tol)) $(m_mark)"
+    return "| $(idrepr_md(ids)) | $(timestr) | $(memstr) |"
+end
+const REGRESS_MARK = ":x:"
+const IMPROVE_MARK = ":white_check_mark:"
+resultmark(sym::Symbol) = sym == :regression ? REGRESS_MARK : (sym == :improvement ? IMPROVE_MARK : "")
+
+# compare against previous timings
+if previous_results !== nothing
+    commit = ENV["BUILDKITE_COMMIT"]
+    comparison = judge(minimum(timings), minimum(previous_results.timings))
+
+    println("Improvements:")
+    println(improvements(comparison))
+
+    println("Regressions:")
+    println(regressions(comparison))
+
+    # comment on PR
+    if github_token !== nothing && get(ENV, "BUILDKITE_PULL_REQUEST", "false") !== "false"
+        auth = GitHub.authenticate(github_token)
+        repo = GitHub.repo("JuliaGPU/GemmKernels.jl"; auth)
+        pr = parse(Int, ENV["BUILDKITE_PULL_REQUEST"])
+
+        # generate a comment
+        io = IOBuffer()
+        print(io, """
+            Benchmark results for commit $commit (comparing to $(previous_results.commit)):
+
+            | ID | time ratio | memory ratio |
+            |----|------------|--------------|
+            """)
+        entries = BenchmarkTools.leaves(comparison)
+        entries = entries[sortperm(map(string∘first, entries))]
+        for (ids, t) in BenchmarkTools.leaves(comparison)
+            if BenchmarkTools.isregression(t) || BenchmarkTools.isimprovement(t)
+                println(io, resultrow(ids, t))
+            end
+        end
+        body = String(take!(io))
+
+        # find a previous comment to edit
+        function find_previous_comment()
+            kwargs = Dict(:auth => auth, :page_limit => 1, :params => (; per_page=100))
+            while true
+                comments, pages = git.luolix.topments(repo, pr; auth)
+                for comment in comments
+                    if startswith(comment.body, "Benchmark results for")
+                        return comment
+                    end
+                end
+                if haskey(pages, "next")
+                    delete!(kwargs, :params)
+                    kwargs[:start_page] = pages["next"]
+                else
+                    return nothing
+                end
+            end
+        end
+        previous_comment = find_previous_comment()
+
+        # submit
+        if previous_comment === nothing
+            GitHub.create_comment(repo, pr, :pr; auth, params=(; body=body))
+        else
+            GitHub.edit_comment(repo, previous_comment, :pr; auth, params=(; body=body))
+        end
+    end
+end