First try on benchmark CI (#315)

(WIP)
TODO: store results and PR label triggering

.github/workflows/benchmarkpr.yml

@@ -0,0 +1,78 @@
+name: performance tracking
+env:
+  JULIA_NUM_THREADS: 2
+on:
+  pull_request:
+    branches:
+      - master
+      - benchx
+  push:
+    branches:
+      - master
+      - benchx
+    tags: '*'
+jobs:
+  benchmark:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.allow_failure }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1'
+        os:
+          - ubuntu-latest
+        arch:
+          - x64
+        include:
+          - version: '1'
+            allow_failure: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: actions/cache@v3
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - uses: julia-actions/julia-buildpkg@latest
+      - name: install dependencies
+        run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"'
+      - name: Run benchmark judge if pull request
+        if: github.event_name == 'pull_request'
+        run: julia -e "
+          using BenchmarkCI, PkgBenchmark; 
+          jd=BenchmarkCI.judge(baseline=\"origin/${GITHUB_BASE_REF}\"); 
+          writeresults(\"targetbenchout.json\", jd.target_results); 
+          writeresults(\"baselinebenchout.json\", jd.baseline_results); 
+          "
+      - name: Post results if pull request
+        if: github.event_name == 'pull_request'
+        run: julia -e 'using BenchmarkCI; BenchmarkCI.postjudge()'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run benchmark if push/merge
+        if: github.event_name == 'push'
+        run: julia --project=benchmark -e "
+          import Pkg;
+          Pkg.develop(path=pwd());
+          Pkg.instantiate();
+          using PkgBenchmark, BenchmarkTools; 
+          br=benchmarkpkg(pwd()); 
+          br_median = BenchmarkResults(br.name, br.commit, median(br.benchmarkgroup), br.date, br.julia_commit, br.vinfo, br.benchmarkconfig);
+          writeresults(\"medianbenchout.json\", br_median); 
+          "
+      - uses: actions/upload-artifact@v3
+        with: 
+          name: Store benchmark result
+          path: ./*benchout.json
+

.gitignore

@@ -6,5 +6,8 @@ docs/build/
 docs/site/
 benchmark/.results/*
 benchmark/.tune.jld
+benchmark/Manifest.toml
+.benchmarkci
 *.cov
 /Manifest.toml
+notes

benchmark/Project.toml

@@ -0,0 +1,7 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[compat]
+BenchmarkTools = "1"

benchmark/benchmarks.jl

@@ -1,18 +1,39 @@
 using BenchmarkTools
 using Graphs
 
-DIGRAPHS = Dict{String,DiGraph}(
+const benchdir = dirname(@__FILE__)
+
+const DIGRAPHS = Dict{String,DiGraph}(
     "complete100" => complete_digraph(100), "path500" => path_digraph(500)
 )
 
-GRAPHS = Dict{String,Graph}(
+const GRAPHS = Dict{String,Graph}(
     "complete100" => complete_graph(100),
     "tutte" => smallgraph(:tutte),
     "path500" => path_graph(500),
 )
 
-suite = BenchmarkGroup()
-include("core.jl")
+serialbenchmarks = [
+    "serial/core.jl",
+    "serial/connectivity.jl",
+    "serial/centrality.jl",
+    "serial/edges.jl",
+    "serial/insertions.jl",
+    "serial/traversals.jl",
+]
+
+const SUITE = BenchmarkGroup()
+
+foreach(serialbenchmarks) do bm
+    include(bm)
+end
+
+parallelbenchmarks = [
+    "parallel/egonets.jl",
+]
+
+foreach(parallelbenchmarks) do bm
+    include(joinpath(benchdir, bm))
+end
 
-tune!(suite);
-results = run(suite; verbose=true, seconds=10)
+nothing

benchmark/parallel/egonets.jl

@@ -1,62 +1,57 @@
 using Graphs
 using BenchmarkTools
-@show Threads.nthreads()
-
-@benchgroup "parallel" begin
-    @benchgroup "egonet" begin
-        function vertex_function(g::Graph, i::Int)
-            a = 0
-            for u in neighbors(g, i)
-                a += degree(g, u)
-            end
-            return a
-        end
-
-        function twohop(g::Graph, i::Int)
-            a = 0
-            for u in neighbors(g, i)
-                for v in neighbors(g, u)
-                    a += degree(g, v)
-                end
-            end
-            return a
-        end
 
-        function mapvertices(f, g::Graph)
-            n = nv(g)
-            a = zeros(Int, n)
-            Threads.@threads for i in 1:n
-                a[i] = f(g, i)
-            end
-            return a
-        end
+SUITE["parallel"] = BenchmarkGroup([],
+                                   "egonet" => BenchmarkGroup([])
+                                   )
 
-        function mapvertices_single(f, g)
-            n = nv(g)
-            a = zeros(Int, n)
-            for i in 1:n
-                a[i] = f(g, i)
-            end
-            return a
-        end
+SUITE["serial"] = BenchmarkGroup([],
+                                   "egonet" => BenchmarkGroup([])
+                                   )
 
-        function comparison(f, g)
-            println("Mulithreaded on $(Threads.nthreads())")
-            b1 = @benchmarkable mapvertices($f, $g)
-            println(b1)
+function vertex_function(g::Graph, i::Int)
+    a = 0
+    for u in neighbors(g, i)
+        a += degree(g, u)
+    end
+    return a
+end
 
-            println("singlethreaded")
-            b2 = @benchmarkable mapvertices_single($f, $g)
-            println(b2)
-            return println("done")
+function twohop(g::Graph, i::Int)
+    a = 0
+    for u in neighbors(g, i)
+        for v in neighbors(g, u)
+            a += degree(g, v)
         end
+    end
+    return a
+end
 
-        nv_ = 10000
-        g = SimpleGraph(nv_, 64 * nv_)
-        f = vertex_function
-        println(g)
+function mapvertices(f, g::Graph)
+    n = nv(g)
+    a = zeros(Int, n)
+    Threads.@threads for i in 1:n
+        a[i] = f(g, i)
+    end
+    return a
+end
 
-        comparison(vertex_function, g)
-        comparison(twohop, g)
+function mapvertices_single(f, g)
+    n = nv(g)
+    a = zeros(Int, n)
+    for i in 1:n
+        a[i] = f(g, i)
     end
+    return a
+end
+
+let
+    nv_ = 10000
+    g = SimpleGraph(nv_, 64 * nv_)
+
+    SUITE["parallel"]["egonet"]["vertexfunction"] = @benchmarkable mapvertices($vertex_function, $g)
+    SUITE["parallel"]["egonet"]["twohop"] = @benchmarkable mapvertices($twohop, $g)
+
+    SUITE["serial"]["egonet"]["vertexfunction"] = @benchmarkable mapvertices_single($vertex_function, $g)
+    SUITE["serial"]["egonet"]["twohop"] = @benchmarkable mapvertices_single($twohop, $g)
 end

benchmark/serial/centrality.jl

@@ -0,0 +1,20 @@
+SUITE["centrality"] = BenchmarkGroup([],
+                               "graphs" => BenchmarkGroup([]), 
+                               "digraphs" => BenchmarkGroup([]), 
+                               )
+
+# graphs
+SUITE["centrality"]["graphs"]["degree_centrality"] = @benchmarkable [Graphs.degree_centrality(g) for (_, g) in $GRAPHS]
+SUITE["centrality"]["graphs"]["closeness_centrality"] = @benchmarkable [Graphs.closeness_centrality(g) for (_, g) in $GRAPHS]
+# if nv(g) < 1000 is needed ?
+SUITE["centrality"]["graphs"]["betweenness_centrality"] = @benchmarkable [Graphs.betweenness_centrality(g) for (_, g) in $GRAPHS]
+SUITE["centrality"]["graphs"]["katz_centrality"] = @benchmarkable [Graphs.katz_centrality(g) for (_, g) in $GRAPHS]
+
+#digraphs
+SUITE["centrality"]["digraphs"]["degree_centrality"] = @benchmarkable [Graphs.degree_centrality(g) for (_, g) in $DIGRAPHS]
+SUITE["centrality"]["digraphs"]["closeness_centrality"] = @benchmarkable [Graphs.closeness_centrality(g) for (_, g) in $DIGRAPHS]
+# if nv(g) < 1000 is needed ?
+SUITE["centrality"]["digraphs"]["betweenness_centrality"] = @benchmarkable [Graphs.betweenness_centrality(g) for (_, g) in $DIGRAPHS]
+SUITE["centrality"]["digraphs"]["katz_centrality"] = @benchmarkable [Graphs.katz_centrality(g) for (_, g) in $DIGRAPHS]
+# if nv(g) < 500 is needed ?
+SUITE["centrality"]["digraphs"]["pagerank"] = @benchmarkable [Graphs.pagerank(g) for (_, g) in $DIGRAPHS]

benchmark/serial/connectivity.jl

@@ -0,0 +1,8 @@
+SUITE["connectivity"] = BenchmarkGroup([],
+                               "graphs" => BenchmarkGroup([]), 
+                               "digraphs" => BenchmarkGroup([]), 
+                               )
+
+SUITE["connectivity"]["digraphs"]["strongly_connected_components"] = @benchmarkable [Graphs.strongly_connected_components(g) for (_, g) in $DIGRAPHS]
+
+SUITE["connectivity"]["graphs"]["connected_components"] = @benchmarkable [Graphs.connected_components(g) for (_, g) in $GRAPHS]