Add benchmarks (#321)

* Add some benchmarks for use with PkgBenchmark

* Add benchmark utilities and add to Travis

* Fix newlines at the end of files

* Fix word wrapping

* Update to 1.2

.gitignore

@@ -5,3 +5,4 @@
 *.cov
 .ipynb_checkpoints/
 Manifest.toml
+benchmark/*.json

.travis.yml

@@ -4,13 +4,25 @@ os:
   - osx
 julia:
   - 1.0
-  - 1.1
+  - 1.2
 notifications:
   email: false
 after_success:
   - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Coveralls.submit(process_folder())'
 jobs:
   include:
+    - name: "Benchmark"
+      julia: 1.2
+      os: linux
+      before_script:
+        - git fetch origin '+refs/heads/master:refs/remotes/origin/master'
+        - git branch master origin/master
+        # Run benchmark outside `script` so that it's hidden by default:
+        - julia --project=benchmark benchmark/runjudge.jl'
+      script:
+        - julia --project=benchmark benchmark/pprintjudge.jl'
+      after_success: skip
+      if: NOT (branch = master)
     - stage: Documentation
       julia: 1.0
       script: julia --project=docs -e '

LICENSE.md

@@ -23,3 +23,51 @@ The Convex.jl package is licensed under the Simplified "2-clause" BSD License:
 > THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 > (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 > OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The file benchmark/benchmarks/benchmark.jl contains some utilities copied from the MathOptInterface.jl package
+(https://github.com/JuliaOpt/MathOptInterface.jl) which is licensed under the MIT License:
+
+>Copyright (c) 2017: Miles Lubin and contributors Copyright (c) 2017: Google Inc.
+>
+>Permission is hereby granted, free of charge, to any person obtaining a copy
+>of this software and associated documentation files (the "Software"), to deal
+>in the Software without restriction, including without limitation the rights
+>to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+>copies of the Software, and to permit persons to whom the Software is
+>furnished to do so, subject to the following conditions:
+>
+>The above copyright notice and this permission notice shall be included in all
+>copies or substantial portions of the Software.
+>
+>THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+>IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+>FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+>AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+>LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+>OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+>SOFTWARE.
+
+The files benchmark/pprinthelper.jl, benchmark/pprintjudge.jl,
+benchmark/pprintresult.jl, benchmark/runbenchmarks.jl, and benchmark/runjudge.jl
+and .travis.yml contain code copied from the Transducers.jl package
+(https://github.com/tkf/Transducers.jl/) which is licensed under the MIT License:
+
+>Copyright (c) 2018 Takafumi Arakaki
+>
+>Permission is hereby granted, free of charge, to any person obtaining a copy
+>of this software and associated documentation files (the "Software"), to deal
+>in the Software without restriction, including without limitation the rights
+>to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+>copies of the Software, and to permit persons to whom the Software is
+>furnished to do so, subject to the following conditions:
+>
+>The above copyright notice and this permission notice shall be included in all
+>copies or substantial portions of the Software.
+>
+>THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+>IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+>FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+>AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+>LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+>OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+>SOFTWARE.

benchmark/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"

benchmark/benchmarks.jl

@@ -0,0 +1,21 @@
+using Pkg
+tempdir = mktempdir()
+Pkg.activate(tempdir)
+Pkg.develop(PackageSpec(path=joinpath(@__DIR__, "..")))
+Pkg.add(["BenchmarkTools", "SCS", "ECOS", "PkgBenchmark"])
+Pkg.resolve()
+
+using Convex: Convex
+using SCS: SCSSolver
+using ECOS: ECOSSolver
+using BenchmarkTools
+
+
+include("benchmarks/benchmarks.jl") # defines module Benchmarks
+
+const SUITE = BenchmarkGroup()
+
+SUITE["SCS"] = Benchmarks.suite(p -> Convex.solve!(p, SCSSolver(verbose=0)))
+SUITE["ECOS"] = Benchmarks.suite(p -> Convex.solve!(p, ECOSSolver(verbose=0));
+    exclude = [r"sdp", r"SDP"])
+SUITE["formulation"] = Benchmarks.suite(Convex.conic_problem)

benchmark/benchmarks/affine_benchmarks.jl

@@ -0,0 +1,164 @@
+@add_benchmark function affine_negate_atom(handle_problem)
+    x = Variable()
+    p = minimize(-x, [x <= 0])
+    handle_problem(p)
+end
+
+@add_benchmark function affine_kron_atom(handle_problem)
+    x = ComplexVariable(3, 3)
+    y = [1.0 2.0; 3.0 4.0]
+    p = satisfy(kron(x,y) == kron(eye(3), y))
+    handle_problem(p)
+end
+
+@add_benchmark function affine_mult_atom(handle_problem)
+    probs=[
+        let
+            x = Variable(1)
+            p = minimize(2.0 * x, [x >= 2, x <= 4])
+        end,
+        let
+            x = Variable(2)
+            A = 1.5 * eye(2)
+            p = minimize([2 2] * x, [A * x >= [1.1; 1.1]])
+        end,
+        let
+            y = Variable(1)
+            x = Variable(3)
+            z = [1.0, 2.0, 3.0] * y
+            k = -y * [1.0, 2.0, 3.0]
+            c = [y <= 3.0, y >= 0.0, x >= ones(3),  k <= x, x <= z]
+            o = 3 * y
+            p = Problem(:minimize, o, c)
+        end,
+        let
+            x = ComplexVariable(2,2)
+            p = minimize(   real( [1.0im, 0.0]' * x * [1.0im, 0.0] ),
+                    [ x == [1.0 0.0; 0.0 1.0] ] )
+        end
+    ]
+    handle_problem.(probs)
+end
+
+@add_benchmark function affine_dot_atom(handle_problem)
+    probs = [
+        let
+            x = Variable(2)
+            p = minimize(dot([2.0; 2.0], x), x >= [1.1; 1.1])
+        end,
+        let
+            x = Variable(2,2)
+            p = minimize(dot(fill(2.0, (2,2)), x), x >= 1.1)
+        end
+    ]
+
+    handle_problem.(probs)
+end
+
+
+@add_benchmark function affine_add_atom(handle_problem)
+
+    probs = [
+        let
+            x = Variable(1)
+            y = Variable(1)
+            p = minimize(x + y, [x >= 3, y >= 2])
+        end,
+        let
+            x = Variable(1)
+            p = minimize(x, [eye(2) + x >= eye(2)])
+        end,
+        let
+            y = Variable()
+            p = minimize(y - 5, y >= -1)
+        end ]
+
+    handle_problem.(probs)
+end
+
+
+
+@add_benchmark function affine_transpose_atom(handle_problem)
+    probs = [
+        let
+            x = Variable(2)
+            c = ones(2, 1)
+            p = minimize(x' * c, x >= 1)
+        end,
+        let
+            X = Variable(2, 2)
+            c = ones(2, 1)
+            p = minimize(c' * X' * c, [X >= ones(2, 2)])
+        end,
+        let
+            rows = 2
+            cols = 3
+            r = rand(rows, cols)
+            r_2 = rand(cols, rows)
+            x = Variable(rows, cols)
+            c = ones(1, cols)
+            d = ones(rows, 1)
+            p = minimize(c * x' * d + d' * x * c' + (c * x''''' * d)',
+                        [x' >= r_2, x >= r, x''' >= r_2, x'' >= r])
+        end ]
+
+        handle_problem.(probs)
+end
+
+@add_benchmark function affine_index_atom(handle_problem)
+        probs = [
+        let
+            x = Variable(2)
+            p = minimize(x[1] + x[2], [x >= 1])
+        end,
+        let
+            x = Variable(3)
+            I = [true true false]
+            p = minimize(sum(x[I]), [x >= 1])
+        end,
+        let
+            rows = 6
+            cols = 8
+            n = 2
+            X = Variable(rows, cols)
+            A = randn(rows, cols)
+            c = rand(1, n)
+            p = minimize(c * X[1:n, 5:5+n-1]' * c', X >= A)
+        end]
+        handle_problem.(probs)
+end
+
+@add_benchmark function affine_sum_atom(handle_problem)
+        probs = [
+        let
+            x = Variable(2,2)
+            p = minimize(sum(x) - 2*x[1,1], x>=1, x[1,1]<=2)
+        end,
+        let
+            x = Variable(2,2)
+            p = minimize(sum(x) - 2*x[1,1], x>=1, x[1,1]<=2)
+        end
+        ]
+    return handle_problem.(probs)
+end
+
+@add_benchmark function affine_diag_atom(handle_problem)
+    probs = [
+        let
+            x = Variable(2,2)
+            p = minimize(sum(diag(x,1)), x >= 1)
+        end,
+        let
+            x = Variable(4, 4)
+            p = minimize(sum(diag(x)), x >= 2)
+        end,
+    ]
+    return handle_problem.(probs)
+
+end
+
+@add_benchmark function affine_tr_atom(handle_problem)
+    x = Variable(2,2)
+    p = minimize(tr(x), x >= 1)
+    handle_problem(p)
+end

benchmark/benchmarks/benchmarks.jl

@@ -0,0 +1,56 @@
+# `BENCHMARKS`, `suite`, and `add_benchmark` were taken from MathOptInterface
+# which is available under an MIT license (see LICENSE).
+module Benchmarks
+
+using BenchmarkTools
+using Convex, LinearAlgebra
+
+const BENCHMARKS = Dict{String, Function}()
+
+"""
+    suite(
+        handle_problem::Function;
+        exclude::Vector{Regex} = Regex[]
+    )
+
+Create a suite of benchmarks. `handle_problem` should be a function that takes one
+argument, a Convex.jl `Problem` and processes it (e.g. `solve!` the problem with
+a specific solver).
+
+Use `exclude` to exclude a subset of benchmarks.
+
+### Examples
+
+```julia
+suite() do p
+    solve!(p, GLPK.Optimizer())
+end
+```
+"""
+function suite(handle_problem::Function; exclude::Vector{Regex} = Regex[])
+    group = BenchmarkGroup()
+    for (name, func) in BENCHMARKS
+        any(occursin.(exclude, Ref(name))) && continue
+        group[name] = @benchmarkable $func($handle_problem) setup=Convex.clearmemory()
+    end
+    return group
+end
+
+###
+### Benchmarks
+###
+
+macro add_benchmark(f)
+    name = f.args[1].args[1]
+    return quote
+        $(esc(f))
+        BENCHMARKS[String($(Base.Meta.quot(name)))] = $(esc(name))
+    end
+end
+
+eye(n) = Matrix(1.0I, n, n)
+include("constraint_benchmarks.jl")
+include("affine_benchmarks.jl")
+include("sdp_benchmarks.jl")
+
+end

benchmark/benchmarks/constraint_benchmarks.jl

@@ -0,0 +1,64 @@
+@add_benchmark function LT_constraints(handle_problem)
+    p = satisfy()
+    x = [Variable() for _ = 1:1000]
+    for (i, xi) in enumerate(x)
+        push!(p.constraints, xi <= 1.0 * i)
+    end
+    return handle_problem(p)
+end
+
+@add_benchmark function LT_constraint(handle_problem)
+    p = satisfy()
+    x = Variable(1000)
+    push!(p.constraints, x <= collect(1.0:1000.0))
+    return handle_problem(p)
+end
+
+@add_benchmark function GT_constraints(handle_problem)
+    p = satisfy()
+    x = [Variable() for _ = 1:1000]
+    for (i, xi) in enumerate(x)
+        push!(p.constraints, xi >= 1.0 * i)
+    end
+    return handle_problem(p)
+end
+
+@add_benchmark function GT_constraint(handle_problem)
+    p = satisfy()
+    x = Variable(1000)
+    push!(p.constraints, x >= collect(1.0:1000.0))
+    return handle_problem(p)
+end
+
+
+@add_benchmark function equality_constraints(handle_problem)
+    p = satisfy()
+    x = [Variable() for _ = 1:1000]
+    for (i, xi) in enumerate(x)
+        push!(p.constraints, xi == 1.0 * i)
+    end
+    return handle_problem(p)
+end
+
+@add_benchmark function equality_constraint(handle_problem)
+    p = satisfy()
+    x = Variable(1000)
+    push!(p.constraints, x == collect(1.0:1000.0))
+    return handle_problem(p)
+end
+
+@add_benchmark function SDP_constraint(handle_problem)
+    p = satisfy()
+    x = Variable(44, 44) # 990 vectorized entries
+    push!(p.constraints, x ⪰ 0)
+    return handle_problem(p)
+end
+
+@add_benchmark function SDP_constraints(handle_problem)
+    p = satisfy()
+    x = [Variable(4, 4) for _ = 1:100] # 1000 total vectorized entries
+    for v in x
+        push!(p.constraints, v ⪰ 0)
+    end
+    return handle_problem(p)
+end