Skip to content

Commit

Permalink
Update from MOI branch: reduce code duplication
Browse files Browse the repository at this point in the history
  • Loading branch information
@ericphanson
ericphanson committed Oct 28, 2019
1 parent 12985f5 commit 980dfd5
Showing 1 changed file with 74 additions and 78 deletions.
152 changes: 74 additions & 78 deletions src/problem_depot/problem_depot.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,131 +42,127 @@ affine_diag_atom (generic function with 1 method)
const PROBLEMS = Dict{String, Dict{String, Function}}()

"""
run_tests(
handle_problem!::Function;
exclude::Vector{Regex} = Regex[],
T=Float64, atol=1e-3, rtol=0.0,
)
Run a set of tests. `handle_problem!` should be a function that takes one
argument, a Convex.jl `Problem` and processes it (e.g. `solve!` the problem with
a specific solver).
Use `exclude` to exclude a subset of sets; automatically excludes `r"benchmark"`.
The test tolerances specified by `atol` and `rtol`. Set `T` to choose a numeric type
for the problem. Currently this option is not respected and all problems are specified
Float64` precision.
### Examples
```julia
run_tests(exclude=[r"mip"]) do p
solve!(p, SCSSolver(verbose=0))
end
```
"""
function run_tests(handle_problem!::Function; exclude::Vector{Regex} = Regex[], T=Float64, atol=1e-3, rtol=0.0)
push!(exclude, r"benchmark")
foreach_problem(;exclude=exclude) do name, problem_func
@testset "$name" begin
problem_func(handle_problem!, Val(true), atol, rtol, T)
end
end
end

"""
foreach_problem(apply::Function; exclude::Vector{Regex} = Regex[])
foreach_problem(apply::Function, [class::String],
problems::Union{Nothing, Vector{String}, Vector{Regex}} = nothing;
exclude::Vector{Regex} = Regex[])
Provides a convience method for iterating over problems in [`PROBLEMS`](@ref).
For each problem in [`PROBLEMS`](@ref), apply the function `apply`, which
takes two arguments: the name of the function associated to the problem,
and the function associated to the problem itself.
### Example
[`run_tests`](@ref) can be implemented just by
```julia
foreach_problem(;exclude=exclude) do name, problem_func
@testset "\$name" begin
problem_func(handle_problem!, Val(true), atol, rtol, T)
end
end
```
Optionally, pass a second argument `class` to only iterate over a class of
problems (`class` should satsify `class ∈ keys(PROBLEMS)`), and pass third
argument `problems` to only allow certain problems (specified by exact names or
regex). Use the `exclude` keyword argument to exclude problems by regex.
"""
function foreach_problem(apply::Function; exclude::Vector{Regex} = Regex[])
for (class, dict) in PROBLEMS
function foreach_problem( apply::Function,
problems::Union{Nothing, Vector{String}, Vector{Regex}} = nothing;
exclude::Vector{Regex} = Regex[])
for class in keys(PROBLEMS)
any(occursin.(exclude, Ref(class))) && continue
@testset "$class" begin
for (name, func) in dict
any(occursin.(exclude, Ref(name))) && continue
apply(name, func)
end
foreach_problem(apply, class, problems; exclude=exclude)
end
end

function foreach_problem( apply::Function,
class::String,
problems::Union{Nothing, Vector{String}, Vector{Regex}} = nothing;
exclude::Vector{Regex} = Regex[])
for (name, func) in PROBLEMS[class]
any(occursin.(exclude, Ref(name))) && continue
if problems !== nothing
problems isa Vector{String} && !(name ∈ problems) && continue
problems isa Vector{Regex} && !any(occursin.(problems, Ref(name))) && continue
end
apply(name, func)
end
end


"""
suite(
run_tests(
handle_problem!::Function;
problems::Union{Nothing, Vector{String}, Vector{Regex}} = nothing;
exclude::Vector{Regex} = Regex[],
test = Val(false),
T=Float64, atol=1e-3, rtol=0.0,
)
Create a benchmark_suite of benchmarks. `handle_problem!` should be a function that takes one
Run a set of tests. `handle_problem!` should be a function that takes one
argument, a Convex.jl `Problem` and processes it (e.g. `solve!` the problem with
a specific solver).
Use `exclude` to exclude a subset of benchmarks. Set `test=true` to also check the
answers, with tolerances specified by `atol` and `rtol`. Set `T` to choose a numeric
type for the problem. Currently this option is not respected and all problems are
specified Float64` precision.
Use `exclude` to exclude a subset of sets; automatically excludes
`r"benchmark"`. Optionally, pass a second argument `problems` to only allow certain problems
(specified by exact names or regex). The test tolerances specified by `atol` and
`rtol`. Set `T` to choose a numeric type for the problem. Currently
this is only used for choosing the type parameter of the underlying
MathOptInterface model, but not for the actual problem data.
### Examples
```julia
benchmark_suite(exclude=[r"mip"]) do p
run_tests(exclude=[r"mip"]) do p
solve!(p, SCSSolver(verbose=0))
end
```
"""
function benchmark_suite(handle_problem!::Function; exclude::Vector{Regex} = Regex[], T=Float64, atol=1e-3, rtol=0.0, test = Val(false))
group = BenchmarkGroup()
for (class, dict) in ProblemDepot.PROBLEMS
function run_tests( handle_problem!::Function,
problems::Union{Nothing, Vector{String}, Vector{Regex}} = nothing;
exclude::Vector{Regex} = Regex[], T=Float64, atol=1e-3, rtol=0.0)
push!(exclude, r"benchmark")
for class in keys(PROBLEMS)
any(occursin.(exclude, Ref(class))) && continue
group[class] = BenchmarkGroup()
for (name, func) in dict
any(occursin.(exclude, Ref(name))) && continue
group[class][name] = @benchmarkable $func($handle_problem!, $test, $atol, $rtol, $T)
@testset "$class" begin
foreach_problem(class, problems; exclude=exclude) do name, problem_func
@testset "$name" begin
problem_func(handle_problem!, Val(true), atol, rtol, T)
end
end
end
end
return group
end


"""
suite(
benchmark_suite(
handle_problem!::Function,
problems::Vector{String};
problems::Union{Nothing, Vector{String}, Vector{Regex}} = nothing;
exclude::Vector{Regex} = Regex[],
test = Val(false),
T=Float64, atol=1e-3, rtol=0.0,
)
Create a benchmark_suite of benchmarks using only the problems specified
in the list `problems`.
Create a benchmark_suite of benchmarks. `handle_problem!` should be a function
that takes one argument, a Convex.jl `Problem` and processes it (e.g. `solve!`
the problem with a specific solver). Pass a second argument `problems` to specify
run benchmarks only with certain problems (specified by exact names or regex).
Use `exclude` to exclude a subset of benchmarks. Optionally, pass a second
argument `problems` to only allow certain problems (specified by exact names or
regex). Set `test=true` to also check the answers, with tolerances specified by
`atol` and `rtol`. Set `T` to choose a numeric type for the problem. Currently
this is only used for choosing the type parameter of the underlying
MathOptInterface model, but not for the actual problem data.
### Examples
```julia
benchmark_suite(exclude=[r"mip"]) do p
solve!(p, SCSSolver(verbose=0))
end
```
"""
function benchmark_suite(handle_problem!::Function, problems::Vector{String}; exclude::Vector{Regex} = Regex[], T=Float64, atol=1e-3, rtol=0.0, test = Val(false))
function benchmark_suite(handle_problem!::Function,
problems::Union{Nothing, Vector{String}, Vector{Regex}} = nothing;
exclude::Vector{Regex} = Regex[],
T=Float64, atol=1e-3, rtol=0.0, test = Val(false))
group = BenchmarkGroup()
for (class, dict) in ProblemDepot.PROBLEMS
for class in keys(PROBLEMS)
any(occursin.(exclude, Ref(class))) && continue
group[class] = BenchmarkGroup()
for (name, func) in dict
any(occursin.(exclude, Ref(name))) && continue
name ∈ problems || continue
group[class][name] = @benchmarkable $func($handle_problem!, $test, $atol, $rtol, $T)
foreach_problem(class, problems; exclude=exclude) do name, problem_func
group[class][name] = @benchmarkable $problem_func($handle_problem!, $test, $atol, $rtol, $T)
end
end
return group
Expand Down

0 comments on commit 980dfd5

Please sign in to comment.