suite(
    new_model::Function;
    exclude::Vector{Regex} = Regex[]
)

Create a suite of benchmarks. new_model should be a function that takes no arguments, and returns a new instance of the optimizer you wish to benchmark.

Use exclude to exclude a subset of benchmarks.

BenchmarkTools

To use this function you must first install and load the BenchmarkTools.jl package.

Example

julia> import BenchmarkTools, GLPK, Gurobi

julia> MOI.Benchmarks.suite(GLPK.Optimizer)

julia> MOI.Benchmarks.suite(; exclude = [r"delete"]) do
           return Gurobi.Optimizer()
       end

create_baseline(suite, name::String; directory::String = ""; kwargs...)

Run all benchmarks in suite and save to files called name in directory.

Extra kwargs are based to BenchmarkTools.run.

BenchmarkTools

To use this function you must first install and load the BenchmarkTools.jl package.

Example

julia> import BenchmarkTools, GLPK

julia> my_suite = MOI.Benchmarks.suite(GLPK.Optimizer);

julia> MOI.Benchmarks.create_baseline(
           my_suite,
           "glpk_master";
           directory = "/tmp",
           verbose = true,
       )

compare_against_baseline(
    suite, name::String; directory::String = "",
    report_filename::String = "report.txt"
)

Run all benchmarks in suite and compare against files called name in directory that were created by a call to create_baseline.

A report summarizing the comparison is written to report_filename in directory.

Extra kwargs are based to BenchmarkTools.run.

BenchmarkTools

To use this function you must first install and load the BenchmarkTools.jl package.

Example

julia> import BenchmarkTools, GLPK

julia> my_suite = MOI.Benchmarks.suite(GLPK.Optimizer);

julia> MOI.Benchmarks.compare_against_baseline(
           my_suite,
           "glpk_master";
           directory = "/tmp",
           verbose = true,
       )