From 883a2a738caf2e58a179339bff026efcbb49ae97 Mon Sep 17 00:00:00 2001 From: skyleaworlder <870033938@qq.com> Date: Wed, 26 Jul 2023 14:14:11 +0000 Subject: [PATCH 1/3] update: change --deps-list format --- README.md | 4 ++-- src/env_utils.jl | 25 +++++++++++-------------- test/env_utils_test.jl | 2 +- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 0e88de9..5e9efaa 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ For specification, `Dependencies List` is a single string that simulates an arra e.g. ```shell -> DEPS_LIST="https://github.com/FluxML/NNlib.jl#backports-0.8.21,https://github.com/skyleaworlder/NNlib.jl#dummy-benchmark-test;Flux,Flux@0.13.12" +> DEPS_LIST="https://github.com/FluxML/NNlib.jl#backports-0.8.21,Flux;https://github.com/skyleaworlder/NNlib.jl#dummy-benchmark-test,Flux@0.13.12" > julia --project=benchmark benchmark/runbenchmarks-cli.jl --deps-list=$DEPS_LIST ``` @@ -93,7 +93,7 @@ More precisely, the granularity of the element of `Enabled Parts` and `Disabled e.g. ```shell -> DEPS_LIST="https://github.com/FluxML/NNlib.jl#backports-0.8.21,https://github.com/skyleaworlder/NNlib.jl#dummy-benchmark-test;Flux,Flux@0.13.12" +> DEPS_LIST="https://github.com/FluxML/NNlib.jl#backports-0.8.21,Flux;https://github.com/skyleaworlder/NNlib.jl#dummy-benchmark-test,Flux@0.13.12" > # Only Flux-MLP and all NNlib > julia --project=benchmark benchmark/runbenchmarks-cli.jl --enable="flux(mlp);nnlib" --deps-list=$DEPS_LIST > # All benchmarks except Flux, NNlib-gemm and NNlib-activations diff --git a/src/env_utils.jl b/src/env_utils.jl index 0af59d8..fb07d16 100644 --- a/src/env_utils.jl +++ b/src/env_utils.jl @@ -231,27 +231,24 @@ end """ parse_deps_list(deps_list::String)::Tuple{Vector{Dependency}, Vector{Dependency}} -is used to parse command argument, "deps-list", to 2 sets of dependencies, -which means parse_deps_list can support difference of multiple dependencies. -Each element separated by a semicolon. Each element consists of two parts: -the first part is baseline dependency, and the second part is target. +is used to parse command argument, "deps-list" represents 2 sets of dependencies. +Each element separated by a semicolon. Each element consists of any number of deps. Now, deps_list only supports FluxML packages. e.g. deps_list can be - "NNlib,https://github.com/skyleaworlder/NNlib.jl#dummy-benchmark-test;Flux,Flux#0.13.12" + "NNlib,Flux;https://github.com/skyleaworlder/NNlib.jl#dummy-benchmark-test,Flux#0.13.12" """ function parse_deps_list(deps_list::String)::Tuple{Vector{Dependency}, Vector{Dependency}} - dep_pairs = filter( - dep_pair_vec -> length(dep_pair_vec) == 2, - map( - dep_pair -> split(dep_pair, ","), - split(deps_list, ";"))) + tmp = split(deps_list, ";") + length(tmp) != 2 && throw(error( + "Must provide 2 sets of dependencies in 'deps-list' argument")) + baseline_deps = map( - baseline_dep -> Dependency(string(baseline_dep)), - map(x -> x[1], dep_pairs)) + dep -> Dependency(string(dep)), + split(tmp[1], ",")) target_deps = map( - target_dep -> Dependency(string(target_dep)), - map(x -> x[2], dep_pairs)) + dep -> Dependency(string(dep)), + split(tmp[2], ",")) return (baseline_deps, target_deps) end diff --git a/test/env_utils_test.jl b/test/env_utils_test.jl index fc53f02..22cd98a 100644 --- a/test/env_utils_test.jl +++ b/test/env_utils_test.jl @@ -40,7 +40,7 @@ end @testset "parse deps list" begin - deps_list = "NNlib,https://github.com/skyleaworlder/NNlib.jl#dummy-benchmark-test;Flux,Flux@0.13.12" + deps_list = "NNlib,Flux;https://github.com/skyleaworlder/NNlib.jl#dummy-benchmark-test,Flux@0.13.12" baseline_deps, target_deps = parse_deps_list(deps_list) @test (length(baseline_deps) == 2 && baseline_deps[1].name == "NNlib" && From dc2043d354655740367266ec7a0fa6983f97b7af Mon Sep 17 00:00:00 2001 From: skyleaworlder <870033938@qq.com> Date: Thu, 27 Jul 2023 16:33:06 +0000 Subject: [PATCH 2/3] feat: update parse_deps_list to support multiple sets of dependencies --- src/env_utils.jl | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/env_utils.jl b/src/env_utils.jl index fb07d16..5b32a03 100644 --- a/src/env_utils.jl +++ b/src/env_utils.jl @@ -229,27 +229,31 @@ end """ - parse_deps_list(deps_list::String)::Tuple{Vector{Dependency}, Vector{Dependency}} + parse_deps(deps)::Vector{Dependency} -is used to parse command argument, "deps-list" represents 2 sets of dependencies. +is used to parse dependencies. + +* deps: suggested to be a string, can be `--target` or `--baseline` + +e.g. deps can be like "NNlib,Flux" or + "https://github.com/skyleaworlder/NNlib.jl#dummy-benchmark-test,Flux#0.13.12" +""" +parse_deps(deps) = map(dep -> Dependency(string(dep)), split(deps, ",")) + + +""" + parse_deps_list(deps_list::String)::Tuple + +is used to parse command argument, "deps-list" represents multiple sets of dependencies. Each element separated by a semicolon. Each element consists of any number of deps. Now, deps_list only supports FluxML packages. e.g. deps_list can be "NNlib,Flux;https://github.com/skyleaworlder/NNlib.jl#dummy-benchmark-test,Flux#0.13.12" """ -function parse_deps_list(deps_list::String)::Tuple{Vector{Dependency}, Vector{Dependency}} - tmp = split(deps_list, ";") - length(tmp) != 2 && throw(error( - "Must provide 2 sets of dependencies in 'deps-list' argument")) - - baseline_deps = map( - dep -> Dependency(string(dep)), - split(tmp[1], ",")) - target_deps = map( - dep -> Dependency(string(dep)), - split(tmp[2], ",")) - return (baseline_deps, target_deps) +function parse_deps_list(deps_list::String)::Tuple + parsed_deps_list = [parse_deps(deps) for deps in split(deps_list, ";")] + return Tuple(parsed_deps_list) end From 6d28bb7a25f699cc5635b210fceb361492198509 Mon Sep 17 00:00:00 2001 From: skyleaworlder <870033938@qq.com> Date: Thu, 27 Jul 2023 16:48:00 +0000 Subject: [PATCH 3/3] refact: change runbenchmarks-cli.jl function, support multiple sets of dependencies, but not judge and generate report.md any longer --- benchmark/runbenchmarks-cli.jl | 74 ++++++++++------------------------ 1 file changed, 21 insertions(+), 53 deletions(-) diff --git a/benchmark/runbenchmarks-cli.jl b/benchmark/runbenchmarks-cli.jl index 35d1cf2..3d19f2b 100644 --- a/benchmark/runbenchmarks-cli.jl +++ b/benchmark/runbenchmarks-cli.jl @@ -1,63 +1,31 @@ using Pkg - -########################################################################### - Pkg.develop(PackageSpec(path = ENV["PWD"])) using FluxMLBenchmarks parsed_args = parse_commandline() retune_arg = parsed_args["retune"] retune_arg || get_tuning_json() -enable_arg = parsed_args["enable"] -disable_arg = parsed_args["disable"] +enable_arg, disable_arg = parsed_args["enable"], parsed_args["disable"] enabled_benchmarks = parse_enabled_benchmarks(enable_arg, disable_arg) deps_list = parsed_args["deps-list"] -baseline_fluxml_deps, target_fluxml_deps = parse_deps_list(deps_list) -time_setup_fluxml_env = @elapsed setup_fluxml_env(baseline_fluxml_deps) -@info "TIME: setup FluxML benchmarking environment (baseline) cost $time_setup_fluxml_env" - -using BenchmarkTools -using PkgBenchmark -group_baseline = benchmarkpkg( - dirname(@__DIR__), - BenchmarkConfig( - env = merge( - Dict("JULIA_NUM_THREADS" => get(ENV, "JULIA_NUM_THREADS", "1")), - enabled_benchmarks - ) - ), - resultfile = joinpath(@__DIR__, "result-baseline.json") -) - -teardown() - -########################################################################### - -Pkg.develop(PackageSpec(path = ENV["PWD"])) -using FluxMLBenchmarks - -time_setup_fluxml_env = @elapsed setup_fluxml_env(target_fluxml_deps) -@info "TIME: setup FluxML benchmarking environment (target) cost $time_setup_fluxml_env" - -using BenchmarkTools -using PkgBenchmark -group_target = benchmarkpkg( - dirname(@__DIR__), - BenchmarkConfig( - env = merge( - Dict("JULIA_NUM_THREADS" => get(ENV, "JULIA_NUM_THREADS", "1")), - enabled_benchmarks - ) - ), - resultfile = joinpath(@__DIR__, "result-target.json"), -) - -teardown() - -########################################################################### - -judgement = judge(group_target, group_baseline) -report_md = markdown_report(judgement) -write(joinpath(@__DIR__, "report.md"), report_md) -display_markdown_report(report_md) +parsed_deps_list = parse_deps_list(deps_list) +for (i, deps) in enumerate(parsed_deps_list) + time_setup_fluxml_env = @elapsed setup_fluxml_env(deps) + @info "($i) TIME: setup FluxML benchmarking environment cost $time_setup_fluxml_env" + + using PkgBenchmark: benchmarkpkg, BenchmarkConfig + time_run_benchmarks = @elapsed begin benchmarkpkg( + dirname(@__DIR__), + BenchmarkConfig( + env = merge( + Dict("JULIA_NUM_THREADS" => get(ENV, "JULIA_NUM_THREADS", "1")), + enabled_benchmarks + ) + ), + resultfile = joinpath(@__DIR__, "result-$i.json") + ) end + @info "($i) TIME: run benchmarks cost $time_run_benchmarks" + + teardown() +end