From 7f4eec5ad9e14648f26329f413b2a2d64f197476 Mon Sep 17 00:00:00 2001 From: skyleaworlder Date: Thu, 17 Aug 2023 09:46:17 +0000 Subject: [PATCH] feat: add 'arch' option --- benchmark/script/mergereports-cli.jl | 2 +- benchmark/script/runbenchmarks-pr.jl | 2 +- src/env_utils.jl | 17 +++++++++++++++++ src/judge_utils.jl | 22 +++++++++++----------- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/benchmark/script/mergereports-cli.jl b/benchmark/script/mergereports-cli.jl index 892afc2..17d2609 100644 --- a/benchmark/script/mergereports-cli.jl +++ b/benchmark/script/mergereports-cli.jl @@ -24,7 +24,7 @@ if parsed_args["push-result"] && suitable_to_use_result_cache(target_url) @info "RESULT: $target_url is suitable to push its result to remote" writeresults(joinpath(@__DIR__, "..", "result-target.json"), target_benchmarkresults) push_result(target_url, joinpath(@__DIR__, "..", "result-target.json") - ; git_push_password = parsed_args["push-password"]) + ; git_push_password = parsed_args["push-password"], arch = parsed_args["arch"]) end # generate report.md as the content of comment diff --git a/benchmark/script/runbenchmarks-pr.jl b/benchmark/script/runbenchmarks-pr.jl index a9e56b8..0eb3168 100644 --- a/benchmark/script/runbenchmarks-pr.jl +++ b/benchmark/script/runbenchmarks-pr.jl @@ -24,7 +24,7 @@ try throw("") end - global group_baseline = get_benchmarkresults_from_branch(baseline_url) + global group_baseline = get_benchmarkresults_from_branch(baseline_url; arch=parsed_args["arch"]) if isnothing(group_baseline) @warn "RESULT: cannot get result file, run benchmarks" throw("") diff --git a/src/env_utils.jl b/src/env_utils.jl index 31cf402..72fbebb 100644 --- a/src/env_utils.jl +++ b/src/env_utils.jl @@ -38,6 +38,13 @@ const FLUXML_AVAILABLE_TOP_LEVEL_BENCHMARKS = [ "flux", "nnlib" ] +""" +SUPPORTED_ARCHITECTURES means the supported architecture for this tool. +""" +const SUPPORTED_ARCHITECTURES = [ + "cpu", "gpu" +] + """ Dependency @@ -256,6 +263,12 @@ function parse_commandline() "--push-password" help = "used to authenticate when pushing" action = :store_arg + + # about architecture + "--arch" + help = "enable gpu benchmarks" + action = :store_arg + default = "cpu" end args = parse_args(s) # script-related arguments cli / pr / merge-report / cache-setup @@ -267,6 +280,10 @@ function parse_commandline() args["push-result"] && isnothing(args["push-password"]) && throw(error("Must provide 'push-password' if you want to 'push-result'")) + # if arch is in SUPPORTED_ARCHITECTURES, allow returning args + !(args["arch"] in SUPPORTED_ARCHITECTURES) && + throw(error("--arch must be in SUPPORTED_ARCHITECTURES")) + # if (deps-list) or (target & baseline) specified, allow returning args !isnothing(args["deps-list"]) && return args !isnothing(args["target"]) && !isnothing(args["baseline"]) && diff --git a/src/judge_utils.jl b/src/judge_utils.jl index 5f976a1..a7ca760 100644 --- a/src/judge_utils.jl +++ b/src/judge_utils.jl @@ -15,7 +15,7 @@ const TARGET_RESULT_FILE_NAME = "result-target.json" """ - gen_result_filename(single_deps_list::String)::String + gen_result_filename(single_deps_list::String; arch = "cpu")::String is used to generate the filename of benchmark results. The name of results doesn't have relationship with BenchmarkResults, @@ -25,11 +25,11 @@ we can use the information of dependencies to specify a file. * single_deps_list: a part of command argument, `deps-list`. `target` and `baseline` also can passed in. """ -function gen_result_filename(single_deps_list::String)::String +function gen_result_filename(single_deps_list::String; arch = "cpu")::String # keep deps in sort, in case impact of different sequence. sorted_deps = reduce((a, b) -> "$a,$b", sort(map(string, split(single_deps_list, ",")))) - return "$(bytes2hex(sha256(sorted_deps))).json" + return "$(arch)-$(bytes2hex(sha256(sorted_deps))).json" end @@ -61,7 +61,7 @@ end """ - get_result_file_from_branch(single_deps_list::String)s + get_result_file_from_branch(single_deps_list::String; arch = "cpu") is used to checkout corresponding result file from benchmark-results branch. If possible, the result will help to skip benchmarks running. @@ -72,8 +72,8 @@ the content of a file checked out by git. It can be "benchmark/result-baseline.j TODO: badly-designed in semantic, due to the usage of `gen_result_filename`` """ -function get_result_file_from_branch(single_deps_list::String, checkout_filename::String) - result_filename = gen_result_filename(single_deps_list) +function get_result_file_from_branch(single_deps_list::String, checkout_filename::String; arch = "cpu") + result_filename = gen_result_filename(single_deps_list; arch=arch) cmd = pipeline(`git show $RESULTS_BRANCH:$result_filename` ; stdout=checkout_filename) try @@ -89,13 +89,13 @@ end """ - get_benchmarkresults_from_branch(single_deps_list::String)::Union{Nothing,BenchmarkResults} + get_benchmarkresults_from_branch(single_deps_list::String; arch = "cpu")::Union{Nothing,BenchmarkResults} is used to get BenchmarkResults from deps-list. """ -function get_benchmarkresults_from_branch(single_deps_list::String)::Union{Nothing,BenchmarkResults} +function get_benchmarkresults_from_branch(single_deps_list::String; arch = "cpu")::Union{Nothing,BenchmarkResults} try - get_result_file_from_branch(single_deps_list, "tmp.json") + get_result_file_from_branch(single_deps_list, "tmp.json"; arch=arch) catch @warn "RESULT: get_result_file_from_branch failed to get result file with $single_deps_list." return @@ -115,7 +115,7 @@ TODO: badly-designed in semantic, due to the usage of `gen_result_filename` TODO: need better way to process git_push_username and git_push_password """ function push_result(single_deps_list::String, result_file_path::String - ; need_password = true, kwargs...) + ; need_password = true, arch = "cpu", kwargs...) # FIXME: a little bit hacked REPO = LibGit2.GitRepo(joinpath(@__DIR__, "..", ".git")) origin_remote = LibGit2.lookup_remote(REPO, "origin") @@ -135,7 +135,7 @@ function push_result(single_deps_list::String, result_file_path::String end result_file_path_in_br_repo = joinpath( - LibGit2.path(br_repo), gen_result_filename(single_deps_list)) + LibGit2.path(br_repo), gen_result_filename(single_deps_list; arch=arch)) mv(result_file_path, result_file_path_in_br_repo) br_origin_remote = LibGit2.lookup_remote(br_repo, "origin") if isnothing(br_origin_remote)