From 6f8e87578bdb34682eec9df5699a632a14b38437 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Tue, 24 Oct 2017 20:42:18 -0700 Subject: [PATCH 1/2] Update Nanosoldier to Julia 0.6 and JSON benchmarks --- .travis.yml | 3 ++- README.md | 2 +- REQUIRE | 5 ++--- src/Nanosoldier.jl | 12 +++++------- src/build.jl | 10 +++++----- src/config.jl | 13 ++++++------- src/jobs/BenchmarkJob.jl | 24 ++++++++++++------------ src/jobs/jobs.jl | 2 +- src/server.jl | 3 +-- src/submission.jl | 30 +++++++++++------------------- test/report.md | 8 ++++---- test/runtests.jl | 4 ++-- 12 files changed, 52 insertions(+), 64 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2140000..f7ec0ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,8 @@ language: julia os: - linux julia: - - release + - 0.6 + - nightly notifications: email: false script: diff --git a/README.md b/README.md index fa6c773..ae3a07f 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Once a `BenchmarkJob` is complete, the results are uploaded to the has its own directory for results. This directory contains the following items: - `report.md` is a markdown report that summarizes the job results -- `data.tar.gz` contains raw timing data in JLD format. To untar this file, run +- `data.tar.gz` contains raw timing data in JSON format. To untar this file, run `tar -xzvf data.tar.gz`. You can analyze this data using the [BenchmarkTools](https://github.com/JuliaCI/BaseBenchmarkReports) package. - `logs` is a directory containing the build logs and benchmark execution logs for the job. diff --git a/REQUIRE b/REQUIRE index 906c810..d919b99 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,7 +1,6 @@ -julia 0.4 -BenchmarkTools 0.0.7 +julia 0.6 +BenchmarkTools 0.2.0 GitHub JSON -JLD HttpCommon Compat 0.8.6 diff --git a/src/Nanosoldier.jl b/src/Nanosoldier.jl index fc1f310..fc807f4 100644 --- a/src/Nanosoldier.jl +++ b/src/Nanosoldier.jl @@ -1,11 +1,9 @@ module Nanosoldier -import GitHub, BenchmarkTools, JLD, JSON, HttpCommon +import GitHub, BenchmarkTools, JSON, HttpCommon using Compat -import Compat.UTF8String - const TRIGGER = r"\@nanosoldier\s*`.*?`" const SHA_SEPARATOR = '@' const BRANCH_SEPARATOR = ':' @@ -26,13 +24,13 @@ gitreset!(path) = cd(gitreset!, path) # error handling # ################## -type NanosoldierError{E<:Exception} <: Exception - url::UTF8String - msg::UTF8String +struct NanosoldierError{E<:Exception} <: Exception + url::String + msg::String err::E end -NanosoldierError{E<:Exception}(msg, err::E) = NanosoldierError{E}("", msg, err) +NanosoldierError(msg, err::E) where {E<:Exception} = NanosoldierError{E}("", msg, err) function Base.show(io::IO, err::NanosoldierError) print(io, "NanosoldierError: ", err.msg, ": ") diff --git a/src/build.jl b/src/build.jl index 07a2e42..0f4fa87 100644 --- a/src/build.jl +++ b/src/build.jl @@ -2,15 +2,15 @@ # BuildRef # ############ -type BuildRef - repo::UTF8String # the build repo - sha::UTF8String # the build + status SHA - vinfo::UTF8String # versioninfo() taken during the build +mutable struct BuildRef + repo::String # the build repo + sha::String # the build + status SHA + vinfo::String # versioninfo() taken during the build end BuildRef(repo, sha) = BuildRef(repo, sha, "retrieving versioninfo() failed") -function @compat(Base.:(==))(a::BuildRef, b::BuildRef) +function Base.:(==)(a::BuildRef, b::BuildRef) return (a.repo == b.repo && a.sha == b.sha && a.vinfo == b.vinfo) diff --git a/src/config.jl b/src/config.jl index e856b2f..d8b4e0c 100644 --- a/src/config.jl +++ b/src/config.jl @@ -1,13 +1,12 @@ - -immutable Config - user::UTF8String # the OS username of the user running the server +struct Config + user::String # the OS username of the user running the server nodes::Vector{Int} # the pids for the nodes on the cluster cpus::Vector{Int} # the indices of the cpus per node auth::GitHub.Authorization # the GitHub authorization used to post statuses/reports - secret::UTF8String # the GitHub secret used to validate webhooks - trackrepo::UTF8String # the main Julia repo tracked by the server - reportrepo::UTF8String # the repo to which result reports are posted - workdir::UTF8String # the server's work directory + secret::String # the GitHub secret used to validate webhooks + trackrepo::String # the main Julia repo tracked by the server + reportrepo::String # the repo to which result reports are posted + workdir::String # the server's work directory testmode::Bool # if true, jobs will run as test jobs function Config(user, nodes, cpus, auth, secret; workdir = pwd(), diff --git a/src/jobs/BenchmarkJob.jl b/src/jobs/BenchmarkJob.jl index 7765b93..89adce9 100644 --- a/src/jobs/BenchmarkJob.jl +++ b/src/jobs/BenchmarkJob.jl @@ -40,9 +40,9 @@ end # BenchmarkJob # ################ -type BenchmarkJob <: AbstractJob +mutable struct BenchmarkJob <: AbstractJob submission::JobSubmission # the original submission - tagpred::UTF8String # predicate string to be fed to @tagged + tagpred::String # predicate string to be fed to @tagged against::Nullable{BuildRef} # the comparison build (if available) date::Dates.Date # the date of the submitted job isdaily::Bool # is the job a daily job? @@ -144,10 +144,10 @@ function retrieve_daily_data!(results, key, cfg, date) try run(`tar -xvzf data.tar.gz`) datafiles = readdir(datapath) - primary_index = findfirst(fname -> endswith(fname, "_primary.jld"), datafiles) + primary_index = findfirst(fname -> endswith(fname, "_primary.json"), datafiles) if primary_index > 0 primary_file = datafiles[primary_index] - results[key] = BenchmarkTools.load(joinpath(datapath, primary_file), "results") + results[key] = BenchmarkTools.load(joinpath(datapath, primary_file))[1] found_previous_date = true end catch err @@ -268,14 +268,14 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol) branchname = cfg.testmode ? "test" : "nanosoldier" oldpwd = pwd() try run(`$juliapath -e 'Pkg.clone("https://github.com/JuliaCI/BaseBenchmarks.jl")'`) end - cd(readstring(`$juliapath -e 'print(Pkg.dir("BaseBenchmarks"))'`)) + cd(read(`$juliapath -e 'print(Pkg.dir("BaseBenchmarks"))'`, String)) run(`git fetch --all --quiet`) run(`git reset --hard --quiet origin/$(branchname)`) cd(oldpwd) # The following code sets up a CPU shield, then spins up a new julia process on the # shielded CPU that runs the benchmarks. The results from this new process are - # then serialized to a JLD file so that we can retrieve them. + # then serialized to a JSON file so that we can retrieve them. # # CPU shielding requires passwordless sudo access to `cset`. To enable this for the # server user, run `sudo visudo` and add the following line: @@ -305,7 +305,7 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol) benchname = string(build.sha, "_", whichbuild) benchout = joinpath(tmplogdir(job), string(benchname, ".out")) bencherr = joinpath(tmplogdir(job), string(benchname, ".err")) - benchresults = joinpath(tmpdatadir(job), string(benchname, ".jld")) + benchresults = joinpath(tmpdatadir(job), string(benchname, ".json")) open(jlscriptpath, "w") do file println(file, """ @@ -316,12 +316,12 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol) # move ourselves onto the first CPU in the shielded set run(`sudo cset proc -m -p \$(getpid()) -t /user/child`) - VERSION < v"0.5.0-dev+4338" ? blas_set_num_threads(1) : BLAS.set_num_threads(1) # ensure BLAS threads do not trample each other + BLAS.set_num_threads(1) # ensure BLAS threads do not trample each other addprocs(1) # add worker that can be used by parallel benchmarks using BaseBenchmarks using BenchmarkTools - using JLD + using JSON println("LOADING SUITE...") BaseBenchmarks.loadall!() @@ -336,7 +336,7 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol) results = run(benchmarks; verbose = true) println("SAVING RESULT...") - BenchmarkTools.save(\"$(benchresults)\", "results", results) + BenchmarkTools.save(\"$(benchresults)\", results) println("DONE!") @@ -365,12 +365,12 @@ function execute_benchmarks!(job::BenchmarkJob, whichbuild::Symbol) run(`sudo cset set -d /user/child`) run(`sudo cset shield --reset`) - results = BenchmarkTools.load(benchresults, "results") + results = BenchmarkTools.load(benchresults)[1] # Get the verbose output of versioninfo for the build, throwing away # environment information that is useless/potentially risky to expose. try - build.vinfo = first(split(readstring(`$(juliapath) -e 'versioninfo(true)'`), "Environment")) + build.vinfo = first(split(read(`$(juliapath) -e 'versioninfo(true)'`, String), "Environment")) catch err build.vinfo = string("retrieving versioninfo() failed: ", sprint(showerror, err)) end diff --git a/src/jobs/jobs.jl b/src/jobs/jobs.jl index ba0c653..cd669e9 100644 --- a/src/jobs/jobs.jl +++ b/src/jobs/jobs.jl @@ -6,7 +6,7 @@ # - `Base.run(job::J)`: execute `job` # - `Base.summary(job::J)`: a short string identifying/describing `job` -abstract AbstractJob +abstract type AbstractJob end reply_status(job::AbstractJob, args...; kwargs...) = reply_status(submission(job), args...; kwargs...) reply_comment(job::AbstractJob, args...; kwargs...) = reply_comment(submission(job), args...; kwargs...) diff --git a/src/server.jl b/src/server.jl index 2b6bf2a..f88ff54 100644 --- a/src/server.jl +++ b/src/server.jl @@ -1,5 +1,4 @@ - -immutable Server +struct Server config::Config jobs::Vector{AbstractJob} listener::GitHub.CommentListener diff --git a/src/submission.jl b/src/submission.jl index 814bf51..eed6bd3 100644 --- a/src/submission.jl +++ b/src/submission.jl @@ -1,14 +1,13 @@ - -type JobSubmission +mutable struct JobSubmission config::Config build::BuildRef - statussha::UTF8String # the SHA to send statuses to (since `build` can mutate) - url::UTF8String # the URL linking to the triggering comment + statussha::String # the SHA to send statuses to (since `build` can mutate) + url::String # the URL linking to the triggering comment fromkind::Symbol # `:pr`, `:review`, or `:commit`? prnumber::Nullable{Int} # the job's PR number, if `fromkind` is `:pr` or `:review` - func::UTF8String - args::Vector{UTF8String} - kwargs::Dict{Symbol,UTF8String} + func::String + args::Vector{String} + kwargs::Dict{Symbol,String} end function JobSubmission(config::Config, event::GitHub.WebhookEvent, submission_string) @@ -21,7 +20,7 @@ function JobSubmission(config::Config, event::GitHub.WebhookEvent, submission_st end end -function @compat(Base.:(==))(a::JobSubmission, b::JobSubmission) +function Base.:(==)(a::JobSubmission, b::JobSubmission) if isnull(a.prnumber) == isnull(b.prnumber) same_prnumber = isnull(a.prnumber) ? true : (get(a.prnumber) == get(b.prnumber)) return (same_prnumber && a.config == b.config && a.build == b.build && @@ -77,22 +76,15 @@ function parse_event(config::Config, event::GitHub.WebhookEvent) end # `x` can only be Expr, Symbol, QuoteNode, T<:Number, or T<:AbstractString -function phrase_argument{T}(x::T) - if T <: Expr || T <: Symbol || T <: QuoteNode - return UTF8String(string(x)) - elseif T <: AbstractString || T <: Number - return UTF8String(repr(x)) - else - error("invalid argument type $(typeof(x))") - end -end +phrase_argument(x::Union{Expr, Symbol, QuoteNode}) = string(x) +phrase_argument(x::Union{AbstractString, Number}) = repr(x) function parse_submission_string(submission_string) fncall = match(r"`.*?`", submission_string).match[2:end-1] argind = searchindex(fncall, "(") name = fncall[1:(argind - 1)] parsed_args = parse(replace(fncall[argind:end], ";", ",")) - args, kwargs = Vector{UTF8String}(), Dict{Symbol,UTF8String}() + args, kwargs = Vector{String}(), Dict{Symbol,String}() if isa(parsed_args, Expr) && parsed_args.head == :tuple started_kwargs = false for x in parsed_args.args @@ -132,7 +124,7 @@ function upload_report_repo!(sub::JobSubmission, markdownpath, message) sha = cd(reportdir(cfg)) do run(`git add -A`) run(`git commit -m $message`) - headsha = chomp(readstring(`git rev-parse HEAD`)) + headsha = readchomp(`git rev-parse HEAD`) run(`git pull -X ours`) run(`git push`) return headsha diff --git a/test/report.md b/test/report.md index af8975a..7d7c9b1 100644 --- a/test/report.md +++ b/test/report.md @@ -29,15 +29,15 @@ benchmark results remained invariant between builds). | ID | time ratio | memory ratio | |----|------------|--------------| -| `["g","h","z"]` | 1.00 (60%) | 5.00 (27%) :x: | -| `["g","h",("y",1)]` | 2.00 (5%) :x: | 0.00 (3%) :white_check_mark: | -| `["g","h",("y",2)]` | 0.50 (5%) :white_check_mark: | 1.00 (1%) | +| `["g", "h", "z"]` | 1.00 (60%) | 5.00 (27%) :x: | +| `["g", "h", ("y", 1)]` | 2.00 (5%) :x: | 0.00 (3%) :white_check_mark: | +| `["g", "h", ("y", 2)]` | 0.50 (5%) :white_check_mark: | 1.00 (1%) | ## Benchmark Group List Here's a list of all the benchmark groups executed by this job: -- `["g","h"]` +- `["g", "h"]` ## Version Info diff --git a/test/runtests.jl b/test/runtests.jl index 579cb4d..bdc8ce7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,5 @@ import GitHub -using Nanosoldier, Base.Test, Compat, BenchmarkTools +using Nanosoldier, Compat, Compat.Test, BenchmarkTools using Nanosoldier: BuildRef, JobSubmission, Config, BenchmarkJob, AbstractJob using BenchmarkTools: TrialEstimate, Parameters @@ -140,6 +140,6 @@ results["judged"] = BenchmarkTools.judge(results["primary"], results["against"]) @test begin mdpath = joinpath(dirname(@__FILE__), "report.md") open(mdpath, "r") do file - readstring(file) == sprint(io -> Nanosoldier.printreport(io, job, results)) + read(file, String) == sprint(io -> Nanosoldier.printreport(io, job, results)) end end From 9e3ea12d10b300d0b9b0882787d37ecb3c575537 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Sat, 28 Oct 2017 15:48:43 -0700 Subject: [PATCH 2/2] Replace dependency on HttpCommon with HTTP --- REQUIRE | 2 +- src/Nanosoldier.jl | 2 +- src/server.jl | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/REQUIRE b/REQUIRE index d919b99..e9a048d 100644 --- a/REQUIRE +++ b/REQUIRE @@ -2,5 +2,5 @@ julia 0.6 BenchmarkTools 0.2.0 GitHub JSON -HttpCommon +HTTP Compat 0.8.6 diff --git a/src/Nanosoldier.jl b/src/Nanosoldier.jl index fc807f4..d718900 100644 --- a/src/Nanosoldier.jl +++ b/src/Nanosoldier.jl @@ -1,6 +1,6 @@ module Nanosoldier -import GitHub, BenchmarkTools, JSON, HttpCommon +import GitHub, BenchmarkTools, JSON, HTTP using Compat diff --git a/src/server.jl b/src/server.jl index f88ff54..029c502 100644 --- a/src/server.jl +++ b/src/server.jl @@ -12,10 +12,10 @@ struct Server handle = (event, phrase) -> begin nodelog(config, 1, "received job submission with phrase $phrase") if event.kind == "issue_comment" && !(haskey(event.payload["issue"], "pull_request")) - return HttpCommon.Response(400, "nanosoldier jobs cannot be triggered from issue comments (only PRs or commits)") + return HTTP.Response(400, "nanosoldier jobs cannot be triggered from issue comments (only PRs or commits)") end if haskey(event.payload, "action") && !(in(event.payload["action"], ("created", "opened"))) - return HttpCommon.Response(204, "no action taken (submission was from an edit, close, or delete)") + return HTTP.Response(204, "no action taken (submission was from an edit, close, or delete)") end submission = JobSubmission(config, event, phrase.match) addedjob = false @@ -33,9 +33,9 @@ struct Server end if !(addedjob) reply_status(submission, "error", "invalid job submission; check syntax") - HttpCommon.Response(400, "invalid job submission") + HTTP.Response(400, "invalid job submission") end - return HttpCommon.Response(202, "received job submission") + return HTTP.Response(202, "received job submission") end listener = GitHub.CommentListener(handle, TRIGGER;