add some latency benchmarks

Manifest.toml

@@ -27,6 +27,9 @@ git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e"
 uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 version = "0.21.0"
 
+[[LibGit2]]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
 [[Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
@@ -46,18 +49,29 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
 [[Parsers]]
 deps = ["Dates", "Test"]
-git-tree-sha1 = "ef0af6c8601db18c282d092ccbd2f01f3f0cd70b"
+git-tree-sha1 = "0139ba59ce9bc680e2925aec5b7db79065d60556"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "0.3.7"
+version = "0.3.10"
+
+[[Pkg]]
+deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [[Printf]]
 deps = ["Unicode"]
 uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
+[[REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
 [[Random]]
 deps = ["Serialization"]
 uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
+[[SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+
 [[Serialization]]
 uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 
@@ -76,5 +90,9 @@ uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
+[[UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
 [[Unicode]]
 uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

Project.toml

@@ -6,6 +6,7 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

src/BaseBenchmarks.jl

@@ -28,7 +28,8 @@ const MODULES = Dict("array" => :ArrayBenchmarks,
                      "sort" => :SortBenchmarks,
                      "sparse" => :SparseBenchmarks,
                      "string" => :StringBenchmarks,
-                     "tuple" => :TupleBenchmarks)
+                     "tuple" => :TupleBenchmarks,
+                     "latency" => :LatencyBenchmarks)
 
 load!(id::AbstractString; kwargs...) = load!(SUITE, id; kwargs...)
 

src/latency/LatencyBenchmarks.jl

@@ -0,0 +1,75 @@
+module LatencyBenchmarks
+
+using BenchmarkTools
+using Random
+using Pkg
+
+const SUITE = BenchmarkGroup()
+
+# Interpolating @__DIR__ into commands is finicky
+__dir__ = @__DIR__
+
+function julia_cmd()
+    julia_path = joinpath(Sys.BINDIR, Base.julia_exename())
+    cmd = `$julia_path --startup-file=no --project=$__dir__`
+end
+
+temp_depot = nothing
+
+function clean_env(reset_compiled=false)
+    global temp_depot
+    if temp_depot==nothing
+        temp_depot = mktempdir()
+        ENV["JULIA_DEPOT_PATH"] = temp_depot
+        # Instantiate in the new depot path
+        run(`$(julia_cmd()) -e 'using Pkg; Pkg.instantiate()'`)
+    end
+    if reset_compiled
+        rm(joinpath(temp_depot, "compiled"); recursive=true, force=true)
+    end
+end
+
+function compile_if_stale(pkg)
+    pkgid = pkg == "DataFrames" ? Base.PkgId(Base.UUID("a93c6f00-e57d-5684-b7b6-d8193f3e46c0"), "DataFrames") :
+            pkg == "Plots"      ? Base.PkgId(Base.UUID("91a5bcdd-55d7-5caf-9e0b-520d859cae80"), "Plots") :
+            pkg == "CSV"        ? Base.PkgId(Base.UUID("336ed68f-0bac-5ca0-87d4-7b16caf5d00b"), "CSV") :
+            error("unknown package")
+    code = """
+    import Base: UUID
+    pkg = Base.PkgId($(repr(pkgid.uuid)), $(repr(pkgid.name)))
+    paths = Base.find_all_in_cache_path(pkg)
+    sourcepath = Base.locate_package(pkg)
+    stale = true
+    for path_to_try in paths
+        staledeps = Base.stale_cachefile(sourcepath, path_to_try)
+        staledeps === true && continue
+        stale = false
+        break
+    end
+    if stale
+        Base.compilecache(pkg, sourcepath)
+    end
+    """
+    run(`$(julia_cmd()) -e $code`)
+end
+
+
+# Julia startup time
+SUITE["julia startup"] = @benchmarkable run(`$(julia_cmd()) -e ''`)
+
+# Precompile DataFrames
+for pkg in ["DataFrames", "CSV", "Plots"]
+    s = "using $pkg"
+    cmd = `$(julia_cmd()) -e $s`
+    SUITE["precompile $pkg"] = @benchmarkable run($cmd) setup=(clean_env(true)) evals=1
+    SUITE["load $pkg"] = @benchmarkable run($cmd) setup=(clean_env(); compile_if_stale($pkg))
+end
+
+SUITE["first plot"] = @benchmarkable run(`$(julia_cmd()) -e 'using Plots; p = plot(rand(5)); 
+    savefig(p, tempname() *  ".png")'`) setup=(clean_env(); compile_if_stale("Plots"))
+
+csv_file = joinpath(__dir__, "test.csv")
+cmd = "using CSV; CSV.read($(repr(csv_file)), silencewarnings=true)"
+SUITE["first csv"] = @benchmarkable run(`$(julia_cmd()) -e $cmd`) setup=(clean_env(); compile_if_stale("CSV"))
+
+end