From 2d33cf676f724ef477dd188980701ae81aa832b0 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 17 Jan 2023 19:01:02 -0800 Subject: [PATCH] Squashed 'packages/ProgressMeter/' content from commit 83e619fa git-subtree-dir: packages/ProgressMeter git-subtree-split: 83e619faed2780ab5d6929777c8f6151e3b88c5e --- .github/workflows/TagBot.yml | 15 + .github/workflows/ci.yml | 55 ++ .gitignore | 1 + LICENSE.txt | 8 + Project.toml | 19 + README.md | 436 ++++++++++++++ src/ProgressMeter.jl | 1050 ++++++++++++++++++++++++++++++++++ test/core.jl | 60 ++ test/runtests.jl | 25 + test/test.jl | 418 ++++++++++++++ test/test_float.jl | 63 ++ test/test_map.jl | 156 +++++ test/test_showvalues.jl | 114 ++++ test/test_threads.jl | 88 +++ 14 files changed, 2508 insertions(+) create mode 100644 .github/workflows/TagBot.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 Project.toml create mode 100644 README.md create mode 100644 src/ProgressMeter.jl create mode 100644 test/core.jl create mode 100644 test/runtests.jl create mode 100644 test/test.jl create mode 100644 test/test_float.jl create mode 100644 test/test_map.jl create mode 100644 test/test_showvalues.jl create mode 100644 test/test_threads.jl diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml new file mode 100644 index 0000000..f49313b --- /dev/null +++ b/.github/workflows/TagBot.yml @@ -0,0 +1,15 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..de446c1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,55 @@ +name: CI +on: + pull_request: + branches: + - master + push: + branches: + - master + tags: '*' +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '0.7' + - '1.0' + - '1' + - 'nightly' + os: + - ubuntu-latest + - macOS-latest + - windows-latest + arch: + - x64 + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + env: + JULIA_NUM_THREADS: 1 + - uses: julia-actions/julia-runtest@v1 + if: ${{ matrix.version != '0.7' && matrix.version != '1.0' }} + env: + JULIA_NUM_THREADS: 2 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v1 + with: + file: lcov.info diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b067edd --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/Manifest.toml diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..679869f --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,8 @@ +The MIT License (MIT) +Copyright (c) 2013 Timothy E. Holy + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..09fb0b9 --- /dev/null +++ b/Project.toml @@ -0,0 +1,19 @@ +name = "ProgressMeter" +uuid = "92933f4c-e287-5a05-a399-4b506db050ca" +version = "1.7.2" + +[deps] +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[compat] +julia = "0.7, 1" + +[extras] +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test", "Random", "Distributed", "InteractiveUtils"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..b9026b7 --- /dev/null +++ b/README.md @@ -0,0 +1,436 @@ +# ProgressMeter.jl + +[![Build Status](https://github.com/timholy/ProgressMeter.jl/workflows/CI/badge.svg)](https://github.com/timholy/ProgressMeter.jl/actions) + +Progress meter for long-running operations in Julia + +## Installation + +Within julia, execute +```julia +using Pkg; Pkg.add("ProgressMeter") +``` + +## Usage + +### Progress meters for tasks with a pre-determined number of steps + +This works for functions that process things in loops or with `map`/`pmap`/`reduce`: + +```julia +using Distributed +using ProgressMeter + +@showprogress 1 "Computing..." for i in 1:50 + sleep(0.1) +end + +@showprogress pmap(1:10) do x + sleep(0.1) + x^2 +end + +@showprogress reduce(1:10) do x, y + sleep(0.1) + x + y +end +``` + +The first incantation will use a minimum update interval of 1 second, and show the ETA and +final duration. If your computation runs so quickly that it never needs to show progress, +no extraneous output will be displayed. + +The `@showprogress` macro wraps a `for` loop, comprehension, `@distributed` for loop, or +`map`/`pmap`/`reduce` as long as the object being iterated over implements the `length` +method and will handle `continue` correctly. + +```julia +using Distributed +using ProgressMeter + +@showprogress @distributed for i in 1:10 + sleep(0.1) +end + +result = @showprogress 1 "Computing..." @distributed (+) for i in 1:10 + sleep(0.1) + i^2 +end +``` + +In the case of a `@distributed` for loop without a reducer, an `@sync` is implied. + +You can also control progress updates and reports manually: + +```julia +function my_long_running_function(filenames::Array) + n = length(filenames) + p = Progress(n, 1) # minimum update interval: 1 second + for f in filenames + # Here's where you do all the hard, slow work + next!(p) + end +end +``` + +For tasks such as reading file data where the progress increment varies between iterations, +you can use `update!`: + +```julia +using ProgressMeter + +function readFileLines(fileName::String) + file = open(fileName,"r") + + seekend(file) + fileSize = position(file) + + seekstart(file) + p = Progress(fileSize, 1) # minimum update interval: 1 second + while !eof(file) + line = readline(file) + # Here's where you do all the hard, slow work + + update!(p, position(file)) + end +end +``` + +The core methods `Progress()`, `ProgressThresh()`, `ProgressUnknown()`, and their updaters +are also thread-safe, so can be used with `Threads.@threads`, `Threads.@spawn` etc.: + +```julia +using ProgressMeter +p = Progress(10) +Threads.@threads for i in 1:10 + sleep(2*rand()) + next!(p) +end +``` + +```julia +using ProgressMeter +n = 10 +p = Progress(n) +tasks = Vector{Task}(undef, n) +for i in 1:n + tasks[i] = Threads.@spawn begin + sleep(2*rand()) + next!(p) + end +end +wait.(tasks) +``` + +### Progress bar style + +Optionally, a description string can be specified which will be prepended to the output, +and a progress meter `M` characters long can be shown. E.g. + +```julia +p = Progress(n, 1, "Computing initial pass...", 50) +``` + +will yield + +``` +Computing initial pass...53%|███████████████████████████ | ETA: 0:09:02 +``` + +in a manner similar to [python-progressbar](https://code.google.com/p/python-progressbar/). + +Also, other properties can be modified through keywords. The glyphs used in the bar may be +specified by passing a `BarGlyphs` object as the keyword argument `barglyphs`. The `BarGlyphs` +constructor can either take 5 characters as arguments or a single 5 character string. E.g. + +```julia +p = Progress(n, dt=0.5, barglyphs=BarGlyphs("[=> ]"), barlen=50, color=:yellow) +``` + +will yield + +``` +Progress: 53%[==========================> ] ETA: 0:09:02 +``` + +It is possible to give a vector of characters that acts like a transition between the empty +character and the fully filled character. For example, definining the progress bar as: + +```julia +p = Progress(n, dt=0.5, + barglyphs=BarGlyphs('|','█', ['▁' ,'▂' ,'▃' ,'▄' ,'▅' ,'▆', '▇'],' ','|',), + barlen=10) +``` + +might show the progress bar as: + +``` +Progress: 34%|███▃ | ETA: 0:00:02 +``` + +where the last bar is not yet fully filled. + +### Progress meters for tasks with a target threshold + +Some tasks only terminate when some criterion is satisfied, for +example to achieve convergence within a specified tolerance. In such +circumstances, you can use the `ProgressThresh` type: + +```julia +prog = ProgressThresh(1e-5, "Minimizing:") +for val in exp10.(range(2, stop=-6, length=20)) + ProgressMeter.update!(prog, val) + sleep(0.1) +end +``` + +### Progress meters for tasks with an unknown number of steps + +Some tasks only terminate when some non-deterministic criterion is satisfied. In such +circumstances, you can use the `ProgressUnknown` type: + +```julia +prog = ProgressUnknown("Titles read:") +for val in ["a" , "b", "c", "d"] + ProgressMeter.next!(prog) + if val == "c" + ProgressMeter.finish!(prog) + break + end + sleep(0.1) +end +``` + +This will display the number of calls to `next!` until `finish!` is called. + +If your counter does not monotonically increases, you can also set the counter by hand. + +```julia +prog = ProgressUnknown("Total length of characters read:") +total_length_characters = 0 +for val in ["aaa" , "bb", "c", "d"] + global total_length_characters += length(val) + ProgressMeter.update!(prog, total_length_characters) + if val == "c" + ProgressMeter.finish!(prog) + break + end + sleep(0.5) +end +``` + +Alternatively, you can display a "spinning ball" symbol +by passing `spinner=true` to the `ProgressUnknown` constructor. +```julia +prog = ProgressUnknown("Working hard:", spinner=true) +while true + ProgressMeter.next!(prog) + rand(1:2*10^8) == 1 && break +end +ProgressMeter.finish!(prog) +``` + +By default, `finish!` changes the spinner to a `✓`, but you can +use a different character by passing a `spinner` keyword +to `finish!`, e.g. passing `spinner='✗'` on a failure condition: +```julia +let found=false + prog = ProgressUnknown("Searching for the Answer:", spinner=true) + for tries = 1:10^8 + ProgressMeter.next!(prog) + if rand(1:2*10^8) == 42 + found=true + break + end + end + ProgressMeter.finish!(prog, spinner = found ? '✓' : '✗') +end +``` + +In fact, you can completely customize the spinner character +by passing a string (or array of characters) to animate as a `spinner` +argument to `next!`: +```julia +prog = ProgressUnknown("Burning the midnight oil:", spinner=true) +while true + ProgressMeter.next!(prog, spinner="🌑🌒🌓🌔🌕🌖🌗🌘") + rand(1:10^8) == 0xB00 && break +end +ProgressMeter.finish!(prog) +``` +(Other interesting-looking spinners include `"⌜⌝⌟⌞"`, `"⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"`, `"🕐🕑🕒🕓🕔🕕🕖🕗🕘🕙🕚🕛"`, `"▖▘▝▗'"`, and `"▁▂▃▄▅▆▇█"`.) + +### Printing additional information + +You can also print and update information related to the computation by using +the `showvalues` keyword. The following example displays the iteration counter +and the value of a dummy variable `x` below the progress meter: + +```julia +x,n = 1,10 +p = Progress(n) +for iter = 1:10 + x *= 2 + sleep(0.5) + ProgressMeter.next!(p; showvalues = [(:iter,iter), (:x,x)]) +end +``` + +In the above example, the data passed to `showvalues` is evaluated even if the progress bar is not updated. +To avoid this unnecessary computation and reduce the overhead, +you can alternatively pass a zero-argument function as a callback to the `showvalues` keyword. + +```julia +x,n = 1,10 +p = Progress(n) +generate_showvalues(iter, x) = () -> [(:iter,iter), (:x,x)] +for iter = 1:10 + x *= 2 + sleep(0.5) +# unlike `showvalues=generate_showvalues(iter, x)()`, this version only evaluate the function when necessary +ProgressMeter.next!(p; showvalues = generate_showvalues(iter, x)) +end +``` + +### Showing average time per iteration + +You can include an average per-iteration duration in your progress meter +by setting the optional keyword argument `showspeed=true` +when constructing a `Progress`, `ProgressUnknown`, or `ProgressThresh`. + +```julia +x,n = 1,10 +p = Progress(n; showspeed=true) +for iter = 1:10 + x *= 2 + sleep(0.5) + ProgressMeter.next!(p; showvalues = [(:iter,iter), (:x,x)]) +end +``` + +will yield something like: + +``` +Progress: XX%|███████████████████████████ | ETA: XX:YY:ZZ (12.34 s/it) +``` + +instead of + +``` +Progress: XX%|███████████████████████████ | ETA: XX:YY:ZZ +``` + +### Conditionally disabling a progress meter + +In addition to the `showspeed` optional keyword argument, +all the progress meters also support the optional `enabled` keyword argument. +You can use this to conditionally disable a progress bar in cases where you want less verbose output +or are using another progress bar to track progress in looping over a function that itself uses a progress bar. + +```julia +function my_awesome_slow_loop(n::Integer; show_progress=true) + p = Progress(n; enabled=show_progress) + for i in 1:n + sleep(0.1) + next!(p) + end +end + +const SHOW_PROGRESS_BARS = parse(Bool, get(ENV, "PROGRESS_BARS", "true")) + +m = 100 +# let environment variable disable outer loop progress bar +p = Progress(m; enabled=SHOW_PROGRESS_BARS) +for i in 1:m + # disable inner loop progress bar since we are tracking progress in the outer loop + my_awesome_slow_loop(i; show_progress=false) + next!(p) +end +``` + +### ProgressMeter with additional information in Jupyter + +Jupyter notebooks/lab does not allow one to overwrite only parts of the output of cell. +In releases up through 1.2, progress bars are printed repeatedly to the output. +Starting with release xx, by default Jupyter clears the output of a cell, but this will +remove **all** output from the cell. You can restore previous behavior by calling +`ProgressMeter.ijulia_behavior(:append)`. You can enable it again by calling `ProgressMeter.ijulia_behavior(:clear)`, +which will also disable the warning message. + +### Tips for parallel programming + +For remote parallelization, when multiple processes or tasks are being used for a computation, +the workers should communicate back to a single task for displaying the progress bar. This +can be accomplished with a `RemoteChannel`: + +```julia +using ProgressMeter +using Distributed + +n_steps = 20 +p = Progress(n_steps) +channel = RemoteChannel(()->Channel{Bool}(), 1) + +# introduce a long-running dummy task to all workers +@everywhere long_task() = sum([ 1/x for x in 1:100_000_000 ]) +@time long_task() # a single execution is about 0.3 seconds + +@sync begin # start two tasks which will be synced in the very end + # the first task updates the progress bar + @async while take!(channel) + next!(p) + end + + # the second task does the computation + @async begin + @distributed (+) for i in 1:n_steps + long_task() + put!(channel, true) # trigger a progress bar update + i^2 + end + put!(channel, false) # this tells the printing task to finish + end +end +``` + +Here, returning some number `i^2` and reducing it somehow `(+)` +is necessary to make the distribution happen. + +### `progress_map` + +More control over the progress bar in a map function can be achieved with the `progress_map` +and `progress_pmap` functions. The keyword argument `progress` can be used to supply a custom progress meter. + +```julia +p = Progress(10, barglyphs=BarGlyphs("[=> ]")) +progress_map(1:10, progress=p) do x + sleep(0.1) + x^2 +end +``` + +### Optional use of the progress meter + +It possible to disable the progress meter when the use is optional. + +```julia +x,n = 1,10 +p = Progress(n; enabled = false) +for iter = 1:10 + x *= 2 + sleep(0.5) + ProgressMeter.next!(p) +end +``` + +In cases where the output is text output such as CI or in an HPC scheduler, the helper function +`is_logging` can be used to disable automatically. + +```julia +is_logging(io) = isa(io, Base.TTY) == false || (get(ENV, "CI", nothing) == "true") +p = Progress(n; output = stderr, enabled = !is_logging(stderr)) +```` + +## Credits + +Thanks to Alan Bahm, Andrew Burroughs, and Jim Garrison for major enhancements to this package. diff --git a/src/ProgressMeter.jl b/src/ProgressMeter.jl new file mode 100644 index 0000000..7c8ad5f --- /dev/null +++ b/src/ProgressMeter.jl @@ -0,0 +1,1050 @@ +module ProgressMeter + +using Printf: @sprintf +using Distributed + +export Progress, ProgressThresh, ProgressUnknown, BarGlyphs, next!, update!, cancel, finish!, @showprogress, progress_map, progress_pmap, ijulia_behavior + +""" +`ProgressMeter` contains a suite of utilities for displaying progress +in long-running computations. The major functions/types in this module +are: + +- `@showprogress`: an easy interface for straightforward situations +- `Progress`: an object for managing progress updates with a predictable number of iterations +- `ProgressThresh`: an object for managing progress updates where termination is governed by a threshold +- `next!` and `update!`: report that progress has been made +- `cancel` and `finish!`: early termination +""" +ProgressMeter + +abstract type AbstractProgress end + +""" +Holds the five characters that will be used to generate the progress bar. +""" +mutable struct BarGlyphs + leftend::Char + fill::Char + front::Union{Vector{Char}, Char} + empty::Char + rightend::Char +end + +""" +String constructor for BarGlyphs - will split the string into 5 chars +""" +function BarGlyphs(s::AbstractString) + glyphs = (s...,) + if !isa(glyphs, NTuple{5,Char}) + error(""" + Invalid string in BarGlyphs constructor. + You supplied "$s". + Note: string argument must be exactly 5 characters long, e.g. "[=> ]". + """) + end + return BarGlyphs(glyphs...) +end + +""" +`prog = Progress(n; dt=0.1, desc="Progress: ", color=:green, +output=stderr, barlen=tty_width(desc), start=0)` creates a progress meter for a +task with `n` iterations or stages starting from `start`. Output will be +generated at intervals at least `dt` seconds apart, and perhaps longer if each +iteration takes longer than `dt`. `desc` is a description of +the current task. Optionally you can disable the progress bar by setting +`enabled=false`. You can also append a per-iteration average duration like +"(12.34 ms/it)" to the description by setting `showspeed=true`. +""" +mutable struct Progress <: AbstractProgress + n::Int + reentrantlocker::Threads.ReentrantLock + dt::Float64 + counter::Int + tinit::Float64 + tsecond::Float64 # ignore the first loop given usually uncharacteristically slow + tlast::Float64 + printed::Bool # true if we have issued at least one status update + desc::String # prefix to the percentage, e.g. "Computing..." + barlen::Union{Int,Nothing} # progress bar size (default is available terminal width) + barglyphs::BarGlyphs # the characters to be used in the bar + color::Symbol # default to green + output::IO # output stream into which the progress is written + offset::Int # position offset of progress bar (default is 0) + numprintedvalues::Int # num values printed below progress in last iteration + start::Int # which iteration number to start from + enabled::Bool # is the output enabled + showspeed::Bool # should the output include average time per iteration + check_iterations::Int + prev_update_count::Int + threads_used::Vector{Int} + + function Progress(n::Integer; + dt::Real=0.1, + desc::AbstractString="Progress: ", + color::Symbol=:green, + output::IO=stderr, + barlen=nothing, + barglyphs::BarGlyphs=BarGlyphs('|','█', Sys.iswindows() ? '█' : ['▏','▎','▍','▌','▋','▊','▉'],' ','|',), + offset::Integer=0, + start::Integer=0, + enabled::Bool = true, + showspeed::Bool = false, + ) + CLEAR_IJULIA[] = clear_ijulia() + reentrantlocker = Threads.ReentrantLock() + counter = start + tinit = tsecond = tlast = time() + printed = false + new(n, reentrantlocker, dt, counter, tinit, tsecond, tlast, printed, desc, barlen, barglyphs, color, output, offset, 0, start, enabled, showspeed, 1, 1, Int[]) + end +end + +Progress(n::Integer, dt::Real, desc::AbstractString="Progress: ", + barlen=nothing, color::Symbol=:green, output::IO=stderr; + offset::Integer=0) = + Progress(n, dt=dt, desc=desc, barlen=barlen, color=color, output=output, offset=offset) + +Progress(n::Integer, desc::AbstractString, offset::Integer=0) = Progress(n, desc=desc, offset=offset) + + +""" +`prog = ProgressThresh(thresh; dt=0.1, desc="Progress: ", +color=:green, output=stderr)` creates a progress meter for a task +which will terminate once a value less than or equal to `thresh` is +reached. Output will be generated at intervals at least `dt` seconds +apart, and perhaps longer if each iteration takes longer than +`dt`. `desc` is a description of the current task. Optionally you can disable +the progress meter by setting `enabled=false`. You can also append a +per-iteration average duration like "(12.34 ms/it)" to the description by +setting `showspeed=true`. +""" +mutable struct ProgressThresh{T<:Real} <: AbstractProgress + thresh::T + reentrantlocker::Threads.ReentrantLock + dt::Float64 + val::T + counter::Int + triggered::Bool + tinit::Float64 + tlast::Float64 + printed::Bool # true if we have issued at least one status update + desc::String # prefix to the percentage, e.g. "Computing..." + color::Symbol # default to green + output::IO # output stream into which the progress is written + numprintedvalues::Int # num values printed below progress in last iteration + offset::Int # position offset of progress bar (default is 0) + enabled::Bool # is the output enabled + showspeed::Bool # should the output include average time per iteration + check_iterations::Int + prev_update_count::Int + threads_used::Vector{Int} + + function ProgressThresh{T}(thresh; + dt::Real=0.1, + desc::AbstractString="Progress: ", + color::Symbol=:green, + output::IO=stderr, + offset::Integer=0, + enabled = true, + showspeed::Bool = false) where T + CLEAR_IJULIA[] = clear_ijulia() + reentrantlocker = Threads.ReentrantLock() + tinit = tlast = time() + printed = false + new{T}(thresh, reentrantlocker, dt, typemax(T), 0, false, tinit, tlast, printed, desc, color, output, 0, offset, enabled, showspeed, 1, 1, Int[]) + end +end +ProgressThresh(thresh::Real; kwargs...) = ProgressThresh{typeof(thresh)}(thresh; kwargs...) + +# Legacy constructor calls +ProgressThresh(thresh::Real, dt::Real, desc::AbstractString="Progress: ", + color::Symbol=:green, output::IO=stderr; + offset::Integer=0) = + ProgressThresh(thresh; dt=dt, desc=desc, color=color, output=output, offset=offset) + +ProgressThresh(thresh::Real, desc::AbstractString, offset::Integer=0) = ProgressThresh(thresh; desc=desc, offset=offset) + +""" +`prog = ProgressUnknown(; dt=0.1, desc="Progress: ", +color=:green, output=stderr)` creates a progress meter for a task +which has a non-deterministic termination criterion. +Output will be generated at intervals at least `dt` seconds +apart, and perhaps longer if each iteration takes longer than +`dt`. `desc` is a description of the current task. Optionally you can disable +the progress meter by setting `enabled=false`. You can also append a +per-iteration average duration like "(12.34 ms/it)" to the description by +setting `showspeed=true`. Instead of displaying a counter, it +can optionally display a spinning ball by passing `spinner=true`. +""" +mutable struct ProgressUnknown <: AbstractProgress + done::Bool + reentrantlocker::Threads.ReentrantLock + dt::Float64 + counter::Int + spincounter::Int + triggered::Bool + tinit::Float64 + tlast::Float64 + printed::Bool # true if we have issued at least one status update + desc::String # prefix to the percentage, e.g. "Computing..." + color::Symbol # default to green + spinner::Bool # show a spinner + output::IO # output stream into which the progress is written + numprintedvalues::Int # num values printed below progress in last iteration + enabled::Bool # is the output enabled + showspeed::Bool # should the output include average time per iteration + check_iterations::Int + prev_update_count::Int + threads_used::Vector{Int} +end + +function ProgressUnknown(;dt::Real=0.1, desc::AbstractString="Progress: ", color::Symbol=:green, spinner::Bool=false, output::IO=stderr, enabled::Bool = true, showspeed::Bool = false) + CLEAR_IJULIA[] = clear_ijulia() + reentrantlocker = Threads.ReentrantLock() + tinit = tlast = time() + printed = false + ProgressUnknown(false, reentrantlocker, dt, 0, 0, false, tinit, tlast, printed, desc, color, spinner, output, 0, enabled, showspeed, 1, 1, Int[]) +end + +ProgressUnknown(dt::Real, desc::AbstractString="Progress: ", + color::Symbol=:green, output::IO=stderr; kwargs...) = + ProgressUnknown(dt=dt, desc=desc, color=color, output=output; kwargs...) + +ProgressUnknown(desc::AbstractString; kwargs...) = ProgressUnknown(desc=desc; kwargs...) + +#...length of percentage and ETA string with days is 29 characters, speed string is always 14 extra characters +function tty_width(desc, output, showspeed::Bool) + full_width = displaysize(output)[2] + desc_width = length(desc) + eta_width = 29 + speed_width = showspeed ? 14 : 0 + return max(0, full_width - desc_width - eta_width - speed_width) +end + +# Package level behavior of IJulia clear output +@enum IJuliaBehavior IJuliaWarned IJuliaClear IJuliaAppend + +const IJULIABEHAVIOR = Ref(IJuliaWarned) + +function ijulia_behavior(b) + @assert b in [:warn, :clear, :append] + b == :warn && (IJULIABEHAVIOR[] = IJuliaWarned) + b == :clear && (IJULIABEHAVIOR[] = IJuliaClear) + b == :append && (IJULIABEHAVIOR[] = IJuliaAppend) +end + +# Whether or not to use IJulia.clear_output +const CLEAR_IJULIA = Ref{Bool}(false) +running_ijulia_kernel() = isdefined(Main, :IJulia) && Main.IJulia.inited +clear_ijulia() = (IJULIABEHAVIOR[] != IJuliaAppend) && running_ijulia_kernel() + +function calc_check_iterations(p, t) + if t == p.tlast + # avoid a NaN which could happen because the print time compensation makes an assumption about how long printing + # takes, therefore it's possible (but rare) for `t == p.tlast` + return p.check_iterations + end + # Adjust the number of iterations that skips time check based on how accurate the last number was + iterations_per_dt = (p.check_iterations / (t - p.tlast)) * p.dt + return round(Int, clamp(iterations_per_dt, 1, p.check_iterations * 10)) +end + +# update progress display +function updateProgress!(p::Progress; showvalues = (), truncate_lines = false, valuecolor = :blue, + offset::Integer = p.offset, keep = (offset == 0), desc::Union{Nothing,AbstractString} = nothing, + ignore_predictor = false) + !p.enabled && return + if p.counter == 2 # ignore the first loop given usually uncharacteristically slow + p.tsecond = time() + end + if desc !== nothing && desc !== p.desc + if p.barlen !== nothing + p.barlen += length(p.desc) - length(desc) #adjust bar length to accommodate new description + end + p.desc = desc + end + p.offset = offset + if p.counter >= p.n + if p.counter == p.n && p.printed + t = time() + barlen = p.barlen isa Nothing ? tty_width(p.desc, p.output, p.showspeed) : p.barlen + percentage_complete = 100.0 * p.counter / p.n + bar = barstring(barlen, percentage_complete, barglyphs=p.barglyphs) + elapsed_time = t - p.tinit + dur = durationstring(elapsed_time) + spacer = endswith(p.desc, " ") ? "" : " " + msg = @sprintf "%s%s%3u%%%s Time: %s" p.desc spacer round(Int, percentage_complete) bar dur + if p.showspeed + sec_per_iter = elapsed_time / (p.counter - p.start) + msg = @sprintf "%s (%s)" msg speedstring(sec_per_iter) + end + !CLEAR_IJULIA[] && print(p.output, "\n" ^ (p.offset + p.numprintedvalues)) + move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues) + printover(p.output, msg, p.color) + printvalues!(p, showvalues; color = valuecolor, truncate = truncate_lines) + if keep + println(p.output) + else + print(p.output, "\r\u1b[A" ^ (p.offset + p.numprintedvalues)) + end + flush(p.output) + end + return nothing + end + if ignore_predictor || predicted_updates_per_dt_have_passed(p) + t = time() + if p.counter > 2 + p.check_iterations = calc_check_iterations(p, t) + end + if t > p.tlast+p.dt + barlen = p.barlen isa Nothing ? tty_width(p.desc, p.output, p.showspeed) : p.barlen + percentage_complete = 100.0 * p.counter / p.n + bar = barstring(barlen, percentage_complete, barglyphs=p.barglyphs) + elapsed_time = t - p.tinit + est_total_time = elapsed_time * (p.n - p.start) / (p.counter - p.start) + if 0 <= est_total_time <= typemax(Int) + eta_sec = round(Int, est_total_time - elapsed_time ) + eta = durationstring(eta_sec) + else + eta = "N/A" + end + spacer = endswith(p.desc, " ") ? "" : " " + msg = @sprintf "%s%s%3u%%%s ETA: %s" p.desc spacer round(Int, percentage_complete) bar eta + if p.showspeed + sec_per_iter = elapsed_time / (p.counter - p.start) + msg = @sprintf "%s (%s)" msg speedstring(sec_per_iter) + end + !CLEAR_IJULIA[] && print(p.output, "\n" ^ (p.offset + p.numprintedvalues)) + move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues) + printover(p.output, msg, p.color) + printvalues!(p, showvalues; color = valuecolor, truncate = truncate_lines) + !CLEAR_IJULIA[] && print(p.output, "\r\u1b[A" ^ (p.offset + p.numprintedvalues)) + flush(p.output) + # Compensate for any overhead of printing. This can be + # especially important if you're running over a slow network + # connection. + p.tlast = t + 2*(time()-t) + p.printed = true + p.prev_update_count = p.counter + end + end + return nothing +end + +function updateProgress!(p::ProgressThresh; showvalues = (), truncate_lines = false, valuecolor = :blue, + offset::Integer = p.offset, keep = (offset == 0), desc = p.desc, ignore_predictor = false) + !p.enabled && return + p.offset = offset + p.desc = desc + if p.val <= p.thresh && !p.triggered + p.triggered = true + if p.printed + t = time() + elapsed_time = t - p.tinit + p.triggered = true + dur = durationstring(elapsed_time) + msg = @sprintf "%s Time: %s (%d iterations)" p.desc dur p.counter + if p.showspeed + sec_per_iter = elapsed_time / p.counter + msg = @sprintf "%s (%s)" msg speedstring(sec_per_iter) + end + print(p.output, "\n" ^ (p.offset + p.numprintedvalues)) + move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues) + printover(p.output, msg, p.color) + printvalues!(p, showvalues; color = valuecolor, truncate = truncate_lines) + if keep + println(p.output) + else + print(p.output, "\r\u1b[A" ^ (p.offset + p.numprintedvalues)) + end + flush(p.output) + end + return + end + + if ignore_predictor || predicted_updates_per_dt_have_passed(p) + t = time() + if p.counter > 2 + p.check_iterations = calc_check_iterations(p, t) + end + if t > p.tlast+p.dt && !p.triggered + msg = @sprintf "%s (thresh = %g, value = %g)" p.desc p.thresh p.val + if p.showspeed + elapsed_time = t - p.tinit + sec_per_iter = elapsed_time / p.counter + msg = @sprintf "%s (%s)" msg speedstring(sec_per_iter) + end + print(p.output, "\n" ^ (p.offset + p.numprintedvalues)) + move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues) + printover(p.output, msg, p.color) + printvalues!(p, showvalues; color = valuecolor, truncate = truncate_lines) + print(p.output, "\r\u1b[A" ^ (p.offset + p.numprintedvalues)) + flush(p.output) + # Compensate for any overhead of printing. This can be + # especially important if you're running over a slow network + # connection. + p.tlast = t + 2*(time()-t) + p.printed = true + p.prev_update_count = p.counter + end + end +end + +const spinner_chars = ['◐','◓','◑','◒'] +const spinner_done = '✓' + +spinner_char(p::ProgressUnknown, spinner::AbstractChar) = spinner +spinner_char(p::ProgressUnknown, spinner::AbstractVector{<:AbstractChar}) = + p.done ? spinner_done : spinner[p.spincounter % length(spinner) + firstindex(spinner)] +spinner_char(p::ProgressUnknown, spinner::AbstractString) = + p.done ? spinner_done : spinner[nextind(spinner, 1, p.spincounter % length(spinner))] + +function updateProgress!(p::ProgressUnknown; showvalues = (), truncate_lines = false, valuecolor = :blue, desc = p.desc, + ignore_predictor = false, spinner::Union{AbstractChar,AbstractString,AbstractVector{<:AbstractChar}} = spinner_chars) + !p.enabled && return + p.desc = desc + if p.done + if p.printed + t = time() + elapsed_time = t - p.tinit + dur = durationstring(elapsed_time) + if p.spinner + msg = @sprintf "%c %s \t Time: %s" spinner_char(p, spinner) p.desc dur + p.spincounter += 1 + else + msg = @sprintf "%s %d \t Time: %s" p.desc p.counter dur + end + if p.showspeed + sec_per_iter = elapsed_time / p.counter + msg = @sprintf "%s (%s)" msg speedstring(sec_per_iter) + end + move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues) + printover(p.output, msg, p.color) + printvalues!(p, showvalues; color = valuecolor, truncate = truncate_lines) + println(p.output) + flush(p.output) + end + return + end + if ignore_predictor || predicted_updates_per_dt_have_passed(p) + t = time() + if p.counter > 2 + p.check_iterations = calc_check_iterations(p, t) + end + if t > p.tlast+p.dt + dur = durationstring(t-p.tinit) + if p.spinner + msg = @sprintf "%c %s \t Time: %s" spinner_char(p, spinner) p.desc dur + p.spincounter += 1 + else + msg = @sprintf "%s %d \t Time: %s" p.desc p.counter dur + end + if p.showspeed + elapsed_time = t - p.tinit + sec_per_iter = elapsed_time / p.counter + msg = @sprintf "%s (%s)" msg speedstring(sec_per_iter) + end + move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues) + printover(p.output, msg, p.color) + printvalues!(p, showvalues; color = valuecolor, truncate = truncate_lines) + flush(p.output) + # Compensate for any overhead of printing. This can be + # especially important if you're running over a slow network + # connection. + p.tlast = t + 2*(time()-t) + p.printed = true + p.prev_update_count = p.counter + return + end + end +end + +predicted_updates_per_dt_have_passed(p::AbstractProgress) = p.counter - p.prev_update_count >= p.check_iterations + +function is_threading(p::AbstractProgress) + Threads.nthreads() == 1 && return false + length(p.threads_used) > 1 && return true + if !in(Threads.threadid(), p.threads_used) + push!(p.threads_used, Threads.threadid()) + end + return length(p.threads_used) > 1 +end + +function lock_if_threading(f::Function, p::AbstractProgress) + if is_threading(p) + lock(p.reentrantlocker) do + f() + end + else + f() + end +end + +# update progress display +""" +`next!(prog, [color], step = 1)` reports that `step` units of progress have been +made. Depending on the time interval since the last update, this may +or may not result in a change to the display. + +You may optionally change the color of the display. See also `update!`. +""" +function next!(p::Union{Progress, ProgressUnknown}; step::Int = 1, options...) + lock_if_threading(p) do + p.counter += step + updateProgress!(p; ignore_predictor = step == 0, options...) + end +end + +function next!(p::Union{Progress, ProgressUnknown}, color::Symbol; step::Int = 1, options...) + lock_if_threading(p) do + p.color = color + p.counter += step + updateProgress!(p; ignore_predictor = step == 0, options...) + end +end + +""" +`update!(prog, counter, [color])` sets the progress counter to +`counter`, relative to the `n` units of progress specified when `prog` +was initialized. Depending on the time interval since the last +update, this may or may not result in a change to the display. + +If `prog` is a `ProgressThresh`, `update!(prog, val, [color])` specifies +the current value. + +You may optionally change the color of the display. See also `next!`. +""" +function update!(p::Union{Progress, ProgressUnknown}, counter::Int=p.counter, color::Symbol=p.color; options...) + lock_if_threading(p) do + counter_changed = p.counter != counter + p.counter = counter + p.color = color + updateProgress!(p; ignore_predictor = !counter_changed, options...) + end +end + +function update!(p::ProgressThresh, val=p.val, color::Symbol=p.color; increment::Bool = true, options...) + lock_if_threading(p) do + p.val = val + if increment + p.counter += 1 + end + p.color = color + updateProgress!(p; options...) + end +end + + +""" +`cancel(prog, [msg], [color=:red])` cancels the progress display +before all tasks were completed. Optionally you can specify the +message printed and its color. + +See also `finish!`. +""" +function cancel(p::AbstractProgress, msg::AbstractString = "Aborted before all tasks were completed", color = :red; showvalues = (), truncate_lines = false, valuecolor = :blue, offset = p.offset, keep = (offset == 0)) + lock_if_threading(p) do + p.offset = offset + if p.printed + print(p.output, "\n" ^ (p.offset + p.numprintedvalues)) + move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues) + printover(p.output, msg, color) + printvalues!(p, showvalues; color = valuecolor, truncate = truncate_lines) + if keep + println(p.output) + else + print(p.output, "\r\u1b[A" ^ (p.offset + p.numprintedvalues)) + end + end + end + return +end + +""" +`finish!(prog)` indicates that all tasks have been completed. + +See also `cancel`. +""" +function finish!(p::Progress; options...) + if p.counter < p.n + update!(p, p.n; options...) + end +end + +function finish!(p::ProgressThresh; options...) + update!(p, p.thresh; options...) +end + +function finish!(p::ProgressUnknown; options...) + lock_if_threading(p) do + p.done = true + updateProgress!(p; options...) + end +end + +# Internal method to print additional values below progress bar +function printvalues!(p::AbstractProgress, showvalues; color = :normal, truncate = false) + length(showvalues) == 0 && return + maxwidth = maximum(Int[length(string(name)) for (name, _) in showvalues]) + + p.numprintedvalues = 0 + + for (name, value) in showvalues + msg = "\n " * rpad(string(name) * ": ", maxwidth+2+1) * string(value) + max_len = (displaysize(p.output)::Tuple{Int,Int})[2] + # I don't understand why the minus 1 is necessary here, but empircally + # it is needed. + msg_lines = ceil(Int, (length(msg)-1) / max_len) + if truncate && msg_lines >= 2 + # For multibyte characters, need to index with nextind. + printover(p.output, msg[1:nextind(msg, 1, max_len-1)] * "…", color) + p.numprintedvalues += 1 + else + printover(p.output, msg, color) + p.numprintedvalues += msg_lines + end + end + p +end + +# Internal method to print additional values below progress bar (lazy-showvalues version) +printvalues!(p::AbstractProgress, showvalues::Function; kwargs...) = printvalues!(p, showvalues(); kwargs...) + +function move_cursor_up_while_clearing_lines(io, numlinesup) + if numlinesup > 0 && CLEAR_IJULIA[] + Main.IJulia.clear_output(true) + if IJULIABEHAVIOR[] == IJuliaWarned + @warn "ProgressMeter by default refresh meters with additional information in IJulia via `IJulia.clear_output`, which clears all outputs in the cell. \n - To prevent this behaviour, do `ProgressMeter.ijulia_behavior(:append)`. \n - To disable this warning message, do `ProgressMeter.ijulia_behavior(:clear)`." + end + else + for _ in 1:numlinesup + print(io, "\r\u1b[K\u1b[A") + end + end +end + +function printover(io::IO, s::AbstractString, color::Symbol = :color_normal) + print(io, "\r") + printstyled(io, s; color=color) + if isdefined(Main, :IJulia) + Main.IJulia.stdio_bytes[] = 0 # issue #76: circumvent IJulia I/O throttling + elseif isdefined(Main, :ESS) || isdefined(Main, :Atom) + else + print(io, "\u1b[K") # clear the rest of the line + end +end + +function compute_front(barglyphs::BarGlyphs, frac_solid::AbstractFloat) + barglyphs.front isa Char && return barglyphs.front + idx = round(Int, frac_solid * (length(barglyphs.front) + 1)) + return idx > length(barglyphs.front) ? barglyphs.fill : + idx == 0 ? barglyphs.empty : + barglyphs.front[idx] +end + +function barstring(barlen, percentage_complete; barglyphs) + bar = "" + if barlen>0 + if percentage_complete == 100 # if we're done, don't use the "front" character + bar = string(barglyphs.leftend, repeat(string(barglyphs.fill), barlen), barglyphs.rightend) + else + n_bars = barlen * percentage_complete / 100 + nsolid = trunc(Int, n_bars) + frac_solid = n_bars - nsolid + nempty = barlen - nsolid - 1 + bar = string(barglyphs.leftend, + repeat(string(barglyphs.fill), max(0,nsolid)), + compute_front(barglyphs, frac_solid), + repeat(string(barglyphs.empty), max(0, nempty)), + barglyphs.rightend) + end + end + bar +end + +function durationstring(nsec) + days = div(nsec, 60*60*24) + r = nsec - 60*60*24*days + hours = div(r,60*60) + r = r - 60*60*hours + minutes = div(r, 60) + seconds = floor(r - 60*minutes) + + hhmmss = @sprintf "%u:%02u:%02u" hours minutes seconds + if days>9 + return @sprintf "%.2f days" nsec/(60*60*24) + elseif days>0 + return @sprintf "%u days, %s" days hhmmss + end + hhmmss +end + +function speedstring(sec_per_iter) + if sec_per_iter == Inf + return " N/A s/it" + end + ns_per_iter = 1_000_000_000 * sec_per_iter + for (divideby, unit) in ( + (1, "ns"), + (1_000, "μs"), + (1_000_000, "ms"), + (1_000_000_000, "s"), + (60 * 1_000_000_000, "m"), + (60 * 60 * 1_000_000_000, "hr"), + (24 * 60 * 60 * 1_000_000_000, "d") + ) + if round(ns_per_iter / divideby) < 100 + return @sprintf "%5.2f %2s/it" (ns_per_iter / divideby) unit + end + end + return " >100 d/it" +end + +function showprogress_process_expr(node, metersym) + if !isa(node, Expr) + node + elseif node.head === :break || node.head === :return + # special handling for break and return statements + quote + ($finish!)($metersym) + $node + end + elseif node.head === :for || node.head === :while + # do not process inner loops + # + # FIXME: do not process break and return statements in inner functions + # either + node + else + # process each subexpression recursively + Expr(node.head, [showprogress_process_expr(a, metersym) for a in node.args]...) + end +end + +struct ProgressWrapper{T} + obj::T + meter::Progress +end + +Base.length(wrap::ProgressWrapper) = Base.length(wrap.obj) + +function Base.iterate(wrap::ProgressWrapper, state...) + ir = iterate(wrap.obj, state...) + + if ir === nothing + finish!(wrap.meter) + elseif !isempty(state) + next!(wrap.meter) + end + + ir +end + +""" +Equivalent of @showprogress for a distributed for loop. +``` +result = @showprogress dt "Computing..." @distributed (+) for i = 1:50 + sleep(0.1) + i^2 +end +``` +""" +function showprogressdistributed(args...) + if length(args) < 1 + throw(ArgumentError("@showprogress @distributed requires at least 1 argument")) + end + progressargs = args[1:end-1] + expr = Base.remove_linenums!(args[end]) + + if expr.head != :macrocall || expr.args[1] != Symbol("@distributed") + throw(ArgumentError("malformed @showprogress @distributed expression")) + end + + distargs = filter(x -> !(x isa LineNumberNode), expr.args[2:end]) + na = length(distargs) + if na == 1 + loop = distargs[1] + elseif na == 2 + reducer = distargs[1] + loop = distargs[2] + else + println("$distargs $na") + throw(ArgumentError("wrong number of arguments to @distributed")) + end + if loop.head !== :for + throw(ArgumentError("malformed @distributed loop")) + end + var = loop.args[1].args[1] + r = loop.args[1].args[2] + body = loop.args[2] + + setup = quote + n = length($(esc(r))) + p = Progress(n, $([esc(arg) for arg in progressargs]...)) + ch = RemoteChannel(() -> Channel{Bool}(n)) + end + + if na == 1 + # would be nice to do this with @sync @distributed but @sync is broken + # https://github.com/JuliaLang/julia/issues/28979 + compute = quote + display = @async let i = 0 + while i < n + take!(ch) + next!(p) + i += 1 + end + end + @distributed for $(esc(var)) = $(esc(r)) + $(esc(body)) + put!(ch, true) + end + nothing + end + else + compute = quote + display = @async while take!(ch) next!(p) end + results = @distributed $(esc(reducer)) for $(esc(var)) = $(esc(r)) + x = $(esc(body)) + put!(ch, true) + x + end + put!(ch, false) + results + end + end + + quote + $setup + results = $compute + wait(display) + results + end +end + +""" +``` +@showprogress dt "Computing..." for i = 1:50 + # computation goes here +end + +@showprogress dt "Computing..." pmap(x->x^2, 1:50) +``` +displays progress in performing a computation. `dt` is the minimum +interval between updates to the user. You may optionally supply a +custom message to be printed that specifies the computation being +performed. + +`@showprogress` works for loops, comprehensions, map, reduce, and pmap. +""" +macro showprogress(args...) + showprogress(args...) +end + +function showprogress(args...) + if length(args) < 1 + throw(ArgumentError("@showprogress requires at least one argument.")) + end + progressargs = args[1:end-1] + expr = args[end] + if expr.head == :macrocall && expr.args[1] == Symbol("@distributed") + return showprogressdistributed(args...) + end + orig = expr = copy(expr) + if expr.args[1] == :|> # e.g. map(x->x^2) |> sum + expr.args[2] = showprogress(progressargs..., expr.args[2]) + return expr + end + metersym = gensym("meter") + mapfuns = (:map, :asyncmap, :reduce, :pmap) + kind = :invalid # :invalid, :loop, or :map + + if isa(expr, Expr) + if expr.head == :for + outerassignidx = 1 + loopbodyidx = lastindex(expr.args) + kind = :loop + elseif expr.head == :comprehension + outerassignidx = lastindex(expr.args) + loopbodyidx = 1 + kind = :loop + elseif expr.head == :typed_comprehension + outerassignidx = lastindex(expr.args) + loopbodyidx = 2 + kind = :loop + elseif expr.head == :call && expr.args[1] in mapfuns + kind = :map + elseif expr.head == :do + call = expr.args[1] + if call.head == :call && call.args[1] in mapfuns + kind = :map + end + end + end + + if kind == :invalid + throw(ArgumentError("Final argument to @showprogress must be a for loop, comprehension, map, reduce, or pmap; got $expr")) + elseif kind == :loop + # As of julia 0.5, a comprehension's "loop" is actually one level deeper in the syntax tree. + if expr.head !== :for + @assert length(expr.args) == loopbodyidx + expr = expr.args[outerassignidx] = copy(expr.args[outerassignidx]) + @assert expr.head === :generator + outerassignidx = lastindex(expr.args) + loopbodyidx = 1 + end + + # Transform the first loop assignment + loopassign = expr.args[outerassignidx] = copy(expr.args[outerassignidx]) + if loopassign.head === :block # this will happen in a for loop with multiple iteration variables + for i in 2:length(loopassign.args) + loopassign.args[i] = esc(loopassign.args[i]) + end + loopassign = loopassign.args[1] = copy(loopassign.args[1]) + end + @assert loopassign.head === :(=) + @assert length(loopassign.args) == 2 + obj = loopassign.args[2] + loopassign.args[1] = esc(loopassign.args[1]) + loopassign.args[2] = :(ProgressWrapper(iterable, $(esc(metersym)))) + + # Transform the loop body break and return statements + if expr.head === :for + expr.args[loopbodyidx] = showprogress_process_expr(expr.args[loopbodyidx], metersym) + end + + # Escape all args except the loop assignment, which was already appropriately escaped. + for i in 1:length(expr.args) + if i != outerassignidx + expr.args[i] = esc(expr.args[i]) + end + end + if orig !== expr + # We have additional escaping to do; this will occur for comprehensions with julia 0.5 or later. + for i in 1:length(orig.args)-1 + orig.args[i] = esc(orig.args[i]) + end + end + + setup = quote + iterable = $(esc(obj)) + $(esc(metersym)) = Progress(length(iterable), $([esc(arg) for arg in progressargs]...)) + end + + if expr.head === :for + return quote + $setup + $expr + end + else + # We're dealing with a comprehension + return quote + begin + $setup + rv = $orig + next!($(esc(metersym))) + rv + end + end + end + else # kind == :map + + # isolate call to map + if expr.head == :do + call = expr.args[1] + else + call = expr + end + + # get args to map to determine progress length + mapargs = collect(Any, filter(call.args[2:end]) do a + return isa(a, Symbol) || !(a.head in (:kw, :parameters)) + end) + if expr.head == :do + insert!(mapargs, 1, :nothing) # to make args for ncalls line up + end + + # change call to progress_map + mapfun = call.args[1] + call.args[1] = :progress_map + + # escape args as appropriate + for i in 2:length(call.args) + call.args[i] = esc(call.args[i]) + end + if expr.head == :do + expr.args[2] = esc(expr.args[2]) + end + + # create appropriate Progress expression + lenex = :(ncalls($(esc(mapfun)), ($([esc(a) for a in mapargs]...),))) + progex = :(Progress($lenex, $([esc(a) for a in progressargs]...))) + + # insert progress and mapfun kwargs + push!(call.args, Expr(:kw, :progress, progex)) + push!(call.args, Expr(:kw, :mapfun, esc(mapfun))) + + return expr + end +end + +""" + progress_map(f, c...; mapfun=map, progress=Progress(...), kwargs...) + +Run a `map`-like function while displaying progress. + +`mapfun` can be any function, but it is only tested with `map`, `reduce` and `pmap`. +""" +function progress_map(args...; mapfun=map, + progress=Progress(ncalls(mapfun, args)), + channel_bufflen=min(1000, ncalls(mapfun, args)), + kwargs...) + f = first(args) + other_args = args[2:end] + channel = RemoteChannel(()->Channel{Bool}(channel_bufflen), 1) + local vals + @sync begin + # display task + @async while take!(channel) + next!(progress) + end + + # map task + @sync begin + vals = mapfun(other_args...; kwargs...) do x... + val = f(x...) + put!(channel, true) + yield() + return val + end + put!(channel, false) + end + end + return vals +end + +""" + progress_pmap(f, [::AbstractWorkerPool], c...; progress=Progress(...), kwargs...) + +Run `pmap` while displaying progress. +""" +progress_pmap(args...; kwargs...) = progress_map(args...; mapfun=pmap, kwargs...) + +""" +Infer the number of calls to the mapped function (i.e. the length of the returned array) given the input arguments to map, reduce or pmap. +""" +function ncalls(mapfun::Function, map_args) + if mapfun == pmap && length(map_args) >= 2 && isa(map_args[2], AbstractWorkerPool) + relevant = map_args[3:end] + else + relevant = map_args[2:end] + end + if isempty(relevant) + error("Unable to determine number of calls in $mapfun. Too few arguments?") + else + return maximum(length(arg) for arg in relevant) + end +end + +end diff --git a/test/core.jl b/test/core.jl new file mode 100644 index 0000000..094b788 --- /dev/null +++ b/test/core.jl @@ -0,0 +1,60 @@ +# test durationstring output at borders +@test ProgressMeter.durationstring(0.9) == "0:00:00" +@test ProgressMeter.durationstring(1.0) == "0:00:01" +@test ProgressMeter.durationstring(59.9) == "0:00:59" +@test ProgressMeter.durationstring(60.0) == "0:01:00" +@test ProgressMeter.durationstring(60*60 - 0.1) == "0:59:59" +@test ProgressMeter.durationstring(60*60) == "1:00:00" +@test ProgressMeter.durationstring(60*60*24 - 0.1) == "23:59:59" +@test ProgressMeter.durationstring(60*60*24) == "1 days, 0:00:00" +@test ProgressMeter.durationstring(60*60*24*10 - 0.1) == "9 days, 23:59:59" +@test ProgressMeter.durationstring(60*60*24*10) == "10.00 days" + +@test ProgressMeter.Progress(5, "Progress:", Int16(5)).offset == 5 +@test ProgressMeter.ProgressThresh(0.2, "Progress:", Int16(5)).offset == 5 + +# test speed string formatting +for ns in [1, 9, 10, 99, 100, 999, 1_000, 9_999, 10_000, 99_000, 100_000, 999_999, 1_000_000, 9_000_000, 10_000_000, 99_999_000, 1_234_567_890, 1_234_567_890 * 10, 1_234_567_890 * 100, 1_234_567_890 * 1_000, 1_234_567_890 * 10_000, 1_234_567_890 * 100_000, 1_234_567_890 * 1_000_000, 1_234_567_890 * 10_000_000] + sec = ns / 1_000_000_000 + try + @test length(ProgressMeter.speedstring(sec)) == 11 + catch + @error "ns = $ns caused $(ProgressMeter.speedstring(sec)) (not length 11)" + throw() + end +end + +# Performance test (from #171) +function prog_perf(n) + prog = Progress(n) + x = 0.0 + for i in 1:n + x += rand() + next!(prog) + end + return x +end + +function noprog_perf(n) + x = 0.0 + for i in 1:n + x += rand() + end + return x +end + +if get(ENV, "GITHUB_ACTIONS", "false") != "true" # CI environment is too unreliable for performance tests + prog_perf(10^7) + noprog_perf(10^7) + @time prog_perf(10^7) + @time noprog_perf(10^7) + @test @elapsed(prog_perf(10^7)) < 9*@elapsed(noprog_perf(10^7)) +end + +# Avoid a NaN due to the estimated print time compensation +# https://github.com/timholy/ProgressMeter.jl/issues/209 +prog = Progress(10) +prog.check_iterations = 999 +t = time() +prog.tlast = t +@test ProgressMeter.calc_check_iterations(prog, t) == 999 diff --git a/test/runtests.jl b/test/runtests.jl new file mode 100644 index 0000000..2f0dd26 --- /dev/null +++ b/test/runtests.jl @@ -0,0 +1,25 @@ +using ProgressMeter +using Test + +if get(ENV, "CI", "false") == "true" + using InteractiveUtils + display(versioninfo()) # among other things, this shows the number of threads +end + +@testset "Core" begin + include("core.jl") + include("test.jl") +end +@testset "Show Values" begin + include("test_showvalues.jl") +end +@testset "Mapping" begin + include("test_map.jl") +end +@testset "Float" begin + include("test_float.jl") +end +@testset "Threading" begin + include("test_threads.jl") +end + diff --git a/test/test.jl b/test/test.jl new file mode 100644 index 0000000..e4c4ecc --- /dev/null +++ b/test/test.jl @@ -0,0 +1,418 @@ +using Random: seed! + +seed!(123) + +function testfunc(n, dt, tsleep) + p = ProgressMeter.Progress(n, dt) + for i = 1:n + sleep(tsleep) + ProgressMeter.next!(p) + end +end +println("Testing original interface...") +testfunc(107, 0.01, 0.01) + + +function testfunc2(n, dt, tsleep, desc, barlen) + p = ProgressMeter.Progress(n, dt, desc, barlen) + for i = 1:n + sleep(tsleep) + ProgressMeter.next!(p) + end +end +println("Testing desc and progress bar") +testfunc2(107, 0.01, 0.01, "Computing...", 50) +println("Testing no desc and no progress bar") +testfunc2(107, 0.01, 0.01, "", 0) + + +function testfunc3(n, tsleep, desc) + p = ProgressMeter.Progress(n, desc) + for i = 1:n + sleep(tsleep) + ProgressMeter.next!(p) + end +end +println("Testing tty width...") +testfunc3(107, 0.02, "Computing (use tty width)...") +println("Testing no description...") +testfunc3(107, 0.02, "") + + + + +function testfunc4() # test "days" format + p = ProgressMeter.Progress(10000000, "Test...") + for i = 1:105 + sleep(0.02) + ProgressMeter.next!(p) + end +end + +println("Testing that not even 1% required...") +testfunc4() + +function testfunc5A(n, dt, tsleep, desc, barlen) + p = ProgressMeter.Progress(n, dt, desc, barlen) + for i = 1:round(Int, floor(n/2)) + sleep(tsleep) + ProgressMeter.next!(p) + end + for i = round(Int, ceil(n/2)):n + sleep(tsleep) + ProgressMeter.next!(p, :red) + end +end + +println("\nTesting changing the bar color") +testfunc5A(107, 0.01, 0.01, "Computing...", 50) + +function testfunc5B(n, dt, tsleep, desc, barlen) + p = ProgressMeter.Progress(n, dt, desc, barlen) + for i = 1:n + sleep(tsleep) + ProgressMeter.next!(p) + if i % 10 == 0 + stepnum = floor(Int, i/10) + 1 + ProgressMeter.update!(p, desc = "Step $stepnum...") + end + end +end + +println("\nTesting changing the description") +testfunc5B(107, 0.01, 0.05, "Step 1...", 50) + + +function testfunc6(n, dt, tsleep) + ProgressMeter.@showprogress dt for i in 1:n + if i == div(n, 2) + break + end + if rand() < 0.7 + sleep(tsleep) + continue + end + end +end + +function testfunc6a(n, dt, tsleep) + ProgressMeter.@showprogress dt for i in 1:n, j in 1:n + if i == div(n, 2) + break + end + if rand() < 0.7 + sleep(tsleep) + continue + end + end +end + +println("Testing @showprogress macro on for loop") +testfunc6(3000, 0.01, 0.002) +testfunc6a(30, 0.01, 0.002) + + +function testfunc7(n, dt, tsleep) + s = ProgressMeter.@showprogress dt "Calculating..." [(sleep(tsleep); z) for z in 1:n] + @test s == [1:n;] +end + +function testfunc7a(n, dt, tsleep) + s = ProgressMeter.@showprogress dt "Calculating..." [(sleep(tsleep); z) for z in 1:n, y in 1:n] + @test s == [z for z in 1:n, y in 1:n] +end + +println("Testing @showprogress macro on comprehension") +testfunc7(25, 0.1, 0.1) +testfunc7a(5, 0.1, 0.1) + + +function testfunc8(n, dt, tsleep) + ProgressMeter.@showprogress dt for i in 1:n + if rand() < 0.7 + sleep(tsleep) + continue + end + for j in 1:10 + if j % 2 == 0 + continue + end + end + while rand(Bool) + continue + end + while true + break + end + end +end + +println("Testing @showprogress macro on a for loop with inner loops containing continue and break statements") +testfunc8(3000, 0.01, 0.002) + + +function testfunc9(n, dt, tsleep) + s = ProgressMeter.@showprogress dt "Calculating..." Float64[(sleep(tsleep); z) for z in 1:n] + @test s == [1:n;] +end + +function testfunc9a(n, dt, tsleep) + s = ProgressMeter.@showprogress dt "Calculating..." Float64[(sleep(tsleep); z) for z in 1:n, y in 1:n] + @test s == [z for z in 1:n, y in 1:n] +end + +println("Testing @showprogress macro on typed comprehension") +testfunc9(100, 0.1, 0.01) +testfunc9a(10, 0.1, 0.01) + + +function testfunc10(n, k, dt, tsleep) + p = ProgressMeter.Progress(n, dt) + for i = 1:k + sleep(tsleep) + ProgressMeter.next!(p) + end + ProgressMeter.finish!(p) +end +println("Testing under-shooting progress with finish!...") +testfunc10(107, 105, 0.01, 0.01) +println("Testing over-shooting progress with finish!...") +testfunc10(107, 111, 0.01, 0.01) + +function testfunc11(n, dt, tsleep) + p = ProgressMeter.Progress(n, dt) + for i = 1:n÷2 + sleep(tsleep) + ProgressMeter.next!(p) + end + sleep(tsleep) + ProgressMeter.update!(p, 0) + for i = 1:n + sleep(tsleep) + ProgressMeter.next!(p) + end +end +println("Testing update! to 0...") +testfunc11(6, 0.01, 0.3) + +function testfunc13() + ProgressMeter.@showprogress 1 for i=1:10 + return + end +end + +println("Testing @showprogress macro on loop ending with return statement") +testfunc13() + +function testfunc13() + n = 30 + # no keyword arguments + p = ProgressMeter.Progress(n) + for i in 1:n + sleep(0.1) + ProgressMeter.next!(p) + end + # full keyword argumetns + start = 15 + p = ProgressMeter.Progress(n, dt=0.01, desc="", color=:red, output=stderr, barlen=40, start = start) + for i in 1:n-start + sleep(0.1) + ProgressMeter.next!(p) + end +end + +println("Testing keyword arguments") +testfunc13() + +function testfunc14(barglyphs) + n = 30 + # with the string constructor + p = ProgressMeter.Progress(n, barglyphs=ProgressMeter.BarGlyphs(barglyphs)) + for i in 1:n + sleep(0.1) + ProgressMeter.next!(p) + end + # with the 5 char constructor + chars = (barglyphs...,) + p = ProgressMeter.Progress(n, barglyphs=ProgressMeter.BarGlyphs(chars...)) + for i in 1:n + sleep(0.1) + ProgressMeter.next!(p) + end + p = ProgressMeter.Progress(n, dt=0.01, desc="", + color=:red, output=stderr, barlen=40, + barglyphs=ProgressMeter.BarGlyphs(barglyphs)) + for i in 1:n + sleep(0.1) + ProgressMeter.next!(p) + end +end + +println("Testing custom bar glyphs") +testfunc14("[=> ]") +@test_throws ErrorException testfunc14("gklelt") + +# Threshold-based progress reports +println("Testing threshold-based progress") +prog = ProgressMeter.ProgressThresh(1e-5, "Minimizing:") +for val in 10 .^ range(2, stop=-6, length=20) + ProgressMeter.update!(prog, val) + sleep(0.1) +end +# issue #166 +@test ProgressMeter.ProgressThresh(1.0f0; desc = "Desc: ") isa ProgressMeter.ProgressThresh{Float32} + +# Threshold-based progress reports with increment=false +println("Testing threshold-based progress") +prog = ProgressMeter.ProgressThresh(1e-5, "Minimizing:") +for val in 10 .^ range(2, stop=-6, length=20) + ProgressMeter.update!(prog, val; increment=false) + @test prog.counter == 0 + sleep(0.1) +end +colors = [:red, :blue, :green] +prog = ProgressMeter.ProgressThresh(1e-5, "Minimizing:") +for val in 10 .^ range(2, stop=-6, length=20) + ProgressMeter.update!(prog, val, rand(colors); increment=false) + @test prog.counter == 0 + sleep(0.1) +end + +# ProgressUnknown progress reports +println("Testing progress unknown") +prog = ProgressMeter.ProgressUnknown("Reading entry:") +for _ in 1:10 + ProgressMeter.next!(prog) + sleep(0.1) +end +ProgressMeter.finish!(prog) + +prog = ProgressMeter.ProgressUnknown("Reading entry:") +for k in 1:2:20 + ProgressMeter.update!(prog, k) + sleep(0.1) +end + +colors = [:red, :blue, :green] +prog = ProgressMeter.ProgressUnknown("Reading entry:") +for k in 1:2:20 + ProgressMeter.update!(prog, k, rand(colors)) + sleep(0.1) +end +ProgressMeter.finish!(prog) + +prog = ProgressMeter.ProgressUnknown("Reading entry:", spinner=true) +for _ in 1:10 + ProgressMeter.next!(prog) + sleep(0.1) +end +ProgressMeter.finish!(prog) + +prog = ProgressMeter.ProgressUnknown("Reading entry:", spinner=true) +for _ in 1:10 + ProgressMeter.next!(prog) + sleep(0.1) +end +ProgressMeter.finish!(prog, spinner='✗') + +myspinner = ['🌑', '🌒', '🌓', '🌔', '🌕', '🌖', '🌗', '🌘'] +prog = ProgressUnknown("Custom spinner:", spinner=true) +for val in 1:10 + ProgressMeter.next!(prog, spinner=myspinner) + sleep(0.1) +end +ProgressMeter.finish!(prog, spinner='🌞') + +prog = ProgressUnknown("Custom spinner:", spinner=true) +for val in 1:10 + ProgressMeter.next!(prog, spinner="⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏") + sleep(0.1) +end +ProgressMeter.finish!(prog) + +println("Testing fractional bars") +for front in (['▏','▎','▍','▌','▋','▊', '▉'], ['▁' ,'▂' ,'▃' ,'▄' ,'▅' ,'▆', '▇'], ['░', '▒', '▓',]) + p = ProgressMeter.Progress(100, dt=0.01, barglyphs=ProgressMeter.BarGlyphs('|','█',front,' ','|'), barlen=10) + for i in 1:100 + ProgressMeter.next!(p) + sleep(0.02) + end +end + +function testfunc15(n, dt, tsleep) + result = ProgressMeter.@showprogress dt @distributed (+) for i in 1:n + if rand() < 0.7 + sleep(tsleep) + end + i ^ 2 + end + @test result == sum(abs2.(1:n)) +end + +println("Testing @showprogress macro on distributed for loop with reducer") +testfunc15(3000, 0.01, 0.002) + +function testfunc16(n, dt, tsleep) + ProgressMeter.@showprogress dt "Description: " @distributed for i in 1:n + if rand() < 0.7 + sleep(tsleep) + end + i ^ 2 + end +end + +println("Testing @showprogress macro on distributed for loop without reducer") +testfunc16(3000, 0.01, 0.002) + +function testfunc17() + n = 30 + p = ProgressMeter.Progress(n, start=15) + for i in 15+1:30 + sleep(0.1) + ProgressMeter.next!(p) + end +end + +println("Testing start offset") +testfunc17() + +# speed display option +function testfunc18A(n, dt, tsleep; start=15) + p = ProgressMeter.Progress(n; dt=dt, start=start, showspeed=true) + for i in start+1:start+n + sleep(tsleep) + ProgressMeter.next!(p) + end +end + +function testfunc18B(n, dt, tsleep) + p = ProgressMeter.ProgressUnknown(n; dt=dt, showspeed=true) + for _ in 1:n + sleep(tsleep) + ProgressMeter.next!(p) + end + ProgressMeter.finish!(p) +end + +function testfunc18C() + p = ProgressMeter.ProgressThresh(1e-5; desc="Minimizing:", showspeed=true) + for val in 10 .^ range(2, stop=-6, length=20) + ProgressMeter.update!(p, val) + sleep(0.1) + end +end + +println("Testing speed display") +testfunc18A(1_000, 0.01, 0.002) +testfunc18B(1_000, 0.01, 0.002) +testfunc18C() + +function testfunc19() + p = ProgressMeter.ProgressThresh(1e-5; desc="Minimizing:", showspeed=true) + for val in 10 .^ range(2, stop=-6, length=20) + ProgressMeter.update!(p, val; increment=false) + sleep(0.1) + end +end +println("Testing speed display with no update") +testfunc19() diff --git a/test/test_float.jl b/test/test_float.jl new file mode 100644 index 0000000..a1445c4 --- /dev/null +++ b/test/test_float.jl @@ -0,0 +1,63 @@ +println("Testing floating normal progress bar (offset 4)") +function testfunc1(n, dt, tsleep, desc, barlen, offset) + p = ProgressMeter.Progress(n, dt, desc, barlen, offset=offset) + for i = 1:n + sleep(tsleep) + ProgressMeter.next!(p) + end + print("\n" ^ 5) +end +testfunc1(50, 0.2, 0.2, "progress ", 70, 4) + +println("Testing floating normal progress bars with values and keep (2 levels)") +function testfunc2(n, dt, tsleep, desc, barlen) + p1 = ProgressMeter.Progress(n, dt, desc, barlen, offset=0) + p2 = ProgressMeter.Progress(n, dt, desc, barlen, offset=5) + for i = 1:n + sleep(tsleep) + ProgressMeter.next!(p1; showvalues = [(:i, i), + (:constant, "foo"), (:isq, i^2), (:large, 2^i)], keep=false) + ProgressMeter.next!(p2; showvalues = [(:i, i), + (:constant, "foo"), (:isq, i^2), (:large, 2^i)]) + end + print("\n" ^ 10) +end +testfunc2(50, 0.2, 0.2, "progress ", 70) + +println("Testing floating normal progress bars with changing offset") +function testfunc3(n, dt, tsleep, desc, barlen) + p1 = ProgressMeter.Progress(n, dt, desc, barlen, offset=0) + p2 = ProgressMeter.Progress(n, dt, desc, barlen, offset=1) + for i = 1:n + sleep(tsleep) + ProgressMeter.next!(p1; showvalues = [(:i, i) for _ in 1:i], keep=false) + ProgressMeter.next!(p2; showvalues = [(:i, i), + (:constant, "foo"), (:isq, i^2), (:large, 2^i)], offset = (p1.offset + p1.numprintedvalues)) + end + print("\n" ^ (10 + 5)) +end +testfunc3(10, 0.2, 0.5, "progress ", 70) + +println("Testing floating thresh progress bar (offset 2)") +function testfunc4(thresh, dt, tsleep, desc, offset) + prog = ProgressMeter.ProgressThresh(thresh, dt, desc, offset=offset) + for val in 10 .^ range(2, stop=-6, length=20) + ProgressMeter.update!(prog, val) + sleep(tsleep) + end + print("\n" ^ 3) +end +testfunc4(1e-5, 0.2, 0.2, "Minimizing: ", 2) + +println("Testing floating in @showprogress macro (3 levels)") +function testfunc5(n, tsleep) + ProgressMeter.@showprogress "Level 0 " for i in 1:n + ProgressMeter.@showprogress " Level 1 " 1 for i2 in 1:n + ProgressMeter.@showprogress " Level 2 " 2 for i3 in 1:n + sleep(tsleep) + end + end + end + print("\n" ^ 2) +end +testfunc5(5, 0.1) diff --git a/test/test_map.jl b/test/test_map.jl new file mode 100644 index 0000000..b7c7fe4 --- /dev/null +++ b/test/test_map.jl @@ -0,0 +1,156 @@ +using Test +using Distributed +procs = addprocs(2) +@everywhere using ProgressMeter + +@testset "map tests" begin + println("Testing map functions") + + # basic + vals = progress_map(1:10) do x + sleep(0.1) + return x^2 + end + @test vals == map(x->x^2, 1:10) + + vals = progress_map(1:10, mapfun=pmap) do x + sleep(0.1) + return x^2 + end + @test vals == map(x->x^2, 1:10) + + val = progress_map(1:10, mapfun=reduce) do x, y + sleep(0.1) + return x+y + end + @test val == reduce((x,y)->x+y, 1:10) + + # errors + @test_throws ErrorException progress_map(1:10) do x + if x > 3 + error("intentional error") + end + return x^2 + end + println() + + @test_throws RemoteException progress_map(1:10, mapfun=pmap) do x + if x > 3 + error("intentional error") + end + return x^2 + end + println() + + @test_throws ErrorException progress_map(1:10, mapfun=reduce) do x, y + if x > 3 + error("intentional error") + end + return x + y + end + println() + + # @showprogress + vals = @showprogress map(1:10) do x + return x^2 + end + @test vals == map(x->x^2, 1:10) + + vals = @showprogress asyncmap(1:10) do x + return x^2 + end + @test vals == map(x->x^2, 1:10) + + vals = @showprogress pmap(1:10) do x + return x^2 + end + @test vals == map(x->x^2, 1:10) + + val = @showprogress reduce(1:10) do x, y + return x + y + end + @test val == reduce((x, y)->x+y, 1:10) + + # function passed by name + function testfun(x) + return x^2 + end + vals = @showprogress map(testfun, 1:10) + @test vals == map(testfun, 1:10) + vals = @showprogress pmap(testfun, 1:10) + @test vals == map(testfun, 1:10) + val = @showprogress reduce(+, 1:10) + @test val == reduce(+, 1:10) + + # #136: make sure mid progress shows up even without sleep + println("Verify that intermediate progress is displayed:") + @showprogress map(1:100) do i + A = rand(10000,1000) + sum(A) + end + + # multiple args + vals = @showprogress pmap((x,y)->x*y, 1:10, 2:11) + @test vals == map((x,y)->x*y, 1:10, 2:11) + + + + # abstract worker pool arg + wp = WorkerPool(procs) + vals = @showprogress pmap(testfun, wp, 1:10) + @test vals == map(testfun, 1:10) + + vals = @showprogress pmap(wp, 1:10) do x + x^2 + end + @test vals == map(testfun, 1:10) + + + + # Progress args + vals = @showprogress 0.1 "Computing" pmap(testfun, 1:10) + @test vals == map(testfun, 1:10) + + + + # named vector arg + a = collect(1:10) + vals = @showprogress pmap(x->x^2, a) + @test vals == map(x->x^2, a) + + + + # global variable in do + C = 10 + vals = @showprogress pmap(1:10) do x + return C*x + end + @test vals == map(x->C*x, 1:10) + + + + # keyword arguments + vals = @showprogress pmap(x->x^2, 1:100, batch_size=10) + @test vals == map(x->x^2, 1:100) + # with semicolon + vals = @showprogress pmap(x->x^2, 1:100; batch_size=10) + @test vals == map(x->x^2, 1:100) + + + # pipes after map + @showprogress map(testfun, 1:10) |> sum |> length + + # pipes after map do block + vals = @showprogress map(1:10) do x + sleep(.1) + return x => x^2 + end |> Dict + @test vals == Dict(x=>x^2 for x in 1:10) + + # pipe + pmap + sumvals = @showprogress pmap(testfun, 1:10) |> sum + @test sumvals == sum(map(testfun, 1:10)) + +end + +rmprocs(procs) diff --git a/test/test_showvalues.jl b/test/test_showvalues.jl new file mode 100644 index 0000000..8ef0175 --- /dev/null +++ b/test/test_showvalues.jl @@ -0,0 +1,114 @@ +for ijulia_behavior in [:warn, :clear, :append] + +ProgressMeter.ijulia_behavior(ijulia_behavior) + +# For testing lazy-showvalue too +lazy_no_lazy(values) = (rand() < 0.5) ? values : () -> values + +println("Testing showvalues with a Dict (2 values)") +function testfunc1(n, dt, tsleep, desc, barlen) + p = ProgressMeter.Progress(n, dt, desc, barlen) + for i = 1:n + sleep(tsleep) + values = Dict(:i => i, :halfdone => (i >= n/2)) + ProgressMeter.next!(p; showvalues = lazy_no_lazy(values)) + end +end +testfunc1(50, 1, 0.2, "progress ", 70) + +println("Testing showvalues with an Array of tuples (4 values)") +function testfunc2(n, dt, tsleep, desc, barlen) + p = ProgressMeter.Progress(n, dt, desc, barlen) + for i = 1:n + sleep(tsleep) + values = [(:i, i), (:constant, "foo"), (:isq, i^2), (:large, 2^i)] + ProgressMeter.next!(p; showvalues = lazy_no_lazy(values)) + end +end +testfunc2(30, 1, 0.2, "progress ", 60) + +println("Testing showvalues when types of names differ (3 values)") +function testfunc3(n, dt, tsleep, desc, barlen) + p = ProgressMeter.Progress(n, dt, desc, barlen) + for i = 1:n + sleep(tsleep) + values = [(:i, i*10), ("constant", "foo"), + ("foobar", round(i*tsleep, digits=4))] + ProgressMeter.next!(p; showvalues = lazy_no_lazy(values)) + end +end +testfunc3(30, 1, 0.2, "progress ", 70) + +println("Testing progress with showing values when num values to print changes between iterations") +function testfunc4(n, dt, tsleep, desc, barlen) + p = ProgressMeter.Progress(n, dt, desc, barlen) + for i = 1:n + sleep(tsleep) + values_pool = [(:i, i*10), ("constant", "foo"), + ("foobar", round(i*tsleep, digits=4))] + values = values_pool[randn(3) .< 0.5] + ProgressMeter.next!(p; showvalues = lazy_no_lazy(values)) + end +end +testfunc4(30, 1, 0.2, "opt steps ", 70) + +println("Testing showvalues with changing number of lines") +prog = ProgressMeter.Progress(50) +for i in 1:50 + values = Dict(:left => 100 - i, + :message => repeat("0123456789", (i%10 + 1)*15), + :final => "this comes after") + ProgressMeter.update!(prog, i; showvalues = lazy_no_lazy(values)) + sleep(0.1) +end + +println("Testing showvalues with a different color (1 value)") +function testfunc5(n, dt, tsleep, desc, barlen) + p = ProgressMeter.Progress(n, dt, desc, barlen) + for i = 1:n + sleep(tsleep) + values = [(:large, 2^i)] + ProgressMeter.next!(p; showvalues = lazy_no_lazy(values), + valuecolor = :yellow) + end +end +testfunc5(10, 1, 0.2, "progress ", 40) + +println("Testing showvalues with threshold-based progress") +prog = ProgressMeter.ProgressThresh(1e-5, "Minimizing:") +for val in 10 .^ range(2, stop=-6, length=20) + values = Dict(:margin => abs(val - 1e-5)) + ProgressMeter.update!(prog, val; showvalues = lazy_no_lazy(values)) + sleep(0.1) +end + +println("Testing showvalues with online progress") +prog = ProgressMeter.ProgressUnknown("Entries read:") +for title in ["a", "b", "c", "d", "e"] + values = Dict(:title => title) + ProgressMeter.next!(prog; showvalues = lazy_no_lazy(values)) + sleep(0.5) +end +ProgressMeter.finish!(prog) + + +println("Testing showvalues with early cancel") +prog = ProgressMeter.Progress(100, 1, "progress: ", 70) +for i in 1:50 + values = Dict(:left => 100 - i) + ProgressMeter.update!(prog, i; showvalues = lazy_no_lazy(values)) + sleep(0.1) +end +ProgressMeter.cancel(prog) + + +println("Testing showvalues with truncate") +prog = ProgressMeter.Progress(50, 1, "progress: ") +for i in 1:50 + values = Dict(:left => 100 - i, :message => repeat("0123456789", i)) + ProgressMeter.update!(prog, i; + showvalues = lazy_no_lazy(values), truncate_lines = true) + sleep(0.1) +end + +end # if \ No newline at end of file diff --git a/test/test_threads.jl b/test/test_threads.jl new file mode 100644 index 0000000..f88a6f1 --- /dev/null +++ b/test/test_threads.jl @@ -0,0 +1,88 @@ + +@testset "ProgressThreads tests" begin + threads = Threads.nthreads() + println("Testing Progress() with Threads.@threads across $threads threads") + (Threads.nthreads() == 1) && @info "Threads.nthreads() == 1, so Threads.@threads test is suboptimal" + n = 20 #per thread + threadsUsed = Int[] + vals = ones(n*threads) + p = ProgressMeter.Progress(n*threads) + Threads.@threads for i = 1:(n*threads) + !in(Threads.threadid(), threadsUsed) && push!(threadsUsed, Threads.threadid()) + vals[i] = 0 + sleep(0.1) + ProgressMeter.next!(p) + end + @test !any(vals .== 1) #Check that all elements have been iterated + @test length(threadsUsed) == threads #Ensure that all threads are used + + + println("Testing ProgressUnknown() with Threads.@threads across $threads threads") + trigger = 100.0 + prog = ProgressMeter.ProgressUnknown("Attepts at exceeding trigger:") + vals = Float64[] + threadsUsed = Int[] + Threads.@threads for _ in 1:1000 + !in(Threads.threadid(), threadsUsed) && push!(threadsUsed, Threads.threadid()) + push!(vals, rand()) + valssum = sum(vals) + if sum(vals) <= trigger + ProgressMeter.next!(prog) + elseif !prog.done + ProgressMeter.finish!(prog) + break + else + break + end + sleep(0.1*rand()) + end + @test sum(vals) > trigger + @test length(threadsUsed) == threads #Ensure that all threads are used + + + println("Testing ProgressThresh() with Threads.@threads across $threads threads") + thresh = 1.0 + prog = ProgressMeter.ProgressThresh(thresh, "Minimizing:") + vals = fill(300.0, 1) + threadsUsed = Int[] + Threads.@threads for _ in 1:100000 + !in(Threads.threadid(), threadsUsed) && push!(threadsUsed, Threads.threadid()) + push!(vals, -rand()) + valssum = sum(vals) + if valssum > thresh + ProgressMeter.update!(prog, valssum) + else + ProgressMeter.finish!(prog) + break + end + sleep(0.1*rand()) + end + @test sum(vals) <= thresh + @test length(threadsUsed) == threads #Ensure that all threads are used + + + @static if VERSION >= v"1.3.0-rc1" #Threads.@spawn not available before 1.3 + if (Threads.nthreads() > 1) + threads = Threads.nthreads() - 1 + println("Testing Progress() with Threads.@spawn across $threads threads") + n = 20 #per thread + tasks = Vector{Task}(undef, threads) + threadsUsed = Int[] + vals = ones(n*threads) + p = ProgressMeter.Progress(n*threads) + for t in 1:threads + tasks[t] = Threads.@spawn for i in 1:n + !in(Threads.threadid(), threadsUsed) && push!(threadsUsed, Threads.threadid()) + vals[(n*(t-1)) + i] = 0 + sleep(0.05 + (rand()*0.1)) + ProgressMeter.next!(p) + end + end + wait.(tasks) + @test !any(vals .== 1) #Check that all elements have been iterated + #@test length(threadsUsed) == threads #Ensure that all threads are used (unreliable for @spawn) + else + @info "Threads.nthreads() == 1, so Threads.@spawn tests cannot be meaningfully tested" + end + end +end