Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Break out doctesting / refactor #774

Merged
merged 23 commits into from
Jul 1, 2019
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add doctest=:only
  • Loading branch information
mortenpi committed Jun 19, 2019
commit fd48f13845ff8309bf0f12ea2c8aa3a46e783cb3
35 changes: 34 additions & 1 deletion src/Builder.jl
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ module Builder

import ..Documenter:
Anchors,
DocTests,
Documents,
Documenter,
Utilities
@@ -24,6 +25,7 @@ using DocStringExtensions
The default document processing "pipeline", which consists of the following actions:

- [`SetupBuildDirectory`](@ref)
- [`Doctest`](@ref)
- [`ExpandTemplates`](@ref)
- [`CrossReferences`](@ref)
- [`CheckDocument`](@ref)
@@ -38,6 +40,11 @@ Creates the correct directory layout within the `build` folder and parses markdo
"""
abstract type SetupBuildDirectory <: DocumentPipeline end

"""
Populates the `.blueprint` field of the [`Documents.Document`](@ref) object.
"""
abstract type Doctest <: DocumentPipeline end

"""
Executes a sequence of actions on each node of the parsed markdown files in turn.
"""
@@ -65,6 +72,7 @@ Writes the document tree to the `build` directory.
abstract type RenderDocument <: DocumentPipeline end

Selectors.order(::Type{SetupBuildDirectory}) = 1.0
Selectors.order(::Type{Doctest}) = 1.1
Selectors.order(::Type{ExpandTemplates}) = 2.0
Selectors.order(::Type{CrossReferences}) = 3.0
Selectors.order(::Type{CheckDocument}) = 4.0
@@ -187,31 +195,48 @@ walk_navpages(ps::Vector, parent, doc) = [walk_navpages(p, parent, doc)::Documen
walk_navpages(src::String, parent, doc) = walk_navpages(true, nothing, src, [], parent, doc)


function Selectors.runner(::Type{Doctest}, doc::Documents.Document)
if doc.user.doctest in [:fix, :only, true]
@info "Doctest: running doctests."
DocTests.doctest(doc.blueprint, doc)
num_errors = length(doc.internal.errors)
if (doc.user.doctest === :only || doc.user.strict) && num_errors > 0
error("`makedocs` encountered $(num_errors > 1 ? "$(num_errors) doctest errors" : "a doctest error"). Terminating build")
end
else
@info "Doctest: skipped."
end
end

function Selectors.runner(::Type{ExpandTemplates}, doc::Documents.Document)
is_doctest_only(doc, "ExpandTemplates") && return
@info "ExpandTemplates: expanding markdown templates."
Documenter.Expanders.expand(doc)
end

function Selectors.runner(::Type{CrossReferences}, doc::Documents.Document)
is_doctest_only(doc, "CrossReferences") && return
@info "CrossReferences: building cross-references."
Documenter.CrossReferences.crossref(doc)
end

function Selectors.runner(::Type{CheckDocument}, doc::Documents.Document)
is_doctest_only(doc, "CheckDocument") && return
@info "CheckDocument: running document checks."
Documenter.DocChecks.missingdocs(doc)
Documenter.DocTests.doctest(doc)
Documenter.DocChecks.footnotes(doc)
Documenter.DocChecks.linkcheck(doc)
end

function Selectors.runner(::Type{Populate}, doc::Documents.Document)
is_doctest_only(doc, "Populate") && return
@info "Populate: populating indices."
Documents.doctest_replace!(doc)
Documents.populate!(doc)
end

function Selectors.runner(::Type{RenderDocument}, doc::Documents.Document)
is_doctest_only(doc, "RenderDocument") && return
count = length(doc.internal.errors)
if doc.user.strict && count > 0
error("`makedocs` encountered $(count > 1 ? "errors" : "an error"). Terminating build")
@@ -223,4 +248,12 @@ end

Selectors.runner(::Type{DocumentPipeline}, doc::Documents.Document) = nothing

function is_doctest_only(doc, stepname)
if doc.user.doctest in [:fix, :only]
@info "Skipped $stepname step (doctest only)."
return true
end
return false
end

end
104 changes: 87 additions & 17 deletions src/DocTests.jl
Original file line number Diff line number Diff line change
@@ -6,49 +6,119 @@ module DocTests
using DocStringExtensions

import ..Documenter:
DocSystem,
Documenter,
Documents,
Expanders,
Utilities
Utilities,
IdDict

import Markdown, REPL
import .Utilities: Markdown2

# Julia code block testing.
# -------------------------

mutable struct MutableMD2CodeBlock
language :: String
code :: String
end
MutableMD2CodeBlock(block :: Markdown2.CodeBlock) = MutableMD2CodeBlock(block.language, block.code)

"""
$(SIGNATURES)

Traverses the document tree and tries to run each Julia code block encountered. Will abort
the document generation when an error is thrown. Use `doctest = false` keyword in
[`Documenter.makedocs`](@ref) to disable doctesting.
Traverses the pages and modules in the documenter blueprint, searching and
executing doctests.

Will abort the document generation when an error is thrown. Use `doctest = false`
keyword in [`Documenter.makedocs`](@ref) to disable doctesting.
"""
function doctest(doc::Documents.Document)
if doc.user.doctest === :fix || doc.user.doctest
@debug "running doctests."
for (src, page) in doc.blueprint.pages
empty!(page.globals.meta)
for element in page.elements
page.globals.meta[:CurrentFile] = page.source
Documents.walk(page.globals.meta, page.mapping[element]) do block
doctest(block, page.globals.meta, doc, page)
end
function doctest(blueprint::Documents.DocumentBlueprint, doc::Documents.Document)
@debug "Running doctests."
# find all the doctest blocks in the pages
for (src, page) in blueprint.pages
doctest(page, doc)
end

# find all the doctest block in all the docstrings (within specified modules)
for mod in blueprint.modules
for (binding, multidoc) in DocSystem.getmeta(mod)
for signature in multidoc.order
doctest(multidoc.docs[signature], doc)
end
end
end
end

function doctest(page::Documents.Page, doc::Documents.Document)
page.globals.meta[:CurrentFile] = page.source
doctest(page.md2ast, page, doc)
end

function doctest(docstr::Docs.DocStr, doc::Documents.Document)
# Note: parsedocs / formatdoc in Base is weird.
# Markdown.MD(Any[Markdown.parse(seekstart(buffer))])
md = DocSystem.parsedoc(docstr)
@assert isa(md, Markdown.MD)
if length(md.content) == 1 && isa(first(md.content), Markdown.MD)
md = first(md.content)
end
md2ast = Markdown2.convert(Markdown2.MD, md)
page = Documents.Page("", "", :build, [], IdDict(), Documents.Globals(), md2ast)
if :path in keys(docstr.data)
page.globals.meta[:CurrentFile] = docstr.data[:path]
else
@debug "skipped doctesting."
page.globals.meta[:CurrentFile] = nothing
end
doctest(md2ast, page, doc)
end

function parse_metablock(block::Markdown2.CodeBlock, page, doc)
@assert startswith(block.language, "@meta")
meta = Dict{Symbol, Any}()
for (ex, str) in Utilities.parseblock(block.code, doc, page)
if Utilities.isassign(ex)
try
meta[ex.args[1]] = Core.eval(Main, ex.args[2])
catch err
push!(doc.internal.errors, :meta_block)
@warn "Failed to evaluate `$(strip(str))` in `@meta` block." err
end
end
end
return meta
end

function doctest(md2ast::Markdown2.MD, page, doc::Documents.Document)
Markdown2.walk(md2ast) do node
isa(node, Markdown2.CodeBlock) || return true
if startswith(node.language, "jldoctest")
doctest(node, page.globals.meta, doc, page)
elseif startswith(node.language, "@meta")
merge!(page.globals.meta, parse_metablock(node, page, doc))
else
return true
end
return false
end
end

function doctest(block::Markdown.Code, meta::Dict, doc::Documents.Document, page)
lang = block.language
doctest(Markdown2._convert_block(block), meta, doc, page)
end

function doctest(block_immutable::Markdown2.CodeBlock, meta::Dict, doc::Documents.Document, page)
lang = block_immutable.language
if startswith(lang, "jldoctest")
# Define new module or reuse an old one from this page if we have a named doctest.
name = match(r"jldoctest[ ]?(.*)$", split(lang, ';', limit = 2)[1])[1]
sym = isempty(name) ? gensym("doctest-") : Symbol("doctest-", name)
sandbox = get!(() -> Expanders.get_new_sandbox(sym), page.globals.meta, sym)

# Normalise line endings.
block = MutableMD2CodeBlock(block_immutable)
block.code = replace(block.code, "\r\n" => "\n")

# parse keyword arguments to doctest
@@ -103,7 +173,7 @@ function doctest(block::Markdown.Code, meta::Dict, doc::Documents.Document, page
```
""")
end
delete!(meta, :LocalDocTestArguments)
delete!(meta, :LocalDocTestArguments)
end
false
end
@@ -117,7 +187,7 @@ end
# Doctest evaluation.

mutable struct Result
block :: Markdown.Code # The entire code block that is being tested.
block :: MutableMD2CodeBlock # The entire code block that is being tested.
input :: String # Part of `block.code` representing the current input.
output :: String # Part of `block.code` representing the current expected output.
file :: String # File in which the doctest is written. Either `.md` or `.jl`.
4 changes: 2 additions & 2 deletions src/Documenter.jl
Original file line number Diff line number Diff line change
@@ -38,10 +38,10 @@ include("Utilities/Utilities.jl")
include("DocSystem.jl")
include("Anchors.jl")
include("Documents.jl")
include("Builder.jl")
include("Expanders.jl")
include("CrossReferences.jl")
include("DocTests.jl")
include("Builder.jl")
include("CrossReferences.jl")
include("DocChecks.jl")
include("Writers/Writers.jl")
include("Deps.jl")
6 changes: 3 additions & 3 deletions test/doctests/doctests.jl
Original file line number Diff line number Diff line change
@@ -136,7 +136,7 @@ rfile(filename) = joinpath(@__DIR__, "stdouts", filename)

run_makedocs(["broken.md", "foobroken.md"]; modules=[FooBroken], strict=true) do result, success, backtrace, output
@test !success
@test_broken is_same_as_file(output, rfile("stdout.6"))
@test is_same_as_file(output, rfile("stdout.6"))
end

run_makedocs(["fooworking.md"]; modules=[FooWorking], strict=true) do result, success, backtrace, output
@@ -145,8 +145,8 @@ rfile(filename) = joinpath(@__DIR__, "stdouts", filename)
end

run_makedocs(["foobroken.md"]; modules=[FooBroken], strict=true) do result, success, backtrace, output
@test_broken !success
@test_broken is_same_as_file(output, rfile("stdout.8"))
@test !success
@test is_same_as_file(output, rfile("stdout.8"))
end

# Here we try the default (strict = false) -- output should say that doctest failed, but
32 changes: 23 additions & 9 deletions test/doctests/fix/tests.jl
Original file line number Diff line number Diff line change
@@ -1,25 +1,39 @@
# Tests for doctest = :fix
#
# DOCUMENTER_TEST_DEBUG= JULIA_DEBUG=all julia test/doctests/fix/tests.jl
#
module DocTestFixTest
using Documenter, Test

println("="^50)
@info("Testing `doctest = :fix`")
mktempdir(@__DIR__) do dir
function test_doctest_fix(dir)
srcdir = mktempdir(dir)
builddir = mktempdir(dir)
@debug "Testing doctest = :fix" srcdir builddir
cp(joinpath(@__DIR__, "broken.md"), joinpath(srcdir, "index.md"))
cp(joinpath(@__DIR__, "broken.jl"), joinpath(srcdir, "src.jl"))
include(joinpath(srcdir, "src.jl"))
@eval using .Foo

# fix up
include(joinpath(srcdir, "src.jl")); @eval import .Foo
@debug "Running doctest/fix doctests with doctest=:fix"
makedocs(sitename="-", modules = [Foo], source = srcdir, build = builddir, doctest = :fix)

# test that strict = true works
include(joinpath(srcdir, "src.jl")); @eval import .Foo
@debug "Running doctest/fix doctests with doctest=true"
makedocs(sitename="-", modules = [Foo], source = srcdir, build = builddir, strict = true)

# also test that we obtain the expected output
@test read(joinpath(srcdir, "index.md"), String) ==
read(joinpath(@__DIR__, "fixed.md"), String)
@test read(joinpath(srcdir, "src.jl"), String) ==
read(joinpath(@__DIR__, "fixed.jl"), String)
@test read(joinpath(srcdir, "index.md"), String) == read(joinpath(@__DIR__, "fixed.md"), String)
@test read(joinpath(srcdir, "src.jl"), String) == read(joinpath(@__DIR__, "fixed.jl"), String)
end

println("="^50)
@info("Testing `doctest = :fix`")
if haskey(ENV, "DOCUMENTER_TEST_DEBUG")
# in this mode the directories remain
test_doctest_fix(mktempdir(@__DIR__))
else
mktempdir(test_doctest_fix, @__DIR__)
end
@info("Done testing `doctest = :fix`")
println("="^50)
1 change: 1 addition & 0 deletions test/doctests/stdouts/stdout.1
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
[ Info: SetupBuildDirectory: setting up build directory.
[ Info: Doctest: running doctests.
[ Info: ExpandTemplates: expanding markdown templates.
[ Info: CrossReferences: building cross-references.
[ Info: CheckDocument: running document checks.
1 change: 1 addition & 0 deletions test/doctests/stdouts/stdout.11
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
[ Info: SetupBuildDirectory: setting up build directory.
[ Info: Doctest: running doctests.
[ Info: ExpandTemplates: expanding markdown templates.
[ Info: CrossReferences: building cross-references.
[ Info: CheckDocument: running document checks.
11 changes: 6 additions & 5 deletions test/doctests/stdouts/stdout.12
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
[ Info: SetupBuildDirectory: setting up build directory.
[ Info: ExpandTemplates: expanding markdown templates.
[ Info: CrossReferences: building cross-references.
[ Info: CheckDocument: running document checks.
┌ Error: doctest failure in ~/Julia/JuliaDocs/Documenter/test/doctests2/tmpfiBBfP/tmpcviARh/broken.md:3-6
[ Info: Doctest: running doctests.
┌ Error: doctest failure in src/broken.md:3-6
│ ```jldoctest
│ julia> 2 + 2
@@ -24,7 +22,10 @@
│ diff =
│ Warning: Diff output requires color.
│ -64
└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:266
└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:336
[ Info: ExpandTemplates: expanding markdown templates.
[ Info: CrossReferences: building cross-references.
[ Info: CheckDocument: running document checks.
[ Info: Populate: populating indices.
[ Info: RenderDocument: rendering document.
[ Info: HTMLWriter: rendering HTML pages.
9 changes: 3 additions & 6 deletions test/doctests/stdouts/stdout.2
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
[ Info: SetupBuildDirectory: setting up build directory.
[ Info: ExpandTemplates: expanding markdown templates.
[ Info: CrossReferences: building cross-references.
[ Info: CheckDocument: running document checks.
┌ Error: doctest failure in ~/Julia/JuliaDocs/Documenter/test/doctests2/tmp71nSFG/tmpUzYK6R/broken.md:3-6
[ Info: Doctest: running doctests.
┌ Error: doctest failure in src/broken.md:3-6
│ ```jldoctest
│ julia> 2 + 2
@@ -24,5 +22,4 @@
│ diff =
│ Warning: Diff output requires color.
│ -64
└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:266
[ Info: Populate: populating indices.
└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:336
1 change: 1 addition & 0 deletions test/doctests/stdouts/stdout.3
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
[ Info: SetupBuildDirectory: setting up build directory.
[ Info: Doctest: running doctests.
[ Info: ExpandTemplates: expanding markdown templates.
[ Info: CrossReferences: building cross-references.
[ Info: CheckDocument: running document checks.
Loading