Add doctest=:only

src/Builder.jl

@@ -9,6 +9,7 @@ module Builder
 
 import ..Documenter:
     Anchors,
+    DocTests,
     Documents,
     Documenter,
     Utilities
@@ -24,6 +25,7 @@ using DocStringExtensions
 The default document processing "pipeline", which consists of the following actions:
 
 - [`SetupBuildDirectory`](@ref)
+- [`Doctest`](@ref)
 - [`ExpandTemplates`](@ref)
 - [`CrossReferences`](@ref)
 - [`CheckDocument`](@ref)
@@ -38,6 +40,11 @@ Creates the correct directory layout within the `build` folder and parses markdo
 """
 abstract type SetupBuildDirectory <: DocumentPipeline end
 
+"""
+Populates the `.blueprint` field of the [`Documents.Document`](@ref) object.
+"""
+abstract type Doctest <: DocumentPipeline end
+
 """
 Executes a sequence of actions on each node of the parsed markdown files in turn.
 """
@@ -65,6 +72,7 @@ Writes the document tree to the `build` directory.
 abstract type RenderDocument <: DocumentPipeline end
 
 Selectors.order(::Type{SetupBuildDirectory})   = 1.0
+Selectors.order(::Type{Doctest})               = 1.1
 Selectors.order(::Type{ExpandTemplates})       = 2.0
 Selectors.order(::Type{CrossReferences})       = 3.0
 Selectors.order(::Type{CheckDocument})         = 4.0
@@ -187,31 +195,48 @@ walk_navpages(ps::Vector, parent, doc) = [walk_navpages(p, parent, doc)::Documen
 walk_navpages(src::String, parent, doc) = walk_navpages(true, nothing, src, [], parent, doc)
 
 
+function Selectors.runner(::Type{Doctest}, doc::Documents.Document)
+    if doc.user.doctest in [:fix, :only, true]
+        @info "Doctest: running doctests."
+        DocTests.doctest(doc.blueprint, doc)
+        num_errors = length(doc.internal.errors)
+        if (doc.user.doctest === :only || doc.user.strict) && num_errors > 0
+            error("`makedocs` encountered $(num_errors > 1 ? "$(num_errors) doctest errors" : "a doctest error"). Terminating build")
+        end
+    else
+        @info "Doctest: skipped."
+    end
+end
+
 function Selectors.runner(::Type{ExpandTemplates}, doc::Documents.Document)
+    is_doctest_only(doc, "ExpandTemplates") && return
     @info "ExpandTemplates: expanding markdown templates."
     Documenter.Expanders.expand(doc)
 end
 
 function Selectors.runner(::Type{CrossReferences}, doc::Documents.Document)
+    is_doctest_only(doc, "CrossReferences") && return
     @info "CrossReferences: building cross-references."
     Documenter.CrossReferences.crossref(doc)
 end
 
 function Selectors.runner(::Type{CheckDocument}, doc::Documents.Document)
+    is_doctest_only(doc, "CheckDocument") && return
     @info "CheckDocument: running document checks."
     Documenter.DocChecks.missingdocs(doc)
-    Documenter.DocTests.doctest(doc)
     Documenter.DocChecks.footnotes(doc)
     Documenter.DocChecks.linkcheck(doc)
 end
 
 function Selectors.runner(::Type{Populate}, doc::Documents.Document)
+    is_doctest_only(doc, "Populate") && return
     @info "Populate: populating indices."
     Documents.doctest_replace!(doc)
     Documents.populate!(doc)
 end
 
 function Selectors.runner(::Type{RenderDocument}, doc::Documents.Document)
+    is_doctest_only(doc, "RenderDocument") && return
     count = length(doc.internal.errors)
     if doc.user.strict && count > 0
         error("`makedocs` encountered $(count > 1 ? "errors" : "an error"). Terminating build")
@@ -223,4 +248,12 @@ end
 
 Selectors.runner(::Type{DocumentPipeline}, doc::Documents.Document) = nothing
 
+function is_doctest_only(doc, stepname)
+    if doc.user.doctest in [:fix, :only]
+        @info "Skipped $stepname step (doctest only)."
+        return true
+    end
+    return false
+end
+
 end

src/DocTests.jl

@@ -6,49 +6,119 @@ module DocTests
 using DocStringExtensions
 
 import ..Documenter:
+    DocSystem,
     Documenter,
     Documents,
     Expanders,
-    Utilities
+    Utilities,
+    IdDict
 
 import Markdown, REPL
+import .Utilities: Markdown2
 
 # Julia code block testing.
 # -------------------------
 
+mutable struct MutableMD2CodeBlock
+    language :: String
+    code :: String
+end
+MutableMD2CodeBlock(block :: Markdown2.CodeBlock) = MutableMD2CodeBlock(block.language, block.code)
+
 """
 $(SIGNATURES)
 
-Traverses the document tree and tries to run each Julia code block encountered. Will abort
-the document generation when an error is thrown. Use `doctest = false` keyword in
-[`Documenter.makedocs`](@ref) to disable doctesting.
+Traverses the pages and modules in the documenter blueprint, searching and
+executing doctests.
+
+Will abort the document generation when an error is thrown. Use `doctest = false`
+keyword in [`Documenter.makedocs`](@ref) to disable doctesting.
 """
-function doctest(doc::Documents.Document)
-    if doc.user.doctest === :fix || doc.user.doctest
-        @debug "running doctests."
-        for (src, page) in doc.blueprint.pages
-            empty!(page.globals.meta)
-            for element in page.elements
-                page.globals.meta[:CurrentFile] = page.source
-                Documents.walk(page.globals.meta, page.mapping[element]) do block
-                    doctest(block, page.globals.meta, doc, page)
-                end
+function doctest(blueprint::Documents.DocumentBlueprint, doc::Documents.Document)
+    @debug "Running doctests."
+    # find all the doctest blocks in the pages
+    for (src, page) in blueprint.pages
+        doctest(page, doc)
+    end
+
+    # find all the doctest block in all the docstrings (within specified modules)
+    for mod in blueprint.modules
+        for (binding, multidoc) in DocSystem.getmeta(mod)
+            for signature in multidoc.order
+                doctest(multidoc.docs[signature], doc)
             end
         end
+    end
+end
+
+function doctest(page::Documents.Page, doc::Documents.Document)
+    page.globals.meta[:CurrentFile] = page.source
+    doctest(page.md2ast, page, doc)
+end
+
+function doctest(docstr::Docs.DocStr, doc::Documents.Document)
+    # Note: parsedocs / formatdoc in Base is weird.
+    # Markdown.MD(Any[Markdown.parse(seekstart(buffer))])
+    md = DocSystem.parsedoc(docstr)
+    @assert isa(md, Markdown.MD)
+    if length(md.content) == 1 && isa(first(md.content), Markdown.MD)
+        md = first(md.content)
+    end
+    md2ast = Markdown2.convert(Markdown2.MD, md)
+    page = Documents.Page("", "", :build, [], IdDict(), Documents.Globals(), md2ast)
+    if :path in keys(docstr.data)
+        page.globals.meta[:CurrentFile] = docstr.data[:path]
     else
         @debug "skipped doctesting."
+        page.globals.meta[:CurrentFile] = nothing
+    end
+    doctest(md2ast, page, doc)
+end
+
+function parse_metablock(block::Markdown2.CodeBlock, page, doc)
+    @assert startswith(block.language, "@meta")
+    meta = Dict{Symbol, Any}()
+    for (ex, str) in Utilities.parseblock(block.code, doc, page)
+        if Utilities.isassign(ex)
+            try
+                meta[ex.args[1]] = Core.eval(Main, ex.args[2])
+            catch err
+                push!(doc.internal.errors, :meta_block)
+                @warn "Failed to evaluate `$(strip(str))` in `@meta` block." err
+            end
+        end
+    end
+    return meta
+end
+
+function doctest(md2ast::Markdown2.MD, page, doc::Documents.Document)
+    Markdown2.walk(md2ast) do node
+        isa(node, Markdown2.CodeBlock) || return true
+        if startswith(node.language, "jldoctest")
+            doctest(node, page.globals.meta, doc, page)
+        elseif startswith(node.language, "@meta")
+            merge!(page.globals.meta, parse_metablock(node, page, doc))
+        else
+            return true
+        end
+        return false
     end
 end
 
 function doctest(block::Markdown.Code, meta::Dict, doc::Documents.Document, page)
-    lang = block.language
+    doctest(Markdown2._convert_block(block), meta, doc, page)
+end
+
+function doctest(block_immutable::Markdown2.CodeBlock, meta::Dict, doc::Documents.Document, page)
+    lang = block_immutable.language
     if startswith(lang, "jldoctest")
         # Define new module or reuse an old one from this page if we have a named doctest.
         name = match(r"jldoctest[ ]?(.*)$", split(lang, ';', limit = 2)[1])[1]
         sym = isempty(name) ? gensym("doctest-") : Symbol("doctest-", name)
         sandbox = get!(() -> Expanders.get_new_sandbox(sym), page.globals.meta, sym)
 
         # Normalise line endings.
+        block = MutableMD2CodeBlock(block_immutable)
         block.code = replace(block.code, "\r\n" => "\n")
 
         # parse keyword arguments to doctest
@@ -103,7 +173,7 @@ function doctest(block::Markdown.Code, meta::Dict, doc::Documents.Document, page
                 ```
                 """)
         end
-       delete!(meta, :LocalDocTestArguments)
+        delete!(meta, :LocalDocTestArguments)
     end
     false
 end
@@ -117,7 +187,7 @@ end
 # Doctest evaluation.
 
 mutable struct Result
-    block  :: Markdown.Code # The entire code block that is being tested.
+    block  :: MutableMD2CodeBlock # The entire code block that is being tested.
     input  :: String # Part of `block.code` representing the current input.
     output :: String # Part of `block.code` representing the current expected output.
     file   :: String # File in which the doctest is written. Either `.md` or `.jl`.

src/Documenter.jl

@@ -38,10 +38,10 @@ include("Utilities/Utilities.jl")
 include("DocSystem.jl")
 include("Anchors.jl")
 include("Documents.jl")
-include("Builder.jl")
 include("Expanders.jl")
-include("CrossReferences.jl")
 include("DocTests.jl")
+include("Builder.jl")
+include("CrossReferences.jl")
 include("DocChecks.jl")
 include("Writers/Writers.jl")
 include("Deps.jl")

test/doctests/doctests.jl

@@ -136,7 +136,7 @@ rfile(filename) = joinpath(@__DIR__, "stdouts", filename)
 
     run_makedocs(["broken.md", "foobroken.md"]; modules=[FooBroken], strict=true) do result, success, backtrace, output
         @test !success
-        @test_broken is_same_as_file(output, rfile("stdout.6"))
+        @test is_same_as_file(output, rfile("stdout.6"))
     end
 
     run_makedocs(["fooworking.md"]; modules=[FooWorking], strict=true) do result, success, backtrace, output
@@ -145,8 +145,8 @@ rfile(filename) = joinpath(@__DIR__, "stdouts", filename)
     end
 
     run_makedocs(["foobroken.md"]; modules=[FooBroken], strict=true) do result, success, backtrace, output
-        @test_broken !success
-        @test_broken is_same_as_file(output, rfile("stdout.8"))
+        @test !success
+        @test is_same_as_file(output, rfile("stdout.8"))
     end
 
     # Here we try the default (strict = false) -- output should say that doctest failed, but

test/doctests/fix/tests.jl

@@ -1,25 +1,39 @@
 # Tests for doctest = :fix
+#
+# DOCUMENTER_TEST_DEBUG= JULIA_DEBUG=all julia test/doctests/fix/tests.jl
+#
 module DocTestFixTest
 using Documenter, Test
 
-println("="^50)
-@info("Testing `doctest = :fix`")
-mktempdir(@__DIR__) do dir
+function test_doctest_fix(dir)
     srcdir = mktempdir(dir)
     builddir = mktempdir(dir)
+    @debug "Testing doctest = :fix" srcdir builddir
     cp(joinpath(@__DIR__, "broken.md"), joinpath(srcdir, "index.md"))
     cp(joinpath(@__DIR__, "broken.jl"), joinpath(srcdir, "src.jl"))
-    include(joinpath(srcdir, "src.jl"))
-    @eval using .Foo
+
     # fix up
+    include(joinpath(srcdir, "src.jl")); @eval import .Foo
+    @debug "Running doctest/fix doctests with doctest=:fix"
     makedocs(sitename="-", modules = [Foo], source = srcdir, build = builddir, doctest = :fix)
+
     # test that strict = true works
+    include(joinpath(srcdir, "src.jl")); @eval import .Foo
+    @debug "Running doctest/fix doctests with doctest=true"
     makedocs(sitename="-", modules = [Foo], source = srcdir, build = builddir, strict = true)
+
     # also test that we obtain the expected output
-    @test read(joinpath(srcdir, "index.md"), String) ==
-          read(joinpath(@__DIR__, "fixed.md"), String)
-    @test read(joinpath(srcdir, "src.jl"), String) ==
-          read(joinpath(@__DIR__, "fixed.jl"), String)
+    @test read(joinpath(srcdir, "index.md"), String) == read(joinpath(@__DIR__, "fixed.md"), String)
+    @test read(joinpath(srcdir, "src.jl"), String) == read(joinpath(@__DIR__, "fixed.jl"), String)
+end
+
+println("="^50)
+@info("Testing `doctest = :fix`")
+if haskey(ENV, "DOCUMENTER_TEST_DEBUG")
+    # in this mode the directories remain
+    test_doctest_fix(mktempdir(@__DIR__))
+else
+    mktempdir(test_doctest_fix, @__DIR__)
 end
 @info("Done testing `doctest = :fix`")
 println("="^50)

test/doctests/stdouts/stdout.1

@@ -1,4 +1,5 @@
 [ Info: SetupBuildDirectory: setting up build directory.
+[ Info: Doctest: running doctests.
 [ Info: ExpandTemplates: expanding markdown templates.
 [ Info: CrossReferences: building cross-references.
 [ Info: CheckDocument: running document checks.

test/doctests/stdouts/stdout.11

@@ -1,4 +1,5 @@
 [ Info: SetupBuildDirectory: setting up build directory.
+[ Info: Doctest: running doctests.
 [ Info: ExpandTemplates: expanding markdown templates.
 [ Info: CrossReferences: building cross-references.
 [ Info: CheckDocument: running document checks.

test/doctests/stdouts/stdout.12

@@ -1,8 +1,6 @@
 [ Info: SetupBuildDirectory: setting up build directory.
-[ Info: ExpandTemplates: expanding markdown templates.
-[ Info: CrossReferences: building cross-references.
-[ Info: CheckDocument: running document checks.
-┌ Error: doctest failure in ~/Julia/JuliaDocs/Documenter/test/doctests2/tmpfiBBfP/tmpcviARh/broken.md:3-6
+[ Info: Doctest: running doctests.
+┌ Error: doctest failure in src/broken.md:3-6
 │ 
 │ ```jldoctest
 │ julia> 2 + 2
@@ -24,7 +22,10 @@
 │   diff =
 │    Warning: Diff output requires color.
 │    -64
-└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:266
+└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:336
+[ Info: ExpandTemplates: expanding markdown templates.
+[ Info: CrossReferences: building cross-references.
+[ Info: CheckDocument: running document checks.
 [ Info: Populate: populating indices.
 [ Info: RenderDocument: rendering document.
 [ Info: HTMLWriter: rendering HTML pages.

test/doctests/stdouts/stdout.2

@@ -1,8 +1,6 @@
 [ Info: SetupBuildDirectory: setting up build directory.
-[ Info: ExpandTemplates: expanding markdown templates.
-[ Info: CrossReferences: building cross-references.
-[ Info: CheckDocument: running document checks.
-┌ Error: doctest failure in ~/Julia/JuliaDocs/Documenter/test/doctests2/tmp71nSFG/tmpUzYK6R/broken.md:3-6
+[ Info: Doctest: running doctests.
+┌ Error: doctest failure in src/broken.md:3-6
 │ 
 │ ```jldoctest
 │ julia> 2 + 2
@@ -24,5 +22,4 @@
 │   diff =
 │    Warning: Diff output requires color.
 │    -64
-└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:266
-[ Info: Populate: populating indices.
+└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:336

test/doctests/stdouts/stdout.3

@@ -1,4 +1,5 @@
 [ Info: SetupBuildDirectory: setting up build directory.
+[ Info: Doctest: running doctests.
 [ Info: ExpandTemplates: expanding markdown templates.
 [ Info: CrossReferences: building cross-references.
 [ Info: CheckDocument: running document checks.