Add tests for doctest=:only

test/doctests/doctests.jl

@@ -160,6 +160,33 @@ rfile(filename) = joinpath(@__DIR__, "stdouts", filename)
         @test success
         @test is_same_as_file(output, rfile("stdout.12"))
     end
+
+    # Tests for doctest = :only. The outout should reflect that the docs themselves do not
+    # get built.
+    run_makedocs(["working.md"]; modules=[FooWorking], doctest = :only) do result, success, backtrace, output
+        @test success
+        @test is_same_as_file(output, rfile("stdout.21"))
+    end
+
+    run_makedocs(["working.md"]; modules=[FooBroken], doctest = :only) do result, success, backtrace, output
+        @test !success
+        @test is_same_as_file(output, rfile("stdout.22"))
+    end
+
+    run_makedocs(["broken.md"]; modules=[FooWorking], doctest = :only) do result, success, backtrace, output
+        @test !success
+        @test is_same_as_file(output, rfile("stdout.23"))
+    end
+
+    run_makedocs(["broken.md"]; modules=[FooBroken], doctest = :only) do result, success, backtrace, output
+        @test !success
+        @test is_same_as_file(output, rfile("stdout.24"))
+    end
+    # strict gets ignored with doctest = :only
+    run_makedocs(["broken.md"]; modules=[FooBroken], doctest = :only, strict=false) do result, success, backtrace, output
+        @test !success
+        @test is_same_as_file(output, rfile("stdout.25"))
+    end
 end
 
 end # module

test/doctests/stdouts/stdout.21

@@ -0,0 +1,7 @@
+[ Info: SetupBuildDirectory: setting up build directory.
+[ Info: Doctest: running doctests.
+[ Info: Skipped ExpandTemplates step (doctest only).
+[ Info: Skipped CrossReferences step (doctest only).
+[ Info: Skipped CheckDocument step (doctest only).
+[ Info: Skipped Populate step (doctest only).
+[ Info: Skipped RenderDocument step (doctest only).

test/doctests/stdouts/stdout.22

@@ -0,0 +1,25 @@
+[ Info: SetupBuildDirectory: setting up build directory.
+[ Info: Doctest: running doctests.
+┌ Error: doctest failure in ~/Julia/JuliaDocs/Documenter/test/doctests/src/FooBroken.jl:6-9
+│ 
+│ ```jldoctest
+│ julia> 2 + 2
+│ 42
+│ ```
+│ 
+│ Subexpression:
+│ 
+│ 2 + 2
+│ 
+│ Evaluated output:
+│ 
+│ 4
+│ 
+│ Expected output:
+│ 
+│ 42
+│ 
+│   diff =
+│    Warning: Diff output requires color.
+│    424
+└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:336

test/doctests/stdouts/stdout.23

@@ -0,0 +1,25 @@
+[ Info: SetupBuildDirectory: setting up build directory.
+[ Info: Doctest: running doctests.
+┌ Error: doctest failure in src/broken.md:3-6
+│ 
+│ ```jldoctest
+│ julia> 2 + 2
+│ -6
+│ ```
+│ 
+│ Subexpression:
+│ 
+│ 2 + 2
+│ 
+│ Evaluated output:
+│ 
+│ 4
+│ 
+│ Expected output:
+│ 
+│ -6
+│ 
+│   diff =
+│    Warning: Diff output requires color.
+│    -64
+└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:336

test/doctests/stdouts/stdout.24

@@ -0,0 +1,48 @@
+[ Info: SetupBuildDirectory: setting up build directory.
+[ Info: Doctest: running doctests.
+┌ Error: doctest failure in src/broken.md:3-6
+│ 
+│ ```jldoctest
+│ julia> 2 + 2
+│ -6
+│ ```
+│ 
+│ Subexpression:
+│ 
+│ 2 + 2
+│ 
+│ Evaluated output:
+│ 
+│ 4
+│ 
+│ Expected output:
+│ 
+│ -6
+│ 
+│   diff =
+│    Warning: Diff output requires color.
+│    -64
+└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:336
+┌ Error: doctest failure in ~/Julia/JuliaDocs/Documenter/test/doctests/src/FooBroken.jl:6-9
+│ 
+│ ```jldoctest
+│ julia> 2 + 2
+│ 42
+│ ```
+│ 
+│ Subexpression:
+│ 
+│ 2 + 2
+│ 
+│ Evaluated output:
+│ 
+│ 4
+│ 
+│ Expected output:
+│ 
+│ 42
+│ 
+│   diff =
+│    Warning: Diff output requires color.
+│    424
+└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:336

test/doctests/stdouts/stdout.25

@@ -0,0 +1,48 @@
+[ Info: SetupBuildDirectory: setting up build directory.
+[ Info: Doctest: running doctests.
+┌ Error: doctest failure in src/broken.md:3-6
+│ 
+│ ```jldoctest
+│ julia> 2 + 2
+│ -6
+│ ```
+│ 
+│ Subexpression:
+│ 
+│ 2 + 2
+│ 
+│ Evaluated output:
+│ 
+│ 4
+│ 
+│ Expected output:
+│ 
+│ -6
+│ 
+│   diff =
+│    Warning: Diff output requires color.
+│    -64
+└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:336
+┌ Error: doctest failure in ~/Julia/JuliaDocs/Documenter/test/doctests/src/FooBroken.jl:6-9
+│ 
+│ ```jldoctest
+│ julia> 2 + 2
+│ 42
+│ ```
+│ 
+│ Subexpression:
+│ 
+│ 2 + 2
+│ 
+│ Evaluated output:
+│ 
+│ 4
+│ 
+│ Expected output:
+│ 
+│ 42
+│ 
+│   diff =
+│    Warning: Diff output requires color.
+│    424
+└ @ Documenter.DocTests ~/Julia/JuliaDocs/Documenter/src/DocTests.jl:336