feat: write at-example outputs over threshold to a file (#2247)

JuliaDocs · Sep 14, 2023 · 1aaf685 · 1aaf685
1 parent df8e97b
commit 1aaf685
Show file tree

Hide file tree

Showing 18 changed files with 1,170 additions and 30 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -134,6 +134,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 * The `doctest` routine can now receive the same `plugins` keyword argument as `makedocs`. This enables `doctest` to run if any plugin with a mandatory `Plugin` object is loaded, e.g., [DocumenterCitations](https://github.com/JuliaDocs/DocumenterCitations.jl). ([#2245])
 
+* The HTML output will automatically write larger `@example`-block outputs to files, to make the generated HTML files smaller. The size threshold can be controlled with the `example_size_threshold` option to `HTML`. ([#2143], [#2247])
+
 ### Fixed
 
 * Line endings in Markdown source files are now normalized to `LF` before parsing, to work around [a bug in the Julia Markdown parser][julia-29344] where parsing is sensitive to line endings, and can therefore cause platform-dependent behavior. ([#1906])

diff --git a/Project.toml b/Project.toml
@@ -20,6 +20,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 RegistryInstances = "2792f1a3-b283-48e8-9a74-f99dce5104f3"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 

diff --git a/src/html/HTMLWriter.jl b/src/html/HTMLWriter.jl
diff --git a/test/examples/images/big.gif b/test/examples/images/big.gif
diff --git a/test/examples/images/big.png b/test/examples/images/big.png
diff --git a/test/examples/images/big.svg b/test/examples/images/big.svg
diff --git a/test/examples/images/big.webp b/test/examples/images/big.webp
diff --git a/test/examples/images/tiny.jpeg b/test/examples/images/tiny.jpeg
diff --git a/test/examples/images/tiny.png b/test/examples/images/tiny.png
diff --git a/test/examples/images/tiny.webp b/test/examples/images/tiny.webp
diff --git a/test/examples/make.jl b/test/examples/make.jl
@@ -1,3 +1,4 @@
+import SHA
 using Documenter
 include("../TestUtilities.jl"); using Main.TestUtilities
 
@@ -142,6 +143,39 @@ module AutoDocs
     end
 end
 
+struct MIMEBytes{M <: MIME}
+    bytes :: Vector{UInt8}
+    hash_slug :: String
+    function MIMEBytes(mime::AbstractString, bytes::AbstractVector{UInt8})
+        hash_slug = bytes2hex(SHA.sha1(bytes))[1:8]
+        new{MIME{Symbol(mime)}}(bytes, hash_slug)
+    end
+end
+Base.show(io::IO, ::M, obj::MIMEBytes{M}) where {M <: MIME} = write(io, obj.bytes)
+
+const AT_EXAMPLE_FILES = Dict(
+    ("png", :big) => MIMEBytes("image/png", read(joinpath(@__DIR__, "images", "big.png"))),
+    ("png", :tiny) => MIMEBytes("image/png", read(joinpath(@__DIR__, "images", "tiny.png"))),
+    ("webp", :big) => MIMEBytes("image/webp", read(joinpath(@__DIR__, "images", "big.webp"))),
+    ("webp", :tiny) => MIMEBytes("image/webp", read(joinpath(@__DIR__, "images", "tiny.webp"))),
+    ("gif", :big) => MIMEBytes("image/gif", read(joinpath(@__DIR__, "images", "big.gif"))),
+    ("jpeg", :tiny) => MIMEBytes("image/jpeg", read(joinpath(@__DIR__, "images", "tiny.jpeg"))),
+)
+SVG_BIG = MIMEBytes("image/svg+xml", read(joinpath(@__DIR__, "images", "big.svg")))
+SVG_HTML = MIMEBytes("text/html", read(joinpath(@__DIR__, "images", "big.svg")))
+
+struct MultiMIMESVG
+    bytes :: Vector{UInt8}
+    hash_slug :: String
+    function MultiMIMESVG(bytes::AbstractVector{UInt8})
+        hash_slug = bytes2hex(SHA.sha1(bytes))[1:8]
+        new(bytes, hash_slug)
+    end
+end
+Base.show(io::IO, ::MIME"image/svg+xml", obj::MultiMIMESVG) = write(io, obj.bytes)
+Base.show(io::IO, ::MIME"text/html", obj::MultiMIMESVG) = write(io, obj.bytes)
+SVG_MULTI = MultiMIMESVG(read(joinpath(@__DIR__, "images", "big.svg")))
+
 # Helper functions
 function withassets(f, assets...)
     src(asset) = joinpath(@__DIR__, asset)
@@ -211,6 +245,10 @@ htmlbuild_pages = Any[
         "editurl/ugly.md",
     ],
     "xrefs.md",
+    "Outputs" => [
+        "outputs/index.md",
+        "outputs/outputs.md",
+    ],
 ]
 
 function html_doc(

diff --git a/test/examples/references/latex_showcase.tex b/test/examples/references/latex_showcase.tex
@@ -2,7 +2,7 @@
 \newcommand{\DocMainTitle}{Documenter LaTeX Showcase}
 \newcommand{\DocVersion}{1.2.3}
 \newcommand{\DocAuthors}{}
-\newcommand{\JuliaVersion}{1.9.2}
+\newcommand{\JuliaVersion}{1.9.3}
 
 % ---- Insert preamble
 \input{preamble.tex}
@@ -1161,7 +1161,7 @@ \chapter{Docstrings}
 
 
 
-\href{https://example.org/Repository.jl/blob/make.jl#L27-31}{\texttt{source}}
+\href{https://example.org/Repository.jl/blob/make.jl#L28-32}{\texttt{source}}
 
 
 \end{adjustwidth}
@@ -1178,7 +1178,7 @@ \chapter{Docstrings}
 
 
 
-\href{https://example.org/Repository.jl/blob/make.jl#L34-38}{\texttt{source}}
+\href{https://example.org/Repository.jl/blob/make.jl#L35-39}{\texttt{source}}
 
 
 \end{adjustwidth}
@@ -1252,7 +1252,7 @@ \section{An index of docstrings}
 
 
 
-\href{https://example.org/Repository.jl/blob/make.jl#L41-71}{\texttt{source}}
+\href{https://example.org/Repository.jl/blob/make.jl#L42-72}{\texttt{source}}
 
 
 \end{adjustwidth}
@@ -1271,7 +1271,7 @@ \section{at-autodocs}
 
 
 
-\href{https://example.org/Repository.jl/blob/make.jl#L75-75}{\texttt{source}}
+\href{https://example.org/Repository.jl/blob/make.jl#L76-76}{\texttt{source}}
 
 
 \end{adjustwidth}
@@ -1283,7 +1283,7 @@ \section{at-autodocs}
 
 
 
-\href{https://example.org/Repository.jl/blob/make.jl#L88-88}{\texttt{source}}
+\href{https://example.org/Repository.jl/blob/make.jl#L89-89}{\texttt{source}}
 
 
 \end{adjustwidth}
@@ -1295,7 +1295,7 @@ \section{at-autodocs}
 
 
 
-\href{https://example.org/Repository.jl/blob/make.jl#L91-91}{\texttt{source}}
+\href{https://example.org/Repository.jl/blob/make.jl#L92-92}{\texttt{source}}
 
 
 \end{adjustwidth}
@@ -1307,7 +1307,7 @@ \section{at-autodocs}
 
 
 
-\href{https://example.org/Repository.jl/blob/make.jl#L85-85}{\texttt{source}}
+\href{https://example.org/Repository.jl/blob/make.jl#L86-86}{\texttt{source}}
 
 
 \end{adjustwidth}
@@ -1319,7 +1319,7 @@ \section{at-autodocs}
 
 
 
-\href{https://example.org/Repository.jl/blob/make.jl#L94-94}{\texttt{source}}
+\href{https://example.org/Repository.jl/blob/make.jl#L95-95}{\texttt{source}}
 
 
 \end{adjustwidth}

diff --git a/test/examples/references/latex_simple.tex b/test/examples/references/latex_simple.tex
@@ -2,7 +2,7 @@
 \newcommand{\DocMainTitle}{Documenter LaTeX Simple Non-Docker}
 \newcommand{\DocVersion}{1.2.3}
 \newcommand{\DocAuthors}{}
-\newcommand{\JuliaVersion}{1.9.2}
+\newcommand{\JuliaVersion}{1.9.3}
 
 % ---- Insert preamble
 \input{preamble.tex}

diff --git a/test/examples/src/example-output.md b/test/examples/src/example-output.md
@@ -9,3 +9,44 @@ Checking that `@example` output is contained in a specific HTML class.
 ```@example
 println("hello")
 ```
+
+## `@example` outputs to file
+
+```@example
+Main.AT_EXAMPLE_FILES[("png", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("png", :tiny)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("webp", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("webp", :tiny)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("gif", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("jpeg", :tiny)]
+```
+
+### SVG output
+
+```@example
+Main.SVG_BIG
+```
+
+### `text/html` fallbacks
+
+SVG with just `text/html` output (in practice, `DataFrame`s and such would fall into this category):
+
+```@example
+Main.SVG_HTML
+```
+
+SVG with both `text/html` and `image/svg+xml` MIME, in which case we expect to pick the image one (because text is too big) and write it to a file.
+
+```@example
+Main.SVG_MULTI
+```
diff --git a/test/examples/src/index.md b/test/examples/src/index.md
@@ -655,3 +655,24 @@ They do not get evaluated.
 
 !!! ukw "Unknown admonition class"
     Admonition with an unknown admonition class.
+
+## `@example` outputs to file
+
+```@example
+Main.AT_EXAMPLE_FILES[("png", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("png", :tiny)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("webp", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("webp", :tiny)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("gif", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("jpeg", :tiny)]
+```
diff --git a/test/examples/src/outputs/index.md b/test/examples/src/outputs/index.md
@@ -0,0 +1,22 @@
+# Output handling (`outputs/index.md`)
+
+## `@example` outputs to file
+
+```@example
+Main.AT_EXAMPLE_FILES[("png", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("png", :tiny)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("webp", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("webp", :tiny)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("gif", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("jpeg", :tiny)]
+```
diff --git a/test/examples/src/outputs/outputs.md b/test/examples/src/outputs/outputs.md
@@ -0,0 +1,22 @@
+# Output handling (`outputs/outputs.md`)
+
+## `@example` outputs to file
+
+```@example
+Main.AT_EXAMPLE_FILES[("png", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("png", :tiny)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("webp", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("webp", :tiny)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("gif", :big)]
+```
+```@example
+Main.AT_EXAMPLE_FILES[("jpeg", :tiny)]
+```
diff --git a/test/examples/tests.jl b/test/examples/tests.jl
@@ -1,5 +1,6 @@
 using Test
 import JSON
+import Base64
 
 # DOCUMENTER_TEST_EXAMPLES can be used to control which builds are performed in
 # make.jl. But for the tests we need to make sure that all the relevant builds
@@ -76,7 +77,7 @@ all_md_files_in_src = let srcdir = joinpath(@__DIR__, "src"), mdfiles = String[]
     end
     mdfiles
 end
-@test length(all_md_files_in_src) == 25
+@test length(all_md_files_in_src) == 27
 
 @testset "Examples" begin
     @testset "HTML: deploy/$name" for (doc, name) in [
@@ -182,6 +183,37 @@ end
                 @test haskey(siteinfo_json["documenter"], "julia_version")
                 @test haskey(siteinfo_json["documenter"], "generation_timestamp")
             end
+
+            @testset "at-example outputs: $fmt/$size" for ((fmt, size), data) in AT_EXAMPLE_FILES
+                if size === :tiny
+                    encoded_data = Base64.base64encode(data.bytes)
+                    @test occursin(encoded_data, read(joinpath(build_dir, "index.html"), String))
+                    @test occursin(encoded_data, read(joinpath(build_dir, "example-output", "index.html"), String))
+                    @test occursin(encoded_data, read(joinpath(build_dir, "outputs", "index.html"), String))
+                    @test occursin(encoded_data, read(joinpath(build_dir, "outputs", "outputs", "index.html"), String))
+                else # size === :big
+                    # From src/index.md
+                    @test isfile(joinpath(build_dir, "index-$(data.hash_slug).$(fmt)"))
+                    @test read(joinpath(build_dir, "index-$(data.hash_slug).$(fmt)")) == data.bytes
+                    # From src/example-output.md
+                    @test isfile(joinpath(build_dir, "example-output", "$(data.hash_slug).$(fmt)"))
+                    @test read(joinpath(build_dir, "example-output", "$(data.hash_slug).$(fmt)")) == data.bytes
+                    # From src/outputs/index.md
+                    @test isfile(joinpath(build_dir, "outputs", "index-$(data.hash_slug).$(fmt)"))
+                    @test read(joinpath(build_dir, "outputs", "index-$(data.hash_slug).$(fmt)")) == data.bytes
+                    # From src/outputs/outputs.md
+                    @test isfile(joinpath(build_dir, "outputs", "outputs", "$(data.hash_slug).$(fmt)"))
+                    @test read(joinpath(build_dir, "outputs", "outputs", "$(data.hash_slug).$(fmt)")) == data.bytes
+                end
+            end
+            # SVG on src/example-output.md
+            @test isfile(joinpath(build_dir, "example-output", "$(SVG_BIG.hash_slug).svg"))
+            @test read(joinpath(build_dir, "example-output", "$(SVG_BIG.hash_slug).svg")) == SVG_BIG.bytes
+            # SVG on src/example-output.md, from Main.SVG_MULTI
+            @test isfile(joinpath(build_dir, "example-output", "$(SVG_BIG.hash_slug)-001.svg"))
+            @test read(joinpath(build_dir, "example-output", "$(SVG_BIG.hash_slug).svg")) == SVG_BIG.bytes
+            # .. but, crucially, Main.SVG_HTML did _not_ get written out.
+            @test !isfile(joinpath(build_dir, "example-output", "$(SVG_BIG.hash_slug)-002.svg"))
         end
     end
 
@@ -209,6 +241,37 @@ end
             documenterjs = String(read(joinpath(build_dir, "assets", "documenter.js")))
             @test occursin("languages/julia.min", documenterjs)
             @test occursin("languages/julia-repl.min", documenterjs)
+
+            @testset "at-example outputs: $fmt/$size" for ((fmt, size), data) in AT_EXAMPLE_FILES
+                if size === :tiny
+                    encoded_data = Base64.base64encode(data.bytes)
+                    @test occursin(encoded_data, read(joinpath(build_dir, "index.html"), String))
+                    @test occursin(encoded_data, read(joinpath(build_dir, "example-output.html"), String))
+                    @test occursin(encoded_data, read(joinpath(build_dir, "outputs", "index.html"), String))
+                    @test occursin(encoded_data, read(joinpath(build_dir, "outputs", "outputs.html"), String))
+                else # size === :big
+                    # From src/index.md
+                    @test isfile(joinpath(build_dir, "index-$(data.hash_slug).$(fmt)"))
+                    @test read(joinpath(build_dir, "index-$(data.hash_slug).$(fmt)")) == data.bytes
+                    # From src/example-output.md
+                    @test isfile(joinpath(build_dir, "example-output-$(data.hash_slug).$(fmt)"))
+                    @test read(joinpath(build_dir, "example-output-$(data.hash_slug).$(fmt)")) == data.bytes
+                    # From src/outputs/index.md
+                    @test isfile(joinpath(build_dir, "outputs", "index-$(data.hash_slug).$(fmt)"))
+                    @test read(joinpath(build_dir, "outputs", "index-$(data.hash_slug).$(fmt)")) == data.bytes
+                    # From src/outputs/outputs.md
+                    @test isfile(joinpath(build_dir, "outputs", "outputs-$(data.hash_slug).$(fmt)"))
+                    @test read(joinpath(build_dir, "outputs", "outputs-$(data.hash_slug).$(fmt)")) == data.bytes
+                end
+            end
+            # SVG on src/example-output.md
+            @test isfile(joinpath(build_dir, "example-output-$(SVG_BIG.hash_slug).svg"))
+            @test read(joinpath(build_dir, "example-output-$(SVG_BIG.hash_slug).svg")) == SVG_BIG.bytes
+            # SVG on src/example-output.md, from Main.SVG_MULTI
+            @test isfile(joinpath(build_dir, "example-output-$(SVG_BIG.hash_slug)-001.svg"))
+            @test read(joinpath(build_dir, "example-output-$(SVG_BIG.hash_slug).svg")) == SVG_BIG.bytes
+            # .. but, crucially, Main.SVG_HTML did _not_ get written out.
+            @test !isfile(joinpath(build_dir, "example-output-$(SVG_BIG.hash_slug)-002.svg"))
         end
     end