LaTeX: Fix newline inside escapeinside

src/Format.jl

@@ -198,17 +198,39 @@ function render(io::IO, mime::MIME"text/html", tokens::TokenIterator)
     println(io, "\n</pre>")
 end
 
+const CONSECUTIVE_WHITESPACE = r"\s+"
+
+function print_formatted(io::IO,mime::MIME"text/latex",str,id,style)
+    id === :t || print(io, "(*@\\HLJL", id, "{")
+    escape(io, mime, str; charescape=(id === :t))
+    id === :t || print(io, "}@*)")
+end
+
+function render_nonwhitespace(io::IO,mime::MIME"text/latex",str,id,style)
+    # Whitespace chars within an escapeinside are not correctly printed in a
+    # lstlisting env. So in order to have the correct highlighting
+    # and correct characters, that are recognized by listings, they need to be
+    # added outside of the escapeinside.
+    offset = 1
+    for m in eachmatch(CONSECUTIVE_WHITESPACE,str)
+        whitespace = m.match
+        nonwhitespace = str[offset:prevind(str,m.offset)]
+        offset = nextind(str,m.offset,length(whitespace))
+        length(nonwhitespace) > 0 && print_formatted(io,mime,nonwhitespace,id,style)
+        print(io,whitespace)
+    end
+    nonwhitespace = str[offset:end]
+    length(nonwhitespace) > 0 && print_formatted(io,mime,str[offset:end],id,style)
+end
+
 function render(io::IO, mime::MIME"text/latex", tokens::TokenIterator)
     println(io, "\\begin{lstlisting}")
     for (str, id, style) in tokens
-        id === :t || print(io, "(*@\\HLJL", id, "{")
-        escape(io, mime, str; charescape=(id === :t))
-        id === :t || print(io, "}@*)")
+        render_nonwhitespace(io,mime,str,id,style)
     end
     println(io, "\n\\end{lstlisting}")
 end
 
-
 # Character escapes.
 
 function escape(io::IO, ::MIME"text/html", str::AbstractString)
@@ -237,11 +259,6 @@ function escape(io::IO, ::MIME"text/latex", str::AbstractString; charescape=fals
         char === '}'   ? printe(io, charescape, "{\\}}") :
         char === '~'   ? printe(io, charescape, "{\\textasciitilde}") :
         char === '"'   ? printe(io, charescape, "\"{}") :
-        # Linebreaks within an escapeinside do not occur if not replaced by proper
-        # LaTeX linebreaks. Also to preserve spaces outside of verbatim they need to
-        # be explicity placed inside an mbox (or hphantom).
-        (char === '\n' && !charescape) ? printe(io, charescape, "{\\newline}") :
-        (char === ' ' && !charescape) ? printe(io, charescape, "{\\mbox{\\space}}") :
             print(io, char)
     end
 end

test/runtests.jl

@@ -264,6 +264,11 @@ end
                 Highlights.Format.escape(buffer,mime,str,charescape=charescape)
                 return Highlights.takebuf_str(buffer)
             end
+            render_nonwhitespace = function(mime,str)
+                buffer = IOBuffer()
+                Highlights.Format.render_nonwhitespace(buffer,mime,str,0,nothing)
+                return Highlights.takebuf_str(buffer)
+            end
             escapeinside(str) = "(*@{"*str*"}@*)"
             let mime = MIME("text/latex")
                 @test render(mime, Themes.Style("fg: 111"))   == "[1]{\\textcolor[RGB]{17,17,17}{#1}}"
@@ -284,10 +289,16 @@ end
                     "   some text\n next line"*escapeinside(ebrace_L)*escapeinside(ebrace_R)
                 # This is escaped so LaTeX automatically removes whitespace characters.
                 # Also no additional escapeinside for special characters needed.
-                n = escape(mime,"\n",charescape=false)
                 s = escape(mime," ",charescape=false)
                 @test escape(mime,"   some text\n next line{}",charescape=false) ==
-                    "$(s)$(s)$(s)some$(s)text$(n)$(s)next$(s)line"*ebrace_L*ebrace_R
+                    "$(s)$(s)$(s)some$(s)text\n$(s)next$(s)line"*ebrace_L*ebrace_R
+                r(x) = "(*@\\HLJL0{$x}@*)"
+                @test render_nonwhitespace(MIME("text/latex"),"\tfoo  \nbar\t") == "\t"*r("foo")*"  \n"*r("bar")*"\t"
+                @test render_nonwhitespace(MIME("text/latex"),"foo") == r("foo")
+                @test render_nonwhitespace(MIME("text/latex"),"\t") == "\t"
+                @test render_nonwhitespace(MIME("text/latex"),"α") == r("α")
+                @test render_nonwhitespace(MIME("text/latex")," α\tβfoo ") == " "*r("α")*"\t"*r("βfoo")*" "
+                @test render_nonwhitespace(MIME("text/latex")," α ") == " "*r("α")*" "
             end
         end
         @testset "Custom Nodes" begin