doc2tex: generate docs to Latex (#17997)

* `doc2tex`: generate docs to Latex

* address some comments

compiler/commands.nim

@@ -444,6 +444,7 @@ proc parseCommand*(command: string): Command =
   of "e": cmdNimscript
   of "doc0": cmdDoc0
   of "doc2", "doc": cmdDoc2
+  of "doc2tex": cmdDoc2tex
   of "rst2html": cmdRst2html
   of "rst2tex": cmdRst2tex
   of "jsondoc0": cmdJsondoc0

compiler/docgen.nim

@@ -169,14 +169,17 @@ proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
   else:
     result = getOutFile(conf, filename, ext)
 
-proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, outExt: string = HtmlExt, module: PSym = nil): PDoc =
+proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex}
+
+proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
+                    outExt: string = HtmlExt, module: PSym = nil): PDoc =
   declareClosures()
   new(result)
   result.module = module
   result.conf = conf
   result.cache = cache
   result.outDir = conf.outDir.string
-  initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex),
+  initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
                    conf.configVars, filename.string,
                    {roSupportRawDirective, roSupportMarkdown,
                     roPreferMarkdown, roNimFile},
@@ -249,7 +252,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
 
 template dispA(conf: ConfigRef; dest: var string, xml, tex: string,
                args: openArray[string]) =
-  if conf.cmd != cmdRst2tex: dest.addf(xml, args)
+  if not conf.isLatexCmd: dest.addf(xml, args)
   else: dest.addf(tex, args)
 
 proc getVarIdx(varnames: openArray[string], id: string): int =
@@ -551,7 +554,7 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string,
         let id = $d.listingCounter
         dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
         var dest2 = ""
-        renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
+        renderNimCode(dest2, code, d.target)
         dest.add dest2
         dest.add(d.config.getOrDefault"doc.listing_end" % id)
         return rsRunnable
@@ -821,7 +824,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
 
   inc(d.id)
   let
-    plainNameEsc = xmltree.escape(plainName.strip)
+    plainNameEsc = esc(d.target, plainName.strip)
+    uniqueName = if k in routineKinds: plainNameEsc else: name
     cleanPlainSymbol = renderPlainSymbolName(nameNode)
     complexSymbol = complexName(k, n, cleanPlainSymbol)
     plainSymbolEnc = encodeUrl(cleanPlainSymbol)
@@ -835,7 +839,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
 
   d.section[k].add(getConfigVar(d.conf, "doc.item") %
-    ["name", name, "header", result, "desc", comm, "itemID", $d.id,
+    ["name", name, "uniqueName", uniqueName,
+     "header", result, "desc", comm, "itemID", $d.id,
      "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
      "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc,
      "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc,
@@ -1199,7 +1204,8 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
     var shouldSort = i in routineKinds and groupedToc
     genSection(d, i, shouldSort)
     toc.add(d.toc[i])
-  if toc != "":
+  if toc != "" or d.target == outLatex:
+    # for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set
     toc = getConfigVar(d.conf, "doc.toc") % ["content", toc]
   for i in TSymKind: code.add(d.section[i])
 
@@ -1217,7 +1223,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
         "\\\\\\vspace{0.5em}\\large $1", [d.meta[metaSubtitle]])
 
   var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
-  let bodyname = if d.hasToc and not d.isPureRst:
+  let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd:
                    groupsection.setLen 0
                    "doc.body_toc_group"
                  elif d.hasToc: "doc.body_toc"

compiler/docgen2.nim

@@ -71,10 +71,15 @@ template myOpenImpl(ext: untyped) {.dirty.} =
 proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(HtmlExt)
 
+proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+  myOpenImpl(TexExt)
+
 proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(JsonExt)
 
 const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
+const docgen2TexPass* = makePass(open = myOpenTex, process = processNode,
+                                 close = close)
 const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
                                   close = closeJson)
 

compiler/main.nim

@@ -69,12 +69,15 @@ proc commandCheck(graph: ModuleGraph) =
     writeRodFiles(graph)
 
 when not defined(leanCompiler):
-  proc commandDoc2(graph: ModuleGraph; json: bool) =
+  proc commandDoc2(graph: ModuleGraph; ext: string) =
     handleDocOutputOptions graph.config
     graph.config.setErrorMaxHighMaybe
     semanticPasses(graph)
-    if json: registerPass(graph, docgen2JsonPass)
-    else: registerPass(graph, docgen2Pass)
+    case ext:
+    of TexExt:  registerPass(graph, docgen2TexPass)
+    of JsonExt: registerPass(graph, docgen2JsonPass)
+    of HtmlExt: registerPass(graph, docgen2Pass)
+    else: doAssert false, $ext
     compileProject(graph)
     finishDoc2Pass(graph.config.projectName)
 
@@ -249,7 +252,8 @@ proc mainCommand*(graph: ModuleGraph) =
       conf.quitOrRaise "compiler wasn't built with documentation generator"
     else:
       wantMainModule(conf)
-      loadConfigs(DocConfig, cache, conf, graph.idgen)
+      let docConf = if conf.cmd == cmdDoc2tex: DocTexConfig else: DocConfig
+      loadConfigs(docConf, cache, conf, graph.idgen)
       defineSymbol(conf.symbols, "nimdoc")
       body
 
@@ -285,7 +289,7 @@ proc mainCommand*(graph: ModuleGraph) =
         # of labels links in doc comments, e.g. for random.rand:
         #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
         #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-      commandDoc2(graph, false)
+      commandDoc2(graph, HtmlExt)
       if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
         commandBuildIndex(conf, $conf.outDir)
   of cmdRst2html:
@@ -299,18 +303,21 @@ proc mainCommand*(graph: ModuleGraph) =
     else:
       loadConfigs(DocConfig, cache, conf, graph.idgen)
       commandRst2Html(cache, conf)
-  of cmdRst2tex:
+  of cmdRst2tex, cmdDoc2tex:
     for warn in [warnRedefinitionOfLabel, warnUnknownSubstitutionX,
                  warnLanguageXNotSupported,
                  warnFieldXNotSupported, warnRstStyle]:
       conf.setNoteDefaults(warn, true)
     when defined(leanCompiler):
       conf.quitOrRaise "compiler wasn't built with documentation generator"
     else:
-      loadConfigs(DocTexConfig, cache, conf, graph.idgen)
-      commandRst2TeX(cache, conf)
+      if conf.cmd == cmdRst2tex:
+        loadConfigs(DocTexConfig, cache, conf, graph.idgen)
+        commandRst2TeX(cache, conf)
+      else:
+        docLikeCmd commandDoc2(graph, TexExt)
   of cmdJsondoc0: docLikeCmd commandJson(cache, conf)
-  of cmdJsondoc: docLikeCmd commandDoc2(graph, true)
+  of cmdJsondoc: docLikeCmd commandDoc2(graph, JsonExt)
   of cmdCtags: docLikeCmd commandTags(cache, conf)
   of cmdBuildindex: docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
   of cmdGendepend: commandGenDepend(graph)

compiler/options.nim

@@ -144,6 +144,7 @@ type
     cmdNimscript # evaluate nimscript
     cmdDoc0
     cmdDoc2
+    cmdDoc2tex
     cmdRst2html # convert a reStructuredText file to HTML
     cmdRst2tex # convert a reStructuredText file to TeX
     cmdJsondoc0
@@ -160,7 +161,8 @@ type
 
 const
   cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun}
-  cmdDocLike* = {cmdDoc0, cmdDoc2, cmdJsondoc0, cmdJsondoc, cmdCtags, cmdBuildindex}
+  cmdDocLike* = {cmdDoc0, cmdDoc2, cmdDoc2tex, cmdJsondoc0, cmdJsondoc,
+                 cmdCtags, cmdBuildindex}
 
 type
   TStringSeq* = seq[string]

compiler/renderverbatim.nim

@@ -1,8 +1,6 @@
 import strutils
-from xmltree import addEscaped
 
 import ast, options, msgs
-import packages/docutils/highlite
 
 const isDebug = false
 when isDebug:
@@ -131,22 +129,3 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
       lastNonemptyPos = result.len
   result.setLen lastNonemptyPos
 
-proc renderNimCode*(result: var string, code: string, isLatex = false) =
-  var toknizr: GeneralTokenizer
-  initGeneralTokenizer(toknizr, code)
-  var buf = ""
-  template append(kind, val) =
-    buf.setLen 0
-    buf.addEscaped(val)
-    let class = tokenClassToStr[kind]
-    if isLatex:
-      result.addf "\\span$1{$2}", [class, buf]
-    else:
-      result.addf  "<span class=\"$1\">$2</span>", [class, buf]
-  while true:
-    getNextToken(toknizr, langNim)
-    case toknizr.kind
-    of gtEof: break  # End Of File (or string)
-    else:
-      # TODO: avoid alloc; maybe toOpenArray
-      append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))

config/nimdoc.cfg

@@ -42,6 +42,7 @@ doc.section.toc2 = """
 # * $itemSymOrID: the symbolic name or the ID if that is not unique.
 # * $itemSymOrIDEnc: quoted version for URLs or attributes.
 # * $name: reduced name of the item.
+# * $uniqueName: name with parameters for routine types or $name for others.
 # * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used).
 
 doc.item = """

config/nimdoc.tex.cfg

@@ -8,22 +8,24 @@ split.item.toc = "20"
 # after this number of characters
 
 doc.section = """
-\chapter{$sectionTitle}\label{$sectionID}
-\begin{description}
+\rsthA{$sectionTitle}\label{$sectionID}
 $content
-\end{description}
 """
 
 doc.section.toc = ""
 # $sectionID $sectionTitleID $sectionTitle $content
 
 doc.item = """
-\item[\texttt{$header}\label{$itemID}]\mbox{~}\\*
+
+\phantomsection\addcontentsline{toc}{subsection}{$uniqueName}
+
+\begin{rstpre}
+$header
+\end{rstpre}
 $desc
 """
 
 doc.item.toc = ""
-#  \item $name\ref{$itemID}
 
 doc.toc = r"\tableofcontents \newpage"
 
@@ -38,6 +40,10 @@ $moduledesc
 $content
 """
 
+# $1 - number of listing in document, $2 - language (e.g. langNim), $3 - anchor
+doc.listing_start = "\\begin{rstpre}\n"
+doc.listing_end = "\n\\end{rstpre}\n"
+
 doc.file = """
 % This file was generated by Nim.
 % Generated: $date $time UTC

lib/packages/docutils/rst.nim

@@ -26,7 +26,8 @@
 ## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and
 ##    `nim rst2html`:cmd: for ``*.rst`` files
 ##
-## .. [#latex] command `nim rst2tex`:cmd: for ``*.rst``.
+## .. [#latex] commands `nim doc2tex`:cmd: for ``*.nim`` and
+##    `nim rst2tex`:cmd: for ``*.rst``.
 ##
 ## If you are new to RST please consider reading the following:
 ##

lib/packages/docutils/rstgen.nim

@@ -179,7 +179,9 @@ proc writeIndexFile*(g: var RstGenerator, outfile: string) =
   ## If the index is empty the file won't be created.
   if g.theIndex.len > 0: writeFile(outfile, g.theIndex)
 
-proc addXmlChar(dest: var string, c: char) =
+proc addHtmlChar(dest: var string, c: char) =
+  # Escapes HTML characters. Note that single quote ' is not escaped as
+  # ' -- unlike XML (for standards pre HTML5 it was even forbidden).
   case c
   of '&': add(dest, "&")
   of '<': add(dest, "&lt;")
@@ -195,26 +197,19 @@ proc addRtfChar(dest: var string, c: char) =
   else: add(dest, c)
 
 proc addTexChar(dest: var string, c: char) =
+  # Escapes 10 special Latex characters. Note that [, ], and ` are not
+  # considered as such. TODO: neither is @, am I wrong?
   case c
-  of '_': add(dest, "\\_")
-  of '{': add(dest, "\\symbol{123}")
-  of '}': add(dest, "\\symbol{125}")
-  of '[': add(dest, "\\symbol{91}")
-  of ']': add(dest, "\\symbol{93}")
-  of '\\': add(dest, "\\symbol{92}")
-  of '$': add(dest, "\\$")
-  of '&': add(dest, "\\&")
-  of '#': add(dest, "\\#")
-  of '%': add(dest, "\\%")
-  of '~': add(dest, "\\symbol{126}")
-  of '@': add(dest, "\\symbol{64}")
-  of '^': add(dest, "\\symbol{94}")
-  of '`': add(dest, "\\symbol{96}")
+  of '_', '{', '}', '$', '&', '#', '%': add(dest, "\\" & c)
+  # \~ and \^ have a special meaning unless they are followed by {}
+  of '~', '^': add(dest, "\\" & c & "{}")
+  # add {} to avoid gobbling up space by \textbackslash
+  of '\\': add(dest, "\\textbackslash{}")
   else: add(dest, c)
 
 proc escChar*(target: OutputTarget, dest: var string, c: char) {.inline.} =
   case target
-  of outHtml:  addXmlChar(dest, c)
+  of outHtml:  addHtmlChar(dest, c)
   of outLatex: addTexChar(dest, c)
 
 proc addSplitter(target: OutputTarget; dest: var string) {.inline.} =
@@ -987,6 +982,25 @@ proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string,
       "</td></tr></tbody></table>" & (
       d.config.getOrDefault"doc.listing_button" % id)
 
+proc renderCodeLang*(result: var string, lang: SourceLanguage, code: string,
+                     target: OutputTarget) =
+  var g: GeneralTokenizer
+  initGeneralTokenizer(g, code)
+  while true:
+    getNextToken(g, lang)
+    case g.kind
+    of gtEof: break
+    of gtNone, gtWhitespace:
+      add(result, substr(code, g.start, g.length + g.start - 1))
+    else:
+      dispA(target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [
+        esc(target, substr(code, g.start, g.length+g.start-1)),
+        tokenClassToStr[g.kind]])
+  deinitGeneralTokenizer(g)
+
+proc renderNimCode*(result: var string, code: string, target: OutputTarget) =
+  renderCodeLang(result, langNim, code, target)
+
 proc renderCode(d: PDoc, n: PRstNode, result: var string) =
   ## Renders a code (code block or inline code), appending it to `result`.
   ##
@@ -1028,19 +1042,7 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) =
       d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr)
     for letter in m.text: escChar(d.target, result, letter)
   else:
-    var g: GeneralTokenizer
-    initGeneralTokenizer(g, m.text)
-    while true:
-      getNextToken(g, params.lang)
-      case g.kind
-      of gtEof: break
-      of gtNone, gtWhitespace:
-        add(result, substr(m.text, g.start, g.length + g.start - 1))
-      else:
-        dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [
-          esc(d.target, substr(m.text, g.start, g.length+g.start-1)),
-          tokenClassToStr[g.kind]])
-    deinitGeneralTokenizer(g)
+    renderCodeLang(result, params.lang, m.text, d.target)
   dispA(d.target, result, blockEnd, blockEnd)
 
 proc renderContainer(d: PDoc, n: PRstNode, result: var string) =

nimdoc/testproject/expected/subdir/subdir_b/utils.html

@@ -208,7 +208,7 @@ <h1><a class="toc-backref" href="#18">Templates</a></h1>
 
 this should be shown in utils.html
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Identifier">assert</span><span class="Whitespace"> </span><span class="DecNumber">3</span><span class="Operator">*</span><span class="DecNumber">2</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">6</span></pre>ditto
+<pre class="listing"><span class="Identifier">assert</span> <span class="DecNumber">3</span><span class="Operator">*</span><span class="DecNumber">2</span> <span class="Operator">==</span> <span class="DecNumber">6</span></pre>ditto
 
 </dd>