Header shifting option for Markdown, Djot, Pandoc AST

classes/markdown.lua

@@ -1,10 +1,21 @@
+--- "Stupid" class so that processing Markdown, Djot and Pandoc AST is directly
+-- available from command line, without having to write a SILE document.
+-- It also aims at overriding the legacy markdown class from SILE, as long as
+-- it is shipped with the core distribution.
+--
 local book = require("classes.book")
 local class = pl.class(book)
 class._name = "markdown"
 
 function class:_init (options)
   book._init(self, options)
+  -- Load all the packages: corresponding inputters are then also registered.
+  -- Since we support switching between formats via "code blocks", we want
+  -- to make it easier for the user and not have him bother about how to load
+  -- the right inputter and appropriate packages.
+  self:loadPackage("djot")
   self:loadPackage("markdown")
+  self:loadPackage("pandocast")
   return self
 end
 

examples/introduction.md

@@ -4,9 +4,9 @@ This collection of modules for the [SILE](https://github.com/sile-typesetter/sil
 system provides a complete redesign of its former native Markdown support, with
 a great set of Pandoc-like extensions and plenty of extra goodies.
 
-- `\autodoc:package{markdown}`{=sile} inputter and package: native support of Markdown files.
-- `\autodoc:package{pandocast}`{=sile} inputter and package: native support of Pandoc JSON AST files.
-- `\autodoc:package{djot}`{=sile} inputter and package: native support of Djot files.
+- `\autodoc:package{markdown}`{=sile} input handler and package: native support of Markdown.
+- `\autodoc:package{djot}`{=sile} input handler and package: native support of Djot.
+- `\autodoc:package{pandocast}`{=sile} input handler and package: native support of Pandoc JSON AST files.
 
 For casual readers, this collection notably aims at easily converting Markdown or Djot documents to print-quality PDFs.
 
@@ -38,29 +38,43 @@ sile -u inputters.markdown somefile.md
 ... Or for a Djot document.
 
 ```
-sile -u inputters.djot -u packages.markdown somefile.dj
+sile -u inputters.djot somefile.dj
 ```
 
-This method directly produces a PDF from the input file, using SILE's standard **book** class.
+This method directly produces a PDF from the input file, using SILE's standard **book** class.[^intro-book-class]
+
+[^intro-book-class]: Actually, it uses a **markdown** class derived from the standard book class and loading the required modules.
+You don't really have to know that, unless you intend to invoke SILE with the `-c` option to specify another class of your choice; in that case you will need to load additional modules explicitly---unless it is a resilient class, of course.
 
 ### Usage with the resilient collection
 
-To unleash the full potential of this package collection, we recommend that
-you also install our [**resilient.sile**](https://github.com/Omikhleia/resilient.sile)
-collection of classes and packages.
+To unleash the full potential of this package collection, we recommend that you also install our [**resilient.sile**](https://github.com/Omikhleia/resilient.sile) collection of classes and packages.
+
+```bash
+luarocks --lua-version 5.4 install --dev resilient.sile
+```
 
 Then, you can automatically benefit from a few advanced features.
-Conversion from command line just requires to load a resilient class, and optionally
-the poetry package. For instance:
+Conversion from command line just requires to load a resilient class, and optionally the poetry package.
+For instance:
 
 ```
 sile -c resilient.book -u inputters.markdown -u packages.resilient.poetry somefile.md
 ```
 
 (And likewise for the Pandoc AST or Djot processing.)
 
-A "resilient style file" is also generated. It can be modified to change many styling
-decisions and adapt the output at convenience.
+A "resilient style file" is also generated.
+It can be modified to change many styling decisions and adapt the output at convenience.
+
+### Advanced usage from existing documents
+
+While direct conversion from the command line may be adequate for very simple workflows, there are a number of things usually best addressed by using some kind of "wrapper" document.
+
+This package collection provides several ways for including Markdown or Djot content in documents written in SIL syntax, the default mark-up language provided by SILE.
+These are described further in this guide.
+
+The resilient collection also introduces a "master document" format, streamlining several usual tasks. Give it a chance, and you may even end up producing a book with SILE without a single statement in SIL.
 
 ### Credits
 

examples/sile-and-djot.dj

@@ -387,3 +387,20 @@ Any option supported by the `\autodoc:package{resilient.epigraph}`{=sile} packag
 
 Be aware that this behavior is currently an extension.
 Other Djot renderers will therefore likely skip the caption.
+
+### Configuration
+
+You can pass additional options to the `\autodoc:command{\include}`{=sile} command or the `\autodoc:environment[check=false]{raw}`{=sile} environment to tune the behavior of the renderer.
+
+The `shift_headings` option can take an integer value and causes headers in included or embedded raw content to be offset (that is, shifted by the given amount).
+For instance:
+
+```
+\include[src=somefile.dj, shift_headings=1]
+```
+
+For document classes supporting it, this feature also allows you to access levels above the default scheme, such as "parts".
+
+```
+\include[src=somefile.dj, shift_headings=-1]
+```

examples/sile-and-markdown-manual.sil

@@ -62,7 +62,7 @@ Creative Commons Attribution, Share-Alike License,
 version 2.0 (http://creativecommons.org/licenses/by-sa/2.0/).
 
 \pagebreak
-\fancytableofcontents[start=2]
+\tableofcontents
 %
 % Introduction chapter
 %
@@ -148,4 +148,7 @@ our package might not support them well. Its main focus is on Markdown parity wi
 \include[src=pandoc-tables.pandoc]
 
 \include[src=sile-and-djot.dj]
+
+\include[src=sile-and-pandoc.dj]
+
 \end{document}

examples/sile-and-markdown.md

@@ -534,9 +534,9 @@ or coordinates (e.g. Oxford is located at 51° 45' 7" N, 1° 15' 27" W).
 
 ### Configuration {#configuration}
 
-Most Markdown syntax extensions are enabled by default. You can pass additional options to
-the `\autodoc:command{\include}`{=sile} command or the `\autodoc:environment[check=false]{raw}`{=sile}
-environment to tune the behavior of the Markdown parser.
+Most Markdown syntax extensions are enabled by default.
+You can pass additional options to
+the `\autodoc:command{\include}`{=sile} command or the `\autodoc:environment[check=false]{raw}`{=sile} environment to tune the behavior of the Markdown parser.
 
 ::: {custom-style=raggedright}
 > Available options are: `smart`, `smart_primes`, `strikeout`, `mark`, `subscript`, `superscript`,
@@ -550,62 +550,18 @@ environment to tune the behavior of the Markdown parser.
 For instance, to disable the smart typography feature:
 
 ```
-\include[src=somefile.pandoc, smart=false]
+\include[src=somefile.md, smart=false]
 ```
 
-## The Pandoc-based converters
-
-Pandoc is a free-software document converter, created by the same John MacFarlane who
-provided the **lunamark** library which empowers SILE’s native markdown package.
-The latter, though, does not offer as many options and extensions as Pandoc does,
-for advanced typesetting.
-
-In the event where the native solution would fall short for you ---e.g. would you need some extension
-it doesn't yet support--- you may want to use Pandoc directly for converting your document to an
-output suitable for SILE.
-
-The following solutions are still experimental proof-of-concepts, but you may give them a chance,
-and help us fill the gaps.
-
-### Using Pandoc's AST with the pandocast package
-
-The experimental `\autodoc:package{pandocast}`{=sile} package allows you to use Pandoc’s JSON AST
-as an input format for documents.
-You can obtain an AST output from Pandoc for any supported source format. Keeping the focus on
-Markdown here:
+The `shift_headings` option can take an integer value and causes headers in included or embedded raw content to be offset (that is, shifted by the given amount).
+For instance:
 
 ```
-pandoc -t json somefile.md -f markdown -o somefile.pandoc
+\include[src=somefile.md, shift_headings=1]
 ```
 
-Once the package is loaded, the `\autodoc:command{\include[src=<file>]}`{=sile} command supports
-reading and processing such a Pandoc AST file, assuming the `.pandoc` extension or specifying
-the `format=pandocast` parameter:
+For document classes supporting it, this feature also allows you to access levels above the default scheme, such as "parts".
 
 ```
-\use[module=packages.pandocast]
-\include[src=somefile.pandoc]
+\include[src=somefile.md, shift_headings=-1]
 ```
-
-This package supports the same advanced features as the native markdown package, e.g. the
-ability to use custom styles, to pass native content through to SILE, etc.
-
-There is a small _caveat_, though: one must use a version of Pandoc which generates
-an AST compatible with our parser ("inputter"). While the Pandoc AST is somewhat stable,
-it may change when new features are introduced in the software.
-
-### Using a Pandoc "custom writer" in Lua
-
-Pandoc also supports "custom writers" developed in Lua^[<https://pandoc.org/custom-writers.html>].
-
-This custom writer API is fairly recent and might change. Actually, besides a "Classic style" API
-(pending deprecation), there's now also a "New style" API...  While such custom writers may have
-some rough edges, the idea is quite appealing nevertheless. After all, SILE is mostly written in Lua,
-so the skills are there in the community.
-
-Again, there is no official solution using this conversion path, but some pretty neat
-experimental results have been
-achieved^[<https://github.com/Omikhleia/omikhleia-sile-packages/blob/main/examples/markdown-sample.pdf>].
-That custom writer targets a specific (non-standard) class and a bunch of specific packages, which might
-not have been ported to the latest version of SILE... This said, you can also certainly help pushing the
-idea forward!

examples/sile-and-pandoc.dj

@@ -0,0 +1,49 @@
+# Alternative route: SILE and Pandoc
+
+Pandoc is a free-software document converter, created by the same John MacFarlane who provided the *lunamark* and *djot* libraries which empower SILE’s native Markdown and Djot package.
+The latter, though, do not offer as many options and extensions as Pandoc does, for advanced typesetting.
+
+In the event where the native solution would fall short for you ---e.g. would you need some extension of feature it doesn't yet support--- you may want to use Pandoc directly for converting your document to an output suitable for SILE.
+
+The following solutions are still experimental proof-of-concepts, but you may give them a chance,and help us fill the gaps.
+
+## Using Pandoc's AST with the pandocast package
+
+The experimental `\autodoc:package{pandocast}`{=sile} package allows you to use Pandoc’s JSON AST as an input format for documents.
+You can obtain an AST output from Pandoc for any supported source format. Keeping the focus on Markdown here:
+
+```
+pandoc -t json somefile.md -f markdown -o somefile.pandoc
+```
+
+Once the package is loaded, the `\autodoc:command{\include[src=<file>]}`{=sile} command supports reading and processing such a Pandoc AST file, assuming the `.pandoc` extension or specifying the `format=pandocast` parameter:
+
+```
+\use[module=packages.pandocast]
+\include[src=somefile.pandoc]
+```
+
+This package supports the same advanced features as the native Markdown solution, e.g. the ability to use custom styles, to pass native content through to SILE, etc.
+
+The `shift_headings` option is also available, as with the Markdown and Djot solutions.
+
+There is a small _caveat_, though: one must use a version of Pandoc which generates an AST compatible with our input handler ("inputter").
+While the Pandoc AST is somewhat stable, it may change when new features are introduced in the software.
+
+## Using a Pandoc "custom writer" in Lua
+
+Pandoc also supports "custom writers" developed in Lua[^pandoc-writer].
+
+[^pandoc-writer]: <https://pandoc.org/custom-writers.html>
+
+This custom writer API is fairly recent and might change.
+Actually, besides a "Classic style" API (deprecated), there's now also a "New style" API...
+While such custom writers may have some rough edges, the idea is quite appealing nevertheless. 
+After all, SILE is mostly written in Lua, so the skills are there in the community.
+
+Again, there is no official solution using this conversion path, but some pretty neat experimental results have been achieved[^omi-writer].
+That custom writer targets a specific (non-standard) class and a bunch of specific packages, which might not have been ported to the latest version of SILE...
+This said, you can also certainly help pushing the idea forward!
+
+[^omi-writer]: <https://github.com/Omikhleia/omikhleia-sile-packages/blob/main/examples/markdown-sample.pdf>
+

inputters/djot.lua

@@ -14,11 +14,12 @@ local djotast = require("djot.ast")
 
 local Renderer = pl.class()
 
-function Renderer:_init()
+function Renderer:_init(options)
   self.references = {}
   self.footnotes = {}
   self.metadata = {}
-  self.tight = false -- We do use it currently, though!
+  self.shift_headings = SU.cast("integer", options.shift_headings or 0)
+  self.tight = false -- We do not use it currently, though!
 end
 
 function Renderer:render(doc)
@@ -125,7 +126,7 @@ function Renderer:heading (node)
   -- At document level, the id is set on the section.
   -- But in nested blocks (e.g. in divs), the id is set on the header.
   options.id = options.id or self.sectionid
-  options.level = node.level
+  options.level = node.level + self.shift_headings
   return utils.createCommand("markdown:internal:header", options, content)
 end
 
@@ -592,10 +593,10 @@ function inputter.appropriate (round, filename, _)
   return false
 end
 
-function inputter.parse (_, doc)
+function inputter:parse (doc)
   local djot = require("djot")
   local ast = djot.parse(doc, false, function (warning) SU.warn(warning.message) end)
-  local renderer = Renderer()
+  local renderer = Renderer(self.options)
   local tree = renderer:render(ast)
 
   -- The "writer" returns a SILE AST.

inputters/markdown.lua

@@ -65,9 +65,10 @@ end
 -- Lunamark writer for SILE
 -- Yay, direct lunamark AST ("ropes") conversion to SILE AST
 
-local function SileAstWriter (options)
+local function SileAstWriter (writerOps, renderOps)
   local generic = require("lunamark.writer.generic")
-  local writer = generic.new(options or {})
+  local writer = generic.new(writerOps or {})
+  local shift_headings = SU.cast("integer", renderOps.shift_headings or 0)
 
   -- Simple one-to-one mappings between lunamark AST and SILE
 
@@ -90,7 +91,7 @@ local function SileAstWriter (options)
 
   writer.header = function (s, level, attr)
     local opts = attr or {} -- passthru (class and key-value pairs)
-    opts.level = level
+    opts.level = level + shift_headings
     return utils.createCommand("markdown:internal:header", opts, s)
   end
 
@@ -418,7 +419,7 @@ function inputter:parse (doc)
   local writer = SileAstWriter({
     layout = "minimize" -- The default layout is to output \n\n as inter-block separator
                         -- Let's cancel it completely, and insert our own \par where needed.
-  })
+  }, self.options)
 
   extensions.alter_syntax = customSyntax(writer, extensions)