Merge pull request #598 from typelevel/plugin/tree-processor

sbt plugin: add new laikaTreeProcessors setting

docs/src/02-running-laika/01-sbt-plugin.md

@@ -289,16 +289,19 @@ import laika.format.Markdown
 import laika.config.SyntaxHighlighting
 
 laikaExtensions := Seq(Markdown.GitHubFlavor, SyntaxHighlighting)
-``` 
+```
+
+The `ExtensionBundle` API provides access to all stages of a transformation.
+You can:
 
 - [Overriding Renderers]: adjust the rendered output for specific AST node types.
 
 - [AST Rewriting]: transform the document AST between parsing and rendering.
 
 - [Implementing Directives]: install custom directives.
 
-- Or use any other hook in [The ExtensionBundle API]).
- 
+- Or use any other hook in [The ExtensionBundle API].
+
 
 ### Configuring Input and Output
 

docs/src/04-customizing-laika/05-ast-rewriting.md

@@ -123,3 +123,60 @@ val newDoc = doc.rewrite(RewriteRules.forBlocks {
   })
 })
 ```
+
+
+### Effectful AST Transformations
+
+The rewrite rules shown in this chapter so far were all applied to individual nodes within parsed documents,
+and had to be pure functions without any side effects.
+
+There is a related functionality called `TreeProcessor`, which is part of the `laika-io` module 
+and the sbt plugin and offers additional options:
+
+* Adding, removing or replacing entire documents from the AST.
+* Applying rewrite rules to specific documents only.
+* Defining rules which perform side effects.
+
+Our example shows how to add a document to the virtual tree for PDF documents only:
+
+@:select(config)
+
+@:choice(sbt)
+
+```scala mdoc:invisible
+import laika.sbt.LaikaPlugin.autoImport._
+```
+
+```scala mdoc:compile-only
+import cats.effect.IO
+import laika.ast._
+import laika.theme.TreeProcessorBuilder
+
+def intro: Document = ??? // e.g. created in-memory
+
+val processor = TreeProcessorBuilder[IO].mapTree { tree =>
+  tree.modifyTree(_.prependContent(intro))
+}
+
+laikaTreeProcessors += LaikaTreeProcessor(processor, OutputContext(PDF))
+```
+
+@:choice(library)
+
+```scala mdoc:compile-only
+import cats.effect.IO
+import laika.api._
+import laika.format._
+import laika.io.syntax._
+
+def intro: Document = ??? // e.g. created in-memory
+
+val transformer = Transformer
+  .from(Markdown)
+  .to(PDF)
+  .parallel[IO]
+  .mapTree(_.modifyTree(_.prependContent(intro)))
+  .build
+```
+
+@:@

sbt/src/main/scala/laika/sbt/LaikaPlugin.scala

@@ -56,6 +56,8 @@ import sbt.*
   * - `laikaTheme`: configuration for the theme to use for all transformations, if not specified the default
   *   `Helium` theme with all default colors and fonts will be used.
   *
+  * - `laikaTreeProcessors`: functions processing the AST between parsing and rendering (empty by default).
+  *
   * - `laikaRenderers`: contains the configurations for all of Laika's built-in renderers and enables users
   *   to install their own or 3rd-party renderers.
   *
@@ -85,7 +87,7 @@ object LaikaPlugin extends AutoPlugin {
   val requirements     = plugins.JvmPlugin
   override val trigger = noTrigger
 
-  object autoImport extends ExtensionBundles {
+  object autoImport extends ExtensionBundles with TreeProcessors {
 
     // settingKey macro does not accept HK types
     implicit class InputTreeBuilder(val delegate: laika.io.model.InputTreeBuilder[IO])
@@ -123,6 +125,11 @@ object LaikaPlugin extends AutoPlugin {
     val laikaTheme =
       settingKey[ThemeProvider]("Configures the theme to use for all transformations")
 
+    val laikaTreeProcessors =
+      settingKey[Seq[LaikaTreeProcessor]](
+        "Functions processing the AST between parsing and rendering"
+      )
+
     val laikaGenerateAPI =
       taskKey[Seq[String]]("Generates API documentation and moves it to the site's API target")
 
@@ -159,6 +166,7 @@ object LaikaPlugin extends AutoPlugin {
     laikaConfig                 := LaikaConfig.defaults,
     laikaPreviewConfig          := LaikaPreviewConfig.defaults,
     laikaTheme                  := Helium.defaults.build,
+    laikaTreeProcessors         := Nil,
     laikaDescribe               := Tasks.describe.value,
     laikaIncludeAPI             := false,
     laikaIncludeEPUB            := Settings.validated(

sbt/src/main/scala/laika/sbt/Tasks.scala

@@ -32,6 +32,7 @@ import cats.data.NonEmptyChain
 import laika.api.builder.{ OperationConfig, ParserBuilder }
 import laika.api.config.Config
 import laika.api.format.MarkupFormat
+import laika.ast.OutputContext
 import laika.ast.Path.Root
 import laika.config.{ LaikaKeys, Selections, Versions }
 import laika.io.config.{ BinaryRendererConfig, TextRendererConfig }
@@ -127,6 +128,16 @@ object Tasks {
       tree
     }
 
+    def prepareTree(
+        treeProcessors: Seq[LaikaTreeProcessor],
+        context: OutputContext
+    ) = {
+      val processor = treeProcessors
+        .map(_.delegate.apply(context))
+        .reduceOption(_.andThen(_))
+      processor.fold(IO.pure(tree))(_.run(tree))
+    }
+
     def renderText(config: TextRendererConfig): Set[File] = {
 
       val target =
@@ -137,19 +148,23 @@ object Tasks {
       else target.mkdirs()
       val dirPath = FilePath.fromJavaFile(target)
 
-      Renderer
-        .of(config.format)
-        .withConfig(Settings.parserConfig.value)
-        .parallel[IO]
-        .withTheme(laikaTheme.value)
-        .build
-        .use(
-          _
-            .from(tree)
-            .toDirectory(dirPath)(userConfig.encoding)
-            .render
-        )
-        .unsafeRunSync()
+      val op = prepareTree(laikaTreeProcessors.value, OutputContext(config.format))
+        .flatMap { tree =>
+          Renderer
+            .of(config.format)
+            .withConfig(Settings.parserConfig.value)
+            .parallel[IO]
+            .withTheme(laikaTheme.value)
+            .build
+            .use(
+              _
+                .from(tree)
+                .toDirectory(dirPath)(userConfig.encoding)
+                .render
+            )
+        }
+
+      op.unsafeRunSync()
 
       streams.value.log.info(Logs.outputs(tree.root, config.alias))
       streams.value.log.info(s"Generated ${config.alias} in $target")
@@ -164,35 +179,38 @@ object Tasks {
 
       val currentVersion = tree.root.config.get[Versions].toOption.map(_.currentVersion)
 
-      val ops = Renderer
-        .of(config.format)
-        .withConfig(Settings.parserConfig.value)
-        .parallel[IO]
-        .withTheme(laikaTheme.value)
-        .build
-        .use { renderer =>
-          val roots =
-            if (config.supportsSeparations) validated(Selections.createCombinations(tree.root))
-            else NonEmptyChain.one(tree.root -> Selections.Classifiers(Nil))
-          roots.traverse { case (root, classifiers) =>
-            val artifactPath = config.artifact.withClassifiers(classifiers.value).fullPath
-            val isVersioned  =
-              currentVersion.isDefined &&
-              tree.root
-                .selectTreeConfig(artifactPath.parent)
-                .get[Boolean](LaikaKeys.versioned)
-                .getOrElse(false)
-            val finalPath    =
-              if (isVersioned) Root / currentVersion.get.pathSegment / artifactPath.relative
-              else artifactPath
-            val file         = siteTarget / finalPath.toString
-            renderer
-              .from(root)
-              .copying(tree.staticDocuments)
-              .toFile(FilePath.fromJavaFile(file))
-              .render
-              .as(file)
-          }
+      val ops = prepareTree(laikaTreeProcessors.value, OutputContext(config.format))
+        .flatMap { tree =>
+          Renderer
+            .of(config.format)
+            .withConfig(Settings.parserConfig.value)
+            .parallel[IO]
+            .withTheme(laikaTheme.value)
+            .build
+            .use { renderer =>
+              val roots =
+                if (config.supportsSeparations) validated(Selections.createCombinations(tree.root))
+                else NonEmptyChain.one(tree.root -> Selections.Classifiers(Nil))
+              roots.traverse { case (root, classifiers) =>
+                val artifactPath = config.artifact.withClassifiers(classifiers.value).fullPath
+                val isVersioned  =
+                  currentVersion.isDefined &&
+                  tree.root
+                    .selectTreeConfig(artifactPath.parent)
+                    .get[Boolean](LaikaKeys.versioned)
+                    .getOrElse(false)
+                val finalPath    =
+                  if (isVersioned) Root / currentVersion.get.pathSegment / artifactPath.relative
+                  else artifactPath
+                val file         = siteTarget / finalPath.toString
+                renderer
+                  .from(root)
+                  .copying(tree.staticDocuments)
+                  .toFile(FilePath.fromJavaFile(file))
+                  .render
+                  .as(file)
+              }
+            }
         }
 
       val res = ops.unsafeRunSync()

sbt/src/main/scala/laika/sbt/TreeProcessors.scala

@@ -0,0 +1,46 @@
+package laika.sbt
+
+import cats.data.Kleisli
+import cats.effect.IO
+import laika.ast.OutputContext
+import laika.io.model.ParsedTree
+import laika.theme.Theme.TreeProcessor
+
+/** API shortcuts for adding tree processors to the the `laikaTreeProcessors` setting.
+  *
+  * Tree processors are effect-ful functions processing the document AST between parsing and rendering.
+  *
+  * One example would be generating a table of content based on inspecting the content of the tree and
+  * insert it as the first document. You can also modify or delete content.
+  *
+  * Processors are usually specific per output format, therefore the API allows to specify the format
+  * each processor should be applied to.
+  *
+  * @author Jens Halm
+  */
+trait TreeProcessors {
+
+  class LaikaTreeProcessor(val delegate: OutputContext => TreeProcessor[IO])
+
+  object LaikaTreeProcessor {
+
+    /** Wraps a function that returns a tree processor depending on the output context passed.
+      *
+      * The returned instance can be added to the `laikaTreeProcessors` setting.
+      */
+    def apply(f: OutputContext => TreeProcessor[IO]): LaikaTreeProcessor =
+      new LaikaTreeProcessor(f)
+
+    /** Adds a function that processes the document tree between parsing and rendering,
+      * to be executed only for the specified output format.
+      *
+      * The returned instance can be added to the `laikaTreeProcessors` setting.
+      */
+    def apply(f: TreeProcessor[IO], context: OutputContext): LaikaTreeProcessor =
+      apply(rendererContext =>
+        if (rendererContext == context) f else Kleisli.ask[IO, ParsedTree[IO]]
+      )
+
+  }
+
+}

sbt/src/sbt-test/site/treeProcessors/build.sbt

@@ -0,0 +1,40 @@
+import cats.effect.IO
+import laika.ast.Path.Root
+import laika.ast._
+import laika.format.{ HTML, PDF }
+import laika.theme.TreeProcessorBuilder
+
+name := "site-sourceDirectories"
+
+version := "0.1"
+
+scalaVersion := "2.12.6"
+
+enablePlugins(LaikaPlugin)
+
+val addedAST = RootElement(
+  Title("Title 3").withId("title").withStyle("title"),
+  Paragraph(
+    Text("Hello "),
+    Emphasized("World 3"),
+    Text(".")
+  )
+)
+
+val removeDoc1 = TreeProcessorBuilder[IO].mapTree { tree =>
+  tree.modifyTree(_.removeContent(_ == Root / "hello1.md"))
+}
+
+val removeDoc2 = TreeProcessorBuilder[IO].mapTree { tree =>
+  tree.modifyTree(_.removeContent(_ == Root / "hello2.md"))
+}
+
+val addDoc3 = TreeProcessorBuilder[IO].mapTree { tree =>
+  tree.modifyTree(_.prependContent(Document(Root / "hello3.md", addedAST)))
+}
+
+laikaTreeProcessors ++= Seq(
+  LaikaTreeProcessor(removeDoc1, OutputContext(HTML)),
+  LaikaTreeProcessor(removeDoc2, OutputContext(PDF)),
+  LaikaTreeProcessor(addDoc3, OutputContext(HTML))
+)