Allow general search and replace in Preprocess (#139)

Allow general search and replace in Preprocess

project/build.properties

@@ -1 +1 @@
-sbt.version=1.2.3
+sbt.version=1.2.8

project/plugins.sbt

@@ -3,6 +3,6 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.2")
 //#sbt-ghpages
 addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.2")
 //#sbt-ghpages
-addSbtPlugin("io.github.jonas" % "sbt-paradox-material-theme" % "0.5.1")
+addSbtPlugin("io.github.jonas" % "sbt-paradox-material-theme" % "0.6.0")
 
 libraryDependencies += "org.scala-sbt" %% "scripted-plugin" % sbtVersion.value

src/main/paradox/preprocess.md

@@ -1,8 +1,8 @@
 # Preprocessing
 
-sbt-site supports basic variable substitution via the `PreprocessPlugin` if for example you need to replace a version string in a source file or a generated HTML file. More advanced preprocessing, such as interpreting code snippets with [tut], is possible via additional configuration as shown below.
+sbt-site supports simple variable and Regular Expression substitution via the `PreprocessPlugin` if for example you need to replace a version string in a source file or a generated HTML file. More advanced preprocessing, such as interpreting code snippets with [tut], is possible via additional configuration as shown below.
 
-## Variable Substitution
+## Substitution
 
 The `PreprocessPlugin` reads files from an input directory, substitute variables and writes them to an output directory. In addition to preprocessing static content it is also possible to use the plugin either before or after invoking a @ref:[site generator](generators/index.md). To enable, add this to your `build.sbt` file:
 
@@ -12,14 +12,20 @@ By default files are read from `src/site-preprocess` but this is configurable by
 
 @@ snip[sourceDirectory](../../sbt-test/preprocess/does-transform-variables/build.sbt) { #sourceDirectory }
 
-Variables are delimited by surrounding the name with `@` symbols (e.g. `@VERSION@`). Values are assigned to variables via the setting `preprocessVars: [Map[String, String]]`. For example:
+Variables are delimited by surrounding the name with `@` symbols (e.g. `@VERSION@`). Values are assigned to variables via the setting `preprocessVars: Map[String, String]`. For example:
 
 @@ snip[preprocessVars](../../sbt-test/preprocess/does-transform-variables/build.sbt) { #preprocessVars }
 
 @@@ note
 The plugin will generate an error if a variable is found in the source file with no matching value in `preprocessVars`.
 @@@
 
+More advanced substitution patterns can be used by providing Regular Expression rules via the setting `preprocessRules: Seq[(Regex, Match => String)]`.
+For example Scaladoc used to (before 2.12.9 and 2.13.0) prepend ".scala" to source links of Java files.
+The following preprocessing rule will find and fix such links:
+
+@@ snip[preprocessRules](../../sbt-test/preprocess/transform-scaladoc/build.sbt) { #preprocessRules }
+
 The setting `preprocessIncludeFilter` is used to define the filename extensions that should be processed when `makeSite` is run.
 
 @@ snip[preprocessIncludeFilter](../../sbt-test/preprocess/does-transform-variables/build.sbt) { #preprocessIncludeFilter }

src/main/scala/com/typesafe/sbt/site/preprocess/PreprocessKeys.scala

@@ -1,6 +1,10 @@
 package com.typesafe.sbt.site.preprocess
 
 import sbt._
+
+import scala.util.matching.Regex
+import scala.util.matching.Regex.Match
+
 /**
  * Keys for the preprocess sub-plugin.
  * @since 10/29/15
@@ -10,5 +14,6 @@ trait PreprocessKeys {
       "preprocessIncludeFilter", "Filter defining set of files to preprocess")
   val preprocessVars = SettingKey[Map[String, String]](
       "preprocessVars", "Replacements for preprocessing.")
+  val preprocessRules = SettingKey[Seq[(Regex, Match => String)]]("preprocessRules", "Rules for preprocessing supporting Regular Expressions")
   val preprocess = TaskKey[File]("preprocess", "Preprocess a directory of files.")
 }

src/main/scala/com/typesafe/sbt/site/preprocess/PreprocessPlugin.scala

@@ -7,6 +7,8 @@ import sbt.Keys._
 import sbt._
 
 import scala.util.matching.Regex
+import scala.util.matching.Regex.Match
+
 /** Provides ability to map values to `@` delimited variables. */
 object PreprocessPlugin extends AutoPlugin {
   override def requires = SitePlugin
@@ -33,12 +35,13 @@ object PreprocessPlugin extends AutoPlugin {
           //#preprocessIncludeFilter
           ,
         preprocessVars := Map("VERSION" -> version.value),
+        preprocessRules := Seq.empty,
         includeFilter in Preprocess := AllPassFilter,
         sourceDirectory := sourceDirectory.value / "site-preprocess",
         target := target.value / Preprocess.name,
         preprocess := simplePreprocess(
           sourceDirectory.value, target.value, streams.value.cacheDirectory, preprocessIncludeFilter.value,
-          preprocessVars.value, streams.value.log),
+          preprocessVars.value, preprocessRules.value, streams.value.log),
         mappings := gatherMappings(preprocess.value, includeFilter.value),
         siteSubdirName := ""
       )
@@ -55,10 +58,17 @@ object PreprocessPlugin extends AutoPlugin {
     cacheFile: File,
     fileFilter: FileFilter,
     replacements: Map[String, String],
+    rules: Seq[(Regex, Match => String)],
     log: Logger): File = {
+
+    def replacement(replacements: Map[String, String])(m: Match): String =
+      ((key: String) => replacements.getOrElse(
+        key,
+        sys.error("No replacement value defined for: " + key)))(m.group(1))
+
     transformDirectory(
       sourceDir, targetDir, fileFilter.accept,
-      SiteHelpers.transformFile(replaceVariable(Variable, replacements ++ defaultReplacements)),
+      SiteHelpers.transformFile(replaceVariable(Seq((Variable, replacement(replacements ++ defaultReplacements)(_))) ++ rules)),
       cacheFile, log)
   }
 
@@ -109,10 +119,11 @@ object PreprocessPlugin extends AutoPlugin {
   /**
    * Simple variable replacement in a string.
    */
-  private[sbt] def replaceVariable(regex: Regex, replacements: Map[String, String])
+  private[sbt] def replaceVariable(rules: Seq[(Regex, Match => String)])
     (input: String): String = {
-    def replacement(key: String): String = replacements.getOrElse(
-      key, sys.error("No replacement value defined for: " + key))
-    regex.replaceAllIn(input, m => replacement(m.group(1)))
+    rules.foldLeft(input) {
+      case (line, (regex, replace)) =>
+        regex.replaceAllIn(line, replace)
+      }
   }
 }

src/sbt-test/preprocess/does-transform-variables/build.sbt

@@ -12,6 +12,10 @@ version := "0.0-ABCD"
 preprocessVars in Preprocess := Map("VERSION" -> version.value, "DATE" -> new Date().toString)
 //#preprocessVars
 
+preprocessRules in Preprocess := Seq(
+  ("Author: ([a-z]+)".r, _.group(0).toUpperCase)
+)
+
 //#preprocessIncludeFilter
 preprocessIncludeFilter := "*.md" | "*.markdown"
 //#preprocessIncludeFilter
@@ -26,4 +30,5 @@ TaskKey[Unit]("checkContent") := {
   assert(readme.exists, s"${readme.getAbsolutePath} did not exist")
   val content = IO.readLines(readme)
   assert(content.exists(_.contains(version.value)), s"Did not find version in:\n${content.mkString("\n")}")
+  assert(content.exists(_.contains("AUTHOR: ME")), s"Did not find 'AUTHOR: ME' in: \n${content.mkString("\n")}")
 }

src/sbt-test/preprocess/does-transform-variables/src/site-preprocess/README.md

@@ -4,3 +4,4 @@
 
 Version: @VERSION@
 Date: @DATE@
+Author: me

src/sbt-test/preprocess/transform-scaladoc/build.sbt

@@ -0,0 +1,27 @@
+scalacOptions in (Compile, doc) ++= Seq(
+  "-sourcepath", baseDirectory.value.toString,
+  "-doc-source-url", {
+    val branch = if (isSnapshot.value) "master" else s"v${version.value}"
+    s"https://github.com/sbt/sbt-site/tree/${branch}/src/sbt-test/preprocess/transform-scaladoc€{FILE_PATH}.scala#L1"
+  }
+)
+
+enablePlugins(PreprocessPlugin)
+
+sourceDirectory in Preprocess := (target in (Compile, doc)).value
+siteSubdirName in Preprocess := "api"
+makeSite := makeSite.dependsOn(doc in Compile).value
+
+//#preprocessRules
+preprocessRules in Preprocess := Seq(
+  ("\\.java\\.scala".r, _ => ".java")
+)
+//#preprocessRules
+
+TaskKey[Unit]("checkContent") := {
+  val dest = (target in makeSite).value / (siteSubdirName in Preprocess).value
+  val app = dest / "App.html"
+  assert(app.exists, s"${app.getAbsolutePath} did not exist")
+  val content = IO.readLines(app)
+  assert(!content.exists(_.contains(".java.scala")), s"Found not replaced file suffix in:\n${content.mkString("\n")}")
+}