Polyglot Code Highlighting (#500)

Add polyglot code syntax highlighting for markdown code fences that specify a language. We support all of the [codemirror languages](https://github.com/codemirror/language-data).

Co-authored-by: Martin Kavalar <martin@nextjournal.com>

CHANGELOG.md

@@ -42,6 +42,8 @@ Changes can be:
 
 * 💫 Support non-evaluated clojure code listings in markdown documents by specifying `{:nextjournal.clerk/code-listing true}` after the language ([#482](https://github.com/nextjournal/clerk/issues/482)).
 
+* 🏳️‍🌈 Syntax highlighting for code listings in all [languages supported by codemirror](https://github.com/codemirror/language-data) ([#500](https://github.com/nextjournal/clerk/issues/500)).
+
 * 🐜 Turn off analyzer pass for validation of `:type` tags, fixes [#488](https://github.com/nextjournal/clerk/issues/488) @craig-latacora
 
 * 🐜 Strip `:type` metadata from forms before printing them to hash, fixes [#489](https://github.com/nextjournal/clerk/issues/489) @craig-latacora

book.clj

@@ -95,7 +95,7 @@
 
 ;; In Emacs, add the following to your config:
 
-;; ```elisp
+;; ```el
 ;; (defun clerk-show ()
 ;;   (interactive)
 ;;   (when-let
@@ -121,7 +121,7 @@
 
 ;; With [neovim](https://neovim.io/) + [conjure](https://github.com/Olical/conjure/) one can use the following vimscript function to save the file and show it with Clerk:
 
-;; ```
+;; ```vimscript
 ;; function! ClerkShow()
 ;; exe "w"
 ;; exe "ConjureEval (nextjournal.clerk/show! \"" . expand("%:p") . "\")"
@@ -251,7 +251,7 @@
 
 ;; ### 🎼 Code
 
-;; The code viewer uses
+;; By default the code viewer uses
 ;; [clojure-mode](https://nextjournal.github.io/clojure-mode/) for
 ;; syntax highlighting.
 (clerk/code (macroexpand '(when test
@@ -262,6 +262,24 @@
 
 (clerk/code "(defn my-fn\n  \"This is a Doc String\"\n  [args]\n  42)")
 
+;; You can specify the language for syntax highlighting via `::clerk/opts`.
+(clerk/code {::clerk/opts {:language "python"}} "
+class Foo(object):
+    def __init__(self):
+        pass
+    def do_this(self):
+        return 1")
+
+;; Or use a code fence with a language in a markdown.
+
+(clerk/md "```c++
+#include <iostream>
+int main() {
+    std::cout << \" Hello, world! \" << std::endl
+    return 0
+}
+```")
+
 ;; ### 🏞 Images
 
 ;; Clerk now has built-in support for the

notebooks/cherry.clj

@@ -70,11 +70,10 @@
           [:div
            [:div.flex
             [:div.viewer-code.flex-auto.w-80.mb-2 [nextjournal.clerk.render.code/editor !input]]
-            [:button.flex-none.bg-slate-100.mb-2.pl-2.pr-2
-             {:on-click click-handler}
-             "Compile!"]]
-           [:div.bg-slate-50
-            [nextjournal.clerk.render/render-code @!compiled]]
+            [:button.flex-none.rounded-md.border.border-slate-200.bg-slate-100.mb-2.pl-2.pr-2.font-sans
+             {:on-click click-handler} "Compile!"]]
+           [:div.bg-slate-100.p-2.border.border-slate-200
+            [nextjournal.clerk.render/render-code @!compiled {:language "js"}]]
            [nextjournal.clerk.render/inspect
             (try (js/eval @!compiled)
                  (catch :default e e))]])))}

notebooks/markdown_fences.md

@@ -1,27 +1,69 @@
 # 🤺 Markdown Fences
+## Handling Clojure blocks
 
 ```
 '(evaluated :and "highlighted")
 ```
 
+```clj
+'(evaluated :and "highlighted" :language clj)
+```
+
 ```clojure
-'(evaluated :and "highlighted")
+'(evaluated :and "highlighted" :language clojure)
 ```
 
+Use `{:nextjournal.clerk/code-listing true}` in the fence info to signal that a block should not be evaluated.
+
 ```clojure {:nextjournal.clerk/code-listing true}
-'(1 2 "not evaluated" :but-still-highlighted)
+(1 2 "not evaluated" :but-still-highlighted)
 ```
 
-```clojure {:nextjournal.clerk/code-listing true}
-'(1 2 "not evaluated" :but-still-highlighted)
+## 🏳️‍🌈 Polyglot Highlighting
+
+EDN
+
+```edn
+(1 2 "not evaluated" :but-still-highlighted)
 ```
 
+Javascript
+
 ```js
 () => {
   if (true) {
     return 'not evaluated'
   } else {
-    return 'what'
+    return 123
   }
 }
 ```
+
+Python
+
+```py
+class Foo(object):
+    def __init__(self):
+        pass
+    def do_this(self):
+        return 1
+```
+
+C++
+
+```c++
+#include <iostream>
+
+int main() {
+    std::cout << "Hello, world!" << std::endl;
+    return 0;
+}
+```
+
+## Indented Code Blocks
+[Indented code blocks](https://spec.commonmark.org/0.30/#indented-code-blocks) default to clojure highlighting
+
+    (no (off) :fence)
+    (but "highlighted")
+
+fin.

notebooks/viewers/code.clj

@@ -36,6 +36,22 @@ with     quite
                 some
    whitespace      ")
 
+;; Code in some other language, say Rust:
+(clerk/code {::clerk/opts {:language "rust"}}
+  "fn calculate_factorial(n: u32, result: &mut u32) {
+    if n == 0 {
+        *result = 1;
+    } else {
+        *result = n * calculate_factorial(n - 1, result);
+    }
+}
+fn main() {
+    let number = 5;
+    let mut result = 0;
+    calculate_factorial(number, &mut result);
+    println!(\"The factorial of {} is: {}\", number, result);
+}")
+
 ;; Editable code viewer
 (clerk/with-viewer
   '(fn [code-str _] [:div.viewer-code [nextjournal.clerk.render.code/editor (reagent.core/atom code-str)]])

notebooks/viewers/markdown.clj

@@ -33,8 +33,8 @@ It's [Markdown](https://daringfireball.net/projects/markdown/), like you know it
 ;; > — Special Forms
 
 ;; ## Code Listings
-
-;; ```
+;; Clojure
+;; ```clj
 ;; {:name :code,
 ;;  :render-fn 'nextjournal.clerk.render/render-code,
 ;;  :transform-fn
@@ -45,7 +45,19 @@ It's [Markdown](https://daringfireball.net/projects/markdown/), like you know it
 ;;      [v]
 ;;      (if (string? v) v (str/trim (with-out-str (pprint/pprint v)))))))}
 ;; ```
-
+;; APL
+;; ```apl
+;; numbers ← 1 2 3 4 5
+;; sum ← 0
+;; n ← ≢numbers  ⍝ Get the number of elements in the array
+;;
+;; :For i :In ⍳n
+;; sum ← sum + numbers[i]
+;; :End
+;;
+;; sum
+;; ```
+;;
 ;; ## Soft vs. Hard Line Breaks
 ;; This one ⇥
 ;; ⇤ is a [soft break](https://spec.commonmark.org/0.30/#soft-line-breaks) and is rendered as a space.

src/nextjournal/clerk/builder.clj

@@ -20,6 +20,7 @@
   (into ["CHANGELOG.md"
          "README.md"
          "notebooks/markdown.md"
+         "notebooks/markdown_fences.md"
          "notebooks/onwards.md"]
         (map #(str "notebooks/" % ".clj"))
         ["cards"

src/nextjournal/clerk/parser.cljc

@@ -357,21 +357,26 @@
          :nodes (rest nodes)
          ::md-slice []))
 
-(defn fenced-clojure-code-block? [{:as block :keys [type info language]}]
+(defn runnable-code-block? [{:as block :keys [info language]}]
   (and (code? block)
        info
        (or (empty? language)
            (re-matches #"clj(c?)|clojure" language))
-       (not (:nextjournal.clerk/code-listing (let [parsed (p/parse-string-all (subs info (count language)))]
-                                               (when (n/sexpr-able? parsed)
-                                                 (n/sexpr parsed)))))))
+       (not (:nextjournal.clerk/code-listing
+             (when-some [parsed (when (and (seq language) (str/starts-with? info language))
+                                  (p/parse-string-all (subs info (count language))))]
+               (when (n/sexpr-able? parsed)
+                 (n/sexpr parsed)))))))
+
+#_(runnable-code-block? {:type :code :language "clojure" :info "clojure"})
+#_(runnable-code-block? {:type :code :language "clojure" :info "clojure {:nextjournal.clerk/code-listing true}"})
 
 (defn parse-markdown-string [{:as opts :keys [doc?]} s]
   (let [{:as ctx :keys [content]} (parse-markdown (markdown-context) s)]
     (loop [{:as state :keys [nodes] ::keys [md-slice]} {:blocks [] ::md-slice [] :nodes content :md-context ctx}]
       (if-some [node (first nodes)]
         (recur
-         (if (fenced-clojure-code-block? node)
+         (if (runnable-code-block? node)
            (-> state
                (update :blocks #(cond-> % (seq md-slice) (conj {:type :markdown :doc {:type :doc :content md-slice}})))
                (parse-markdown-cell opts))

src/nextjournal/clerk/render.cljs

@@ -932,9 +932,9 @@
 
 (defn render-code-block [code-string {:as opts :keys [id]}]
   [:div.viewer.code-viewer.w-full.max-w-wide {:data-block-id id}
-   [code/render-code code-string opts]])
+   [code/render-code code-string (assoc opts :language "clojure")]])
 
-(defn render-folded-code-block [code-string {:keys [id]}]
+(defn render-folded-code-block [code-string {:as opts :keys [id]}]
   (let [!hidden? (hooks/use-state true)]
     (if @!hidden?
       [:div.relative.pl-12.font-sans.text-slate-400.cursor-pointer.flex.overflow-y-hidden.group
@@ -967,7 +967,7 @@
          {:class "text-[10px]"}
          "evaluated in 0.2s"]]
        [:div.code-viewer.mb-2.relative.code-viewer.w-full.max-w-wide {:data-block-id id :style {:margin-top 0}}
-        [render-code code-string]]])))
+        [render-code code-string (assoc opts :language "clojure")]]])))
 
 
 (defn url-for [{:as src :keys [blob-id]}]

src/nextjournal/clerk/render/code.cljs

@@ -1,13 +1,14 @@
 (ns nextjournal.clerk.render.code
-  (:require ["@codemirror/language" :refer [HighlightStyle syntaxHighlighting]]
-            ["@codemirror/state" :refer [EditorState RangeSetBuilder Text]]
+  (:require ["@codemirror/language" :refer [HighlightStyle syntaxHighlighting LanguageDescription]]
+            ["@codemirror/state" :refer [EditorState RangeSet RangeSetBuilder Text]]
             ["@codemirror/view" :refer [EditorView Decoration]]
             ["@lezer/highlight" :refer [tags highlightTree]]
             ["@nextjournal/lang-clojure" :refer [clojureLanguage]]
             [applied-science.js-interop :as j]
             [clojure.string :as str]
             [nextjournal.clerk.render.hooks :as hooks]
-            [nextjournal.clojure-mode :as clojure-mode]))
+            [nextjournal.clojure-mode :as clojure-mode]
+            [shadow.esm]))
 
 (def highlight-style
   (.define HighlightStyle
@@ -87,23 +88,47 @@
                   (< pos to)
                   (concat [(.sliceString text pos to)]))))))))
 
-(defn lang->deco-range [lang code]
-  (let [builder (RangeSetBuilder.)]
-    (when lang
-      (highlightTree (.. lang -parser (parse code)) highlight-style
-                     (fn [from to style]
-                       (.add builder from to (.mark Decoration (j/obj :class style))))))
-    (.finish builder)))
+(defn import-matching-language-parser [language]
+  (.. (shadow.esm/dynamic-import "https://cdn.skypack.dev/@codemirror/language-data@6.1.0")
+      (then (fn [^js mod]
+              (when-some [langs (.-languages mod)]
+                (when-some [^js matching (or (.matchLanguageName LanguageDescription langs language)
+                                             (.matchFilename LanguageDescription langs (str "code." language)))]
+                  (.load matching)))))
+      (then (fn [^js lang-support] (when lang-support (.. lang-support -language -parser))))
+      (catch (fn [err] (js/console.warn (str "Cannot load language parser for: " language) err)))))
 
-(defn render-code [^String code {:keys [language]}]
+(defn add-style-ranges! [range-builder syntax-tree]
+  (highlightTree syntax-tree highlight-style
+                 (fn [from to style]
+                   (.add range-builder from to (.mark Decoration (j/obj :class style))))))
+
+(defn clojure-style-rangeset [code]
+  (.finish (doto (RangeSetBuilder.)
+             (add-style-ranges! (.. ^js clojureLanguage -parser (parse code))))))
+
+(defn syntax-highlight [{:keys [code style-rangeset]}]
   (let [text (.of Text (.split code "\n"))]
-    [:div.cm-editor
-     [:cm-scroller
-      (into [:div.cm-content.whitespace-pre]
-            (map (partial style-line
-                          ;; TODO: use-promise hook resolving to language data according to @codemirror/language-data
-                          (lang->deco-range (when (= "clojure" language) clojureLanguage) code) text))
-            (range 1 (inc (.-lines text))))]]))
+    (into [:div.cm-content.whitespace-pre]
+          (map (partial style-line style-rangeset text))
+          (range 1 (inc (.-lines text))))))
+
+(defn highlight-imported-language [{:keys [code language]}]
+  (let [^js builder (RangeSetBuilder.)
+        ^js parser (hooks/use-promise (import-matching-language-parser language))]
+    (when parser (add-style-ranges! builder (.parse parser code)))
+    [syntax-highlight {:code code :style-rangeset (.finish builder)}]))
+
+(defn render-code [^String code {:keys [language]}]
+  [:div.cm-editor
+   [:cm-scroller
+    (cond
+      (not language)
+      [syntax-highlight {:code code :style-rangeset (.-empty RangeSet)}]
+      (#{"clojure" "clojurescript" "clj" "cljs" "cljc" "edn"} language)
+      [syntax-highlight {:code code :style-rangeset (clojure-style-rangeset code)}]
+      :else
+      [highlight-imported-language {:code code :language language}])]])
 
 ;; editable code viewer
 (def theme

src/nextjournal/clerk/viewer.cljc

@@ -598,8 +598,7 @@
               code?
               (conj (with-viewer (if fold? `folded-code-block-viewer `code-block-viewer)
                       {:nextjournal/opts (assoc (select-keys cell [:loc])
-                                                :id (processed-block-id (str id "-code"))
-                                                :language "clojure")}
+                                                :id (processed-block-id (str id "-code")))}
                       (dissoc cell :result)))
               (or result? eval?)
               (conj (cond-> (ensure-wrapped (-> cell (assoc ::doc doc) (set/rename-keys {:result ::result})))
@@ -708,7 +707,7 @@
     :transform-fn (update-val #(with-viewer `html-viewer
                                  [:div.code-viewer.code-listing
                                   (with-viewer `code-viewer
-                                    {:nextjournal/opts (select-keys % [:language])}
+                                    {:nextjournal/opts {:language (:language % "clojure")}}
                                     (str/trim-newline (md.transform/->text %)))]))}
 
    ;; marks
@@ -914,7 +913,11 @@
                                               (with-md-viewer)))})
 
 (def code-viewer
-  {:name `code-viewer :render-fn 'nextjournal.clerk.render/render-code :transform-fn (comp mark-presented (update-val (fn [v] (if (string? v) v (str/trim (with-out-str (pprint/pprint v)))))))})
+  {:name `code-viewer
+   :render-fn 'nextjournal.clerk.render/render-code
+   :transform-fn (comp mark-presented
+                       #(update-in % [:nextjournal/opts :language] (fn [lang] (or lang "clojure")))
+                       (update-val (fn [v] (if (string? v) v (str/trim (with-out-str (pprint/pprint v)))))))})
 
 (def reagent-viewer
   {:name `reagent-viewer :render-fn 'nextjournal.clerk.render/render-reagent :transform-fn mark-presented})