Fix issue with unparagraphed text following code indent (#178)

* Fix issue with unparagraphed text following code indent

Text that immediately follows a code block sometimes does not
get transformed, but instead is stuck as plain text behind the
ending code- and pre-tags

* cleanups

Co-authored-by: Odd Andreas Sørsæther <odd.andreas.sorsether@nav.no>

.gitignore

@@ -8,3 +8,5 @@ target
 .cpcache
 cljs-test-runner-out
 node_modules
+.clj-kondo/
+.calva/

src/clj/markdown/core.clj

@@ -1,17 +1,16 @@
 (ns markdown.core
   (:require [clojure.java.io :as io]
-            [clojure.string :as string]
             [markdown.common
-             :refer [*substring* *inhibit-separator*]]
+             :refer [*inhibit-separator*]]
             [markdown.links
-             :refer [parse-reference parse-reference-link parse-footnote-link]]
+             :refer [parse-reference-link parse-footnote-link]]
             [markdown.transformers
              :refer [transformer-vector footer parse-metadata-headers]])
   (:import [java.io BufferedReader
-                    BufferedWriter
-                    StringReader
-                    StringWriter
-                    Writer]))
+            BufferedWriter
+            StringReader
+            StringWriter
+            Writer]))
 
 (defn- write [^Writer writer ^String text]
   (doseq [c text] (.write writer (int c))))

src/cljc/markdown/common.cljc

@@ -176,7 +176,7 @@
 
 (defn heading-level [text]
   (let [num-hashes (count (filter #(not= \space %) (take-while #(or (= \# %) (= \space %)) (seq text))))]
-    (if (pos? num-hashes) num-hashes)))
+    (when (pos? num-hashes) num-hashes)))
 
 (defn make-heading [text heading-anchors]
   (when-let [heading (heading-level text)]

src/cljc/markdown/transformers.cljc

@@ -15,7 +15,6 @@
              [escape-code
               escaped-chars
               freeze-string
-              separator
               thaw-strings
               strong
               bold
@@ -34,7 +33,7 @@
 
 (defn heading? [text type]
   (when-not (every? #{\space} (take 4 text))
-    (let [trimmed (if text (string/trim text))]
+    (let [trimmed (some-> text string/trim)]
       (and (not-empty trimmed) (every? #{type} trimmed)))))
 
 (defn h1? [text]
@@ -154,39 +153,51 @@
     [text state]))
 
 (defn close-paragraph [text {:keys [next-line paragraph] :as state}]
-  (if (and paragraph (= [\` \` \`] (take-last 3 (some-> next-line string/trim))))
+  (if (and paragraph (some-> next-line string/trim (string/ends-with? "```")))
     [(str text "</p>") (dissoc state :paragraph)]
     [text state]))
 
 (defn paragraph [text state]
   (apply close-paragraph (open-paragraph text state)))
 
-(defn code [text {:keys [eof lists code codeblock paragraph] :as state}]
-  (cond
-    (or lists codeblock)
-    [text state]
+(defn code [text {:keys [eof indent-code-end next-line lists code codeblock paragraph] :as state}]
+  (let [should-close? (or eof
+                          (not (or (string/blank? next-line)
+                                   (string/starts-with? next-line "    "))))]
+    (cond
+      (or lists codeblock)
+      [text state]
 
-    code
-    (if (or eof (not= "    " (string/join (take 4 text))))
-      [(str "</code></pre>" text) (dissoc state :indented-code :code :last-line-empty?)]
-      [(str "\n" (escape-code (string/replace-first text #"    " ""))) state])
+      indent-code-end
+      [text (-> state
+                (dissoc :code :indent-code-end :indented-code)
+                (assoc :last-line-empty? true))]
 
-    paragraph
-    [text state]
+      code
+      [(str (escape-code (string/replace-first text #"    " "\n"))
+            (when should-close? "</code></pre>"))
+       (cond-> state
+         should-close? (assoc :indent-code-end true))]
 
-    (empty? (string/trim text))
-    [text state]
+      paragraph
+      [text state]
 
-    :default
-    (let [num-spaces (count (take-while (partial = \space) text))]
-      (if (> num-spaces 3)
-        [(str "<pre><code>" (escape-code (string/replace-first text #"    " "")))
-         (assoc state :code true :indented-code true)]
-        [text state]))))
+      (empty? (string/trim text))
+      [text state]
+
+      :default
+      (let [num-spaces (count (take-while (partial = \space) text))]
+        (if (>= num-spaces 4)
+          [(str "<pre><code>"
+                (escape-code (string/replace-first text #"    " ""))
+                (when should-close? "</code></pre>"))
+           (cond-> (assoc state :code true :indented-code true)
+             should-close? (assoc :indent-code-end true))]
+          [text state])))))
 
 (defn codeblock [text {:keys [codeblock codeblock-end indented-code next-line lists] :as state}]
   (let [trimmed           (string/trim text)
-        next-line-closes? (= [\` \` \`] (take-last 3 (some-> next-line string/trim)))]
+        next-line-closes? (some-> next-line string/trim (string/ends-with? "```"))]
     (cond
       (and lists codeblock-end)
       ["" (dissoc state :code :codeblock :codeblock-end)]
@@ -197,19 +208,19 @@
                 (dissoc :code :codeblock :codeblock-end))]
 
       (and next-line-closes? codeblock)
-      [(str (escape-code (str text "\n" (apply str (first (string/split next-line #"```"))))) "</code></pre>")
+      [(str (escape-code (str text \newline (apply str (first (string/split next-line #"```"))))) "</code></pre>")
        (assoc state :skip-next-line? (not lists)
                     :codeblock-end true
                     :last-line-empty? (not lists))]
 
       (and
         (not indented-code)
-        (= [\` \` \`] (take 3 trimmed)))
+        (string/starts-with? trimmed "```"))
       (let [[lang code] (split-with (partial not= \newline) (drop 3 trimmed))
             lang      (string/trim (string/join lang))
             s         (apply str (rest code))
             formatter (:code-style state)]
-        [(str "<pre><code" (if (not-empty lang)
+        [(str "<pre><code" (when (seq lang)
                              (str " "
                                   (if formatter
                                     (formatter lang)
@@ -241,7 +252,7 @@
   "Check for blockquotes and signal to blockquote-2 function with
   states blockquote-start and blockquote-end so that tags can be added.
   This approach enables lists to be included in blockquotes."
-  [text {:keys [eof code codeblock lists] :as state}]
+  [text {:keys [eof code codeblock] :as state}]
   (let [trimmed (string/trim text)]
     (cond
       (or code codeblock)
@@ -323,7 +334,7 @@
     (loop [acc      {}
            remain   metadata
            prev-key nil]
-      (if (not (empty? remain))
+      (if (seq remain)
         (let [data     (first remain)
               [key val] (if (sequential? data) data [prev-key data])
               prev-val (get acc key [])

test/markdown/md_test.cljc

@@ -202,12 +202,22 @@
          (entry-function "    foo")))
   (is (= "<pre><code>foo</code></pre><p>bar</p>"
          (entry-function "    foo\n\nbar")))
-  (is (= "<pre><code>foo</code></pre>bar"
+  (is (= "<pre><code>foo</code></pre><p>bar</p>"
          (entry-function "    foo\nbar")))
   (is (= "<p>baz     foo</p><p>bar</p>"
          (entry-function "baz\n    foo\n\nbar")))
   (is (= "<p><div class=\"grid-container\">   <div class=\"child1\">     <p>Element #1</p>   </div> </div></p>"
-         (entry-function "<div class=\"grid-container\">\n  <div class=\"child1\">\n    <p>Element #1</p>\n  </div>\n</div>"))))
+         (entry-function "<div class=\"grid-container\">\n  <div class=\"child1\">\n    <p>Element #1</p>\n  </div>\n</div>")))
+  (is (= "<p>Random text. Random text.</p>"
+         (entry-function "Random text.\nRandom text.\n"))
+      "A single newline is interpreted as space in the same paragraph")
+  (is (= "<p>Random text.</p><pre><code>code block</code></pre><p>Random text. Random text.</p>"
+         (entry-function "Random text.\n\n    code block\n\nRandom text.\nRandom text.\n"))
+      "Two newlines after a code block is interpreted as starting a new paragraph")
+  (is (= "<p>Random text.</p><pre><code>code block\n \nwith spaces\n </code></pre><p>Random text. Random text.</p>"
+         (entry-function
+          "Random text.\n\n    code block\n     \n    with spaces\n     \nRandom text.\nRandom text.\n"))
+      "Two newlines after a indented code block where one contains spaces is still interpreted as a new paragraph"))
 
 (deftest strikethrough
   (is (= "<p><del>foo</del></p>"