Pandoc haskell filter

.github/workflows/test.yml

@@ -40,10 +40,9 @@ jobs:
 
             - name: rustic-doc prerequisites
               run: |
-                   wget https://github.com/jgm/pandoc/releases/download/2.17.0.1/pandoc-2.17.0.1-1-amd64.deb
-                   sudo dpkg -i pandoc-2.17.0.1-1-amd64.deb
                    mkdir -p ~/.local/bin
                    mkdir -p ~/.local/share/emacs/rustic-doc/std/
+                   sudo apt install -y libc6
                    cargo install fd-find
                    cargo install ripgrep
                    rustup component add rust-docs

.gitignore

@@ -2,4 +2,5 @@
 /.cask
 /config.mk
 *autoloads.el
-test/test-project/target
+.stack-work/
+test/test-project/target

README.md

@@ -782,7 +782,7 @@ example](img/rustic-doc.png)
 
 Required:
 
-- [pandoc](https://pandoc.org/installing.html) preferably at least version 2.11, as it will give somewhat nicer generated documentation. Versions older than 2.9 may not work - if you're on a debian based distro installing through your regular repo might not work out.
+- [pandoc](https://pandoc.org/installing.html) Installed automatically.
 - [cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html)
 - [cargo-makedocs](https://github.com/Bunogi/cargo-makedocs)
 - [fd-find](https://github.com/sharkdp/fd) Old versions, especially before 2.x, may not work. Install through Cargo if you're having issues.

rustic-doc.el

@@ -31,16 +31,12 @@
     (require 'xdg)
     (fset 'rustic-doc--xdg-data-home 'xdg-data-home)))
 
-(defvar rustic-doc-lua-filter (concat (file-name-as-directory (getenv "HOME"))
-                                      ".local/bin/rustic-doc-filter.lua")
-  "Save location for the rustic-doc lua filter.")
-
-(defvar rustic-doc-convert-prog (concat (file-name-as-directory (getenv "HOME"))
-                                        ".local/bin/rustic-doc-convert.sh")
-  "Save location for the rustic-doc conversion script.")
-
-(defvar rustic-doc-source-repo
-  "https://raw.githubusercontent.com/brotzeit/rustic/master/rustic-doc/")
+(setq rustic-doc-bin-location (f-join  (file-name-as-directory (getenv "HOME")) ".local/bin/rustic-doc/"))
+(setq rustic-doc-filter (f-join rustic-doc-bin-location "filter"))
+(setq rustic-doc-convert-prog  (f-join rustic-doc-bin-location "convert.sh"))
+(setq rustic-doc-pandoc  (f-join rustic-doc-bin-location "pandoc"))
+(setq rustic-doc-pandoc-tar (f-join rustic-doc-bin-location "pandoc.tar.gz"))
+(setq rustic-doc-source-repo "https://raw.githubusercontent.com/samhedin/rustic/pandoc-haskell-filter/rustic-doc/")
 
 (defvar rustic-doc-current-project nil
   "Location to search for documentation.
@@ -49,13 +45,23 @@ All projects and std by default, otherwise last open project and std.")
 (defvar rustic-doc-save-loc (concat (rustic-doc--xdg-data-home)
                                     "/emacs/rustic-doc"))
 
-(defvar rustic-doc-resources
-  `((,rustic-doc-convert-prog
-     (:exec)
-     ,(concat rustic-doc-source-repo "convert.sh"))
-    (,rustic-doc-lua-filter
-     ()
-     ,(concat rustic-doc-source-repo "filter.lua"))))
+;; todo: fix rustic-doc-filter link after merge.
+(setq rustic-doc-resources
+      `((,rustic-doc-convert-prog
+         (:exec)
+         ,(concat rustic-doc-source-repo "convert.sh"))
+        (,rustic-doc-filter
+         (:exec)
+         "https://github.com/samhedin/rustic/blob/pandoc-haskell-filter/rustic-doc/pandoc_filter/rustdoc-to-org-exe?raw=true")))
+
+(defun rustic-doc--install-pandoc ()
+  "Install a compatible version of pandoc. Does not modify potential existing system installation of pandoc."
+  (unless (file-exists-p rustic-doc-pandoc)
+    (if   (url-copy-file "https://github.com/jgm/pandoc/releases/download/2.17.0.1/pandoc-2.17.0.1-linux-amd64.tar.gz" rustic-doc-pandoc-tar t)
+        (progn
+          (shell-command (format "tar -xf %s " rustic-doc-pandoc-tar ))
+          (shell-command (format "cp pandoc-2.17.0.1/bin/pandoc %s" rustic-doc-pandoc)))
+      (message "Couldn't install pandoc"))))
 
 (defun rustic-doc-default-rg-search-command ()
   "The default search command when using helm-ag.
@@ -94,15 +100,12 @@ Search for SEARCH-TERM inside SEARCH-DIR"
    (t
     (grep (format "grep -RPIni '%s' %s" search-term search-dir)))))
 
-
 (defcustom rustic-doc-search-function 'rustic-doc-default-search-function
   "Function to use for searching documentation.
 The function should take search-dir and search-term as arguments."
   :type 'function
   :group 'rustic-doc)
 
-
-
 (defun rustic-doc--install-resources ()
   "Install or update the rustic-doc resources."
   (dolist (resource rustic-doc-resources)
@@ -253,58 +256,52 @@ If the user has not visited a project, returns the main doc directory."
   (unless (file-directory-p rustic-doc-save-loc)
     (rustic-doc-setup)
     (message "Running first time setup."))
+  (rustic-doc--update-current-project)
   (if rustic-doc-current-project
       (progn
         (message "Converting documentation for %s "
                  rustic-doc-current-project)
-        (if (/= 0 (call-process "cargo" nil "*cargo-makedocs*" nil "makedocs"))
-            (message "\
-cargo makedocs could not generate docs for the current package. \
-See buffer *cargo-makedocs* for more info")
-          (let* ((docs-src
-                  (concat (file-name-as-directory rustic-doc-current-project)
-                          "target/doc"))
-                 (finish-func (lambda (_p)
-                                (message "Finished converting docs for %s"
-                                         rustic-doc-current-project))))
-            (rustic-doc-create-project-dir)
-            (rustic-doc--start-process "rustic-doc-convert"
-                                       rustic-doc-convert-prog
-                                       finish-func
-                                       docs-src
-                                       (rustic-doc--project-doc-dest)))))
+        ;; Spawn a new shell that cd's into `rustic-doc-current-project' and converts the packages.
+        (message (format "cargo makedocs: %s" (shell-command-to-string
+                                       (format  "sh -c \"cd %s; %s makedocs\"" rustic-doc-current-project (rustic-cargo-bin)))))
+        (let* ((docs-src
+                (concat (file-name-as-directory rustic-doc-current-project)
+                        "target/doc"))
+               (finish-func (lambda (_p)
+                              (message "Finished converting docs for dependencies of %s. Std conversion might still be running."
+                                       rustic-doc-current-project))))
+          (rustic-doc-create-project-dir)
+          (rustic-doc--start-process "rustic-doc-convert"
+                                     rustic-doc-convert-prog
+                                     finish-func
+                                     docs-src
+                                     (rustic-doc--project-doc-dest))))
     (message "Activate rustic-doc-mode to run `rustic-doc-convert-current-package")))
 
 (defun rustic-doc--confirm-dep-versions (missing-fd)
   "Verify that dependencies are not too old."
   (when (not missing-fd)
     (when  (> 8 (string-to-number
-                  (substring (shell-command-to-string "fd --version") 3 4)))
-      (message "Your version of fd is too old, please install a recent version, maybe through cargo.")))
-
-  (when (>= 11 (string-to-number
-                (substring (shell-command-to-string "pandoc --version") 9 11)))
-    (message "Your version of pandoc is too old, please install a more recent version. See their github for more info.")))
-
+                 (substring (shell-command-to-string "fd --version") 3 4)))
+      (message "Your version of fd is too old, please install a recent version, maybe through cargo."))))
 
 (defun rustic-doc-install-deps (&optional noconfirm)
   "Install dependencies with Cargo.
 If NOCONFIRM is non-nil, install all dependencies without prompting user."
-  (if (not (executable-find "cargo"))
-      (message "You need to have cargo installed to use rustic-doc")
-    (let ((missing-rg (not (executable-find "rg")))
-          (missing-fd (and  (not (executable-find "fd") )))
-          (missing-makedocs (not (executable-find "cargo-makedocs"))))
-      (rustic-doc--confirm-dep-versions missing-fd)
-      (when (and (or missing-fd missing-makedocs missing-rg)
-                 (or noconfirm (y-or-n-p "Missing some dependencies for rustic doc, install them? ")))
-        (when missing-fd
-          (rustic-doc--start-process "install-fd" "cargo" nil "install" "fd-find"))
-        (when missing-rg
-          (rustic-doc--start-process "install-rg" "cargo" nil "install" "ripgrep"))
-        (when missing-makedocs
-          (rustic-doc--start-process "install-makedocs" "cargo" nil
-                                     "install" "cargo-makedocs"))))))
+  (rustic-doc--install-pandoc)
+  (let ((missing-rg (not (executable-find "rg")))
+        (missing-fd (and  (not (executable-find "fd") )))
+        (missing-makedocs (not (executable-find "cargo-makedocs"))))
+    (rustic-doc--confirm-dep-versions missing-fd)
+    (when (and (or missing-fd missing-makedocs missing-rg)
+               (or noconfirm (y-or-n-p "Missing some dependencies for rustic doc, install them? ")))
+      (when missing-fd
+        (rustic-doc--start-process "install-fd" "cargo" nil "install" "fd-find"))
+      (when missing-rg
+        (rustic-doc--start-process "install-rg" "cargo" nil "install" "ripgrep"))
+      (when missing-makedocs
+        (rustic-doc--start-process "install-makedocs" "cargo" nil
+                                   "install" "cargo-makedocs")))))
 
 ;;;###autoload
 (defun rustic-doc-setup (&optional no-dl noconfirm)
@@ -325,7 +322,8 @@ If NOCONFIRM is non-nil, install all dependencies without prompting user."
                              (lambda (_p)
                                (message "Finished converting docs for std"))
                              "std")
-  (if rustic-doc-current-project
+
+    (if rustic-doc-current-project
       (rustic-doc-convert-current-package)
     (message "Setup is converting std. If you want to convert local dependencies, activate rustic-doc-mode when you are in a rust project and run `rustic-doc-convert-current-package")))
 
@@ -337,12 +335,19 @@ If NOCONFIRM is non-nil, install all dependencies without prompting user."
      proc (lambda (proc event)
             (let ((buf (process-buffer proc)))
               (if (string-match-p (regexp-quote "abnormally") event)
-                  (message "Could not finish process: %s. \
-See the *Messages* buffer or %s for more info." event (concat "*" name "*"))
+                  (progn
+                    (with-current-buffer "*rustic-doc-std-conversion*"
+                      (print (format "STD-CONVERSION: %s" (buffer-string)) #'external-debugging-output))
+                    (with-current-buffer "*messages*"
+                      (print (format "MESSAGES: %s" (buffer-string)) #'external-debugging-output))
+                    (message "Could not finish process: %s. \
+See the *Messages* buffer or %s for more info." event (concat "*" name "*")))
                 (when finish-func
                   (funcall finish-func proc))
-                (when (buffer-live-p buf)
-                  (kill-buffer buf))))))
+                ;; (when (buffer-live-p buf)
+                ;;   (kill-buffer buf))
+                (print )
+                ))))
     proc))
 
 (defun rustic-doc--thing-at-point ()

rustic-doc/Makefile

@@ -1,41 +1,31 @@
 # Used for development
-standard:
+standard: create_filter
 	cd debug_files ;\
-	pandoc enum.Option.html  --lua-filter ../filter.lua -t native -o filterednative ;\
+	pandoc enum.Option.html -t json | ../pandoc_filter/rustdoc-to-org-exe | pandoc -f json -t native -o filterednative ;\
 	pandoc -f native filterednative -o option.org
 
-standard_unfiltered:
+trait: create_filter
 	cd debug_files ;\
-	pandoc enum.Option.html -t native -o filterednative ;\
-	pandoc -f native filterednative -o option.org
-
-
-primitive:
-	cd debug_files ;\
-	pandoc primitive.i16.html  --lua-filter ../filter.lua -t native -o filterednative ;\
-	pandoc -f native filterednative -o primitive.i16.org
-
-
-primitive_unfiltered:
-	cd debug_files ;\
-	pandoc primitive.i16.html -t native -o filterednative ;\
-	pandoc -f native filterednative -o primitive.i16.org
-
-trait:
-	cd debug_files ;\
-	pandoc trait.AsRef.html --lua-filter ../filter.lua -t native -o filterednative ;\
+	pandoc trait.AsRef.html -t json | ../pandoc_filter/rustdoc-to-org-exe | pandoc -f json -t native -o filterednative ;\
 	pandoc -f native filterednative -o AsRef.org
 
-code:
+code: create_filter
 	cd debug_files ;\
-	pandoc shared.rs.html --lua-filter ../filter.lua -t native -o filterednative ;\
+	pandoc shared.rs.html -t json | ../pandoc_filter/rustdoc-to-org-exe | pandoc -f json -t native -o filterednative ;\
 	pandoc -f native filterednative -o shared.rs.org
 
-constant:
+constant: create_filter
 	cd debug_files ;\
-	pandoc constant.ARCH.html --lua-filter ../filter.lua -t native -o filterednative ;\
+	pandoc constant.ARCH.html -t json | ../pandoc_filter/rustdoc-to-org-exe | pandoc -f json -t native -o filterednative ;\
 	pandoc -f native filterednative -o constant.arch.org
 
-overwrite_filter:
-	rm ~/.local/bin/rustic-doc-filter.lua
-	cp ./filter.lua ~/.local/bin/rustic-doc-filter.lua
+create_filter:
+	cd pandoc_filter; \
+	stack install --local-bin-path .; \
+	cd ..
+
+overwrite_local:
+	cd pandoc_filter; \
+	stack install --local-bin-path ~/.local/bin/rustic-doc-filter; \
+	cd ..
+	cp convert.sh ~/.local/bin/rustic-doc-convert.sh

rustic-doc/README.org

@@ -1,8 +1,8 @@
 #+TITLE: Readme
 The way I develop the filter is:
 1. Select an html-file to focus on, such as [[./debug_files/enum.Option.html]].
-2. Open [[./filter.lua]].
+2. Open the haskell filter in [[./pandoc_filter/app/Main.hs]]
 3. Open [[./debug_files/option.org]].
 4. Open [[./debug_files/filterednative]].
-5. Running `make` will now update =option.org= with the org-representation and =filterednative= with the pandoc representation. If anything looks weird in the org-version, more details can be found in filterednative.
+5. Running `make standard` will now update =option.org= with the org-representation and =filterednative= with the pandoc representation. If anything looks weird in the org-version, more details can be found in filterednative.
    6. See [[./Makefile]] for the alternatives. =make standard= uses the default, whereas =make primitive= uses HTML for the type =i16=, which is formatted differently since it is a primitive.

rustic-doc/convert.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
-LUA_FILTER="$HOME/.local/bin/rustic-doc-filter.lua"
+FILTER="$HOME/.local/bin/rustic-doc/filter"
+PANDOC="$HOME/.local/bin/rustic-doc/pandoc"
 function get_toolchain {
     rustup show | sed -nr 's/(.*) \(default\)/\1/p' | head -n 1
 }
@@ -57,6 +58,6 @@ fd . \
     -ehtml \
     --ignore-file "$ignore_file" \
     -j"$cores" \
-    -x pandoc '{}' \
-    --lua-filter "$LUA_FILTER" \
+    -x $PANDOC '{}' \
+    --filter "$FILTER" \
     -o "$DEST_DIR/{.}.org"

rustic-doc/pandoc_filter/ChangeLog.md

@@ -0,0 +1,3 @@
+# Changelog for rustdoc-to-org
+
+## Unreleased changes

rustic-doc/pandoc_filter/Setup.hs

@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain