New doc generation script

doc2md.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+"""
+Convert parser doc string to markdown
+"""
+import sys
+import importlib
+from inspect import isfunction, signature, cleandoc
+import yapf
+import os
+import sys
+
+ignore_lib_functions = [
+    'cast',
+    'wraps',
+    'lru_cache',
+    'namedtuple'
+]
+
+sys.path.append(os.getcwd() + '/jc')
+mod_path = sys.argv[1]
+mod_name = mod_path.split('.')[-1]
+module = importlib.import_module(f'{mod_path}')
+
+######## HEADER ########
+header = f'''[Home](https://kellyjonbrazil.github.io/jc/)
+<a id="{mod_path}"></a>
+
+# {mod_path}
+'''
+
+summary = module.__doc__
+
+functions = []
+for attribute in dir(module):
+    if isfunction(getattr(module, attribute)) \
+        and not getattr(module, attribute).__name__.startswith('_'):
+
+        if 'jc.parsers.' in mod_path and not 'universal' in mod_path:
+            if attribute == 'parse':
+                functions.append(attribute)
+
+        else:
+            if not attribute in ignore_lib_functions:
+                functions.append(attribute)
+
+######## TABLE OF CONTENTS ########
+toc = f'# Table of Contents\n\n*[{mod_path}](#{mod_path})\n'
+for api in functions:
+    toc = f'{toc}  *[{api}](#{mod_path}.{api})\n'
+
+######## API DOCS ########
+api_docs = ''
+for api in functions:
+    api_function = getattr(module, api)
+
+    this_header = f'<a id="{mod_path}.{api}"></a>\n\n### {api}\n'
+    this_sig = str(signature(api_function))
+    formatted_sig = yapf.yapf_api.FormatCode(f'def {api_function.__name__}{this_sig}:\n    pass' )
+    formatted_sig = formatted_sig[0].split(':\n    pass')[0]
+    this_name_and_sig = f'{this_header}\n```python\n{formatted_sig}\n```'
+
+    this_doc = cleandoc(api_function.__doc__)
+    api_docs = api_docs + this_name_and_sig + '\n\n' + this_doc + '\n\n'
+
+######## FOOTER ########
+footer = ''
+if 'jc.parsers.' in mod_path and not 'universal' in mod_path:
+    footer = '### Parser Information\n'
+    comp = ', '.join(module.info.compatible)
+    ver = module.info.version
+    author = module.info.author
+    author_email = module.info.author_email
+    slurpable = 'slurpable' in module.info.tags
+    footer = footer + f'Compatibility: {comp}\n\n'
+    footer = footer + f'Source: [`jc/parsers/{mod_name}.py`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/{mod_name}.py)\n\n'
+    if slurpable:
+        footer = footer + 'This parser can be used with the `--slurp` command-line option.\n\n'
+    footer = footer + f'Version {ver} by {author} ({author_email})\n'
+
+if 'jc.parsers.' in mod_path and not 'universal' in mod_path:
+    final_doc = header + '\n' + summary + '\n\n' + api_docs + '\n' + footer
+elif mod_path == 'jc':
+    final_doc = header + '\n' + summary
+else:
+    final_doc = header + '\n' + toc + '\n' + summary + '\n\n' + api_docs
+
+print(final_doc)
+
+
+
+

docgen.sh

@@ -1,107 +1,34 @@
 #!/bin/bash
-# Generate docs.md
-# requires pydoc-markdown 4.6.1
+# Generate markdown document files (*.md)
 
-# use ./docgen all to generate all docs
-
-readme_config=$(cat <<'EOF'
-{
-    "processors": [
-        {
-            "type": "filter"
-        },
-        {
-            "type": "pydocmd"
-        }
-    ],
-    "renderer": {
-        "type": "markdown",
-        "header_level_by_type": {
-            "Module": 1,
-            "Class": 3,
-            "Method": 3,
-            "Function": 3,
-            "Variable": 3
-        }
-    }
-}
-EOF
-)
-
-toc_config=$(cat <<'EOF'
-{
-    "processors": [
-        {
-            "type": "filter"
-        },
-        {
-            "type": "pydocmd"
-        }
-    ],
-    "renderer": {
-        "type": "markdown",
-        "render_toc": true,
-        "header_level_by_type": {
-            "Module": 1,
-            "Class": 3,
-            "Method": 3,
-            "Function": 3,
-            "Variable": 3
-        }
-    }
-}
-EOF
-)
+# Requires the yapf python library
 
-parser_config=$(cat <<'EOF'
-{
-    "processors": [
-        {
-            "type": "filter",
-            "expression": "not name == \"info\" and not name.startswith(\"_\") and default()"
-        },
-        {
-            "type": "pydocmd"
-        }
-    ],
-    "renderer": {
-        "type": "markdown",
-        "header_level_by_type": {
-            "Module": 1,
-            "Class": 3,
-            "Method": 3,
-            "Function": 3,
-            "Variable": 3
-        }
-    }
-}
-EOF
-)
+# use ./docgen all to generate all docs
 
 cd jc
 (
     echo Building docs for: package
-    pydoc-markdown -m jc "${readme_config}" > ../docs/readme.md; echo "+++ package docs complete"
+    ../doc2md.py jc > ../docs/readme.md && echo "+++ package docs complete" || echo "*** PACKAGE DOCS FAILED"
 ) &
 
 (
     echo Building docs for: lib
-    pydoc-markdown -m jc.lib "${toc_config}" > ../docs/lib.md; echo "+++ lib docs complete"
+    ../doc2md.py jc.lib > ../docs/lib.md && echo "+++ lib docs complete" || echo "*** LIB DOCS FAILED"
 ) &
 
 (
     echo Building docs for: utils
-    pydoc-markdown -m jc.utils "${toc_config}" > ../docs/utils.md; echo "+++ utils docs complete"
+    ../doc2md.py jc.utils > ../docs/utils.md && echo "+++ utils docs complete" || echo "*** UTILS DOCS FAILED"
 ) &
 
 (
     echo Building docs for: streaming
-    pydoc-markdown -m jc.streaming "${toc_config}" > ../docs/streaming.md; echo "+++ streaming docs complete"
+    ../doc2md.py jc.streaming > ../docs/streaming.md && echo "+++ streaming docs complete" || echo "*** STREAMING DOCS FAILED"
 ) &
 
 (
     echo Building docs for: universal parser
-    pydoc-markdown -m jc.parsers.universal "${toc_config}" > ../docs/parsers/universal.md; echo "+++ universal parser docs complete"
+    ../doc2md.py jc.parsers.universal > ../docs/parsers/universal.md && echo "+++ universal parser docs complete" || echo "*** UNIVERSAL PARSER DOCS FAILED"
 ) &
 
 # a bit of inception here... jc is being used to help
@@ -119,27 +46,8 @@ for parser in "${parsers[@]}"; do
     parser_name=$(jq -r '.name' <<< "$parser")
         {
             if [[ $1 == "all" ]] || ! git diff --quiet --exit-code HEAD~5 -- "parsers/${parser_name}.py"; then
-                compatible=$(jq -r '.compatible | join(", ")' <<< "$parser")
-                version=$(jq -r '.version' <<< "$parser")
-                author=$(jq -r '.author' <<< "$parser")
-                author_email=$(jq -r '.author_email' <<< "$parser")
-
                 echo "Building docs for: ${parser_name}"
-                echo "[Home](https://kellyjonbrazil.github.io/jc/)" > ../docs/parsers/"${parser_name}".md
-                pydoc-markdown -m jc.parsers."${parser_name}" "${parser_config}" >> ../docs/parsers/"${parser_name}".md
-                echo "### Parser Information" >> ../docs/parsers/"${parser_name}".md
-                echo "Compatibility:  ${compatible}" >> ../docs/parsers/"${parser_name}".md
-                echo >> ../docs/parsers/"${parser_name}".md
-                echo "Source: [\`jc/parsers/${parser_name}.py\`](https://github.com/kellyjonbrazil/jc/blob/master/jc/parsers/${parser_name}.py)" >> ../docs/parsers/"${parser_name}".md
-                echo >> ../docs/parsers/"${parser_name}".md
-
-                if $(jq -e '.tags | contains(["slurpable"])' <<< "$parser"); then
-                    echo "This parser can be used with the \`--slurp\` command-line option." >> ../docs/parsers/"${parser_name}".md
-                    echo >> ../docs/parsers/"${parser_name}".md
-                fi
-
-                echo "Version ${version} by ${author} (${author_email})" >> ../docs/parsers/"${parser_name}".md
-                echo "+++ ${parser_name} docs complete"
+                ../doc2md.py jc.parsers."${parser_name}" > ../docs/parsers/"${parser_name}".md && echo "+++ ${parser_name} docs complete" || echo "*** ${parser_name} DOCS FAILED"
             fi
         } &
 done