From 0358dd8a474f310a2248a6ade14a4f239c6b01cb Mon Sep 17 00:00:00 2001 From: Ihor Kalnytskyi Date: Wed, 1 Nov 2023 23:54:39 +0200 Subject: [PATCH] Use general approach for rendering in commonmark Instead of supporting graphviz and any other possible text-to-diagram providers directly, let's build a general approach instead where a user can specify whatever executable they want to pipe the content of the code block through it, and replace the whole node w/ its output. This drops direct graphviz support in favor of built-in 'exec' pipe. --- .github/workflows/ci.yml | 3 -- pyproject.toml | 3 +- src/holocron/_processors/commonmark.py | 49 +++++++++++----------- tests/_processors/test_commonmark.py | 56 ++------------------------ 4 files changed, 30 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57bfc6d..84560be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,8 +32,5 @@ jobs: with: python-version: "3.10" - - name: Set up prerequisites - run: sudo apt-get install graphviz libgraphviz-dev - - name: Run tests run: pipx run -- hatch run test:run diff --git a/pyproject.toml b/pyproject.toml index 9434cd3..78621af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] dependencies = [ "Jinja2 >= 3.1", @@ -33,7 +34,6 @@ dependencies = [ "Markdown >= 3.4", "docutils >= 0.19", "feedgen >= 0.9", - "pygraphviz >= 1.10", "termcolor >= 1.1", "colorama >= 0.4", "markdown-it-py >= 2.1", @@ -58,7 +58,6 @@ source = "vcs" sources = ["src"] [tool.hatch.envs.test] -python = "3.10" dependencies = [ "mock >= 4.0", "coverage >= 6.4", diff --git a/src/holocron/_processors/commonmark.py b/src/holocron/_processors/commonmark.py index 8b0f6a3..da5e3c1 100644 --- a/src/holocron/_processors/commonmark.py +++ b/src/holocron/_processors/commonmark.py @@ -2,7 +2,8 @@ import json import logging -import pathlib +import subprocess +import typing as t import markdown_it import markdown_it.renderer @@ -11,13 +12,10 @@ import pygments.formatters.html import pygments.lexers import pygments.util -import pygraphviz from mdit_py_plugins.container import container_plugin from mdit_py_plugins.deflist import deflist_plugin from mdit_py_plugins.footnote import footnote_plugin -import holocron - from ._misc import parameters _LOGGER = logging.getLogger("holocron") @@ -28,22 +26,35 @@ def __init__(self, parser): super().__init__(parser) self._parser = parser - def fence(self, tokens, idx, options, env): + def fence(self, tokens, idx, options, env) -> str: token = tokens[idx] + match token.info.split(maxsplit=1): - case ["dot", params]: - env.setdefault("diagrams", []) + case [language, params]: params = json.loads(params) - diagram_name = f"diagram-{len(env['diagrams'])}.svg" - diagram_data = pygraphviz.AGraph(token.content).draw( - format=params["format"], - prog=params.get("engine", "dot"), - ) - env["diagrams"].append((diagram_name, diagram_data)) - return self._parser.render(f"![]({diagram_name})") + if "exec" in params: + return _exec_pipe(params["exec"], token.content.encode("UTF-8")).decode("UTF-8") + return super().fence(tokens, idx, options, env) +def _exec_pipe(args: t.List[str], input_: t.ByteString, timeout: int = 1000) -> bytes: + try: + completed_process = subprocess.run( + args, + shell=True, + input=input_, + capture_output=True, + timeout=timeout, + check=True + ) + except subprocess.TimeoutExpired: + return b"timed out executing the command" + except subprocess.CalledProcessError as exc: + return exc.stderr + return completed_process.stdout + + def _pygmentize(code: str, language: str, _: str) -> str: if not language: return code @@ -118,13 +129,3 @@ def process( item["content"] = commonmark.renderer.render(tokens, commonmark.options, env) item["destination"] = item["destination"].with_suffix(".html") yield item - - for diagram_name, diagram_bytes in env.get("diagrams", []): - yield holocron.WebSiteItem( - { - "source": pathlib.Path("dot://", str(item["source"]), diagram_name), - "destination": item["destination"].with_name(diagram_name), - "baseurl": app.metadata["url"], - "content": diagram_bytes, - } - ) diff --git a/tests/_processors/test_commonmark.py b/tests/_processors/test_commonmark.py index 26cdf3e..62fb3d2 100644 --- a/tests/_processors/test_commonmark.py +++ b/tests/_processors/test_commonmark.py @@ -407,7 +407,7 @@ def test_args_pygmentize_unknown_language(testapp, language): ] -def test_item_dot_render(testapp): +def test_item_exec(testapp): """Commonmark has to render DOT snippets into SVG if asked to render.""" stream = commonmark.process( @@ -417,10 +417,8 @@ def test_item_dot_render(testapp): { "content": textwrap.dedent( """ - ```dot {"format": "svg"} - graph yoda { - a -- b -- c - } + ```text {"exec": "sed --expression 's/ a / the /g'"} + yoda, a jedi grandmaster ``` """ ), @@ -435,57 +433,11 @@ def test_item_dot_render(testapp): assert list(stream) == [ holocron.Item( { - "content": '

\n', + "content": "yoda, the jedi grandmaster\n", "source": pathlib.Path("1.md"), "destination": pathlib.Path("1.html"), }, ), - holocron.WebSiteItem( - { - "content": _pytest_regex(rb".*\s*", re.DOTALL | re.MULTILINE), - "source": pathlib.Path("dot://1.md/diagram-0.svg"), - "destination": pathlib.Path("diagram-0.svg"), - "baseurl": "https://yoda.ua", - } - ), - ] - - -def test_item_dot_not_rendered(testapp): - """Commonmark has to preserve DOT snippet if not asked to render.""" - - stream = commonmark.process( - testapp, - [ - holocron.Item( - { - "content": textwrap.dedent( - """ - ```dot - graph yoda { - a -- b -- c - } - ``` - """ - ), - "destination": pathlib.Path("1.md"), - } - ) - ], - ) - - assert isinstance(stream, collections.abc.Iterable) - assert list(stream) == [ - holocron.Item( - { - "content": ( - '
graph yoda {\n'
-                    "    a -- b -- c\n"
-                    "}\n
\n" - ), - "destination": pathlib.Path("1.html"), - } - ) ]