From 560f641237fae735b2203ae06c6301e3f533699e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 10 Dec 2021 21:45:41 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20NEW:=20Add=20Docutils=20MyST=20conf?= =?UTF-8?q?ig=20and=20CLI=20(#426)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit updates the docutils only `Parser`, to include relevant MyST configuration options, which can be used with the docutils API, CLI or via a `docutils.conf` file. CLI commands are also exposed for docutils writers: `myst-docutils-html`, `myst-docutils-html5`, `myst-docutils-latex`, `myst-docutils-pseudoxml`, `myst-docutils-xml`. In addition, the commit also stops mathjax classes being added to sections, when parsing in docutils mode, and adds help metadata to the `MdParserConfig`attributes (for programmatic documentation). Co-authored-by: Chris Sewell --- docs/api/parsers.md | 25 ++- docs/api/reference.rst | 10 + docs/conf.py | 32 ++- docs/docutils.md | 79 +++++++ docs/index.md | 1 + docs/sphinx/index.md | 2 + docs/sphinx/use.md | 14 ++ myst_parser/docutils_.py | 201 ++++++++++++++++-- myst_parser/docutils_renderer.py | 23 +- myst_parser/main.py | 73 +++++-- myst_parser/sphinx_.py | 2 +- setup.cfg | 5 + tests/test_docutils.py | 96 ++++++++- .../sourcedirs/include_from_rst/conf.py | 2 + .../sourcedirs/include_from_rst/include.md | 5 + .../sourcedirs/include_from_rst/index.rst | 5 + tests/test_sphinx/test_sphinx_builds.py | 33 +++ .../test_include_from_rst.xml | 10 + 18 files changed, 569 insertions(+), 49 deletions(-) create mode 100644 docs/docutils.md create mode 100644 tests/test_sphinx/sourcedirs/include_from_rst/conf.py create mode 100644 tests/test_sphinx/sourcedirs/include_from_rst/include.md create mode 100644 tests/test_sphinx/sourcedirs/include_from_rst/index.rst create mode 100644 tests/test_sphinx/test_sphinx_builds/test_include_from_rst.xml diff --git a/docs/api/parsers.md b/docs/api/parsers.md index 581dc563..7b499844 100644 --- a/docs/api/parsers.md +++ b/docs/api/parsers.md @@ -9,28 +9,34 @@ The MyST Parser comes bundled with some helper functions to quickly parse MyST Markdown and render its output. +:::{important} +These APIs are primarily intended for testing and development purposes. +For proper parsing see {ref}`myst-sphinx` and {ref}`myst-docutils`. +::: + ### Parse MyST Markdown to HTML -For example, the following code parses markdown and renders as HTML: +The following code parses markdown and renders as HTML using only the markdown-it parser +(i.e. no sphinx or docutils specific processing is done): ```python from myst_parser.main import to_html -to_html("some *text*") +to_html("some *text* {literal}`a`") ``` ```html -'

some text

\n' +'

some text {literal}[a]

\n' ``` ### Parse MyST Markdown to docutils -The following function renders your text as **docutils objects** (for example, for use with the Sphinx ecosystem): +The following function renders your text as **docutils AST objects** (for example, for use with the Sphinx ecosystem): ```python from myst_parser.main import to_docutils -print(to_docutils("some *text*").pformat()) +print(to_docutils("some *text* {literal}`a`").pformat()) ``` ```xml @@ -39,8 +45,17 @@ print(to_docutils("some *text*").pformat()) some text + + + a ``` +:::{note} +This function only performs the initial parse of the AST, +without applying any transforms or post-processing. +See for example the [Sphinx core events](https://www.sphinx-doc.org/en/master/extdev/appapi.html?highlight=config-inited#sphinx-core-events). +::: + ### Parse MyST Markdown as `markdown-it` tokens The MyST Parser uses `markdown-it-py` tokens as an intermediate representation of your text. diff --git a/docs/api/reference.rst b/docs/api/reference.rst index 9fde1952..ad395cd6 100644 --- a/docs/api/reference.rst +++ b/docs/api/reference.rst @@ -81,6 +81,16 @@ Additional Methods .. autofunction:: myst_parser.sphinx_renderer.mock_sphinx_env +.. _api/docutils_parser: + +Docutils Parser Reference +------------------------- + +.. autoclass:: myst_parser.docutils_.Parser + :members: parse + :undoc-members: + :member-order: bysource + :show-inheritance: .. _api/sphinx_parser: diff --git a/docs/conf.py b/docs/conf.py index 8d1da277..9f244fd8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,6 +4,9 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +from sphinx.application import Sphinx +from sphinx.util.docutils import SphinxDirective + from myst_parser import __version__ # -- Project information ----------------------------------------------------- @@ -167,7 +170,34 @@ def run_apidoc(app): ] -def setup(app): +def setup(app: Sphinx): """Add functions to the Sphinx setup.""" + + class DocutilsCliHelpDirective(SphinxDirective): + """Directive to print the docutils CLI help.""" + + has_content = False + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = False + option_spec = {} + + def run(self): + """Run the directive.""" + import io + + from docutils import nodes + from docutils.frontend import OptionParser + + from myst_parser.docutils_ import Parser as DocutilsParser + + stream = io.StringIO() + OptionParser( + components=(DocutilsParser,), + usage="myst-docutils- [options] [ []]", + ).print_help(stream) + return [nodes.literal_block("", stream.getvalue())] + # app.connect("builder-inited", run_apidoc) app.add_css_file("custom.css") + app.add_directive("docutils-cli-help", DocutilsCliHelpDirective) diff --git a/docs/docutils.md b/docs/docutils.md new file mode 100644 index 00000000..48e36697 --- /dev/null +++ b/docs/docutils.md @@ -0,0 +1,79 @@ +(myst-docutils)= + +# MyST with Docutils + +Sphinx, and thus MyST-Parser, is built on top of the [Docutils](https://docutils.sourceforge.io/docs/) package. +MyST-Parser offers a renderer, parser and CLI-interface for working directly with Docutils, independent of Sphinx, as described below. + +:::{note} +Since these tools are independent of Sphinx, this means they cannot parse any Sphinx or Sphinx extensions specific roles or directives. +::: + +On installing MyST-Parser, the following CLI-commands are made available: + +- `myst-docutils-html`: converts MyST to HTML +- `myst-docutils-html5`: converts MyST to HTML5 +- `myst-docutils-latex`: converts MyST to LaTeX +- `myst-docutils-xml`: converts MyST to docutils-native XML +- `myst-docutils-pseudoxml`: converts MyST to pseudo-XML (to visualise the AST structure) + +Each command can be piped stdin or take a file path as an argument: + +```console +$ myst-docutils-html --help +$ echo "Hello World" | myst-docutils-html +$ myst-docutils-html hello-world.md +``` + +The commands are based on the [Docutils Front-End Tools](https://docutils.sourceforge.io/docs/user/tools.html), and so follow the same argument and options structure, included many of the MyST specific options detailed in [](sphinx/config-options). + +:::{dropdown} Shared Docutils CLI Options +```{docutils-cli-help} +``` +::: + +The CLI commands can also utilise the [`docutils.conf` configuration file](https://docutils.sourceforge.io/docs/user/config.html) to configure the behaviour of the CLI commands. For example: + +``` +# These entries affect all processing: +[general] +myst-enable-extensions: deflist,linkify +myst-footnote-transition: no + +# These entries affect specific HTML output: +[html writers] +embed-stylesheet: no + +[html5 writer] +stylesheet-dirs: path/to/html5_polyglot/ +stylesheet-path: minimal.css, responsive.css +``` + +You can also use the {py:class}`myst_parser.docutils_.Parser` class programmatically with the [Docutils publisher API](https://docutils.sourceforge.io/docs/api/publisher.html): + +```python +from docutils.core import publish_string +from myst_parser.docutils_ import Parser + +source = "hallo world\n: Definition" +output = publish_string( + source=source, + writer_name="html5", + settings_overrides={ + "myst_enable_extensions": ["deflist"], + "embed_stylesheet": False, + }, + parser=Parser(), +) +``` + +Finally, you can include MyST Markdown files within a RestructuredText file, using the [`include` directive](https://docutils.sourceforge.io/docs/ref/rst/directives.html#include): + +```rst +.. include:: include.md + :parser: myst_parser.docutils_ +``` + +```{important} +The `parser` option requires `docutils>=0.17` +``` diff --git a/docs/index.md b/docs/index.md index 08eeed08..93d0fa67 100644 --- a/docs/index.md +++ b/docs/index.md @@ -100,6 +100,7 @@ syntax/reference :caption: Topic Guides explain/index.md sphinx/index.md +docutils.md api/index.md develop/index.md ``` diff --git a/docs/sphinx/index.md b/docs/sphinx/index.md index 216e08af..decb14bb 100644 --- a/docs/sphinx/index.md +++ b/docs/sphinx/index.md @@ -1,3 +1,5 @@ +(myst-sphinx)= + # MyST with Sphinx The MyST Parser comes bundled with a Sphinx extension that allows you to use write Sphinx documentation entirely in MyST (or, in a combination of rST and MyST). diff --git a/docs/sphinx/use.md b/docs/sphinx/use.md index 3612940c..8b6df5df 100644 --- a/docs/sphinx/use.md +++ b/docs/sphinx/use.md @@ -32,6 +32,20 @@ To include rST, we must first "wrap" the directive in the [eval-rst directive](s .. include:: snippets/include-rst.rst ``` +(howto/include-md)= +## Include Markdown files into an rST file + +To include a MyST file within a ReStructuredText file, we can use the `parser` option of the `include` directive: + +```rst +.. include:: include.md + :parser: myst_parser.sphinx_ +``` + +```{important} +The `parser` option requires `docutils>=0.17` +``` + ## Use MyST in Jupyter Notebooks The [MyST-NB](https://myst-nb.readthedocs.io) tool provides a Sphinx extension for parsing **Jupyter Notebooks written with MyST Markdown**. It includes features like automatically executing notebooks during documentation builds, storing notebook cell outputs in order to insert them elsewhere in your documentation, and more. See the [MyST-NB documentation](https://myst-nb.readthedocs.io) for more information. diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 9c7c918e..269a6ec2 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -1,36 +1,161 @@ """A module for compatibility with the docutils>=0.17 `include` directive, in RST documents:: - .. include:: + .. include:: path/to/file.md :parser: myst_parser.docutils_ """ -from typing import Tuple +from typing import Any, Callable, Iterable, List, Optional, Tuple, Union -from docutils import nodes +from attr import Attribute +from docutils import frontend, nodes +from docutils.core import default_description, publish_cmdline from docutils.parsers.rst import Parser as RstParser from markdown_it.token import Token from myst_parser.main import MdParserConfig, default_parser -class Parser(RstParser): - """Docutils parser for Markedly Structured Text (MyST).""" +def _validate_int( + setting, value, option_parser, config_parser=None, config_section=None +) -> int: + """Validate an integer setting.""" + return int(value) + + +def _create_validate_tuple(length: int) -> Callable[..., Tuple[str, ...]]: + """Create a validator for a tuple of length `length`.""" + + def _validate( + setting, value, option_parser, config_parser=None, config_section=None + ): + string_list = frontend.validate_comma_separated_list( + setting, value, option_parser, config_parser, config_section + ) + if len(string_list) != length: + raise ValueError( + f"Expecting {length} items in {setting}, got {len(string_list)}." + ) + return tuple(string_list) + + return _validate + + +class Unset: + """A sentinel class for unset settings.""" + + def __repr__(self): + return "UNSET" + + +DOCUTILS_UNSET = Unset() +"""Sentinel for arguments not set through docutils.conf.""" + + +DOCUTILS_EXCLUDED_ARGS = ( + # docutils.conf can't represent callables + "heading_slug_func", + # docutils.conf can't represent dicts + "html_meta", + "substitutions", + # we can't add substitutions so not needed + "sub_delimiters", + # heading anchors are currently sphinx only + "heading_anchors", + # sphinx.ext.mathjax only options + "update_mathjax", + "mathjax_classes", + # We don't want to set the renderer from docutils.conf + "renderer", +) +"""Names of settings that cannot be set in docutils.conf.""" + + +def _docutils_optparse_options_of_attribute( + at: Attribute, default: Any +) -> Tuple[dict, str]: + """Convert an ``MdParserConfig`` attribute into a Docutils optparse options dict.""" + if at.type is int: + return {"validator": _validate_int}, f"(type: int, default: {default})" + if at.type is bool: + return { + "validator": frontend.validate_boolean + }, f"(type: bool, default: {default})" + if at.type is str: + return {}, f"(type: str, default: '{default}')" + if at.type == Iterable[str] or at.name == "url_schemes": + return { + "validator": frontend.validate_comma_separated_list + }, f"(type: comma-delimited, default: '{','.join(default)}')" + if at.type == Tuple[str, str]: + return { + "validator": _create_validate_tuple(2) + }, f"(type: str,str, default: '{','.join(default)}')" + if at.type == Union[int, type(None)] and at.default is None: + return { + "validator": _validate_int, + "default": None, + }, f"(type: null|int, default: {default})" + if at.type == Union[Iterable[str], type(None)] and at.default is None: + return { + "validator": frontend.validate_comma_separated_list, + "default": None, + }, f"(type: comma-delimited, default: '{default or ','.join(default)}')" + raise AssertionError( + f"Configuration option {at.name} not set up for use in docutils.conf." + f"Either add {at.name} to docutils_.DOCUTILS_EXCLUDED_ARGS," + "or add a new entry in _docutils_optparse_of_attribute." + ) + + +def _docutils_setting_tuple_of_attribute( + attribute: Attribute, default: Any +) -> Tuple[str, Any, Any]: + """Convert an ``MdParserConfig`` attribute into a Docutils setting tuple.""" + name = f"myst_{attribute.name}" + flag = "--" + name.replace("_", "-") + options = {"dest": name, "default": DOCUTILS_UNSET} + at_options, type_str = _docutils_optparse_options_of_attribute(attribute, default) + options.update(at_options) + help_str = attribute.metadata.get("help", "") if attribute.metadata else "" + return (f"{help_str} {type_str}", [flag], options) - supported: Tuple[str, ...] = ("md", "markdown", "myst") - """Aliases this parser supports.""" - settings_spec = RstParser.settings_spec - """Runtime settings specification. +def _myst_docutils_setting_tuples(): + """Return a list of Docutils setting for the MyST section.""" + defaults = MdParserConfig() + return tuple( + _docutils_setting_tuple_of_attribute(at, getattr(defaults, at.name)) + for at in MdParserConfig.get_fields() + if at.name not in DOCUTILS_EXCLUDED_ARGS + ) - Defines runtime settings and associated command-line options, as used by - `docutils.frontend.OptionParser`. This is a concatenation of tuples of: - - Option group title (string or `None` which implies no group, just a list - of single options). +def create_myst_config(settings: frontend.Values): + """Create a ``MdParserConfig`` from the given settings.""" + values = {} + for attribute in MdParserConfig.get_fields(): + if attribute.name in DOCUTILS_EXCLUDED_ARGS: + continue + setting = f"myst_{attribute.name}" + val = getattr(settings, setting, DOCUTILS_UNSET) + if val is not DOCUTILS_UNSET: + values[attribute.name] = val + values["renderer"] = "docutils" + return MdParserConfig(**values) - - Description (string or `None`). - - A sequence of option tuples - """ +class Parser(RstParser): + """Docutils parser for Markedly Structured Text (MyST).""" + + supported: Tuple[str, ...] = ("md", "markdown", "myst") + """Aliases this parser supports.""" + + settings_spec = ( + *RstParser.settings_spec, + "MyST options", + None, + _myst_docutils_setting_tuples(), + ) + """Runtime settings specification.""" config_section = "myst parser" config_section_dependencies = ("parsers",) @@ -41,9 +166,12 @@ def parse(self, inputstring: str, document: nodes.document) -> None: :param inputstring: The source string to parse :param document: The root docutils node to add AST elements to - """ - config = MdParserConfig(renderer="docutils") + try: + config = create_myst_config(document.settings) + except (TypeError, ValueError) as error: + document.reporter.error(f"myst configuration invalid: {error.args[0]}") + config = MdParserConfig(renderer="docutils") parser = default_parser(config) parser.options["document"] = document env: dict = {} @@ -53,3 +181,40 @@ def parse(self, inputstring: str, document: nodes.document) -> None: # specified in the sphinx configuration tokens = [Token("front_matter", "", 0, content="{}", map=[0, 0])] + tokens parser.renderer.render(tokens, parser.options, env) + + +def _run_cli(writer_name: str, writer_description: str, argv: Optional[List[str]]): + """Run the command line interface for a particular writer.""" + publish_cmdline( + parser=Parser(), + writer_name=writer_name, + description=( + f"Generates {writer_description} from standalone MyST sources.\n{default_description}" + ), + argv=argv, + ) + + +def cli_html(argv: Optional[List[str]] = None) -> None: + """Cmdline entrypoint for converting MyST to HTML.""" + _run_cli("html", "(X)HTML documents", argv) + + +def cli_html5(argv: Optional[List[str]] = None): + """Cmdline entrypoint for converting MyST to HTML5.""" + _run_cli("html5", "HTML5 documents", argv) + + +def cli_latex(argv: Optional[List[str]] = None): + """Cmdline entrypoint for converting MyST to LaTeX.""" + _run_cli("latex", "LaTeX documents", argv) + + +def cli_xml(argv: Optional[List[str]] = None): + """Cmdline entrypoint for converting MyST to XML.""" + _run_cli("xml", "Docutils-native XML", argv) + + +def cli_pseudoxml(argv: Optional[List[str]] = None): + """Cmdline entrypoint for converting MyST to pseudo-XML.""" + _run_cli("pseudoxml", "pseudo-XML", argv) diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py index d0dc4b2c..4ae60f4d 100644 --- a/myst_parser/docutils_renderer.py +++ b/myst_parser/docutils_renderer.py @@ -52,9 +52,9 @@ from .utils import is_external_url -def make_document(source_path="notset") -> nodes.document: - """Create a new docutils document.""" - settings = OptionParser(components=(RSTParser,)).get_default_values() +def make_document(source_path="notset", parser_cls=RSTParser) -> nodes.document: + """Create a new docutils document, with the parser classes' default settings.""" + settings = OptionParser(components=(parser_cls,)).get_default_values() return new_document(source_path, settings=settings) @@ -474,6 +474,15 @@ def render_fence(self, token: SyntaxTreeNode) -> None: self.add_line_and_source_path(node, token) self.current_node.append(node) + @property + def blocks_mathjax_processing(self) -> bool: + """Only add mathjax ignore classes if using sphinx and myst_update_mathjax is True.""" + return ( + self.sphinx_env is not None + and "myst_update_mathjax" in self.sphinx_env.config + and self.sphinx_env.config.myst_update_mathjax + ) + def render_heading(self, token: SyntaxTreeNode) -> None: if self.md_env.get("match_titles", None) is False: @@ -494,13 +503,7 @@ def render_heading(self, token: SyntaxTreeNode) -> None: self.add_line_and_source_path(title_node, token) new_section = nodes.section() - if level == 1 and ( - self.sphinx_env is None - or ( - "myst_update_mathjax" in self.sphinx_env.config - and self.sphinx_env.config.myst_update_mathjax - ) - ): + if level == 1 and self.blocks_mathjax_processing: new_section["classes"].extend(["tex2jax_ignore", "mathjax_ignore"]) self.add_line_and_source_path(new_section, token) new_section.append(title_node) diff --git a/myst_parser/main.py b/myst_parser/main.py index be5d5991..7af3f0f0 100644 --- a/myst_parser/main.py +++ b/myst_parser/main.py @@ -38,19 +38,46 @@ class MdParserConfig: renderer: str = attr.ib( default="sphinx", validator=in_(["sphinx", "html", "docutils"]) ) - commonmark_only: bool = attr.ib(default=False, validator=instance_of(bool)) - enable_extensions: Iterable[str] = attr.ib(factory=lambda: ["dollarmath"]) + commonmark_only: bool = attr.ib( + default=False, + validator=instance_of(bool), + metadata={"help": "Use strict CommonMark parser"}, + ) + enable_extensions: Iterable[str] = attr.ib( + factory=lambda: ["dollarmath"], metadata={"help": "Enable extensions"} + ) - dmath_allow_labels: bool = attr.ib(default=True, validator=instance_of(bool)) - dmath_allow_space: bool = attr.ib(default=True, validator=instance_of(bool)) - dmath_allow_digits: bool = attr.ib(default=True, validator=instance_of(bool)) - dmath_double_inline: bool = attr.ib(default=False, validator=instance_of(bool)) + dmath_allow_labels: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={"help": "Parse `$$...$$ (label)`"}, + ) + dmath_allow_space: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={"help": "dollarmath: allow initial/final spaces in `$ ... $`"}, + ) + dmath_allow_digits: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={"help": "dollarmath: allow initial/final digits `1$ ...$2`"}, + ) + dmath_double_inline: bool = attr.ib( + default=False, + validator=instance_of(bool), + metadata={"help": "dollarmath: parse inline `$$ ... $$`"}, + ) - update_mathjax: bool = attr.ib(default=True, validator=instance_of(bool)) + update_mathjax: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={"help": "Update sphinx.ext.mathjax configuration"}, + ) mathjax_classes: str = attr.ib( default="tex2jax_process|mathjax_process|math|output_area", validator=instance_of(str), + metadata={"help": "MathJax classes to add to math HTML"}, ) @enable_extensions.validator @@ -79,29 +106,40 @@ def check_extensions(self, attribute, value): disable_syntax: Iterable[str] = attr.ib( factory=list, validator=deep_iterable(instance_of(str), instance_of((list, tuple))), + metadata={"help": "Disable syntax elements"}, ) # see https://en.wikipedia.org/wiki/List_of_URI_schemes url_schemes: Optional[Iterable[str]] = attr.ib( default=cast(Optional[Iterable[str]], ("http", "https", "mailto", "ftp")), validator=optional(deep_iterable(instance_of(str), instance_of((list, tuple)))), + metadata={"help": "URL schemes to allow in links"}, ) heading_anchors: Optional[int] = attr.ib( - default=None, validator=optional(in_([1, 2, 3, 4, 5, 6, 7])) + default=None, + validator=optional(in_([1, 2, 3, 4, 5, 6, 7])), + metadata={"help": "Heading level depth to assign HTML anchors"}, ) heading_slug_func: Optional[Callable[[str], str]] = attr.ib( - default=None, validator=optional(is_callable()) + default=None, + validator=optional(is_callable()), + metadata={"help": "Function for creating heading anchors"}, ) html_meta: Dict[str, str] = attr.ib( factory=dict, validator=deep_mapping(instance_of(str), instance_of(str), instance_of(dict)), repr=lambda v: str(list(v)), + metadata={"help": "HTML meta tags"}, ) - footnote_transition: bool = attr.ib(default=True, validator=instance_of(bool)) + footnote_transition: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={"help": "Place a transition before any footnotes"}, + ) substitutions: Dict[str, Union[str, int, float]] = attr.ib( factory=dict, @@ -109,11 +147,18 @@ def check_extensions(self, attribute, value): instance_of(str), instance_of((str, int, float)), instance_of(dict) ), repr=lambda v: str(list(v)), + metadata={"help": "Substitutions"}, ) - sub_delimiters: Tuple[str, str] = attr.ib(default=("{", "}")) + sub_delimiters: Tuple[str, str] = attr.ib( + default=("{", "}"), metadata={"help": "Substitution delimiters"} + ) - words_per_minute: int = attr.ib(default=200, validator=instance_of(int)) + words_per_minute: int = attr.ib( + default=200, + validator=instance_of(int), + metadata={"help": "For reading speed calculations"}, + ) @sub_delimiters.validator def check_sub_delimiters(self, attribute, value): @@ -270,6 +315,10 @@ def to_docutils( def to_html(text: str, env=None, config: Optional[MdParserConfig] = None): + """Render text to HTML directly using markdown-it-py. + + This is mainly for test purposes only. + """ config = config or MdParserConfig() config.renderer = "html" md = default_parser(config) diff --git a/myst_parser/sphinx_.py b/myst_parser/sphinx_.py index e59ae62a..e7979c80 100644 --- a/myst_parser/sphinx_.py +++ b/myst_parser/sphinx_.py @@ -1,6 +1,6 @@ """A module for compatibility with the docutils>=0.17 `include` directive, in RST documents:: - .. include:: + .. include:: path/to/file.md :parser: myst_parser.sphinx_ """ from myst_parser.sphinx_parser import MystParser as Parser # noqa: F401 diff --git a/setup.cfg b/setup.cfg index 7c440a98..4f7ddcbe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,11 @@ zip_safe = True [options.entry_points] console_scripts = myst-anchors = myst_parser.cli:print_anchors + myst-docutils-html = myst_parser.docutils_:cli_html + myst-docutils-html5 = myst_parser.docutils_:cli_html5 + myst-docutils-latex = myst_parser.docutils_:cli_latex + myst-docutils-xml = myst_parser.docutils_:cli_xml + myst-docutils-pseudoxml = myst_parser.docutils_:cli_pseudoxml [options.extras_require] code_style = diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 4cbb4fc9..52d32810 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -1,12 +1,104 @@ -from myst_parser.docutils_ import Parser +import io +from textwrap import dedent + +import pytest +from docutils import VersionInfo, __version_info__ + +from myst_parser.docutils_ import ( + Parser, + cli_html, + cli_html5, + cli_latex, + cli_pseudoxml, + cli_xml, +) from myst_parser.docutils_renderer import make_document def test_parser(): + """Test calling `Parser.parse` directly.""" parser = Parser() - document = make_document() + document = make_document(parser_cls=Parser) parser.parse("something", document) assert ( document.pformat().strip() == '\n \n something' ) + + +def test_cli_html(monkeypatch, capsys): + monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) + cli_html([]) + captured = capsys.readouterr() + assert not captured.err + assert "text" in captured.out + + +def test_cli_html5(monkeypatch, capsys): + monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) + cli_html5([]) + captured = capsys.readouterr() + assert not captured.err + assert "text" in captured.out + + +def test_cli_latex(monkeypatch, capsys): + monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) + cli_latex([]) + captured = capsys.readouterr() + assert not captured.err + assert "text" in captured.out + + +def test_cli_xml(monkeypatch, capsys): + monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) + cli_xml([]) + captured = capsys.readouterr() + assert not captured.err + assert "text" in captured.out + + +def test_cli_pseudoxml(monkeypatch, capsys): + monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) + cli_pseudoxml([]) + captured = capsys.readouterr() + assert not captured.err + assert "text" in captured.out + + +def test_help_text(): + """Test retrieving settings help text.""" + from docutils.frontend import OptionParser + + stream = io.StringIO() + OptionParser(components=(Parser,)).print_help(stream) + assert "MyST options" in stream.getvalue() + + +@pytest.mark.skipif( + __version_info__ < VersionInfo(0, 17, 0, "final", 0, True), + reason="parser option added in docutils 0.17", +) +def test_include_from_rst(tmp_path): + """Test including a MyST file from within an RST file.""" + from docutils.parsers.rst import Parser as RSTParser + + include_path = tmp_path.joinpath("include.md") + include_path.write_text("# Title") + + parser = RSTParser() + document = make_document(parser_cls=RSTParser) + parser.parse( + f".. include:: {include_path}\n :parser: myst_parser.docutils_", document + ) + assert ( + document.pformat().strip() + == dedent( + """\ + +
+ + Title + """ + ).strip() + ) diff --git a/tests/test_sphinx/sourcedirs/include_from_rst/conf.py b/tests/test_sphinx/sourcedirs/include_from_rst/conf.py new file mode 100644 index 00000000..a743e3c9 --- /dev/null +++ b/tests/test_sphinx/sourcedirs/include_from_rst/conf.py @@ -0,0 +1,2 @@ +extensions = ["myst_parser"] +exclude_patterns = ["_build", "include.md"] diff --git a/tests/test_sphinx/sourcedirs/include_from_rst/include.md b/tests/test_sphinx/sourcedirs/include_from_rst/include.md new file mode 100644 index 00000000..20f34b9c --- /dev/null +++ b/tests/test_sphinx/sourcedirs/include_from_rst/include.md @@ -0,0 +1,5 @@ +# Markdown + +[target] + +[target]: http://example.com/ diff --git a/tests/test_sphinx/sourcedirs/include_from_rst/index.rst b/tests/test_sphinx/sourcedirs/include_from_rst/index.rst new file mode 100644 index 00000000..0bb31777 --- /dev/null +++ b/tests/test_sphinx/sourcedirs/include_from_rst/index.rst @@ -0,0 +1,5 @@ +Title +===== + +.. include:: include.md + :parser: myst_parser.sphinx_ diff --git a/tests/test_sphinx/test_sphinx_builds.py b/tests/test_sphinx/test_sphinx_builds.py index f0c5ebbd..716772b7 100644 --- a/tests/test_sphinx/test_sphinx_builds.py +++ b/tests/test_sphinx/test_sphinx_builds.py @@ -12,6 +12,7 @@ import pytest import sphinx +from docutils import VersionInfo, __version_info__ SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "sourcedirs")) @@ -269,6 +270,38 @@ def test_includes( ) +@pytest.mark.skipif( + __version_info__ < VersionInfo(0, 17, 0, "final", 0, True), + reason="parser option added in docutils 0.17", +) +@pytest.mark.sphinx( + buildername="html", + srcdir=os.path.join(SOURCE_DIR, "include_from_rst"), + freshenv=True, +) +def test_include_from_rst( + app, + status, + warning, + get_sphinx_app_doctree, + get_sphinx_app_output, + remove_sphinx_builds, +): + """Test of include directive inside RST file.""" + app.build() + + assert "build succeeded" in status.getvalue() # Build succeeded + warnings = warning.getvalue().strip() + assert warnings == "" + + get_sphinx_app_doctree( + app, + docname="index", + regress=True, + regress_ext=".xml", + ) + + @pytest.mark.sphinx( buildername="html", srcdir=os.path.join(SOURCE_DIR, "footnotes"), freshenv=True ) diff --git a/tests/test_sphinx/test_sphinx_builds/test_include_from_rst.xml b/tests/test_sphinx/test_sphinx_builds/test_include_from_rst.xml new file mode 100644 index 00000000..6f325067 --- /dev/null +++ b/tests/test_sphinx/test_sphinx_builds/test_include_from_rst.xml @@ -0,0 +1,10 @@ +<document source="index.rst"> + <section ids="title" names="title"> + <title> + Title + <section classes="tex2jax_ignore mathjax_ignore" ids="markdown" names="markdown"> + <title> + Markdown + <paragraph> + <reference refuri="http://example.com/"> + target