gen_single.py

#  Authors: Sylvain MARIE <sylvain.marie@se.com>
#            + All contributors to <https://github.com/smarie/mkdocs-gallery>
#
#  Original idea and code: sphinx-gallery, <https://sphinx-gallery.github.io>
#  License: 3-clause BSD, <https://github.com/smarie/mkdocs-gallery/blob/master/LICENSE>
"""
Generator for a single script example in a gallery.
"""

from __future__ import absolute_import, division, print_function

import ast
import codeop
import contextlib
import copy
import gc
import importlib
import os
import pickle
import re
import subprocess
import sys
import traceback
import warnings
from copy import deepcopy
from functools import partial
from io import StringIO
from pathlib import Path
from shutil import copyfile
from textwrap import indent
from time import time
from typing import List, Set, Tuple

from tqdm import tqdm

from . import glr_path_static, mkdocs_compatibility
from .backreferences import _thumbnail_div, _write_backreferences, identify_names
from .binder import check_binder_conf, gen_binder_md
from .errors import ExtensionError
from .gen_data_model import GalleryBase, GalleryScript, GalleryScriptResults
from .notebook import jupyter_notebook, save_notebook
from .py_source_parser import remove_config_comments, split_code_and_text_blocks
from .scrapers import ImageNotFoundError, _find_image_ext, clean_modules, save_figures
from .utils import _new_file, _replace_by_new_if_needed, optipng, rescale_image

logger = mkdocs_compatibility.getLogger("mkdocs-gallery")


###############################################################################


class _LoggingTee(object):
    """A tee object to redirect streams to the logger."""

    def __init__(self, src_filename: Path):
        self.logger = logger
        self.src_filename = src_filename
        self.logger_buffer = ""
        self.set_std_and_reset_position()

    def set_std_and_reset_position(self):
        if not isinstance(sys.stdout, _LoggingTee):
            self.origs = (sys.stdout, sys.stderr)
        sys.stdout = sys.stderr = self
        self.first_write = True
        self.output = StringIO()
        return self

    def restore_std(self):
        sys.stdout.flush()
        sys.stderr.flush()
        sys.stdout, sys.stderr = self.origs

    def write(self, data):
        self.output.write(data)

        if self.first_write:
            self.logger.verbose("Output from %s", self.src_filename)  # color='brown')
            self.first_write = False

        data = self.logger_buffer + data
        lines = data.splitlines()
        if data and data[-1] not in "\r\n":
            # Wait to write last line if it's incomplete. It will write next
            # time or when the LoggingTee is flushed.
            self.logger_buffer = lines[-1]
            lines = lines[:-1]
        else:
            self.logger_buffer = ""

        for line in lines:
            self.logger.verbose("%s", line)

    def flush(self):
        self.output.flush()
        if self.logger_buffer:
            self.logger.verbose("%s", self.logger_buffer)
            self.logger_buffer = ""

    # When called from a local terminal seaborn needs it in Python3
    def isatty(self):
        return self.output.isatty()

    # When called in gen_single, conveniently use context managing
    def __enter__(self):
        return self

    def __exit__(self, type_, value, tb):
        self.restore_std()


###############################################################################
# The following strings are used when we have several pictures: we use
# an html div tag that our CSS uses to turn the lists into horizontal
# lists.
HLIST_HEADER = """
{: .mkd-glr-horizontal }

"""

HLIST_IMAGE_TEMPLATE = """
    *

      .. image:: /%s
            {: .mkd-glr-multi-img }
"""

SINGLE_IMAGE = """
.. image:: /%s
    {: .mkd-glr-single-img }
"""

# Note: since this seems to be a one-liner, we use inline code. TODO check
CODE_OUTPUT = """Out:
{{: .mkd-glr-script-out }}

```{{.shell .mkd-glr-script-out-disp }}
{0}
```
\n"""

TIMING_CONTENT = """
**Total running time of the script:** ({0: .0f} minutes {1: .3f} seconds)
"""  # Strange enough: this CSS class does not actually exist in sphinx-gallery {{: .mkd-glr-timing }}

# TODO only if html ?   .. only:: html
MKD_GLR_SIG = """\n
[Gallery generated by mkdocs-gallery](https://mkdocs-gallery.github.io){: .mkd-glr-signature }
"""

# Header used to include raw html from data _repr_html_
HTML_HEADER = """<div class="output_subarea output_html rendered_html output_result">
{0}
</div>
"""


def codestr2md(codestr, lang: str = "python", lineno=None, is_exc: bool = False):
    """Return markdown code block from code string."""

    # if lineno is not None:
    #     # Sphinx only starts numbering from the first non-empty line.
    #     blank_lines = codestr.count('\n', 0, -len(codestr.lstrip()))
    #     lineno = '   :lineno-start: {0}\n'.format(lineno + blank_lines)
    # else:
    #     lineno = ''
    # code_directive = ".. code-block:: {0}\n{1}\n".format(lang, lineno)
    # indented_block = indent(codestr, ' ' * 4)
    # return code_directive + indented_block
    style = " .mkd-glr-script-err-disp" if is_exc else ""
    if lineno is not None:
        # Sphinx only starts numbering from the first non-empty line. TODO do we need this too ?
        #     blank_lines = codestr.count('\n', 0, -len(codestr.lstrip()))
        return f'```{{.{lang} {style} linenums="{lineno}"}}\n{codestr}```\n'
    else:
        return f"```{{.{lang} {style}}}\n{codestr}```\n"


def _regroup(x):
    x = x.groups()
    return x[0] + x[1].split(".")[-1] + x[2]


def _sanitize_md(string):
    """Use regex to remove at least some sphinx directives.

    TODO is this still needed ?
    """
    # :class:`a.b.c <thing here>`, :ref:`abc <thing here>` --> thing here
    p, e = r"(\s|^):[^:\s]+:`", r"`(\W|$)"
    string = re.sub(p + r"\S+\s*<([^>`]+)>" + e, r"\1\2\3", string)
    # :class:`~a.b.c` --> c
    string = re.sub(p + r"~([^`]+)" + e, _regroup, string)
    # :class:`a.b.c` --> a.b.c
    string = re.sub(p + r"([^`]+)" + e, r"\1\2\3", string)

    # ``whatever thing`` --> whatever thing
    p = r"(\s|^)`"
    string = re.sub(p + r"`([^`]+)`" + e, r"\1\2\3", string)
    # `whatever thing` --> whatever thing
    string = re.sub(p + r"([^`]+)" + e, r"\1\2\3", string)
    return string


# Find RST/Markdown title chars,
# i.e. lines that consist of (3 or more of the same) 7-bit non-ASCII chars.
# This conditional is not perfect but should hopefully be good enough.
RE_3_OR_MORE_NON_ASCII = r"([\W _])\1{3,}"  # 3 or more identical chars

RST_TITLE_MARKER = re.compile(rf"^[ ]*{RE_3_OR_MORE_NON_ASCII}[ ]*$")
MD_TITLE_MARKER = re.compile(r"^[ ]*[#]+[ ]*(.*)[ ]*$")  # One or more starting hash with optional whitespaces before.
FIRST_NON_MARKER_WITHOUT_HASH = re.compile(rf"^[# ]*(?!{RE_3_OR_MORE_NON_ASCII})[# ]*(.+)", re.MULTILINE)


def extract_readme_title(file: Path, contents: str) -> str:
    """Same as `extract_intro_and_title` for the readme files in galleries, but does not return the introduction.

    Parameters
    ----------
    file : Path
        The readme file path (used for error messages only).

    contents : str
        The already parsed readme contents

    Returns
    -------
    title : str
        The readme title
    """
    # Remove html comments.
    contents = re.sub("(<!--.*?-->)", "", contents, flags=re.DOTALL)

    match = FIRST_NON_MARKER_WITHOUT_HASH.search(contents)
    if match is None:
        raise ExtensionError(f"Could not find a title in readme file: {file}")

    title = match.group(2).strip()
    return title


def extract_readme_last_subtitle(file: Path, contents: str) -> str:
    """Same as `extract_intro_and_title` for the readme files in galleries, but does not return the introduction.

    Parameters
    ----------
    file : Path
        The readme file path (used for error messages only).

    contents : str
        The already parsed readme contents

    Returns
    -------
    last_subtitle : str
        The readme last title, or None.
    """
    paragraphs = extract_paragraphs(contents)

    # iterate from last paragraph
    last_subtitle = None
    for p in reversed(paragraphs):
        current_is_good = False
        for line in reversed(p.splitlines()):
            if current_is_good:
                last_subtitle = line
                break
            # Does this line contain a title ?
            # - md style
            md_match = MD_TITLE_MARKER.search(line)
            if md_match:
                last_subtitle = md_match.group(1)
                break

            # - rst style
            rst_match = RST_TITLE_MARKER.search(line)
            if rst_match:
                current_is_good = True

        if last_subtitle:
            break

    return last_subtitle


def extract_paragraphs(doc: str) -> List[str]:
    # lstrip is just in case docstring has a '\n\n' at the beginning
    paragraphs = doc.lstrip().split("\n\n")

    # remove comments and other syntax like `.. _link:`
    paragraphs = [p for p in paragraphs if not p.startswith(".. ") and len(p) > 0]

    return paragraphs


def extract_intro_and_title(docstring: str, script: GalleryScript) -> Tuple[str, str]:
    """Extract and clean the first paragraph of module-level docstring.

    The title is not saved in the `script` object in this process, users have to do it explicitly.

    Parameters
    ----------
    docstring : str
        The docstring extracted from the top of the script.

    script : GalleryScript
        The script where the docstring was extracted from (used for error messages only).

    Returns
    -------
    title : str
        The title

    introduction : str
        The introduction
    """
    # Extract paragraphs from the text
    paragraphs = extract_paragraphs(docstring)
    if len(paragraphs) == 0:
        raise ExtensionError(
            f"Example docstring should have a header for the example title. "
            f"Please check the example file:\n {script.script_file}\n"
        )

    # Title is the first paragraph with any RST/Markdown title chars
    # removed, i.e. lines that consist of (3 or more of the same) 7-bit
    # non-ASCII chars.
    # This conditional is not perfect but should hopefully be good enough.
    title_paragraph = paragraphs[0]
    match = FIRST_NON_MARKER_WITHOUT_HASH.search(title_paragraph)
    if match is None:
        raise ExtensionError(f"Could not find a title in first paragraph:\n{title_paragraph}")

    title = match.group(2).strip()

    # Use the title if no other paragraphs are provided
    intro_paragraph = title if len(paragraphs) < 2 else paragraphs[1]

    # Concatenate all lines of the first paragraph
    intro = re.sub("\n", " ", intro_paragraph)
    intro = _sanitize_md(intro)

    # Truncate at 95 chars
    if len(intro) > 95:
        intro = intro[:95] + "..."

    return title, intro


def create_thumb_from_image(script: GalleryScript, src_image_path: Path) -> Path:
    """Create a thumbnail image from the `src_image_path`.

    Parameters
    ----------
    script : GalleryScript
        The gallery script.

    src_image_path : Path
        The source image path, with some flexibility about the extension.
        TODO do we actually need this flexibility here ?

    Returns
    -------
    actual_thumb_file : Path
        The actual thumbnail file generated.
    """
    try:
        # Find the image, with flexibility about the actual extenstion ('png', 'svg', 'jpg', 'gif' are supported)
        src_image_path, ext = _find_image_ext(src_image_path)
    except ImageNotFoundError:
        # The source image does not exist !
        try:
            # Does a thumbnail already exist ? with extenstion ('png', 'svg', 'jpg', 'gif')
            thumb_file, ext = _find_image_ext(script.get_thumbnail_file(".png"))
            # Yes - let's assume this one will suit the needs
            return thumb_file
        except ImageNotFoundError:
            # Create something to replace the thumbnail
            default_thumb_path = script.gallery_conf.get("default_thumb_file")
            if default_thumb_path is None:
                default_thumb_path = os.path.join(glr_path_static(), "no_image.png")

            src_image_path, ext = _find_image_ext(Path(default_thumb_path))

    # Now let's create the thumbnail.
    # - First Make sure the thumb dir exists
    script.gallery.make_thumb_dir()

    # - Then create the thum file by copying the src image, possibly rescaling it.
    thumb_file = script.get_thumbnail_file(ext)
    if ext in (".svg", ".gif"):
        # No need to rescale image
        copyfile(src_image_path, thumb_file)
    else:
        # Need to rescale image
        max_width, max_hegiht = script.gallery_conf["thumbnail_size"]
        rescale_image(
            in_file=src_image_path,
            out_file=thumb_file,
            max_width=max_width,
            max_height=max_hegiht,
        )
        if "thumbnails" in script.gallery_conf["compress_images"]:
            optipng(thumb_file, script.gallery_conf["compress_images_args"])

    return thumb_file


def generate(gallery: GalleryBase, seen_backrefs: Set) -> Tuple[str, str, str, List[GalleryScriptResults]]:
    """
    Generate the gallery md for an example directory, including the index.

    Parameters
    ----------
    gallery : GalleryBase
        The gallery or subgallery to process

    seen_backrefs : Set
        Backrefs seen so far.

    Returns
    -------
    title : str
        The gallery title, that is, the title of the readme file.

    root_subtitle : str
        The gallery suptitle that will be used in case the gallery has subsections.

    index_md : str
        The markdown to include in the global gallery readme.

    results : List[GalleryScriptResults]
        A list of processing results for all scripts in this gallery.
    """
    # Read the gallery readme and add it to the index
    readme_contents = gallery.readme_file.read_text(encoding="utf-8")
    readme_title = extract_readme_title(gallery.readme_file, readme_contents)
    if gallery.has_subsections():
        # parse and try to also extract the last subtitle
        last_readme_subtitle = extract_readme_last_subtitle(gallery.readme_file, readme_contents)
    else:
        # Dont look for the last subtitle
        last_readme_subtitle = None

    # Create the destination dir if needed
    gallery.make_generated_dir()

    all_thumbnail_entries = []
    results = []

    for script in tqdm(gallery.scripts, desc=f"generating gallery for {gallery.generated_dir}... "):
        # Generate all files related to this example: download file, jupyter notebook, pickle, markdown...
        script_results = generate_file_md(script=script, seen_backrefs=seen_backrefs)
        results.append(script_results)

        # Create the thumbnails-containing div <div class="mkd-glr-thumbcontainer" ...> to place in the readme
        thumb_div = _thumbnail_div(script_results)
        all_thumbnail_entries.append(thumb_div)

    # Write the gallery summary index.md
    index_md = f"""<!-- {str(gallery.generated_dir_rel_project).replace(os.path.sep, '_')} -->

{readme_contents}

{"".join(all_thumbnail_entries)}
<div class="mkd-glr-clear"></div>


"""
    # Note: the "clear" is to disable floating elements again, now that the gallery section is over.

    return readme_title, last_readme_subtitle, index_md, results


def is_failing_example(script: GalleryScript):
    return script.src_py_file in script.gallery_conf["failing_examples"]


def handle_exception(exc_info, script: GalleryScript):
    """Trim and format exception, maybe raise error, etc."""
    from .gen_gallery import _expected_failing_examples

    etype, exc, tb = exc_info
    stack = traceback.extract_tb(tb)
    # The full traceback will look something like:
    #
    #   File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/gen_single.py...
    #     mem_max, _ = gallery_conf['call_memory'](
    #   File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/gen_galler...
    #     mem, out = memory_usage(func, max_usage=True, retval=True,
    #   File "/home/larsoner/.local/lib/python3.8/site-packages/memory_profi...
    #     returned = f(*args, **kw)
    #   File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/gen_single.py...
    #     exec(self.code, self.fake_main.__dict__)
    #   File "/home/larsoner/python/mkdocs-gallery/sphinx_gallery/tests/tiny...
    #     raise RuntimeError('some error')
    # RuntimeError: some error
    #
    # But we should trim these to just the relevant trace at the user level,
    # so we inspect the traceback to find the start and stop points.
    start = 0
    stop = len(stack)
    root = os.path.dirname(__file__) + os.sep
    for ii, s in enumerate(stack, 1):
        # Trim our internal stack
        if s.filename.startswith(root + "gen_gallery.py") and s.name == "call_memory":
            start = max(ii, start)
        elif s.filename.startswith(root + "gen_single.py"):
            # SyntaxError
            if s.name == "execute_code_block" and ("compile(" in s.line or "save_figures" in s.line):
                start = max(ii, start)
            # Any other error
            elif s.name == "__call__":
                start = max(ii, start)
            # Our internal input() check
            elif s.name == "_check_input" and ii == len(stack):
                stop = ii - 1
    stack = stack[start:stop]

    formatted_exception = "Traceback (most recent call last):\n" + "".join(
        traceback.format_list(stack) + traceback.format_exception_only(etype, exc)
    )

    src_file = script.src_py_file
    expected = src_file in _expected_failing_examples(
        gallery_conf=script.gallery_conf,
        mkdocs_conf=script.gallery.all_info.mkdocs_conf,
    )
    if expected:
        # func, color = logger.info, 'blue'
        func = logger.info
    else:
        # func, color = logger.warning, 'red'
        func = logger.warning
    func(f"{src_file} failed to execute correctly: {formatted_exception}")  # , color=color)

    except_md = codestr2md(formatted_exception, lang="pytb", is_exc=True)

    # Ensure it's marked as our style: this is now already done in codestr2md
    # except_md = "{: .mkd-glr-script-out }\n\n" + except_md
    return except_md, formatted_exception


# Adapted from github.com/python/cpython/blob/3.7/Lib/warnings.py
def _showwarning(message, category, filename, lineno, file=None, line=None):
    if file is None:
        file = sys.stderr
        if file is None:
            # sys.stderr is None when run with pythonw.exe:
            # warnings get lost
            return
    text = warnings.formatwarning(message, category, filename, lineno, line)
    try:
        file.write(text)
    except OSError:
        # the file (probably stderr) is invalid - this warning gets lost.
        pass


@contextlib.contextmanager
def patch_warnings():
    """Patch warnings.showwarning to actually write out the warning."""
    # Sphinx or logging or someone is patching warnings, but we want to
    # capture them, so let's patch over their patch...
    orig_showwarning = warnings.showwarning
    try:
        warnings.showwarning = _showwarning
        yield
    finally:
        warnings.showwarning = orig_showwarning


class _exec_once(object):
    """Deal with memory_usage calling functions more than once (argh)."""

    def __init__(self, code, fake_main):
        self.code = code
        self.fake_main = fake_main
        self.run = False

    def __call__(self):
        if not self.run:
            self.run = True
            old_main = sys.modules.get("__main__", None)
            with patch_warnings():
                sys.modules["__main__"] = self.fake_main
                try:
                    exec(self.code, self.fake_main.__dict__)  # noqa  # our purpose is to execute code :)
                finally:
                    if old_main is not None:
                        sys.modules["__main__"] = old_main


def _get_memory_base(gallery_conf):
    """Get the base amount of memory used by running a Python process."""
    if not gallery_conf["plot_gallery"]:
        return 0.0
    # There might be a cleaner way to do this at some point
    from memory_profiler import memory_usage

    if sys.platform in ("win32", "darwin"):
        sleep, timeout = (1, 2)
    else:
        sleep, timeout = (0.5, 1)
    proc = subprocess.Popen(
        [sys.executable, "-c", "import time, sys; time.sleep(%s); sys.exit(0)" % sleep],
        close_fds=True,
    )
    memories = memory_usage(proc, interval=1e-3, timeout=timeout)
    kwargs = dict(timeout=timeout) if sys.version_info >= (3, 5) else {}
    proc.communicate(**kwargs)
    # On OSX sometimes the last entry can be None
    memories = [mem for mem in memories if mem is not None] + [0.0]
    memory_base = max(memories)
    return memory_base


def _ast_module():
    """Get ast.Module function, dealing with:
    https://bugs.python.org/issue35894"""
    if sys.version_info >= (3, 8):
        ast_Module = partial(ast.Module, type_ignores=[])
    else:
        ast_Module = ast.Module
    return ast_Module


def _check_reset_logging_tee(src_file: Path):
    # Helper to deal with our tests not necessarily calling parse_and_execute
    # but rather execute_code_block directly
    if isinstance(sys.stdout, _LoggingTee):
        logging_tee = sys.stdout
    else:
        logging_tee = _LoggingTee(src_file)
    logging_tee.set_std_and_reset_position()
    return logging_tee


def _exec_and_get_memory(compiler, ast_Module, code_ast, script: GalleryScript):
    """Execute ast, capturing output if last line is expression and get max memory usage."""

    src_file = script.src_py_file.as_posix()

    # capture output if last line is expression
    is_last_expr = False

    if len(code_ast.body) and isinstance(code_ast.body[-1], ast.Expr):
        is_last_expr = True
        last_val = code_ast.body.pop().value
        # exec body minus last expression
        mem_body, _ = script.gallery_conf["call_memory"](
            _exec_once(compiler(code_ast, src_file, "exec"), script.run_vars.fake_main)
        )
        # exec last expression, made into assignment
        body = [ast.Assign(targets=[ast.Name(id="___", ctx=ast.Store())], value=last_val)]
        last_val_ast = ast_Module(body=body)
        ast.fix_missing_locations(last_val_ast)
        mem_last, _ = script.gallery_conf["call_memory"](
            _exec_once(compiler(last_val_ast, src_file, "exec"), script.run_vars.fake_main)
        )
        mem_max = max(mem_body, mem_last)
    else:
        mem_max, _ = script.gallery_conf["call_memory"](
            _exec_once(compiler(code_ast, src_file, "exec"), script.run_vars.fake_main)
        )

    return is_last_expr, mem_max


def _get_last_repr(gallery_conf, ___):
    """Get a repr of the last expression, using first method in 'capture_repr'
    available for the last expression."""
    for meth in gallery_conf["capture_repr"]:
        try:
            last_repr = getattr(___, meth)()
            # for case when last statement is print()
            if last_repr is None or last_repr == "None":
                repr_meth = None
            else:
                repr_meth = meth
        except Exception:
            last_repr = None
            repr_meth = None
        else:
            if isinstance(last_repr, str):
                break
    return last_repr, repr_meth


def _get_code_output(is_last_expr, script: GalleryScript, logging_tee, images_md):
    """Obtain standard output and html output in md."""

    example_globals = script.run_vars.example_globals
    gallery_conf = script.gallery_conf

    last_repr = None
    repr_meth = None
    if is_last_expr:
        # capture the last repr variable
        ___ = example_globals["___"]
        ignore_repr = False
        if gallery_conf["ignore_repr_types"]:
            ignore_repr = re.search(gallery_conf["ignore_repr_types"], str(type(___)))
        if gallery_conf["capture_repr"] != () and not ignore_repr:
            last_repr, repr_meth = _get_last_repr(gallery_conf, ___)

    captured_std = logging_tee.output.getvalue().expandtabs()

    # normal string output
    if repr_meth in ["__repr__", "__str__"] and last_repr:
        captured_std = f"{captured_std}\n{last_repr}"

    if captured_std and not captured_std.isspace():
        captured_std = CODE_OUTPUT.format(captured_std)
    else:
        captured_std = ""

    # give html output its own header
    if repr_meth == "_repr_html_":
        captured_html = HTML_HEADER.format(indent(last_repr, " " * 4))
    else:
        captured_html = ""

    code_output = f"""
{images_md}

{captured_std}

{captured_html}

"""
    return code_output


def _reset_cwd_syspath(cwd, path_to_remove):
    """Reset current working directory to `cwd` and remove `path_to_remove` from `sys.path`."""
    if path_to_remove in sys.path:
        sys.path.remove(path_to_remove)
    os.chdir(cwd)


def execute_code_block(compiler, block, script: GalleryScript):
    """Execute the code block of the example file.

    Parameters
    ----------
    compiler : codeop.Compile
        Compiler to compile AST of code block.

    block : List[Tuple[str, str, int]]
        List of Tuples, each Tuple contains label ('text' or 'code'),
        the corresponding content string of block and the leading line number.

    script: GalleryScript
        The gallery script

    Returns
    -------
    code_output : str
        Output of executing code in md.
    """
    # if script.run_vars.example_globals is None:  # testing shortcut
    #     script.run_vars.example_globals = script.run_vars.fake_main.__dict__

    blabel, bcontent, lineno = block

    # If example is not suitable to run anymore, skip executing its blocks
    if script.run_vars.stop_executing or blabel == "text":
        return ""

    cwd = os.getcwd()
    # Redirect output to stdout
    src_file = script.src_py_file
    logging_tee = _check_reset_logging_tee(src_file)
    assert isinstance(logging_tee, _LoggingTee)  # noqa

    # First cd in the original example dir, so that any file
    # created by the example get created in this directory
    os.chdir(src_file.parent)

    # Add the example dir to the path temporarily (will be removed after execution)
    new_path = os.getcwd()
    sys.path.append(new_path)

    # Save figures unless there is a `mkdocs_gallery_defer_figures` flag
    match = re.search(r"^[\ \t]*#\s*mkdocs_gallery_defer_figures[\ \t]*\n?", bcontent, re.MULTILINE)
    need_save_figures = match is None

    try:
        ast_Module = _ast_module()
        code_ast = ast_Module([bcontent])
        flags = ast.PyCF_ONLY_AST | compiler.flags
        code_ast = compile(bcontent, src_file, "exec", flags, dont_inherit=1)
        ast.increment_lineno(code_ast, lineno - 1)

        is_last_expr, mem_max = _exec_and_get_memory(compiler, ast_Module, code_ast, script=script)
        script.run_vars.memory_used_in_blocks.append(mem_max)

        # This should be inside the try block, e.g., in case of a savefig error
        logging_tee.restore_std()
        if need_save_figures:
            need_save_figures = False
            images_md = save_figures(block, script)
        else:
            images_md = ""

    except Exception:
        logging_tee.restore_std()
        except_md, formatted_exception = handle_exception(sys.exc_info(), script)

        # Breaks build on first example error
        if script.gallery_conf["abort_on_example_error"]:
            raise

        # Stores failing file
        script.gallery_conf["failing_examples"][src_file] = formatted_exception

        # Stop further execution on that script
        script.run_vars.stop_executing = True

        code_output = "\n{0}\n\n\n\n".format(except_md)
        # still call this even though we won't use the images so that
        # figures are closed
        if need_save_figures:
            save_figures(block, script)
    else:
        _reset_cwd_syspath(cwd, new_path)

        code_output = _get_code_output(is_last_expr, script, logging_tee, images_md)
    finally:
        _reset_cwd_syspath(cwd, new_path)
        logging_tee.restore_std()

    # Sanitize ANSI escape characters from MD output
    ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
    code_output = ansi_escape.sub("", code_output)

    return code_output


def _check_input(prompt=None):
    raise ExtensionError("Cannot use input() builtin function in mkdocs-gallery examples")


def parse_and_execute(script: GalleryScript, script_blocks):
    """Execute and capture output from python script already in block structure

    Parameters
    ----------
    script : GalleryScript
        The script

    script_blocks : list
        (label, content, line_number)
        List where each element is a tuple with the label ('text' or 'code'),
        the corresponding content string of block and the leading line number

    Returns
    -------
    output_blocks : list
        List of strings where each element is the restructured text
        representation of the output of each block

    time_elapsed : float
        Time elapsed during execution

    memory_used : float
        Memory used during execution
    """
    # Examples may contain if __name__ == '__main__' guards for in example scikit-learn if the example uses
    # multiprocessing. Here we create a new __main__ module, and temporarily change sys.modules when running our example
    fake_main = importlib.util.module_from_spec(importlib.util.spec_from_loader("__main__", None))
    script.run_vars.fake_main = fake_main

    example_globals = fake_main.__dict__
    example_globals.update(
        {
            # A lot of examples contains 'print(__doc__)' for example in
            # scikit-learn so that running the example prints some useful
            # information. Because the docstring has been separated from
            # the code blocks in mkdocs-gallery, __doc__ is actually
            # __builtin__.__doc__ in the execution context and we do not
            # want to print it
            "__doc__": "",
            # Don't ever support __file__: Issues #166 #212
            # Don't let them use input()
            "input": _check_input,
        }
    )
    script.run_vars.example_globals = example_globals

    # Manipulate the sys.argv before running the example
    # See https://github.com/sphinx-gallery/sphinx-gallery/pull/252

    # Remember the original argv so that we can put them back after run
    argv_orig = sys.argv[:]

    # Remember the original sys.path so that we can reset it after run
    sys_path_orig = deepcopy(sys.path)

    # Python file is the original one (not the copy for download)
    sys.argv[0] = script.src_py_file.as_posix()

    # Allow users to provide additional args through the 'reset_argv' option
    sys.argv[1:] = script.gallery_conf["reset_argv"](script)

    # Perform a garbage collection before starting so that perf kpis are accurate (memory and time)
    gc.collect()

    # Initial memory used
    memory_start, _ = script.gallery_conf["call_memory"](lambda: None)
    script.run_vars.memory_used_in_blocks = [memory_start]  # include at least one entry to avoid max() ever failing

    t_start = time()
    compiler = codeop.Compile()

    # Execute block by block
    output_blocks = list()
    with _LoggingTee(script.src_py_file) as logging_tee:
        for block in script_blocks:
            logging_tee.set_std_and_reset_position()
            output_blocks.append(execute_code_block(compiler, block, script))

    # Compute the elapsed time
    time_elapsed = time() - t_start

    # Set back the sys argv
    sys.argv = argv_orig

    # Set back the sys path
    sys.path = sys_path_orig

    # Write md5 checksum if the example was meant to run (no-plot shall not cache md5sum) and has built correctly
    script.write_final_md5_file()

    # Declare the example as "passing"
    script.gallery_conf["passing_examples"].append(script)

    script.run_vars.memory_delta = max(script.run_vars.memory_used_in_blocks) - memory_start
    memory_used = script.gallery_conf["memory_base"] + script.run_vars.memory_delta

    return output_blocks, time_elapsed, memory_used


def generate_file_md(script: GalleryScript, seen_backrefs=None) -> GalleryScriptResults:
    """Generate the md file for a given example.

    Parameters
    ----------
    script : GalleryScript
        The script to process

    seen_backrefs : set
        The seen backreferences.

    Returns
    -------
    result: FileResult
        The result of running this script
    """
    seen_backrefs = set() if seen_backrefs is None else seen_backrefs

    # Extract the contents of the script
    file_conf, script_blocks, node = split_code_and_text_blocks(script.src_py_file, return_node=True)

    # Extract the title and introduction from the module docstring and save the title in the object
    script.title, intro = extract_intro_and_title(docstring=script_blocks[0][1], script=script)

    # Copy source python script to target folder if it is not there/up to date, so that it can be served/downloaded
    # Note: surprisingly this uses a md5 too, but not the final .md5 persisted on disk.
    script.make_dwnld_py_file()

    # Can the script be entirely skipped (both doc generation and execution) ?
    if not script.has_changed_wrt_persisted_md5():
        # A priori we can...
        skip_and_return = True

        # ...however for executables (not shared modules) we might need to run anyway because of config
        if script.is_executable_example():
            if script.gallery_conf["run_stale_examples"]:
                # Run anyway because config says so.
                skip_and_return = False
            else:
                # Add the example to the "stale examples" before returning
                script.gallery_conf["stale_examples"].append(script.dwnld_py_file)
                # If expected to fail, let's remove it from the 'expected_failing_examples' list,
                # assuming it did when previously executed
                if script.src_py_file in script.gallery_conf["expected_failing_examples"]:
                    script.gallery_conf["expected_failing_examples"].remove(script.src_py_file)

        if skip_and_return:
            # Return with 0 exec time and mem usage, and the existing thumbnail
            thumb_source_path = script.get_thumbnail_source(file_conf)
            thumb_file = create_thumb_from_image(script, thumb_source_path)
            return GalleryScriptResults(script=script, intro=intro, exec_time=0.0, memory=0.0, thumb=thumb_file)

    # Reset matplotlib, seaborn, etc. if needed
    if script.is_executable_example():
        clean_modules(gallery_conf=script.gallery_conf, file=script.src_py_file)

    # Init the runtime vars. Create the images directory and init the image files template
    script.init_before_processing()

    if script.is_executable_example():
        # Note: this writes the md5 checksum if the example was meant to run
        output_blocks, time_elapsed, memory_used = parse_and_execute(script, script_blocks)
        logger.debug(f"{script.src_py_file} ran in : {time_elapsed:.2g} seconds\n")
    else:
        output_blocks = [""] * len(script_blocks)
        time_elapsed = memory_used = 0.0  # don't let the output change
        logger.debug(f"{script.src_py_file} parsed (not executed)\n")

        # Create as many dummy images as required if needed (default none) so that references to script images
        # Can still work, even if the script was not executed (in development mode typically, to go fast).
        # See https://sphinx-gallery.github.io/stable/configuration.html#generating-dummy-images
        nb_dummy_images_to_generate = file_conf.get("dummy_images", None)
        if nb_dummy_images_to_generate is not None:
            if type(nb_dummy_images_to_generate) is not int:
                raise ExtensionError("mkdocs_gallery: 'dummy_images' setting is not a number, got {dummy_image!r}")

            stock_img = os.path.join(glr_path_static(), "no_image.png")
            script.generate_n_dummy_images(img=stock_img, nb=nb_dummy_images_to_generate)

    # Remove the mkdocs-gallery configuration comments from the script if needed
    if script.gallery_conf["remove_config_comments"]:
        script_blocks = [
            (label, remove_config_comments(content), line_number) for label, content, line_number in script_blocks
        ]

    # Remove final empty block, which can occur after config comments are removed
    if script_blocks[-1][1].isspace():
        script_blocks = script_blocks[:-1]
        output_blocks = output_blocks[:-1]

    # Generate the markdown string containing the script prose, code and output.
    example_md = generate_md_from_blocks(script_blocks, output_blocks, file_conf, script.gallery_conf)

    # Write the generated markdown file
    md_header, md_footer = get_example_md_wrapper(script, time_elapsed, memory_used)
    full_md = md_header + example_md + md_footer
    script.save_md_example(full_md)

    # Create the image thumbnail for the gallery summary
    if is_failing_example(script):
        # Failing example thumbnail
        thumb_source_path = Path(os.path.join(glr_path_static(), "broken_example.png"))
    else:
        # Get the thumbnail source image, possibly from config
        thumb_source_path = script.get_thumbnail_source(file_conf)

    thumb_file = create_thumb_from_image(script, thumb_source_path)

    # Generate the jupyter notebook
    example_nb = jupyter_notebook(script, script_blocks)
    ipy_file = _new_file(script.ipynb_file)
    save_notebook(example_nb, ipy_file)
    _replace_by_new_if_needed(ipy_file, md5_mode="t")

    # Write names
    if script.gallery_conf["inspect_global_variables"]:
        global_variables = script.run_vars.example_globals
    else:
        global_variables = None

    # TODO dig in just in case
    example_code_obj = identify_names(script_blocks, global_variables, node)
    if example_code_obj:
        # Write a pickle file (.pickle) containing `example_code_obj`
        codeobj_fname = _new_file(script.codeobj_file)
        with open(codeobj_fname, "wb") as fid:
            pickle.dump(example_code_obj, fid, pickle.HIGHEST_PROTOCOL)
        _replace_by_new_if_needed(codeobj_fname)

    backrefs = set(
        "{module_short}.{name}".format(**cobj)
        for cobjs in example_code_obj.values()
        for cobj in cobjs
        if cobj["module"].startswith(script.gallery_conf["doc_module"])
    )

    # Create results object
    res = GalleryScriptResults(
        script=script,
        intro=intro,
        exec_time=time_elapsed,
        memory=memory_used,
        thumb=thumb_file,
    )

    # Write backreferences if required
    if script.gallery_conf["backreferences_dir"] is not None:
        _write_backreferences(backrefs, seen_backrefs, script_results=res)

    return res


# TODO the note should only appear in html mode. (.. only:: html)
# TODO maybe remove as much as possible the css for now?
EXAMPLE_HEADER = """
<!--
 DO NOT EDIT.
 THIS FILE WAS AUTOMATICALLY GENERATED BY mkdocs-gallery.
 TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
 "{pyfile_to_edit}"
 LINE NUMBERS ARE GIVEN BELOW.
-->

!!! note

    Click [here](#download_links)
    to download the full example code{opt_binder_text}

"""  # TODO there was a {{: .mkd-glr-example-title }} for the title but is it useful ?
MD_BLOCK_HEADER = """\
<!-- GENERATED FROM PYTHON SOURCE LINES {0}-{1} -->

"""


def generate_md_from_blocks(script_blocks, output_blocks, file_conf, gallery_conf) -> str:
    """Generate the md string containing the script prose, code and output.

    Parameters
    ----------
    script_blocks : list
        (label, content, line_number)
        List where each element is a tuple with the label ('text' or 'code'),
        the corresponding content string of block and the leading line number

    output_blocks : list
        List of strings where each element is the restructured text
        representation of the output of each block

    file_conf : dict
        File-specific settings given in source file comments as:
        ``# mkdocs_gallery_<name> = <value>``

    gallery_conf : dict
        Contains the configuration of mkdocs-gallery

    Returns
    -------
    out : str
        The resulting markdown page.
    """

    # A simple example has two blocks: one for the
    # example introduction/explanation and one for the code
    is_example_notebook_like = len(script_blocks) > 2
    example_md = ""
    for bi, ((blabel, bcontent, lineno), code_output) in enumerate(zip(script_blocks, output_blocks)):
        # do not add comment to the title block (bi=0), otherwise the linking does not work properly
        if bi > 0:
            example_md += MD_BLOCK_HEADER.format(lineno, lineno + bcontent.count("\n"))

        if blabel == "code":
            if not file_conf.get("line_numbers", gallery_conf.get("line_numbers", False)):
                lineno = None

            code_md = codestr2md(bcontent, lang=gallery_conf["lang"], lineno=lineno) + "\n"
            if is_example_notebook_like:
                example_md += code_md
                example_md += code_output
            else:
                example_md += code_output
                if "mkd-glr-script-out" in code_output:
                    # Add some vertical space after output
                    example_md += "\n\n<br />\n\n"  # "|\n\n"
                example_md += code_md
        else:
            block_separator = "\n\n" if not bcontent.endswith("\n") else "\n"
            example_md += bcontent + block_separator

    return example_md


def get_example_md_wrapper(script: GalleryScript, time_elapsed: float, memory_used: float) -> Tuple[str, str]:
    """Creates the headers and footers for the example markdown. Returns a template

    Parameters
    ----------
    script : GalleryScript
        The script for which to generate the md.

    time_elapsed : float
        Time elapsed in seconds while executing file

    memory_used : float
        Additional memory used during the run.

    Returns
    -------
    md_before : str
        Part of the final markdown that goes before the notebook / python script.

    md_after : str
        Part of the final markdown that goes after the notebook / python script.
    """
    # Check binder configuration
    binder_conf = check_binder_conf(script.gallery_conf.get("binder"))
    use_binder = len(binder_conf) > 0

    # Write header
    src_relative = script.src_py_file_rel_project.as_posix()
    binder_text = " or to run this example in your browser via Binder" if use_binder else ""
    md_before = EXAMPLE_HEADER.format(pyfile_to_edit=src_relative, opt_binder_text=binder_text)

    # Footer
    md_after = ""
    # Report Time and Memory
    if time_elapsed >= script.gallery_conf["min_reported_time"]:
        time_m, time_s = divmod(time_elapsed, 60)
        md_after += TIMING_CONTENT.format(time_m, time_s)

    if script.gallery_conf["show_memory"]:
        md_after += f"**Estimated memory usage:** {memory_used:.0f} MB\n\n"

    # Download buttons
    # - Generate a binder URL if specified
    binder_badge_md = gen_binder_md(script, binder_conf) if use_binder else ""
    # - Rely on mkdocs-material for the icon
    icon = ":fontawesome-solid-download:"
    # - Generate the download buttons
    # TODO why aren't they centered actually ? does .center work ?
    md_after += f"""
<div id="download_links"></div>

{binder_badge_md}

[{icon} Download Python source code: {script.dwnld_py_file.name}](./{script.dwnld_py_file.name}){{ .md-button .center}}

[{icon} Download Jupyter notebook: {script.ipynb_file.name}](./{script.ipynb_file.name}){{ .md-button .center}}
"""

    # Add the "generated by mkdocs-gallery" footer
    md_after += MKD_GLR_SIG

    return md_before, md_after