Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add support for async galleries #90

Merged
merged 18 commits into from
Jan 16, 2024
48 changes: 48 additions & 0 deletions docs/examples/plot_10_async.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""
# Async support
"""

import asyncio
import time

start = time.time()
await asyncio.sleep(1)
stop = time.time()
f"I waited for {stop - start} seconds!"


# %%
# ## Async Iterator

class AsyncIterator:
async def __aiter__(self):
for chunk in "I'm an async iterator!".split():
yield chunk


async for chunk in AsyncIterator():
print(chunk, end=" ")

# %%
# ## Async comprehensions

" ".join([chunk async for chunk in AsyncIterator()])

# %%
# ## Async content manager


class AsyncContextManager:
async def __aenter__(self):
print("Entering ...")
return self

async def __aexit__(self, *exc_info):
print("Exiting ...")

def __str__(self):
return "I'm an async context manager!"


async with AsyncContextManager() as acm:
print(acm)
66 changes: 62 additions & 4 deletions src/mkdocs_gallery/gen_single.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from io import StringIO
from pathlib import Path
from shutil import copyfile
from textwrap import indent
from textwrap import indent, dedent
from time import time
from typing import List, Set, Tuple

Expand Down Expand Up @@ -739,6 +739,66 @@ def _reset_cwd_syspath(cwd, path_to_remove):
os.chdir(cwd)


def _parse_code(bcontent, src_file, *, compiler_flags):
code_ast = compile(bcontent, src_file, "exec", compiler_flags | ast.PyCF_ONLY_AST, dont_inherit=1)
if _needs_async_handling(bcontent, compiler_flags=compiler_flags):
code_ast = _apply_async_handling(code_ast, compiler_flags=compiler_flags)
return code_ast


def _needs_async_handling(bcontent, *, compiler_flags) -> bool:
try:
compile(bcontent, "<_needs_async_handling()>", "exec", compiler_flags, dont_inherit=1)
except SyntaxError as error:
# FIXME
# bool(re.match(r"'(await|async for|async with)' outside( async)? function", str(error)))
# asynchronous comprehension outside of an asynchronous function
return "async" in str(error)
pmeier marked this conversation as resolved.
Show resolved Hide resolved
except Exception:
return False
else:
return False


def _apply_async_handling(code_ast, *, compiler_flags):
async_handling = compile(
dedent(
"""
async def __async_wrapper__():
# original AST goes here
return locals()
import asyncio as __asyncio__
__async_wrapper_locals__ = __asyncio__.run(__async_wrapper__())
__async_wrapper_result__ = __async_wrapper_locals__.pop("__async_wrapper_result__", None)
globals().update(__async_wrapper_locals__)
__async_wrapper_result__
"""
),
"<_apply_async_handling()>",
"exec",
compiler_flags | ast.PyCF_ONLY_AST,
dont_inherit=1,
)

*original_body, last_node = code_ast.body
if isinstance(last_node, ast.Expr):
# FIXME
# ast.Assign(targets=[ast.Name(id="__async_wrapper_result__", ctx=ast.Store())], value=last_node)
last_node = compile(
f"__async_wrapper_result__ = ({ast.unparse(last_node)})",
"<_apply_async_handling()>",
"exec",
compiler_flags | ast.PyCF_ONLY_AST,
dont_inherit=1,
).body[0]
pmeier marked this conversation as resolved.
Show resolved Hide resolved
original_body.append(last_node)

async_wrapper = async_handling.body[0]
async_wrapper.body = [*original_body, *async_wrapper.body]

return ast.fix_missing_locations(async_handling)


def execute_code_block(compiler, block, script: GalleryScript):
"""Execute the code block of the example file.

Expand Down Expand Up @@ -788,9 +848,7 @@ def execute_code_block(compiler, block, script: GalleryScript):

try:
ast_Module = _ast_module()
code_ast = ast_Module([bcontent])
Copy link
Owner

@smarie smarie Dec 21, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was surprised to see this but it is indeed present in sphinx-gallery: https://github.com/sphinx-gallery/sphinx-gallery/blob/master/sphinx_gallery/gen_rst.py#L965

They probably use this as a pre-validator of good syntax. Should we keep it too ? In that case I suggest to keep these original lines and simply add your if _needs_async_handling extra two lines below (dropping _parse_code). That way the change will be easier to spot when comparing with sphinx-gallery code. Of course no hurry with this one, as this is a nitpick

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They probably use this as a pre-validator of good syntax. Should we keep it too ?

That is what we basically have right now. If we find a SyntaxError there, we need to check whether this comes from using async stuff and can't just exit the try branch pre-maturely. In the light of #90 (comment), we could just bubble up a SyntaxError if our async handling doesn't resolve it.

Copy link
Contributor Author

@pmeier pmeier Dec 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've implemented my suggestion in 5e0e636. Please have a look.

flags = ast.PyCF_ONLY_AST | compiler.flags
code_ast = compile(bcontent, src_file, "exec", flags, dont_inherit=1)
code_ast = _parse_code(bcontent, src_file, compiler_flags=compiler.flags)
ast.increment_lineno(code_ast, lineno - 1)

is_last_expr, mem_max = _exec_and_get_memory(compiler, ast_Module, code_ast, script=script)
Expand Down
Loading