Add copyclosure functions

Resolves #731.
evhub · Apr 21, 2023 · 6be6e90 · 6be6e90
1 parent 62db4b0
commit 6be6e90
Show file tree

Hide file tree

Showing 14 changed files with 321 additions and 137 deletions.
diff --git a/DOCS.md b/DOCS.md
@@ -2146,7 +2146,7 @@ print(binexp(5))
 
 Coconut pattern-matching functions are just normal functions, except where the arguments are patterns to be matched against instead of variables to be assigned to. The syntax for pattern-matching function definition is
 ```coconut
-[async] [match] def <name>(<arg>, <arg>, ... [if <cond>]) [-> <return_type>]:
+[match] def <name>(<arg>, <arg>, ... [if <cond>]) [-> <return_type>]:
     <body>
 ```
 where `<arg>` is defined as
@@ -2161,7 +2161,7 @@ In addition to supporting pattern-matching in their arguments, pattern-matching
 - If pattern-matching function definition fails, it will raise a [`MatchError`](#matcherror) (just like [destructuring assignment](#destructuring-assignment)) instead of a `TypeError`.
 - All defaults in pattern-matching function definition are late-bound rather than early-bound. Thus, `match def f(xs=[]) = xs` will instantiate a new list for each call where `xs` is not given, unlike `def f(xs=[]) = xs`, which will use the same list for all calls where `xs` is unspecified.
 
-_Note: Pattern-matching function definition can be combined with assignment and/or infix function definition._
+Pattern-matching function definition can also be combined with `async` functions, [`copyclosure` functions](#copyclosure-functions), [`yield` functions](#explicit-generators), [infix function definition](#infix-functions), and [assignment function syntax](#assignment-functions). The various keywords in front of the `def` can be put in any order.
 
 ##### Example
 
@@ -2204,16 +2204,65 @@ addpattern def factorial(n) = n * factorial(n - 1)
 **Python:**
 _Can't be done without a complicated decorator definition and a long series of checks for each pattern-matching. See the compiled code for the Python syntax._
 
+### `copyclosure` Functions
+
+Coconut supports the syntax
+```
+copyclosure def <name>(<args>):
+    <body>
+```
+to define a function that uses as its closure a shallow copy of its enclosing scopes at the time that the function is defined, rather than a reference to those scopes (as with normal Python functions).
+
+For example,`in
+```coconut
+def outer_func():
+    funcs = []
+    for x in range(10):
+        copyclosure def inner_func():
+            return x
+        funcs.append(inner_func)
+    return funcs
+```
+the resulting `inner_func`s will each return a _different_ `x` value rather than all the same `x` value, since they look at what `x` was bound to at function definition time rather than during function execution.
+
+`copyclosure` functions can also be combined with `async` functions, [`yield` functions](#explicit-generators), [pattern-matching functions](#pattern-matching-functions), [infix function definition](#infix-functions), and [assignment function syntax](#assignment-functions). The various keywords in front of the `def` can be put in any order.
+
+##### Example
+
+**Coconut:**
+```coconut
+def outer_func():
+    funcs = []
+    for x in range(10):
+        copyclosure def inner_func():
+            return x
+        funcs.append(inner_func)
+    return funcs
+```
+
+**Python:**
+```coconut_python
+from functools import partial
+
+def outer_func():
+    funcs = []
+    for x in range(10):
+        def inner_func(_x):
+            return _x
+        funcs.append(partial(inner_func, x))
+    return funcs
+```
+
 ### Explicit Generators
 
 Coconut supports the syntax
 ```
-[async] yield def <name>(<args>):
+yield def <name>(<args>):
     <body>
 ```
-to denote that you are explicitly defining a generator function. This is useful to ensure that, even if all the `yield`s in your function are removed, it'll always be a generator function. Note that the `async` and `yield` keywords can be in any order.
+to denote that you are explicitly defining a generator function. This is useful to ensure that, even if all the `yield`s in your function are removed, it'll always be a generator function.
 
-Explicit generator functions also support [pattern-matching syntax](#pattern-matching-functions), [infix function definition](#infix-functions), and [assignment function syntax](#assignment-functions) (though note that assignment function syntax here creates a generator return).
+Explicit generator functions can also be combined with `async` functions, [`copyclosure` functions](#copyclosure-functions), [pattern-matching functions](#pattern-matching-functions), [infix function definition](#infix-functions), and [assignment function syntax](#assignment-functions) (though note that assignment function syntax here creates a generator return). The various keywords in front of the `def` can be put in any order.
 
 ##### Example
 

diff --git a/_coconut/__init__.pyi b/_coconut/__init__.pyi
@@ -155,6 +155,7 @@ iter = iter
 len: _t.Callable[..., int] = ...  # pattern-matching needs an untyped _coconut.len to avoid type errors
 list = list
 locals = locals
+globals = globals
 map = map
 min = min
 max = max

diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py
@@ -427,6 +427,7 @@ class Compiler(Grammar, pickleable_obj):
     ]
 
     reformatprocs = [
+        # deferred_code_proc must come first
         lambda self: self.deferred_code_proc,
         lambda self: self.reind_proc,
         lambda self: self.endline_repl,
@@ -670,6 +671,10 @@ def bind(cls):
         cls.string_atom <<= trace_attach(cls.string_atom_ref, cls.method("string_atom_handle"))
         cls.f_string_atom <<= trace_attach(cls.f_string_atom_ref, cls.method("string_atom_handle"))
 
+        # handle all keyword funcdefs with keyword_funcdef_handle
+        cls.keyword_funcdef <<= trace_attach(cls.keyword_funcdef_ref, cls.method("keyword_funcdef_handle"))
+        cls.async_keyword_funcdef <<= trace_attach(cls.async_keyword_funcdef_ref, cls.method("keyword_funcdef_handle"))
+
         # standard handlers of the form name <<= trace_attach(name_tokens, method("name_handle")) (implies name_tokens is reused)
         cls.function_call <<= trace_attach(cls.function_call_tokens, cls.method("function_call_handle"))
         cls.testlist_star_namedexpr <<= trace_attach(cls.testlist_star_namedexpr_tokens, cls.method("testlist_star_expr_handle"))
@@ -756,6 +761,10 @@ def adjust(self, ln, skips=None):
                 adj_ln = i
         return adj_ln + need_unskipped
 
+    def reformat_post_deferred_code_proc(self, snip):
+        """Do post-processing that comes after deferred_code_proc."""
+        return self.apply_procs(self.reformatprocs[1:], snip, reformatting=True, log=False)
+
     def reformat(self, snip, *indices, **kwargs):
         """Post process a preprocessed snippet."""
         internal_assert("ignore_errors" in kwargs, "reformat() missing required keyword argument: 'ignore_errors'")
@@ -1841,19 +1850,25 @@ def transform_returns(self, original, loc, raw_lines, tre_return_grammar=None, i
     def proc_funcdef(self, original, loc, decorators, funcdef, is_async, in_method, is_stmt_lambda):
         """Determines if TCO or TRE can be done and if so does it,
         handles dotted function names, and universalizes async functions."""
-        # process tokens
         raw_lines = list(logical_lines(funcdef, True))
         def_stmt = raw_lines.pop(0)
-        out = ""
-
-        # detect addpattern functions
-        if def_stmt.startswith("addpattern def"):
-            def_stmt = def_stmt[len("addpattern "):]
-            addpattern = True
-        elif def_stmt.startswith("def"):
-            addpattern = False
-        else:
-            raise CoconutInternalException("invalid function definition statement", funcdef)
+        out = []
+
+        # detect addpattern/copyclosure functions
+        addpattern = False
+        copyclosure = False
+        done = False
+        while not done:
+            if def_stmt.startswith("addpattern "):
+                def_stmt = def_stmt[len("addpattern "):]
+                addpattern = True
+            elif def_stmt.startswith("copyclosure "):
+                def_stmt = def_stmt[len("copyclosure "):]
+                copyclosure = True
+            elif def_stmt.startswith("def"):
+                done = True
+            else:
+                raise CoconutInternalException("invalid function definition statement", funcdef)
 
         # extract information about the function
         with self.complain_on_err():
@@ -1919,18 +1934,20 @@ def proc_funcdef(self, original, loc, decorators, funcdef, is_async, in_method,
                 raise CoconutInternalException("could not find name in addpattern function definition", def_stmt)
             # binds most tightly, except for TCO
             addpattern_decorator = self.get_temp_var("addpattern")
-            out += handle_indentation(
-                """
+            out.append(
+                handle_indentation(
+                    """
 try:
     {addpattern_decorator} = _coconut_addpattern({func_name}) {type_ignore}
 except _coconut.NameError:
     {addpattern_decorator} = lambda f: f
                 """,
-                add_newline=True,
-            ).format(
-                func_name=func_name,
-                addpattern_decorator=addpattern_decorator,
-                type_ignore=self.type_ignore_comment(),
+                    add_newline=True,
+                ).format(
+                    func_name=func_name,
+                    addpattern_decorator=addpattern_decorator,
+                    type_ignore=self.type_ignore_comment(),
+                ),
             )
             decorators += "@" + addpattern_decorator + "\n"
 
@@ -2064,29 +2081,48 @@ def {mock_var}({mock_paramdef}):
 
         # handle dotted function definition
         if undotted_name is not None:
-            out += handle_indentation(
-                '''
+            out.append(
+                handle_indentation(
+                    '''
 {decorators}{def_stmt}{func_code}
 {def_name}.__name__ = _coconut_py_str("{undotted_name}")
 {temp_var} = _coconut.getattr({def_name}, "__qualname__", None)
 if {temp_var} is not None:
     {def_name}.__qualname__ = _coconut_py_str("{func_name}" if "." not in {temp_var} else {temp_var}.rsplit(".", 1)[0] + ".{func_name}")
 {func_name} = {def_name}
+                ''',
+                    add_newline=True,
+                ).format(
+                    def_name=def_name,
+                    decorators=decorators,
+                    def_stmt=def_stmt,
+                    func_code=func_code,
+                    func_name=func_name,
+                    undotted_name=undotted_name,
+                    temp_var=self.get_temp_var("qualname"),
+                ),
+            )
+        else:
+            out += [decorators, def_stmt, func_code]
+
+        # handle copyclosure functions
+        if copyclosure:
+            return handle_indentation(
+                '''
+{vars_var} = _coconut.globals().copy()
+{vars_var}.update(_coconut.locals().copy())
+_coconut_exec({func_code_str}, {vars_var})
+{func_name} = {vars_var}["{def_name}"]
                 ''',
                 add_newline=True,
             ).format(
-                def_name=def_name,
-                decorators=decorators,
-                def_stmt=def_stmt,
-                func_code=func_code,
                 func_name=func_name,
-                undotted_name=undotted_name,
-                temp_var=self.get_temp_var("qualname"),
+                def_name=def_name,
+                vars_var=self.get_temp_var("func_vars"),
+                func_code_str=self.wrap_str_of(self.reformat_post_deferred_code_proc("".join(out))),
             )
         else:
-            out += decorators + def_stmt + func_code
-
-        return out
+            return "".join(out)
 
     def deferred_code_proc(self, inputstring, add_code_at_start=False, ignore_names=(), ignore_errors=False, **kwargs):
         """Process all forms of previously deferred code. All such deferred code needs to be handled here so we can properly handle nested deferred code."""
@@ -3227,13 +3263,16 @@ def set_letter_literal_handle(self, tokens):
 
     def stmt_lambdef_handle(self, original, loc, tokens):
         """Process multi-line lambdef statements."""
-        kwds, params, stmts_toks = tokens
+        got_kwds, params, stmts_toks = tokens
 
         is_async = False
-        for kwd in kwds:
+        add_kwds = []
+        for kwd in got_kwds:
             if kwd == "async":
                 self.internal_assert(not is_async, original, loc, "duplicate stmt_lambdef async keyword", kwd)
                 is_async = True
+            elif kwd == "copyclosure":
+                add_kwds.append(kwd)
             else:
                 raise CoconutInternalException("invalid stmt_lambdef keyword", kwd)
 
@@ -3265,6 +3304,8 @@ def stmt_lambdef_handle(self, original, loc, tokens):
                 + body
             )
 
+        funcdef = " ".join(add_kwds + [funcdef])
+
         self.add_code_before[name] = self.decoratable_funcdef_stmt_handle(original, loc, [decorators, funcdef], is_async, is_stmt_lambda=True)
 
         return name
@@ -3829,6 +3870,25 @@ def impl_call_handle(self, loc, tokens):
         else:
             return "_coconut_call_or_coefficient(" + ", ".join(tokens) + ")"
 
+    def keyword_funcdef_handle(self, tokens):
+        """Process function definitions with keywords in front."""
+        keywords, funcdef = tokens
+        for kwd in keywords:
+            if kwd == "yield":
+                funcdef += handle_indentation(
+                    """
+if False:
+    yield
+                    """,
+                    add_newline=True,
+                    extra_indent=1,
+                )
+            else:
+                # new keywords here must be replicated in def_regex and handled in proc_funcdef
+                internal_assert(kwd in ("addpattern", "copyclosure"), "unknown deferred funcdef keyword", kwd)
+                funcdef = kwd + " " + funcdef
+        return funcdef
+
 # end: HANDLERS
 # -----------------------------------------------------------------------------------------------------------------------
 # CHECKING HANDLERS: