Use typing_extensions whenever possible

Resolves   #752.

DOCS.md

@@ -323,16 +323,17 @@ If the `--strict` (`-s` for short) flag is enabled, Coconut will perform additio
 
 The style issues which will cause `--strict` to throw an error are:
 
-- mixing of tabs and spaces (without `--strict` will show a warning),
-- use of `from __future__` imports (Coconut does these automatically) (without `--strict` will show a warning),
-- inheriting from `object` in classes (Coconut does this automatically) (without `--strict` will show a warning),
-- semicolons at end of lines (without `--strict` will show a warning),
-- use of `u` to denote Unicode strings (all Coconut strings are Unicode strings) (without `--strict` will show a warning),
-- missing new line at end of file,
-- trailing whitespace at end of lines,
-- use of the Python-style `lambda` statement (use [Coconut's lambda syntax](#lambdas) instead),
-- use of backslash continuation (use [parenthetical continuation](#enhanced-parenthetical-continuation) instead),
-- Python-3.10/PEP-634-style dotted names in pattern-matching (Coconut style is to preface these with `==`), and
+- mixing of tabs and spaces (without `--strict` will show a warning).
+- use of `from __future__` imports (Coconut does these automatically) (without `--strict` will show a warning).
+- inheriting from `object` in classes (Coconut does this automatically) (without `--strict` will show a warning).
+- semicolons at end of lines (without `--strict` will show a warning).
+- use of `u` to denote Unicode strings (all Coconut strings are Unicode strings) (without `--strict` will show a warning).
+- commas after [statement lambdas](#statement-lambdas) (not recommended as it can be unclear whether the comma is inside or outside the lambda) (without `--strict` will show a warning).
+- missing new line at end of file.
+- trailing whitespace at end of lines.
+- use of the Python-style `lambda` statement (use [Coconut's lambda syntax](#lambdas) instead).
+- use of backslash continuation (use [parenthetical continuation](#enhanced-parenthetical-continuation) instead).
+- Python-3.10/PEP-634-style dotted names in pattern-matching (Coconut style is to preface these with `==`).
 - use of `:` instead of `<:` to specify upper bounds in [Coconut's type parameter syntax](#type-parameter-syntax).
 
 ## Integrations
@@ -1613,7 +1614,7 @@ If the last `statement` (not followed by a semicolon) in a statement lambda is a
 
 Statement lambdas also support implicit lambda syntax such that `def -> _` is equivalent to `def (_=None) -> _` as well as explicitly marking them as pattern-matching such that `match def (x) -> x` will be a pattern-matching function.
 
-Note that statement lambdas have a lower precedence than normal lambdas and thus capture things like trailing commas.
+Note that statement lambdas have a lower precedence than normal lambdas and thus capture things like trailing commas. To avoid confusion, statement lambdas should always be wrapped in their own set of parentheses.
 
 ##### Example
 
@@ -1779,7 +1780,7 @@ mod(5, 3)
 
 Since Coconut syntax is a superset of Python 3 syntax, it supports [Python 3 function type annotation syntax](https://www.python.org/dev/peps/pep-0484/) and [Python 3.6 variable type annotation syntax](https://www.python.org/dev/peps/pep-0526/). By default, Coconut compiles all type annotations into Python-2-compatible type comments. If you want to keep the type annotations instead, simply pass a `--target` that supports them.
 
-Since not all supported Python versions support the [`typing`](https://docs.python.org/3/library/typing.html) module, Coconut provides the [`TYPE_CHECKING`](#type_checking) built-in for hiding your `typing` imports and `TypeVar` definitions from being executed at runtime. Coconut will also automatically use [`typing_extensions`](https://pypi.org/project/typing-extensions/) over `typing` when importing objects not available in `typing` on the current Python version.
+Since not all supported Python versions support the [`typing`](https://docs.python.org/3/library/typing.html) module, Coconut provides the [`TYPE_CHECKING`](#type_checking) built-in for hiding your `typing` imports and `TypeVar` definitions from being executed at runtime. Coconut will also automatically use [`typing_extensions`](https://pypi.org/project/typing-extensions/) over `typing` objects at runtime when importing them from `typing`, even when they aren't natively supported on the current Python version (this works even if you just do `import typing` and then `typing.<Object>`).
 
 Furthermore, when compiling type annotations to Python 3 versions without [PEP 563](https://www.python.org/dev/peps/pep-0563/) support, Coconut wraps annotation in strings to prevent them from being evaluated at runtime (note that `--no-wrap-types` disables all wrapping, including via PEP 563 support).
 

coconut/compiler/compiler.py

@@ -87,6 +87,7 @@
     all_builtins,
     in_place_op_funcs,
     match_first_arg_var,
+    import_existing,
 )
 from coconut.util import (
     pickleable_obj,
@@ -195,8 +196,27 @@ def set_to_tuple(tokens):
         raise CoconutInternalException("invalid set maker item", tokens[0])
 
 
-def import_stmt(imp_from, imp, imp_as):
+def import_stmt(imp_from, imp, imp_as, raw=False):
     """Generate an import statement."""
+    if not raw:
+        module_path = (imp if imp_from is None else imp_from).split(".", 1)
+        existing_imp = import_existing.get(module_path[0])
+        if existing_imp is not None:
+            return handle_indentation(
+                """
+if _coconut.typing.TYPE_CHECKING:
+    {raw_import}
+else:
+    try:
+        {imp_name} = {imp_lookup}
+    except _coconut.AttributeError as _coconut_imp_err:
+        raise _coconut.ImportError(_coconut.str(_coconut_imp_err))
+                """,
+            ).format(
+                raw_import=import_stmt(imp_from, imp, imp_as, raw=True),
+                imp_name=imp_as if imp_as is not None else imp,
+                imp_lookup=".".join([existing_imp] + module_path[1:] + ([imp] if imp_from is not None else [])),
+            )
     return (
         ("from " + imp_from + " " if imp_from is not None else "")
         + "import " + imp
@@ -3072,9 +3092,7 @@ def single_import(self, path, imp_as, type_ignore=False):
             imp_from += imp.rsplit("." + imp_as, 1)[0]
             imp, imp_as = imp_as, None
 
-        if imp_from is None and imp == "sys":
-            out.append((imp_as if imp_as is not None else imp) + " = _coconut_sys")
-        elif imp_as is not None and "." in imp_as:
+        if imp_as is not None and "." in imp_as:
             import_as_var = self.get_temp_var("import")
             out.append(import_stmt(imp_from, imp, import_as_var))
             fake_mods = imp_as.split(".")
@@ -3375,7 +3393,12 @@ def set_letter_literal_handle(self, tokens):
 
     def stmt_lambdef_handle(self, original, loc, tokens):
         """Process multi-line lambdef statements."""
-        got_kwds, params, stmts_toks = tokens
+        got_kwds, params, stmts_toks, followed_by = tokens
+
+        if followed_by == ",":
+            self.strict_err_or_warn("found statement lambda followed by comma; this isn't recommended as it can be unclear whether the comma is inside or outside the lambda (just wrap the lambda in parentheses)", original, loc)
+        else:
+            internal_assert(followed_by == "", "invalid stmt_lambdef followed_by", followed_by)
 
         is_async = False
         add_kwds = []

coconut/compiler/grammar.py

@@ -1592,7 +1592,13 @@ class Grammar(object):
         + arrow.suppress()
         + stmt_lambdef_body
     )
-    stmt_lambdef_ref = general_stmt_lambdef | match_stmt_lambdef
+    stmt_lambdef_ref = (
+        general_stmt_lambdef
+        | match_stmt_lambdef
+    ) + (
+        fixto(FollowedBy(comma), ",")
+        | fixto(always_match, "")
+    )
 
     lambdef <<= addspace(lambdef_base + test) | stmt_lambdef
     lambdef_no_cond = trace(addspace(lambdef_base + test_no_cond))

coconut/compiler/header.py

@@ -534,92 +534,93 @@ async def __anext__(self):
         underscore_imports="{tco_comma}{call_set_names_comma}{handle_cls_args_comma}_namedtuple_of, _coconut, _coconut_Expected, _coconut_MatchError, _coconut_SupportsAdd, _coconut_SupportsMinus, _coconut_SupportsMul, _coconut_SupportsPow, _coconut_SupportsTruediv, _coconut_SupportsFloordiv, _coconut_SupportsMod, _coconut_SupportsAnd, _coconut_SupportsXor, _coconut_SupportsOr, _coconut_SupportsLshift, _coconut_SupportsRshift, _coconut_SupportsMatmul, _coconut_SupportsInv, _coconut_iter_getitem, _coconut_base_compose, _coconut_forward_compose, _coconut_back_compose, _coconut_forward_star_compose, _coconut_back_star_compose, _coconut_forward_dubstar_compose, _coconut_back_dubstar_compose, _coconut_pipe, _coconut_star_pipe, _coconut_dubstar_pipe, _coconut_back_pipe, _coconut_back_star_pipe, _coconut_back_dubstar_pipe, _coconut_none_pipe, _coconut_none_star_pipe, _coconut_none_dubstar_pipe, _coconut_bool_and, _coconut_bool_or, _coconut_none_coalesce, _coconut_minus, _coconut_map, _coconut_partial, _coconut_get_function_match_error, _coconut_base_pattern_func, _coconut_addpattern, _coconut_sentinel, _coconut_assert, _coconut_raise, _coconut_mark_as_match, _coconut_reiterable, _coconut_self_match_types, _coconut_dict_merge, _coconut_exec, _coconut_comma_op, _coconut_multi_dim_arr, _coconut_mk_anon_namedtuple, _coconut_matmul, _coconut_py_str, _coconut_flatten, _coconut_multiset, _coconut_back_none_pipe, _coconut_back_none_star_pipe, _coconut_back_none_dubstar_pipe, _coconut_forward_none_compose, _coconut_back_none_compose, _coconut_forward_none_star_compose, _coconut_back_none_star_compose, _coconut_forward_none_dubstar_compose, _coconut_back_none_dubstar_compose, _coconut_call_or_coefficient, _coconut_in, _coconut_not_in".format(**format_dict),
         import_typing=pycondition(
             (3, 5),
-            if_ge="import typing",
+            if_ge='''
+import typing as _typing
+for _name in dir(_typing):
+    if not hasattr(typing, _name):
+        setattr(typing, _name, getattr(_typing, _name))
+            ''',
             if_lt='''
-class typing_mock{object}:
-    """The typing module is not available at runtime in Python 3.4 or earlier;
-    try hiding your typedefs behind an 'if TYPE_CHECKING:' block."""
-    TYPE_CHECKING = False
+if not hasattr(typing, "TYPE_CHECKING"):
+    typing.TYPE_CHECKING = False
+if not hasattr(typing, "Any"):
     Any = Ellipsis
-    def cast(self, t, x):
+if not hasattr(typing, "cast"):
+    def cast(t, x):
         """typing.cast[T](t: Type[T], x: Any) -> T = x"""
         return x
-    def __getattr__(self, name):
-        raise _coconut.ImportError("the typing module is not available at runtime in Python 3.4 or earlier; try hiding your typedefs behind an 'if TYPE_CHECKING:' block")
+    typing.cast = cast
+    cast = staticmethod(cast)
+if not hasattr(typing, "TypeVar"):
     def TypeVar(name, *args, **kwargs):
         """Runtime mock of typing.TypeVar for Python 3.4 and earlier."""
         return name
+    typing.TypeVar = TypeVar
+    TypeVar = staticmethod(TypeVar)
+if not hasattr(typing, "Generic"):
     class Generic_mock{object}:
         """Runtime mock of typing.Generic for Python 3.4 and earlier."""
         __slots__ = ()
         def __getitem__(self, vars):
             return _coconut.object
-    Generic = Generic_mock()
-typing = typing_mock()
+    typing.Generic = Generic_mock()
             '''.format(**format_dict),
             indent=1,
         ),
         # all typing_extensions imports must be added to the _coconut stub file
         import_typing_36=pycondition(
             (3, 6),
             if_lt='''
-def NamedTuple(name, fields):
-    return _coconut.collections.namedtuple(name, [x for x, t in fields])
-typing.NamedTuple = NamedTuple
-NamedTuple = staticmethod(NamedTuple)
+if not hasattr(typing, "NamedTuple"):
+    def NamedTuple(name, fields):
+        return _coconut.collections.namedtuple(name, [x for x, t in fields])
+    typing.NamedTuple = NamedTuple
+    NamedTuple = staticmethod(NamedTuple)
             ''',
             indent=1,
             newline=True,
         ),
         import_typing_38=pycondition(
             (3, 8),
             if_lt='''
-try:
-    from typing_extensions import Protocol
-except ImportError:
+if not hasattr(typing, "Protocol"):
     class YouNeedToInstallTypingExtensions{object}:
         __slots__ = ()
         def __init__(self):
             raise _coconut.TypeError('Protocols cannot be instantiated')
-    Protocol = YouNeedToInstallTypingExtensions
-typing.Protocol = Protocol
+    typing.Protocol = YouNeedToInstallTypingExtensions
             '''.format(**format_dict),
             indent=1,
             newline=True,
         ),
         import_typing_310=pycondition(
             (3, 10),
             if_lt='''
-try:
-    from typing_extensions import ParamSpec, TypeAlias, Concatenate
-except ImportError:
+if not hasattr(typing, "ParamSpec"):
     def ParamSpec(name, *args, **kwargs):
         """Runtime mock of typing.ParamSpec for Python 3.9 and earlier."""
         return _coconut.typing.TypeVar(name)
+    typing.ParamSpec = ParamSpec
+if not hasattr(typing, "TypeAlias") or not hasattr(typing, "Concatenate"):
     class you_need_to_install_typing_extensions{object}:
         __slots__ = ()
-    TypeAlias = Concatenate = you_need_to_install_typing_extensions()
-typing.ParamSpec = ParamSpec
-typing.TypeAlias = TypeAlias
-typing.Concatenate = Concatenate
+    typing.TypeAlias = typing.Concatenate = you_need_to_install_typing_extensions()
             '''.format(**format_dict),
             indent=1,
             newline=True,
         ),
         import_typing_311=pycondition(
             (3, 11),
             if_lt='''
-try:
-    from typing_extensions import TypeVarTuple, Unpack
-except ImportError:
+if not hasattr(typing, "TypeVarTuple"):
     def TypeVarTuple(name, *args, **kwargs):
         """Runtime mock of typing.TypeVarTuple for Python 3.10 and earlier."""
         return _coconut.typing.TypeVar(name)
+    typing.TypeVarTuple = TypeVarTuple
+if not hasattr(typing, "Unpack"):
     class you_need_to_install_typing_extensions{object}:
         __slots__ = ()
-    Unpack = you_need_to_install_typing_extensions()
-typing.TypeVarTuple = TypeVarTuple
-typing.Unpack = Unpack
+    typing.Unpack = you_need_to_install_typing_extensions()
             '''.format(**format_dict),
             indent=1,
             newline=True,

coconut/compiler/templates/header.py_template

@@ -20,8 +20,23 @@ class _coconut{object}:{COMMENT.EVERYTHING_HERE_MUST_BE_COPIED_TO_STUB_FILE}
 {import_pickle}
 {import_OrderedDict}
 {import_collections_abc}
+    typing = types.ModuleType("typing")
+    try:
+        import typing_extensions
+    except ImportError:
+        typing_extensions = None
+    else:
+        for _name in dir(typing_extensions):
+            if not _name.startswith("__"):
+                setattr(typing, _name, getattr(typing_extensions, _name))
+    typing.__doc__ = "Coconut version of typing that makes use of typing.typing_extensions when possible.\n\n" + (getattr(typing, "__doc__") or "The typing module is not available at runtime in Python 3.4 or earlier; try hiding your typedefs behind an 'if TYPE_CHECKING:' block.")
 {import_typing}
-{import_typing_36}{import_typing_38}{import_typing_310}{import_typing_311}{set_zip_longest}
+{import_typing_36}{import_typing_38}{import_typing_310}{import_typing_311}
+    def _typing_getattr(name):
+        raise _coconut.AttributeError("typing.%s is not available on the current Python version and couldn't be looked up in typing_extensions; try hiding your typedefs behind an 'if TYPE_CHECKING:' block" % (name,))
+    typing.__getattr__ = _typing_getattr
+    _typing_getattr = staticmethod(_typing_getattr)
+{set_zip_longest}
     try:
         import numpy
     except ImportError:

coconut/constants.py

@@ -430,6 +430,8 @@ def get_bool_env_var(env_var, default=False):
     # third-party backports
     "asyncio": ("trollius", (3, 4)),
     "enum": ("aenum", (3, 4)),
+
+    # typing_extensions
     "typing.AsyncContextManager": ("typing_extensions./AsyncContextManager", (3, 6)),
     "typing.AsyncGenerator": ("typing_extensions./AsyncGenerator", (3, 6)),
     "typing.AsyncIterable": ("typing_extensions./AsyncIterable", (3, 6)),
@@ -482,6 +484,10 @@ def get_bool_env_var(env_var, default=False):
     "typing.Unpack": ("typing_extensions./Unpack", (3, 11)),
 }
 
+import_existing = {
+    "typing": "_coconut.typing",
+}
+
 self_match_types = (
     "bool",
     "bytearray",

coconut/root.py

@@ -26,7 +26,7 @@
 VERSION = "3.0.1"
 VERSION_NAME = None
 # False for release, int >= 1 for develop
-DEVELOP = 7
+DEVELOP = 8
 ALPHA = False  # for pre releases rather than post releases
 
 assert DEVELOP is False or DEVELOP >= 1, "DEVELOP must be False or an int >= 1"

coconut/tests/src/cocotest/agnostic/primary.coco

@@ -1602,4 +1602,5 @@ def primary_test() -> bool:
     n = [0]
     assert n[0] == 0
     assert_raises(-> m{{1:2,2:3}}, TypeError)
+    assert_raises((def -> from typing import blah), ImportError)  # NOQA
     return True

coconut/tests/src/cocotest/agnostic/specific.coco

@@ -180,6 +180,7 @@ def py37_spec_test() -> bool:
     assert l == list(range(10))
     class HasVarGen[*Ts]  # type: ignore
     assert HasVarGen `issubclass` object
+    assert typing.Protocol.__module__ == "typing_extensions"
     return True
 
 

coconut/tests/src/cocotest/agnostic/suite.coco

@@ -1051,6 +1051,7 @@ forward 2""") == 900
     really_long_var = 10
     assert ret_args_kwargs(...=really_long_var) == ((), {"really_long_var": 10}) == ret_args_kwargs$(...=really_long_var)()
     assert ret_args_kwargs(123, ...=really_long_var, abc="abc") == ((123,), {"really_long_var": 10, "abc": "abc"}) == ret_args_kwargs$(123, ...=really_long_var, abc="abc")()
+    assert "Coconut version of typing" in typing.__doc__
 
     # must come at end
     assert fibs_calls[0] == 1

coconut/tests/src/cocotest/agnostic/util.coco

@@ -22,7 +22,7 @@ class AccessCounter():
             self.counts[attr] += 1
         return super(AccessCounter, self).__getattribute__(attr)
 
-def assert_raises(c, exc=Exception):
+def assert_raises(c, exc):
     """Test whether callable c raises an exception of type exc."""
     try:
         c()
@@ -231,9 +231,8 @@ addpattern def x! if x = False  # type: ignore
 addpattern def x! = True  # type: ignore
 
 # Type aliases:
+import typing
 if sys.version_info >= (3, 5) or TYPE_CHECKING:
-    import typing
-
     type list_or_tuple = list | tuple
 
     type func_to_int = -> int

coconut/tests/src/extras.coco

@@ -100,7 +100,7 @@ def test_setup_none() -> bool:
     assert version("tag")
     assert version("-v")
     assert_raises(-> version("other"), CoconutException)
-    assert_raises(def -> raise CoconutException("derp").syntax_err(), SyntaxError)
+    assert_raises((def -> raise CoconutException("derp").syntax_err()), SyntaxError)
     assert coconut_eval("x -> x + 1")(2) == 3
     assert coconut_eval("addpattern")
 
@@ -316,6 +316,7 @@ else:
     match x:
         pass"""), CoconutStyleError, err_has="case x:")
     assert_raises(-> parse("obj."), CoconutStyleError, err_has="getattr")
+    assert_raises(-> parse("def x -> pass, 1"), CoconutStyleError, err_has="statement lambda")
 
     setup(strict=True, target="sys")
     assert_raises(-> parse("await f x"), CoconutParseError, err_has='invalid use of the keyword "await"')