From 8c8b1dd785bff474b190f46fabc6ea84b6bf4de6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 01:36:49 -0700 Subject: [PATCH 01/35] Unify UnionType and Union --- Lib/typing.py | 102 +++------------------------------ Objects/unionobject.c | 129 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 95 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 27497741b24887..dcb73d38b5a542 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -29,7 +29,13 @@ import re as stdlib_re # Avoid confusion with the typing.re namespace on <=3.11 import sys import types -from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias +from types import ( + WrapperDescriptorType, + MethodWrapperType, + MethodDescriptorType, + GenericAlias, + UnionType as Union, +) from _typing import ( _idfunc, @@ -315,22 +321,6 @@ def _deduplicate(params): return params -def _remove_dups_flatten(parameters): - """Internal helper for Union creation and substitution. - - Flatten Unions among parameters, then remove duplicates. - """ - # Flatten out Union[Union[...], ...]. - params = [] - for p in parameters: - if isinstance(p, (_UnionGenericAlias, types.UnionType)): - params.extend(p.__args__) - else: - params.append(p) - - return tuple(_deduplicate(params)) - - def _flatten_literal_params(parameters): """Internal helper for Literal creation: flatten Literals among parameters.""" params = [] @@ -657,50 +647,6 @@ class FastConnector(Connection): item = _type_check(parameters, f'{self} accepts only single type.') return _GenericAlias(self, (item,)) -@_SpecialForm -def Union(self, parameters): - """Union type; Union[X, Y] means either X or Y. - - On Python 3.10 and higher, the | operator - can also be used to denote unions; - X | Y means the same thing to the type checker as Union[X, Y]. - - To define a union, use e.g. Union[int, str]. Details: - - The arguments must be types and there must be at least one. - - None as an argument is a special case and is replaced by - type(None). - - Unions of unions are flattened, e.g.:: - - assert Union[Union[int, str], float] == Union[int, str, float] - - - Unions of a single argument vanish, e.g.:: - - assert Union[int] == int # The constructor actually returns int - - - Redundant arguments are skipped, e.g.:: - - assert Union[int, str, int] == Union[int, str] - - - When comparing unions, the argument order is ignored, e.g.:: - - assert Union[int, str] == Union[str, int] - - - You cannot subclass or instantiate a union. - - You can use Optional[X] as a shorthand for Union[X, None]. - """ - if parameters == (): - raise TypeError("Cannot take a Union of no types.") - if not isinstance(parameters, tuple): - parameters = (parameters,) - msg = "Union[arg, ...]: each arg must be a type." - parameters = tuple(_type_check(p, msg) for p in parameters) - parameters = _remove_dups_flatten(parameters) - if len(parameters) == 1: - return parameters[0] - if len(parameters) == 2 and type(None) in parameters: - return _UnionGenericAlias(self, parameters, name="Optional") - return _UnionGenericAlias(self, parameters) - def _make_union(left, right): """Used from the C implementation of TypeVar. @@ -1526,40 +1472,6 @@ def __getitem__(self, params): return self.copy_with(params) -class _UnionGenericAlias(_NotIterable, _GenericAlias, _root=True): - def copy_with(self, params): - return Union[params] - - def __eq__(self, other): - if not isinstance(other, (_UnionGenericAlias, types.UnionType)): - return NotImplemented - return set(self.__args__) == set(other.__args__) - - def __hash__(self): - return hash(frozenset(self.__args__)) - - def __repr__(self): - args = self.__args__ - if len(args) == 2: - if args[0] is type(None): - return f'typing.Optional[{_type_repr(args[1])}]' - elif args[1] is type(None): - return f'typing.Optional[{_type_repr(args[0])}]' - return super().__repr__() - - def __instancecheck__(self, obj): - return self.__subclasscheck__(type(obj)) - - def __subclasscheck__(self, cls): - for arg in self.__args__: - if issubclass(cls, arg): - return True - - def __reduce__(self): - func, (origin, args) = super().__reduce__() - return func, (Union, args) - - def _value_and_type_iter(parameters): return ((p, type(p)) for p in parameters) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index f509a161bb9564..544f5246ba6848 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -368,6 +368,134 @@ _Py_union_args(PyObject *self) return ((unionobject *) self)->args; } +static PyObject * +call_typing_func_object(const char *name, PyObject **args, size_t nargs) +{ + PyObject *typing = PyImport_ImportModule("typing"); + if (typing == NULL) { + return NULL; + } + PyObject *func = PyObject_GetAttrString(typing, name); + if (func == NULL) { + Py_DECREF(typing); + return NULL; + } + PyObject *result = PyObject_Vectorcall(func, args, nargs, NULL); + Py_DECREF(func); + Py_DECREF(typing); + return result; +} + +static PyObject * +type_check(PyObject *arg, const char *msg) +{ + // Calling typing.py here leads to bootstrapping problems + if (Py_IsNone(arg)) { + return Py_NewRef(Py_TYPE(arg)); + } + // Fast path to avoid calling into typing.py + if (is_unionable(arg)) { + return Py_NewRef(arg); + } + PyObject *message_str = PyUnicode_FromString(msg); + if (message_str == NULL) { + return NULL; + } + PyObject *args[2] = {arg, message_str}; + PyObject *result = call_typing_func_object("_type_check", args, 2); + Py_DECREF(message_str); + return result; +} + +static int +add_object_to_union_args(PyObject *args_list, PyObject *args_set, PyObject *obj) +{ + if (Py_IS_TYPE(obj, &_PyUnion_Type)) { + PyObject *args = ((unionobject *) obj)->args; + Py_ssize_t size = PyTuple_GET_SIZE(args); + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *arg = PyTuple_GET_ITEM(args, i); + if (add_object_to_union_args(args_list, args_set, arg) < 0) { + return -1; + } + } + return 0; + } + PyObject *type = type_check(obj, "Union[arg, ...]: each arg must be a type."); + if (type == NULL) { + return -1; + } + if (PySet_Contains(args_set, type)) { + Py_DECREF(type); + return 0; + } + if (PyList_Append(args_list, type) < 0) { + Py_DECREF(type); + return -1; + } + if (PySet_Add(args_set, type) < 0) { + Py_DECREF(type); + return -1; + } + Py_DECREF(type); + return 0; +} + +static PyObject * +union_class_getitem(PyObject *cls, PyObject *args) +{ + PyObject *args_list = PyList_New(0); + if (args_list == NULL) { + return NULL; + } + PyObject *args_set = PySet_New(NULL); + if (args_set == NULL) { + Py_DECREF(args_list); + return NULL; + } + if (!PyTuple_CheckExact(args)) { + if (add_object_to_union_args(args_list, args_set, args) < 0) { + Py_DECREF(args_list); + Py_DECREF(args_set); + return NULL; + } + } + else { + Py_ssize_t size = PyTuple_GET_SIZE(args); + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *arg = PyTuple_GET_ITEM(args, i); + if (add_object_to_union_args(args_list, args_set, arg) < 0) { + Py_DECREF(args_list); + Py_DECREF(args_set); + return NULL; + } + } + } + Py_DECREF(args_set); + if (PyList_GET_SIZE(args_list) == 0) { + Py_DECREF(args_list); + PyErr_SetString(PyExc_TypeError, "Cannot take a Union of no types."); + return NULL; + } + else if (PyList_GET_SIZE(args_list) == 1) { + PyObject *result = PyList_GET_ITEM(args_list, 0); + Py_INCREF(result); + Py_DECREF(args_list); + return result; + } + PyObject *args_tuple = PyList_AsTuple(args_list); + Py_DECREF(args_list); + if (args_tuple == NULL) { + return NULL; + } + return make_union(args_tuple); +} + +static PyMethodDef union_methods[] = { + {"__class_getitem__", union_class_getitem, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {0} +}; + PyTypeObject _PyUnion_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "types.UnionType", @@ -383,6 +511,7 @@ PyTypeObject _PyUnion_Type = { .tp_hash = union_hash, .tp_getattro = union_getattro, .tp_members = union_members, + .tp_methods = union_methods, .tp_richcompare = union_richcompare, .tp_as_mapping = &union_as_mapping, .tp_as_number = &union_as_number, From aba63eb1aedee6ca1729866d1c27889ce39d3571 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 07:47:00 -0700 Subject: [PATCH 02/35] test_typing succeeds --- Lib/test/test_typing.py | 61 +++++++++++++++++++---------------------- Objects/unionobject.c | 21 ++++++++++++++ 2 files changed, 49 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 432fc88b1c072e..7a54f6ba4b19f3 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -491,7 +491,7 @@ def test_cannot_instantiate_vars(self): def test_bound_errors(self): with self.assertRaises(TypeError): - TypeVar('X', bound=Union) + TypeVar('X', bound=Optional) with self.assertRaises(TypeError): TypeVar('X', str, float, bound=Employee) with self.assertRaisesRegex(TypeError, @@ -531,7 +531,7 @@ def test_var_substitution(self): def test_bad_var_substitution(self): T = TypeVar('T') bad_args = ( - (), (int, str), Union, + (), (int, str), Optional, Generic, Generic[T], Protocol, Protocol[T], Final, Final[int], ClassVar, ClassVar[int], ) @@ -1708,10 +1708,8 @@ def test_basics(self): self.assertNotEqual(u, Union) def test_subclass_error(self): - with self.assertRaises(TypeError): - issubclass(int, Union) - with self.assertRaises(TypeError): - issubclass(Union, int) + self.assertNotIsSubclass(int, Union) + self.assertNotIsSubclass(Union, int) with self.assertRaises(TypeError): issubclass(Union[int, str], int) @@ -1756,29 +1754,28 @@ def test_union_union(self): self.assertEqual(v, Union[int, float, Employee]) def test_repr(self): - self.assertEqual(repr(Union), 'typing.Union') u = Union[Employee, int] - self.assertEqual(repr(u), 'typing.Union[%s.Employee, int]' % __name__) + self.assertEqual(repr(u), f'{__name__}.Employee | int') u = Union[int, Employee] - self.assertEqual(repr(u), 'typing.Union[int, %s.Employee]' % __name__) + self.assertEqual(repr(u), f'int | {__name__}.Employee') T = TypeVar('T') u = Union[T, int][int] self.assertEqual(repr(u), repr(int)) u = Union[List[int], int] - self.assertEqual(repr(u), 'typing.Union[typing.List[int], int]') + self.assertEqual(repr(u), 'typing.List[int] | int') u = Union[list[int], dict[str, float]] - self.assertEqual(repr(u), 'typing.Union[list[int], dict[str, float]]') + self.assertEqual(repr(u), f'list[int] | dict[str, float]') u = Union[int | float] - self.assertEqual(repr(u), 'typing.Union[int, float]') + self.assertEqual(repr(u), 'int | float') u = Union[None, str] - self.assertEqual(repr(u), 'typing.Optional[str]') + self.assertEqual(repr(u), 'None | str') u = Union[str, None] - self.assertEqual(repr(u), 'typing.Optional[str]') + self.assertEqual(repr(u), 'str | None') u = Union[None, str, int] - self.assertEqual(repr(u), 'typing.Union[NoneType, str, int]') + self.assertEqual(repr(u), 'None | str | int') u = Optional[str] - self.assertEqual(repr(u), 'typing.Optional[str]') + self.assertEqual(repr(u), 'str | None') def test_dir(self): dir_items = set(dir(Union[str, int])) @@ -1790,14 +1787,11 @@ def test_dir(self): def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, - r'Cannot subclass typing\.Union'): + r"type 'types\.UnionType' is not an acceptable base type"): class C(Union): pass - with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class C(type(Union)): - pass with self.assertRaisesRegex(TypeError, - r'Cannot subclass typing\.Union\[int, str\]'): + r"cannot create 'types\.UnionType' instances"): class C(Union[int, str]): pass @@ -1843,7 +1837,7 @@ def f(x: u): ... def test_function_repr_union(self): def fun() -> int: ... - self.assertEqual(repr(Union[fun, int]), 'typing.Union[fun, int]') + self.assertEqual(repr(Union[fun, int]), f'{__name__}.{fun.__qualname__} | int') def test_union_str_pattern(self): # Shouldn't crash; see http://bugs.python.org/issue25390 @@ -4212,11 +4206,11 @@ class Derived(Base): ... def test_extended_generic_rules_repr(self): T = TypeVar('T') self.assertEqual(repr(Union[Tuple, Callable]).replace('typing.', ''), - 'Union[Tuple, Callable]') + 'Tuple | Callable') self.assertEqual(repr(Union[Tuple, Tuple[int]]).replace('typing.', ''), - 'Union[Tuple, Tuple[int]]') + 'Tuple | Tuple[int]') self.assertEqual(repr(Callable[..., Optional[T]][int]).replace('typing.', ''), - 'Callable[..., Optional[int]]') + 'Callable[..., int | None]') self.assertEqual(repr(Callable[[], List[T]][int]).replace('typing.', ''), 'Callable[[], List[int]]') @@ -4314,9 +4308,9 @@ def __contains__(self, item): with self.assertRaises(TypeError): issubclass(Tuple[int, ...], typing.Iterable) - def test_fail_with_bare_union(self): + def test_fail_with_special_forms(self): with self.assertRaises(TypeError): - List[Union] + List[Final] with self.assertRaises(TypeError): Tuple[Optional] with self.assertRaises(TypeError): @@ -4779,8 +4773,6 @@ def test_subclass_special_form(self): for obj in ( ClassVar[int], Final[int], - Union[int, float], - Optional[int], Literal[1, 2], Concatenate[int, ParamSpec("P")], TypeGuard[int], @@ -8828,7 +8820,6 @@ def test_special_attrs(self): typing.TypeAlias: 'TypeAlias', typing.TypeGuard: 'TypeGuard', typing.TypeVar: 'TypeVar', - typing.Union: 'Union', typing.Self: 'Self', # Subscribed special forms typing.Annotated[Any, "Annotation"]: 'Annotated', @@ -8839,7 +8830,7 @@ def test_special_attrs(self): typing.Literal[Any]: 'Literal', typing.Literal[1, 2]: 'Literal', typing.Literal[True, 2]: 'Literal', - typing.Optional[Any]: 'Optional', + typing.Optional[Any]: 'Union', typing.TypeGuard[Any]: 'TypeGuard', typing.Union[Any]: 'Any', typing.Union[int, float]: 'Union', @@ -8854,11 +8845,15 @@ def test_special_attrs(self): with self.subTest(cls=cls): self.assertEqual(cls.__name__, name, str(cls)) self.assertEqual(cls.__qualname__, name, str(cls)) - self.assertEqual(cls.__module__, 'typing', str(cls)) + mod = 'types' if isinstance(cls, types.UnionType) else 'typing' + self.assertEqual(cls.__module__, mod, str(cls)) for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(cls, proto) loaded = pickle.loads(s) - self.assertIs(cls, loaded) + if isinstance(cls, types.UnionType): + self.assertEqual(cls, loaded) + else: + self.assertIs(cls, loaded) TypeName = typing.NewType('SpecialAttrsTests.TypeName', Any) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 544f5246ba6848..367d9244b86655 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -13,6 +13,7 @@ typedef struct { PyObject_HEAD PyObject *args; PyObject *parameters; + PyObject *weakreflist; } unionobject; static void @@ -21,6 +22,9 @@ unionobject_dealloc(PyObject *self) unionobject *alias = (unionobject *)self; _PyObject_GC_UNTRACK(self); + if (alias->weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject *)alias); + } Py_XDECREF(alias->args); Py_XDECREF(alias->parameters); @@ -330,7 +334,22 @@ union_parameters(PyObject *self, void *Py_UNUSED(unused)) return Py_NewRef(alias->parameters); } +static PyObject * +union_name(PyObject *Py_UNUSED(self), void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("Union"); +} + +static PyObject * +union_origin(PyObject *Py_UNUSED(self), void *Py_UNUSED(ignored)) +{ + return Py_NewRef(&_PyUnion_Type); +} + static PyGetSetDef union_properties[] = { + {"__name__", union_name, NULL, "Name of the type", NULL}, + {"__qualname__", union_name, NULL, "Qualified name of the type", NULL}, + {"__origin__", union_origin, NULL, "Always returns the type", NULL}, {"__parameters__", union_parameters, (setter)NULL, "Type variables in the types.UnionType.", NULL}, {0} }; @@ -517,6 +536,7 @@ PyTypeObject _PyUnion_Type = { .tp_as_number = &union_as_number, .tp_repr = union_repr, .tp_getset = union_properties, + .tp_weaklistoffset = offsetof(unionobject, weakreflist), }; static PyObject * @@ -531,6 +551,7 @@ make_union(PyObject *args) result->parameters = NULL; result->args = Py_NewRef(args); + result->weakreflist = NULL; _PyObject_GC_TRACK(result); return (PyObject*)result; } From e75282ff2ee381db4ac1c4f1f43ecf1d21ac04a7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 07:56:13 -0700 Subject: [PATCH 03/35] blurb --- .../next/Library/2023-06-08-07-56-05.gh-issue-105499.7jV6cP.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-06-08-07-56-05.gh-issue-105499.7jV6cP.rst diff --git a/Misc/NEWS.d/next/Library/2023-06-08-07-56-05.gh-issue-105499.7jV6cP.rst b/Misc/NEWS.d/next/Library/2023-06-08-07-56-05.gh-issue-105499.7jV6cP.rst new file mode 100644 index 00000000000000..46d39751759fc0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-08-07-56-05.gh-issue-105499.7jV6cP.rst @@ -0,0 +1,2 @@ +Make :data:`typing.Union` an alias for :class:`types.UnionType`. Patch by +Jelle Zijlstra. From 540b04b2b8ba9f1ddd6bc7e352ed77121054c3ef Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 07:56:19 -0700 Subject: [PATCH 04/35] Fix test_types --- Lib/test/test_types.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 81744940f25b82..47350e6fde9602 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -698,10 +698,6 @@ def test_or_types_operator(self): y = int | bool with self.assertRaises(TypeError): x < y - # Check that we don't crash if typing.Union does not have a tuple in __args__ - y = typing.Union[str, int] - y.__args__ = [str, int] - self.assertEqual(x, y) def test_hash(self): self.assertEqual(hash(int | str), hash(str | int)) @@ -890,7 +886,7 @@ def forward_before(x: ForwardBefore[int]) -> None: ... self.assertEqual(typing.get_args(typing.get_type_hints(forward_after)['x']), (int, Forward)) self.assertEqual(typing.get_args(typing.get_type_hints(forward_before)['x']), - (int, Forward)) + (Forward, int)) def test_or_type_operator_with_Protocol(self): class Proto(typing.Protocol): From ae8fa625d30a70af7384285fdafd67345b73d0bb Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 07:59:09 -0700 Subject: [PATCH 05/35] Documentation --- Doc/library/typing.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 0a7b77f4415aa3..3fc1599c4d23c8 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -941,6 +941,9 @@ These can be used as types in annotations using ``[]``, each having a unique syn Unions can now be written as ``X | Y``. See :ref:`union type expressions`. + .. versionchanged:: 3.13 + :data:`Union` is now implemented as an alias of :class:`types.UnionType`. + .. data:: Optional Optional type. From 7569c484ad4dfe44dda746f24e3f54df08647075 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 08:09:46 -0700 Subject: [PATCH 06/35] stray f --- Lib/test/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 7a54f6ba4b19f3..459d13361b85ff 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1764,7 +1764,7 @@ def test_repr(self): u = Union[List[int], int] self.assertEqual(repr(u), 'typing.List[int] | int') u = Union[list[int], dict[str, float]] - self.assertEqual(repr(u), f'list[int] | dict[str, float]') + self.assertEqual(repr(u), 'list[int] | dict[str, float]') u = Union[int | float] self.assertEqual(repr(u), 'int | float') From 8bcb930ac59569157f98efc4508c9f13f2e11503 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 08:24:50 -0700 Subject: [PATCH 07/35] Fix tests --- Lib/test/test_dataclasses.py | 2 +- Lib/test/test_functools.py | 2 +- Lib/test/test_inspect.py | 4 ++-- Lib/test/test_pydoc.py | 14 +++++++------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 6669f1c57e2e78..1a8449efb363e1 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2267,7 +2267,7 @@ def test_docstring_one_field_with_default_none(self): class C: x: Union[int, type(None)] = None - self.assertDocStrEqual(C.__doc__, "C(x:Optional[int]=None)") + self.assertDocStrEqual(C.__doc__, "C(x:int|None=None)") def test_docstring_list_field(self): @dataclass diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index d668fa4c3adf5c..add672a76a25ea 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2774,7 +2774,7 @@ def _(arg: typing.Union[int, typing.Iterable[str]]): "Invalid annotation for 'arg'." )) self.assertTrue(str(exc.exception).endswith( - 'typing.Union[int, typing.Iterable[str]] not all arguments are classes.' + 'int | typing.Iterable[str] not all arguments are classes.' )) def test_invalid_positional_argument(self): diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index d89953ab60f022..b5d66ba20f08b6 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1507,8 +1507,8 @@ def wrapper(a, b): class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): from test.typinganndata.ann_module9 import ann, ann1 - self.assertEqual(inspect.formatannotation(ann), 'Union[List[str], int]') - self.assertEqual(inspect.formatannotation(ann1), 'Union[List[testModule.typing.A], int]') + self.assertEqual(inspect.formatannotation(ann), 'typing.List[str] | int') + self.assertEqual(inspect.formatannotation(ann1), 'typing.List[testModule.typing.A] | int') class TestIsDataDescriptor(unittest.TestCase): diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index cefc71cb5a7f54..9b3721b0d51fc7 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -123,7 +123,7 @@ class C(builtins.object) c_alias = test.pydoc_mod.C[int] list_alias1 = typing.List[int] list_alias2 = list[int] - type_union1 = typing.Union[int, str] + type_union1 = int | str type_union2 = int | str VERSION @@ -208,7 +208,7 @@ class C(builtins.object) c_alias = test.pydoc_mod.C[int] list_alias1 = typing.List[int] list_alias2 = list[int] - type_union1 = typing.Union[int, str] + type_union1 = int | str type_union2 = int | str Author @@ -1055,17 +1055,17 @@ def test_generic_alias(self): self.assertIn(list.__doc__.strip().splitlines()[0], doc) def test_union_type(self): - self.assertEqual(pydoc.describe(typing.Union[int, str]), '_UnionGenericAlias') + self.assertEqual(pydoc.describe(typing.Union[int, str]), 'UnionType') doc = pydoc.render_doc(typing.Union[int, str], renderer=pydoc.plaintext) - self.assertIn('_UnionGenericAlias in module typing', doc) - self.assertIn('Union = typing.Union', doc) + self.assertIn('UnionType in module types', doc) + self.assertIn('class UnionType(builtins.object)', doc) if typing.Union.__doc__: self.assertIn(typing.Union.__doc__.strip().splitlines()[0], doc) self.assertEqual(pydoc.describe(int | str), 'UnionType') doc = pydoc.render_doc(int | str, renderer=pydoc.plaintext) - self.assertIn('UnionType in module types object', doc) - self.assertIn('\nclass UnionType(builtins.object)', doc) + self.assertIn('UnionType in module types', doc) + self.assertIn('class UnionType(builtins.object)', doc) self.assertIn(types.UnionType.__doc__.strip().splitlines()[0], doc) def test_special_form(self): From 291953eef1a7303a9953a3126b0bea915c59cfac Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 09:23:42 -0700 Subject: [PATCH 08/35] No more typing._make_union --- Include/internal/pycore_unionobject.h | 1 + Lib/typing.py | 9 --------- Objects/typevarobject.c | 10 ++++++---- Objects/unionobject.c | 6 +++--- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_unionobject.h b/Include/internal/pycore_unionobject.h index 87264635b6e1cf..c7a90997f0fa88 100644 --- a/Include/internal/pycore_unionobject.h +++ b/Include/internal/pycore_unionobject.h @@ -16,6 +16,7 @@ extern PyObject *_Py_union_type_or(PyObject *, PyObject *); extern PyObject *_Py_subs_parameters(PyObject *, PyObject *, PyObject *, PyObject *); extern PyObject *_Py_make_parameters(PyObject *); extern PyObject *_Py_union_args(PyObject *self); +extern PyObject *_Py_union_class_getitem(PyObject *cls, PyObject *args); #ifdef __cplusplus } diff --git a/Lib/typing.py b/Lib/typing.py index 4b69218b0baf2c..60f291b3597a9d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -647,15 +647,6 @@ class FastConnector(Connection): item = _type_check(parameters, f'{self} accepts only single type.') return _GenericAlias(self, (item,)) -def _make_union(left, right): - """Used from the C implementation of TypeVar. - - TypeVar.__or__ calls this instead of returning types.UnionType - because we want to allow unions between TypeVars and strings - (forward references). - """ - return Union[left, right] - @_SpecialForm def Optional(self, parameters): """Optional[X] is equivalent to Union[X, None].""" diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 0b7d84c706d94e..78e19df0897807 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -2,7 +2,7 @@ #include "Python.h" #include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK #include "pycore_typevarobject.h" -#include "pycore_unionobject.h" // _Py_union_type_or +#include "pycore_unionobject.h" // _Py_union_type_or, _Py_union_class_getitem #include "structmember.h" /*[clinic input] @@ -99,9 +99,11 @@ type_check(PyObject *arg, const char *msg) static PyObject * make_union(PyObject *self, PyObject *other) { - PyObject *args[2] = {self, other}; - PyObject *result = call_typing_func_object("_make_union", args, 2); - return result; + PyObject *args = PyTuple_Pack(2, self, other); + if (args == NULL) { + return NULL; + } + return _Py_union_class_getitem((PyObject *)&_PyUnion_Type, args); } static PyObject * diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 367d9244b86655..4d10ef484dd948 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -460,8 +460,8 @@ add_object_to_union_args(PyObject *args_list, PyObject *args_set, PyObject *obj) return 0; } -static PyObject * -union_class_getitem(PyObject *cls, PyObject *args) +PyObject * +_Py_union_class_getitem(PyObject *cls, PyObject *args) { PyObject *args_list = PyList_New(0); if (args_list == NULL) { @@ -511,7 +511,7 @@ union_class_getitem(PyObject *cls, PyObject *args) } static PyMethodDef union_methods[] = { - {"__class_getitem__", union_class_getitem, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", _Py_union_class_getitem, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {0} }; From f7ca8d443253dba858cafe5d53ac04c91c61d036 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 18:25:04 -0700 Subject: [PATCH 09/35] Add tp_new --- Include/internal/pycore_unionobject.h | 2 +- Lib/test/test_types.py | 8 ++++++++ Objects/typevarobject.c | 4 ++-- Objects/unionobject.c | 20 ++++++++++++++++++-- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_unionobject.h b/Include/internal/pycore_unionobject.h index c7a90997f0fa88..f4b65c3c9a3e2a 100644 --- a/Include/internal/pycore_unionobject.h +++ b/Include/internal/pycore_unionobject.h @@ -16,7 +16,7 @@ extern PyObject *_Py_union_type_or(PyObject *, PyObject *); extern PyObject *_Py_subs_parameters(PyObject *, PyObject *, PyObject *, PyObject *); extern PyObject *_Py_make_parameters(PyObject *); extern PyObject *_Py_union_args(PyObject *self); -extern PyObject *_Py_union_class_getitem(PyObject *cls, PyObject *args); +extern PyObject *_Py_union_from_tuple(PyObject *args); #ifdef __cplusplus } diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 47350e6fde9602..c4561ef0860742 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1025,6 +1025,14 @@ def test_or_type_operator_reference_cycle(self): self.assertLessEqual(sys.gettotalrefcount() - before, leeway, msg='Check for union reference leak.') + def test_instantiation(self): + with self.assertRaises(TypeError): + types.UnionType() + self.assertIs(int, types.UnionType(int)) + self.assertIs(int, types.UnionType(int, int)) + self.assertEqual(int | str, types.UnionType(int, str)) + self.assertEqual(int | typing.ForwardRef("str"), types.UnionType(int, "str")) + class MappingProxyTests(unittest.TestCase): mappingproxy = types.MappingProxyType diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 78e19df0897807..4810e124b829a6 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -2,7 +2,7 @@ #include "Python.h" #include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK #include "pycore_typevarobject.h" -#include "pycore_unionobject.h" // _Py_union_type_or, _Py_union_class_getitem +#include "pycore_unionobject.h" // _Py_union_type_or, _Py_union_from_tuple #include "structmember.h" /*[clinic input] @@ -103,7 +103,7 @@ make_union(PyObject *self, PyObject *other) if (args == NULL) { return NULL; } - return _Py_union_class_getitem((PyObject *)&_PyUnion_Type, args); + return _Py_union_from_tuple(args); } static PyObject * diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 4d10ef484dd948..2bd73a7a5cc89e 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -387,6 +387,15 @@ _Py_union_args(PyObject *self) return ((unionobject *) self)->args; } +static PyObject * +union_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + if (!_PyArg_NoKeywords("UnionType", kwds)) { + return NULL; + } + return _Py_union_from_tuple(args); +} + static PyObject * call_typing_func_object(const char *name, PyObject **args, size_t nargs) { @@ -461,7 +470,7 @@ add_object_to_union_args(PyObject *args_list, PyObject *args_set, PyObject *obj) } PyObject * -_Py_union_class_getitem(PyObject *cls, PyObject *args) +_Py_union_from_tuple(PyObject *args) { PyObject *args_list = PyList_New(0); if (args_list == NULL) { @@ -510,8 +519,14 @@ _Py_union_class_getitem(PyObject *cls, PyObject *args) return make_union(args_tuple); } +static PyObject * +union_class_getitem(PyObject *cls, PyObject *args) +{ + return _Py_union_from_tuple(args); +} + static PyMethodDef union_methods[] = { - {"__class_getitem__", _Py_union_class_getitem, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", union_class_getitem, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {0} }; @@ -531,6 +546,7 @@ PyTypeObject _PyUnion_Type = { .tp_getattro = union_getattro, .tp_members = union_members, .tp_methods = union_methods, + .tp_new = union_new, .tp_richcompare = union_richcompare, .tp_as_mapping = &union_as_mapping, .tp_as_number = &union_as_number, From 0f1ab182fe0717fb40b9554c31db22b89938b501 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 18:27:14 -0700 Subject: [PATCH 10/35] fix a refleak --- Objects/typevarobject.c | 4 +++- Objects/unionobject.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 4810e124b829a6..61066afe250404 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -103,7 +103,9 @@ make_union(PyObject *self, PyObject *other) if (args == NULL) { return NULL; } - return _Py_union_from_tuple(args); + PyObject *u = _Py_union_from_tuple(args); + Py_DECREF(args); + return u; } static PyObject * diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 2bd73a7a5cc89e..1f33ad666011e3 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -516,7 +516,9 @@ _Py_union_from_tuple(PyObject *args) if (args_tuple == NULL) { return NULL; } - return make_union(args_tuple); + PyObject *u = make_union(args_tuple); + Py_DECREF(args_tuple); + return u; } static PyObject * From eb47a0b858ebce84cac0755ea59bc6265d84adc1 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 8 Jun 2023 19:40:38 -0700 Subject: [PATCH 11/35] Fix test --- Lib/test/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 459d13361b85ff..b2bb34f9261222 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1791,7 +1791,7 @@ def test_cannot_subclass(self): class C(Union): pass with self.assertRaisesRegex(TypeError, - r"cannot create 'types\.UnionType' instances"): + r"Union\[arg, \.\.\.\]: each arg must be a type\."): class C(Union[int, str]): pass From eaa4e7967f6d34e8e08c9b81878cef345ec45e41 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 9 Jun 2023 07:26:58 -0700 Subject: [PATCH 12/35] Update Lib/test/test_typing.py Co-authored-by: Alex Waygood --- Lib/test/test_typing.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b2bb34f9261222..67915e2fbad783 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1708,8 +1708,6 @@ def test_basics(self): self.assertNotEqual(u, Union) def test_subclass_error(self): - self.assertNotIsSubclass(int, Union) - self.assertNotIsSubclass(Union, int) with self.assertRaises(TypeError): issubclass(Union[int, str], int) From c71e8c3f8cf1f80322cf4d95f1dc86f1f5d6979f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 12 Jun 2023 19:03:29 -0700 Subject: [PATCH 13/35] Add back _UnionGenericAlias --- Doc/whatsnew/3.13.rst | 4 ++++ Lib/test/test_typing.py | 28 ++++++++++++++++++++++++++++ Lib/typing.py | 27 +++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e6504b0152acc2..d17243ffc3c6ff 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -140,6 +140,10 @@ Deprecated Use the ``'w'`` format code instead. (contributed by Hugo van Kemenade in :gh:`80480`) +* The internal class ``typing._UnionGenericAlias`` is no longer used to implement + :data:`typing.Union`. To preserve compatibility with users using this private + class, a compatibility shim will be provided until Python 3.15. + Removed ======= diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 67915e2fbad783..2554277d3327d9 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -9095,5 +9095,33 @@ def test_is_not_instance_of_iterable(self): self.assertNotIsInstance(type_to_test, collections.abc.Iterable) +class UnionGenericAliasTests(BaseTestCase): + def test_constructor(self): + # Used e.g. in typer, pydantic + with self.assertWarns(DeprecationWarning): + inst = typing._UnionGenericAlias(typing.Union, (int, str)) + self.assertEqual(inst, int | str) + with self.assertWarns(DeprecationWarning): + # name is accepted but ignored + inst = typing._UnionGenericAlias(typing.Union, (int, None), name="Optional") + self.assertEqual(inst, int | None) + + def test_isinstance(self): + # Used e.g. in pydantic + with self.assertWarns(DeprecationWarning): + self.assertTrue(isinstance(Union[int, str], typing._UnionGenericAlias)) + with self.assertWarns(DeprecationWarning): + self.assertFalse(isinstance(int, typing._UnionGenericAlias)) + + def test_eq(self): + # type(t) == _UnionGenericAlias is used in vyos + with self.assertWarns(DeprecationWarning): + self.assertEqual(Union, typing._UnionGenericAlias) + with self.assertWarns(DeprecationWarning): + self.assertEqual(typing._UnionGenericAlias, typing._UnionGenericAlias) + with self.assertWarns(DeprecationWarning): + self.assertNotEqual(int, typing._UnionGenericAlias) + + if __name__ == '__main__': main() diff --git a/Lib/typing.py b/Lib/typing.py index 0d9410ebc61e55..ae4d81a1181cba 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -36,6 +36,7 @@ GenericAlias, UnionType as Union, ) +import warnings from _typing import ( _idfunc, @@ -1463,6 +1464,32 @@ def __getitem__(self, params): return self.copy_with(params) +class _UnionGenericAliasMeta(type): + def __instancecheck__(self, inst: type) -> bool: + warnings._deprecated("_UnionGenericAlias", remove=(3, 15)) + return isinstance(inst, types.UnionType) + + def __eq__(self, other): + warnings._deprecated("_UnionGenericAlias", remove=(3, 15)) + if other is Union or other is _UnionGenericAlias: + return True + return NotImplemented + + +class _UnionGenericAlias(metaclass=_UnionGenericAliasMeta): + """Compatibility hack. + + A class named _UnionGenericAlias used to be used to implement + typing.Union. This class exists to serve as a shim to preserve + the meaning of some code that used to use _UnionGenericAlias + directly. + + """ + def __new__(cls, self_cls, parameters, /, *, name=None): + warnings._deprecated("_UnionGenericAlias", remove=(3, 15)) + return Union[parameters] + + def _value_and_type_iter(parameters): return ((p, type(p)) for p in parameters) From bb50899004518eba72ac861e3896cccf27b506c8 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 15 Jun 2023 17:29:59 -0700 Subject: [PATCH 14/35] Remove unnecessary NewRef --- Objects/unionobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 1f33ad666011e3..43a9737e4e2ba7 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -417,9 +417,9 @@ call_typing_func_object(const char *name, PyObject **args, size_t nargs) static PyObject * type_check(PyObject *arg, const char *msg) { - // Calling typing.py here leads to bootstrapping problems if (Py_IsNone(arg)) { - return Py_NewRef(Py_TYPE(arg)); + // NoneType is immortal, so don't need an INCREF + return Py_TYPE(arg); } // Fast path to avoid calling into typing.py if (is_unionable(arg)) { From c564672f0590b1f30424a4c2b64ddde3d0d43406 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 15 Jun 2023 17:42:43 -0700 Subject: [PATCH 15/35] Make typing.Union the canonical name --- Lib/test/test_inspect.py | 4 ++-- Lib/test/test_typing.py | 5 ++--- Objects/unionobject.c | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index b5d66ba20f08b6..0dda2f552cd328 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1507,8 +1507,8 @@ def wrapper(a, b): class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): from test.typinganndata.ann_module9 import ann, ann1 - self.assertEqual(inspect.formatannotation(ann), 'typing.List[str] | int') - self.assertEqual(inspect.formatannotation(ann1), 'typing.List[testModule.typing.A] | int') + self.assertEqual(inspect.formatannotation(ann), 'List[str] | int') + self.assertEqual(inspect.formatannotation(ann1), 'List[testModule.typing.A] | int') class TestIsDataDescriptor(unittest.TestCase): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 48fbb7e50edf5e..b5684dc3be8cc7 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1785,7 +1785,7 @@ def test_dir(self): def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, - r"type 'types\.UnionType' is not an acceptable base type"): + r"type 'typing\.Union' is not an acceptable base type"): class C(Union): pass with self.assertRaisesRegex(TypeError, @@ -9007,8 +9007,7 @@ def test_special_attrs(self): with self.subTest(cls=cls): self.assertEqual(cls.__name__, name, str(cls)) self.assertEqual(cls.__qualname__, name, str(cls)) - mod = 'types' if isinstance(cls, types.UnionType) else 'typing' - self.assertEqual(cls.__module__, mod, str(cls)) + self.assertEqual(cls.__module__, 'typing', str(cls)) for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(cls, proto) loaded = pickle.loads(s) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 43a9737e4e2ba7..4f3d60283e7c82 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -534,8 +534,8 @@ static PyMethodDef union_methods[] = { PyTypeObject _PyUnion_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "types.UnionType", - .tp_doc = PyDoc_STR("Represent a PEP 604 union type\n" + .tp_name = "typing.Union", + .tp_doc = PyDoc_STR("Represent a union type\n" "\n" "E.g. for int | str"), .tp_basicsize = sizeof(unionobject), From 9f58421b613440ae2670e72c7b0928feeacba0ac Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 15 Jun 2023 17:47:00 -0700 Subject: [PATCH 16/35] docs --- Doc/library/stdtypes.rst | 10 ++++------ Doc/library/types.rst | 4 ++++ Doc/library/typing.rst | 5 ++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 0caa725f75e642..d20201adfea811 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5212,16 +5212,14 @@ enables cleaner type hinting syntax compared to :data:`typing.Union`. TypeError: isinstance() argument 2 cannot contain a parameterized generic The user-exposed type for the union object can be accessed from -:data:`types.UnionType` and used for :func:`isinstance` checks. An object cannot be +:data:`typing.Union` and used for :func:`isinstance` checks. An object cannot be instantiated from the type:: >>> import types - >>> isinstance(int | str, types.UnionType) + >>> isinstance(int | str, typing.Union) True - >>> types.UnionType() - Traceback (most recent call last): - File "", line 1, in - TypeError: cannot create 'types.UnionType' instances + >>> types.UnionType(int, str) + int | str .. note:: The :meth:`__or__` method for type objects was added to support the syntax diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 8cbe17df16f107..e9755b018fe50c 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -365,6 +365,10 @@ Standard names are defined for the following types: .. versionadded:: 3.10 + .. versionchanged:: 3.13 + + This is now an alias for :data:`typing.Union`. + .. class:: TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno) The type of traceback objects such as found in ``sys.exception().__traceback__``. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 6b136477d91857..80c6f882c4b222 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -948,7 +948,10 @@ These can be used as types in annotations using ``[]``, each having a unique syn :ref:`union type expressions`. .. versionchanged:: 3.13 - :data:`Union` is now implemented as an alias of :class:`types.UnionType`. + :data:`Union` and :class:`types.UnionType` are now aliases, and both + ``Union[int, str]`` and ``int | str`` create instances of the same class. + To check whether an object is a ``Union`` at runtime, use + ``isinstance(obj, Union)``. .. data:: Optional From 8884a940be83d7ca63b99208ec3bf5e660e99758 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 15 Jun 2023 19:55:05 -0700 Subject: [PATCH 17/35] fix test_pydoc --- Lib/test/test_pydoc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 9b3721b0d51fc7..caa87f93e85a83 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -1055,17 +1055,17 @@ def test_generic_alias(self): self.assertIn(list.__doc__.strip().splitlines()[0], doc) def test_union_type(self): - self.assertEqual(pydoc.describe(typing.Union[int, str]), 'UnionType') + self.assertEqual(pydoc.describe(typing.Union[int, str]), 'Union') doc = pydoc.render_doc(typing.Union[int, str], renderer=pydoc.plaintext) - self.assertIn('UnionType in module types', doc) - self.assertIn('class UnionType(builtins.object)', doc) + self.assertIn('Union in module typing', doc) + self.assertIn('class Union(builtins.object)', doc) if typing.Union.__doc__: self.assertIn(typing.Union.__doc__.strip().splitlines()[0], doc) - self.assertEqual(pydoc.describe(int | str), 'UnionType') + self.assertEqual(pydoc.describe(int | str), 'Union') doc = pydoc.render_doc(int | str, renderer=pydoc.plaintext) - self.assertIn('UnionType in module types', doc) - self.assertIn('class UnionType(builtins.object)', doc) + self.assertIn('Union in module typing', doc) + self.assertIn('class Union(builtins.object)', doc) self.assertIn(types.UnionType.__doc__.strip().splitlines()[0], doc) def test_special_form(self): From bcc2e6a19426b127bc4ad68e8bb1799934174944 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 16 Jun 2023 07:54:07 -0700 Subject: [PATCH 18/35] Update Objects/unionobject.c Co-authored-by: Ken Jin --- Objects/unionobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 4f3d60283e7c82..9b91d011481311 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -419,7 +419,7 @@ type_check(PyObject *arg, const char *msg) { if (Py_IsNone(arg)) { // NoneType is immortal, so don't need an INCREF - return Py_TYPE(arg); + return (PyObject *)Py_TYPE(arg); } // Fast path to avoid calling into typing.py if (is_unionable(arg)) { From c4b217b1e91c8c11339ccbbb8187b784ae90d8b6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 16 Jun 2023 18:06:41 -0700 Subject: [PATCH 19/35] Update Doc/library/typing.rst Co-authored-by: Carl Meyer --- Doc/library/typing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 80c6f882c4b222..2630f7fe09ab28 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -948,7 +948,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn :ref:`union type expressions`. .. versionchanged:: 3.13 - :data:`Union` and :class:`types.UnionType` are now aliases, and both + :class:`types.UnionType` is now an alias for :data:`Union`, and both ``Union[int, str]`` and ``int | str`` create instances of the same class. To check whether an object is a ``Union`` at runtime, use ``isinstance(obj, Union)``. From 5eb2a0c6195518e33b882507eb07617e8a14c863 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 16 Jun 2023 18:06:52 -0700 Subject: [PATCH 20/35] Update Doc/library/stdtypes.rst Co-authored-by: Carl Meyer --- Doc/library/stdtypes.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index d20201adfea811..7fce90db6ea2de 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5218,8 +5218,6 @@ instantiated from the type:: >>> import types >>> isinstance(int | str, typing.Union) True - >>> types.UnionType(int, str) - int | str .. note:: The :meth:`__or__` method for type objects was added to support the syntax From 64ac293a7a67827b200cd75ccdd275682021cb12 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 16 Jun 2023 18:07:04 -0700 Subject: [PATCH 21/35] Update Doc/library/stdtypes.rst Co-authored-by: Carl Meyer --- Doc/library/stdtypes.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 7fce90db6ea2de..e2fd44723e116a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5212,8 +5212,7 @@ enables cleaner type hinting syntax compared to :data:`typing.Union`. TypeError: isinstance() argument 2 cannot contain a parameterized generic The user-exposed type for the union object can be accessed from -:data:`typing.Union` and used for :func:`isinstance` checks. An object cannot be -instantiated from the type:: +:data:`typing.Union` and used for :func:`isinstance` checks:: >>> import types >>> isinstance(int | str, typing.Union) From aacf2b0df7472b9ee7a5bd60e5fad09cfdbc612b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 16 Jun 2023 19:15:43 -0700 Subject: [PATCH 22/35] Add __mro_entries__ --- Lib/test/test_typing.py | 3 +-- Objects/unionobject.c | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a6714d7f28163e..86901cc56d709a 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1788,8 +1788,7 @@ def test_cannot_subclass(self): r"type 'typing\.Union' is not an acceptable base type"): class C(Union): pass - with self.assertRaisesRegex(TypeError, - r"Union\[arg, \.\.\.\]: each arg must be a type\."): + with self.assertRaisesRegex(TypeError, r"Cannot subclass int | str"): class C(Union[int, str]): pass diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 9b91d011481311..c88d45bb10be19 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -527,7 +527,15 @@ union_class_getitem(PyObject *cls, PyObject *args) return _Py_union_from_tuple(args); } +static PyObject * +union_mro_entries(PyObject *self, PyObject *args) +{ + return PyErr_Format(PyExc_TypeError, + "Cannot subclass %R", self); +} + static PyMethodDef union_methods[] = { + {"__mro_entries__", union_mro_entries, METH_O}, {"__class_getitem__", union_class_getitem, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {0} }; From f46c0c6e725efd9e40cff162c5ce366d8581958c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 16 Jun 2023 19:29:17 -0700 Subject: [PATCH 23/35] Improve docs, expose it in _typing --- Doc/library/functools.rst | 6 +++--- Doc/library/stdtypes.rst | 7 ++++--- Doc/library/types.rst | 2 +- Doc/library/typing.rst | 6 +++--- Doc/whatsnew/3.10.rst | 4 ++-- Doc/whatsnew/3.11.rst | 4 ++-- Doc/whatsnew/3.13.rst | 2 +- Lib/functools.py | 12 ++++-------- Lib/test/test_typing.py | 8 ++++---- Lib/typing.py | 16 ++++++++-------- ...023-06-08-07-56-05.gh-issue-105499.7jV6cP.rst | 5 +++-- Modules/_typingmodule.c | 4 ++++ Objects/unionobject.c | 2 +- 13 files changed, 40 insertions(+), 38 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 40f43f8b3519cd..2f7c7111d9aa18 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -462,7 +462,7 @@ The :mod:`functools` module defines the following functions: ... for i, elem in enumerate(arg): ... print(i, elem) - :data:`types.UnionType` and :data:`typing.Union` can also be used:: + :class:`typing.Union` can also be used:: >>> @fun.register ... def _(arg: int | float, verbose=False): @@ -579,8 +579,8 @@ The :mod:`functools` module defines the following functions: The :func:`register` attribute now supports using type annotations. .. versionchanged:: 3.11 - The :func:`register` attribute now supports :data:`types.UnionType` - and :data:`typing.Union` as type annotations. + The :func:`register` attribute now supports + and :class:`typing.Union` as type annotations. .. class:: singledispatchmethod(func) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index e2fd44723e116a..f4caaa53794793 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5150,7 +5150,7 @@ Union Type A union object holds the value of the ``|`` (bitwise or) operation on multiple :ref:`type objects `. These types are intended primarily for :term:`type annotations `. The union type expression -enables cleaner type hinting syntax compared to :data:`typing.Union`. +enables cleaner type hinting syntax compared to subscripting :class:`typing.Union`. .. describe:: X | Y | ... @@ -5186,9 +5186,10 @@ enables cleaner type hinting syntax compared to :data:`typing.Union`. int | str == str | int - * It is compatible with :data:`typing.Union`:: + * It creates instances of :class:`typing.Union`:: int | str == typing.Union[int, str] + type(int | str) is typing.Union * Optional types can be spelled as a union with ``None``:: @@ -5212,7 +5213,7 @@ enables cleaner type hinting syntax compared to :data:`typing.Union`. TypeError: isinstance() argument 2 cannot contain a parameterized generic The user-exposed type for the union object can be accessed from -:data:`typing.Union` and used for :func:`isinstance` checks:: +:class:`typing.Union` and used for :func:`isinstance` checks:: >>> import types >>> isinstance(int | str, typing.Union) diff --git a/Doc/library/types.rst b/Doc/library/types.rst index e9755b018fe50c..5d03d0291d65ac 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -367,7 +367,7 @@ Standard names are defined for the following types: .. versionchanged:: 3.13 - This is now an alias for :data:`typing.Union`. + This is now an alias for :class:`typing.Union`. .. class:: TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 2630f7fe09ab28..7c5d82651da573 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -24,7 +24,7 @@ -------------- This module provides runtime support for type hints. The most fundamental -support consists of the types :data:`Any`, :data:`Union`, :data:`Callable`, +support consists of the types :data:`Any`, :class:`Union`, :data:`Callable`, :class:`TypeVar`, and :class:`Generic`. For a specification, please see :pep:`484`. For a simplified introduction to type hints, see :pep:`483`. @@ -912,7 +912,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn :class:`builtins.tuple ` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. -.. data:: Union +.. class:: Union Union type; ``Union[X, Y]`` is equivalent to ``X | Y`` and means either X or Y. @@ -948,7 +948,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn :ref:`union type expressions`. .. versionchanged:: 3.13 - :class:`types.UnionType` is now an alias for :data:`Union`, and both + :class:`types.UnionType` is now an alias for :class:`Union`, and both ``Union[int, str]`` and ``int | str`` create instances of the same class. To check whether an object is a ``Union`` at runtime, use ``isinstance(obj, Union)``. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index ab030db5b3ffaa..c6ff5e0d5e2d2d 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -718,10 +718,10 @@ PEP 604: New Type Union Operator A new type union operator was introduced which enables the syntax ``X | Y``. This provides a cleaner way of expressing 'either type X or type Y' instead of -using :data:`typing.Union`, especially in type hints. +using :class:`typing.Union`, especially in type hints. In previous versions of Python, to apply a type hint for functions accepting -arguments of multiple types, :data:`typing.Union` was used:: +arguments of multiple types, :class:`typing.Union` was used:: def square(number: Union[int, float]) -> Union[int, float]: return number ** 2 diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 45194130c993a0..90dc51c1c53c03 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -737,8 +737,8 @@ fractions functools --------- -* :func:`functools.singledispatch` now supports :data:`types.UnionType` - and :data:`typing.Union` as annotations to the dispatch argument.:: +* :func:`functools.singledispatch` now supports :class:`types.UnionType` + and :class:`typing.Union` as annotations to the dispatch argument.:: >>> from functools import singledispatch >>> @singledispatch diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 4c9a2e49c63ee6..3c0cf8dc1b5b82 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -163,7 +163,7 @@ Deprecated (Contributed by Hugo van Kemenade in :gh:`80480`) * The internal class ``typing._UnionGenericAlias`` is no longer used to implement - :data:`typing.Union`. To preserve compatibility with users using this private + :class:`typing.Union`. To preserve compatibility with users using this private class, a compatibility shim will be provided until Python 3.15. (Contributed by Jelle Zijlstra in :gh:`105499`.) diff --git a/Lib/functools.py b/Lib/functools.py index 4d5e2709007843..0aa3adf7617de8 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -19,7 +19,7 @@ # import types, weakref # Deferred to single_dispatch() from reprlib import recursive_repr from _thread import RLock -from types import GenericAlias +from types import GenericAlias, UnionType ################################################################################ @@ -840,15 +840,13 @@ def dispatch(cls): return impl def _is_union_type(cls): - from typing import get_origin, Union - return get_origin(cls) in {Union, types.UnionType} + return isinstance(cls, UnionType) def _is_valid_dispatch_type(cls): if isinstance(cls, type): return True - from typing import get_args return (_is_union_type(cls) and - all(isinstance(arg, type) for arg in get_args(cls))) + all(isinstance(arg, type) for arg in cls.__args__)) def register(cls, func=None): """generic_func.register(cls, func) -> func @@ -891,9 +889,7 @@ def register(cls, func=None): ) if _is_union_type(cls): - from typing import get_args - - for arg in get_args(cls): + for arg in cls.__args__: registry[arg] = func else: registry[cls] = func diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 86901cc56d709a..c7c66d64d83cea 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4939,7 +4939,7 @@ class A: __parameters__ = (T,) # Bare classes should be skipped for a in (List, list): - for b in (A, int, TypeVar, TypeVarTuple, ParamSpec, types.GenericAlias, types.UnionType): + for b in (A, int, TypeVar, TypeVarTuple, ParamSpec, types.GenericAlias, Union): with self.subTest(generic=a, sub=b): with self.assertRaisesRegex(TypeError, '.* is not a generic class'): a[b][str] @@ -4958,7 +4958,7 @@ class A: for s in (int, G, A, List, list, TypeVar, TypeVarTuple, ParamSpec, - types.GenericAlias, types.UnionType): + types.GenericAlias, Union): for t in Tuple, tuple: with self.subTest(tuple=t, sub=s): @@ -6386,7 +6386,7 @@ class C(Generic[T]): pass self.assertIs(get_origin(Callable), collections.abc.Callable) self.assertIs(get_origin(list[int]), list) self.assertIs(get_origin(list), None) - self.assertIs(get_origin(list | str), types.UnionType) + self.assertIs(get_origin(list | str), Union) self.assertIs(get_origin(P.args), P) self.assertIs(get_origin(P.kwargs), P) self.assertIs(get_origin(Required[int]), Required) @@ -9084,7 +9084,7 @@ def test_special_attrs(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(cls, proto) loaded = pickle.loads(s) - if isinstance(cls, types.UnionType): + if isinstance(cls, Union): self.assertEqual(cls, loaded) else: self.assertIs(cls, loaded) diff --git a/Lib/typing.py b/Lib/typing.py index 395dfe2c82226b..8740c9f0a0dd6d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -34,7 +34,6 @@ MethodWrapperType, MethodDescriptorType, GenericAlias, - UnionType as Union, ) import warnings @@ -47,6 +46,7 @@ ParamSpecKwargs, TypeAliasType, Generic, + Union, ) # Please keep __all__ alphabetized within each category. @@ -378,7 +378,7 @@ def _eval_type(t, globalns, localns, recursive_guard=frozenset()): """ if isinstance(t, ForwardRef): return t._evaluate(globalns, localns, recursive_guard) - if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): + if isinstance(t, (_GenericAlias, GenericAlias, Union)): if isinstance(t, GenericAlias): args = tuple( ForwardRef(arg) if isinstance(arg, str) else arg @@ -396,7 +396,7 @@ def _eval_type(t, globalns, localns, recursive_guard=frozenset()): return t if isinstance(t, GenericAlias): return GenericAlias(t.__origin__, ev_args) - if isinstance(t, types.UnionType): + if isinstance(t, Union): return functools.reduce(operator.or_, ev_args) else: return t.copy_with(ev_args) @@ -1469,7 +1469,7 @@ def __getitem__(self, params): class _UnionGenericAliasMeta(type): def __instancecheck__(self, inst: type) -> bool: warnings._deprecated("_UnionGenericAlias", remove=(3, 15)) - return isinstance(inst, types.UnionType) + return isinstance(inst, Union) def __eq__(self, other): warnings._deprecated("_UnionGenericAlias", remove=(3, 15)) @@ -2176,7 +2176,7 @@ def _strip_annotations(t): if stripped_args == t.__args__: return t return GenericAlias(t.__origin__, stripped_args) - if isinstance(t, types.UnionType): + if isinstance(t, Union): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: return t @@ -2209,8 +2209,8 @@ def get_origin(tp): return tp.__origin__ if tp is Generic: return Generic - if isinstance(tp, types.UnionType): - return types.UnionType + if isinstance(tp, Union): + return Union return None @@ -2234,7 +2234,7 @@ def get_args(tp): if _should_unflatten_callable_args(tp, res): res = (list(res[:-1]), res[-1]) return res - if isinstance(tp, types.UnionType): + if isinstance(tp, Union): return tp.__args__ return () diff --git a/Misc/NEWS.d/next/Library/2023-06-08-07-56-05.gh-issue-105499.7jV6cP.rst b/Misc/NEWS.d/next/Library/2023-06-08-07-56-05.gh-issue-105499.7jV6cP.rst index 46d39751759fc0..5240f4aa7d1e9c 100644 --- a/Misc/NEWS.d/next/Library/2023-06-08-07-56-05.gh-issue-105499.7jV6cP.rst +++ b/Misc/NEWS.d/next/Library/2023-06-08-07-56-05.gh-issue-105499.7jV6cP.rst @@ -1,2 +1,3 @@ -Make :data:`typing.Union` an alias for :class:`types.UnionType`. Patch by -Jelle Zijlstra. +Make :class:`types.UnionType` an alias for :class:`typing.Union`. Both +``int | str`` and ``Union[int, str]`` now create instances of the same +type. Patch by Jelle Zijlstra. diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 39a124a26adf31..88b46ac3cea603 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -7,6 +7,7 @@ #include "Python.h" #include "internal/pycore_interp.h" #include "internal/pycore_typevarobject.h" +#include "internal/pycore_unionobject.h" // _PyUnion_Type #include "clinic/_typingmodule.c.h" /*[clinic input] @@ -62,6 +63,9 @@ _typing_exec(PyObject *m) if (PyModule_AddObjectRef(m, "TypeAliasType", (PyObject *)&_PyTypeAlias_Type) < 0) { return -1; } + if (PyModule_AddObjectRef(m, "Union", (PyObject *)&_PyUnion_Type) < 0) { + return -1; + } return 0; } diff --git a/Objects/unionobject.c b/Objects/unionobject.c index c88d45bb10be19..6e1c007c116e48 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -1,4 +1,4 @@ -// types.UnionType -- used to represent e.g. Union[int, str], int | str +// typing.Union -- used to represent e.g. Union[int, str], int | str #include "Python.h" #include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK #include "pycore_typevarobject.h" // _PyTypeAlias_Type From 8c0444116a8532db380c22dfa214fea38ebee348 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 20 Jun 2023 09:25:30 -0700 Subject: [PATCH 24/35] Update Doc/library/functools.rst Co-authored-by: Carl Meyer --- Doc/library/functools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 2f7c7111d9aa18..8a3590b45ba269 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -580,7 +580,7 @@ The :mod:`functools` module defines the following functions: .. versionchanged:: 3.11 The :func:`register` attribute now supports - and :class:`typing.Union` as type annotations. + :class:`typing.Union` as a type annotation. .. class:: singledispatchmethod(func) From 98b634a7986cfa56dd74fa3cbf7f66299718ee16 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 10 Oct 2023 21:46:08 -0700 Subject: [PATCH 25/35] Post-merge cleanup --- Doc/library/stdtypes.rst | 2 +- Doc/library/typing.rst | 3 ++- Lib/test/test_typing.py | 5 +---- Modules/_typingmodule.c | 1 + Objects/unionobject.c | 11 +++++++---- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index ac2cac40ab04ea..017b0e5cc15fd9 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5224,7 +5224,7 @@ enables cleaner type hinting syntax compared to subscripting :class:`typing.Unio The user-exposed type for the union object can be accessed from :class:`typing.Union` and used for :func:`isinstance` checks:: - >>> import types + >>> import typing >>> isinstance(int | str, typing.Union) True diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 79dbad9d938592..822dd8623b5c25 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -27,6 +27,7 @@ This module provides runtime support for type hints. For the original specification of the typing system, see :pep:`484`. For a simplified introduction to type hints, see :pep:`483`. + The function below takes and returns a string and is annotated as follows:: def greeting(name: str) -> str: @@ -1063,7 +1064,7 @@ Special forms These can be used as types in annotations. They all support subscription using ``[]``, but each has a unique syntax. -.. data:: Union +.. class:: Union Union type; ``Union[X, Y]`` is equivalent to ``X | Y`` and means either X or Y. diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 016285efe6619e..2dcc936cd3301f 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1798,11 +1798,8 @@ def test_cannot_subclass(self): r"type 'typing\.Union' is not an acceptable base type"): class C(Union): pass - with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): - class D(type(Union)): - pass with self.assertRaisesRegex(TypeError, - r'Cannot subclass typing\.Union\[int, str\]'): + r'Cannot subclass int \| str'): class E(Union[int, str]): pass diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 92db5a7559d976..0de96969ab0c47 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -8,6 +8,7 @@ #include "internal/pycore_interp.h" #include "internal/pycore_typevarobject.h" #include "internal/pycore_unionobject.h" // _PyUnion_Type +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "clinic/_typingmodule.c.h" /*[clinic input] diff --git a/Objects/unionobject.c b/Objects/unionobject.c index fb970f96959ec9..8794a7027be4b3 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -340,11 +340,14 @@ union_origin(PyObject *Py_UNUSED(self), void *Py_UNUSED(ignored)) } static PyGetSetDef union_properties[] = { - {"__name__", union_name, NULL, "Name of the type", NULL}, - {"__qualname__", union_name, NULL, "Qualified name of the type", NULL}, - {"__origin__", union_origin, NULL, "Always returns the type", NULL}, + {"__name__", union_name, NULL, + PyDoc_STR("Name of the type"), NULL}, + {"__qualname__", union_name, NULL, + PyDoc_STR("Qualified name of the type"), NULL}, + {"__origin__", union_origin, NULL, + PyDoc_STR("Always returns the type"), NULL}, {"__parameters__", union_parameters, (setter)NULL, - "Type variables in the types.UnionType.", NULL}, + PyDoc_STR("Type variables in the types.UnionType."), NULL}, {0} }; From f2f961bd753e15a196506f19d6469c7ac07fbc58 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 11 Oct 2023 19:15:26 -0700 Subject: [PATCH 26/35] No need for tp_new --- Lib/test/test_types.py | 8 ++++---- Objects/unionobject.c | 10 ---------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 875de7761b18e0..9945b3c63c158d 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1032,10 +1032,10 @@ def test_or_type_operator_reference_cycle(self): def test_instantiation(self): with self.assertRaises(TypeError): types.UnionType() - self.assertIs(int, types.UnionType(int)) - self.assertIs(int, types.UnionType(int, int)) - self.assertEqual(int | str, types.UnionType(int, str)) - self.assertEqual(int | typing.ForwardRef("str"), types.UnionType(int, "str")) + self.assertIs(int, types.UnionType[int]) + self.assertIs(int, types.UnionType[int, int]) + self.assertEqual(int | str, types.UnionType[int, str]) + self.assertEqual(int | typing.ForwardRef("str"), types.UnionType[int, "str"]) class MappingProxyTests(unittest.TestCase): diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 8794a7027be4b3..031ce609a80ee8 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -384,15 +384,6 @@ _Py_union_args(PyObject *self) return ((unionobject *) self)->args; } -static PyObject * -union_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - if (!_PyArg_NoKeywords("UnionType", kwds)) { - return NULL; - } - return _Py_union_from_tuple(args); -} - static PyObject * call_typing_func_object(const char *name, PyObject **args, size_t nargs) { @@ -553,7 +544,6 @@ PyTypeObject _PyUnion_Type = { .tp_getattro = union_getattro, .tp_members = union_members, .tp_methods = union_methods, - .tp_new = union_new, .tp_richcompare = union_richcompare, .tp_as_mapping = &union_as_mapping, .tp_as_number = &union_as_number, From 9ae102db875ae82863248e10d6b4fd6ad6859000 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 18 Feb 2024 06:54:29 -0800 Subject: [PATCH 27/35] Add back doctest --- Doc/library/stdtypes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 76fc3e63f0326c..26add86c108e2b 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5232,6 +5232,10 @@ The user-exposed type for the union object can be accessed from >>> import typing >>> isinstance(int | str, typing.Union) True + >>> typing.Union() + Traceback (most recent call last): + File "", line 1, in + TypeError: cannot create 'typing.Union' instances .. note:: The :meth:`!__or__` method for type objects was added to support the syntax From 8a80fe0a297b373c86b17833548b95f8e67b01d6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 25 Apr 2024 22:10:01 -0700 Subject: [PATCH 28/35] Fix two issues --- Lib/typing.py | 12 +++--------- Objects/unionobject.c | 7 ++++++- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 5f62cd33139e3b..a134d106b7cc79 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1604,15 +1604,9 @@ def __instancecheck__(self, inst: type) -> bool: def __eq__(self, other): warnings._deprecated("_UnionGenericAlias", remove=(3, 15)) - if not isinstance(other, (_UnionGenericAlias, types.UnionType)): - return NotImplemented - try: # fast path - return set(self.__args__) == set(other.__args__) - except TypeError: # not hashable, slow path - return _compare_args_orderless(self.__args__, other.__args__) - - def __hash__(self): - return hash(frozenset(self.__args__)) + if other is _UnionGenericAlias or other is Union: + return True + return NotImplemented class _UnionGenericAlias(metaclass=_UnionGenericAliasMeta): diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 031ce609a80ee8..f3e4575e242e17 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -441,7 +441,12 @@ add_object_to_union_args(PyObject *args_list, PyObject *args_set, PyObject *obj) if (type == NULL) { return -1; } - if (PySet_Contains(args_set, type)) { + int contains = PySet_Contains(args_set, type); + if (contains < 0) { + Py_DECREF(type); + return -1; + } + else if (contains == 1) { Py_DECREF(type); return 0; } From f910e8869d0a4a151de75e4f90c69db0e696e6bd Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 25 Apr 2024 22:25:32 -0700 Subject: [PATCH 29/35] in progress: hashable unions --- Objects/unionobject.c | 74 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index f3e4575e242e17..c5a21e9fe02cdb 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -424,14 +424,15 @@ type_check(PyObject *arg, const char *msg) } static int -add_object_to_union_args(PyObject *args_list, PyObject *args_set, PyObject *obj) +add_object_to_union_args(PyObject *args_list, PyObject *args_set, + PyObject *unhashable_args, PyObject *obj) { if (Py_IS_TYPE(obj, &_PyUnion_Type)) { PyObject *args = ((unionobject *) obj)->args; Py_ssize_t size = PyTuple_GET_SIZE(args); for (Py_ssize_t i = 0; i < size; i++) { PyObject *arg = PyTuple_GET_ITEM(args, i); - if (add_object_to_union_args(args_list, args_set, arg) < 0) { + if (add_object_to_union_args(args_list, args_set, unhashable_args, arg) < 0) { return -1; } } @@ -443,8 +444,17 @@ add_object_to_union_args(PyObject *args_list, PyObject *args_set, PyObject *obj) } int contains = PySet_Contains(args_set, type); if (contains < 0) { + if (!PyErr_ExceptionMatches(PyExc_TypeError)) { + Py_DECREF(type); + return -1; + } + PyErr_Clear(); + if (PyList_Append(unhashable_args, obj) < 0) { + Py_DECREF(type); + return -1; + } Py_DECREF(type); - return -1; + return 0; } else if (contains == 1) { Py_DECREF(type); @@ -474,10 +484,17 @@ _Py_union_from_tuple(PyObject *args) Py_DECREF(args_list); return NULL; } + PyObject *unhashable_args = PyList_New(0); + if (unhashable_args == NULL) { + Py_DECREF(args_list); + Py_DECREF(args_set); + return NULL; + } if (!PyTuple_CheckExact(args)) { - if (add_object_to_union_args(args_list, args_set, args) < 0) { + if (add_object_to_union_args(args_list, args_set, unhashable_args, args) < 0) { Py_DECREF(args_list); Py_DECREF(args_set); + Py_DECREF(unhashable_args); return NULL; } } @@ -485,14 +502,61 @@ _Py_union_from_tuple(PyObject *args) Py_ssize_t size = PyTuple_GET_SIZE(args); for (Py_ssize_t i = 0; i < size; i++) { PyObject *arg = PyTuple_GET_ITEM(args, i); - if (add_object_to_union_args(args_list, args_set, arg) < 0) { + if (add_object_to_union_args(args_list, args_set, unhashable_args, arg) < 0) { Py_DECREF(args_list); Py_DECREF(args_set); + Py_DECREF(unhashable_args); return NULL; } } } Py_DECREF(args_set); + Py_ssize_t num_unhashable = PyList_Size(unhashable_args); + if (num_unhashable < 0) { + Py_DECREF(args_list); + Py_DECREF(unhashable_args); + return NULL; + } + if (num_unhashable > 0) { + PyObject *new_unhashable = PyList_New(0); + if (new_unhashable == NULL) { + Py_DECREF(args_list); + Py_DECREF(unhashable_args); + Py_DECREF(new_unhashable); + return NULL; + } + for (Py_ssize_t i = 0; i < num_unhashable; i++) { + PyObject *obj = PyList_GetItemRef(unhashable_args, i); + if (obj == NULL) { + Py_DECREF(args_list); + Py_DECREF(unhashable_args); + Py_DECREF(new_unhashable); + return NULL; + } + int contains = PySequence_Contains(new_unhashable, obj); + if (contains < 0) { + Py_DECREF(args_list); + Py_DECREF(unhashable_args); + Py_DECREF(new_unhashable); + Py_DECREF(obj); + return NULL; + } + if (contains == 1) { + Py_DECREF(obj); + continue; + } + if (PyList_Append(args_list, obj) < 0) { + Py_DECREF(args_list); + Py_DECREF(unhashable_args); + Py_DECREF(new_unhashable); + Py_DECREF(obj); + return NULL; + } + Py_DECREF(obj); + } + Py_DECREF(new_unhashable); + } + Py_DECREF(unhashable_args); if (PyList_GET_SIZE(args_list) == 0) { Py_DECREF(args_list); PyErr_SetString(PyExc_TypeError, "Cannot take a Union of no types."); From 65da3f13e53d1f7d38335c6d485240ab3e386ad0 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 28 Sep 2024 13:19:21 -0700 Subject: [PATCH 30/35] fixup --- Doc/deprecations/pending-removal-in-future.rst | 5 +++++ Doc/library/types.rst | 2 +- Doc/library/typing.rst | 2 +- Doc/whatsnew/3.13.rst | 2 -- Lib/typing.py | 6 +++--- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 3f9cf6f208221a..6f1014bd9dee66 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -122,6 +122,11 @@ although there is currently no date scheduled for their removal. * :class:`typing.Text` (:gh:`92332`). +* The internal class ``typing._UnionGenericAlias`` is no longer used to implement + :class:`typing.Union`. To preserve compatibility with users using this private + class, a compatibility shim will be provided until at least Python 3.17. (Contributed by + Jelle Zijlstra in :gh:`105499`.) + * :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value that is not ``None`` from a test case. diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 9ffa0e4f0e7ae3..540da4dd88cf1e 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -359,7 +359,7 @@ Standard names are defined for the following types: .. versionadded:: 3.10 - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 This is now an alias for :class:`typing.Union`. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 619e0cde8fab6e..c349307618b274 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1121,7 +1121,7 @@ These can be used as types in annotations. They all support subscription using Unions can now be written as ``X | Y``. See :ref:`union type expressions`. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 :class:`types.UnionType` is now an alias for :class:`Union`, and both ``Union[int, str]`` and ``int | str`` create instances of the same class. To check whether an object is a ``Union`` at runtime, use diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 905c1c01148c97..169d18c4e5d15e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1546,8 +1546,6 @@ Optimizations Removed Modules And APIs ======================== -======= - .. _whatsnew313-pep594: PEP 594: Remove "dead batteries" from the standard library diff --git a/Lib/typing.py b/Lib/typing.py index 9be347e2b65bb0..7281a625e1e31f 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1649,11 +1649,11 @@ def __getitem__(self, params): class _UnionGenericAliasMeta(type): def __instancecheck__(self, inst: type) -> bool: - warnings._deprecated("_UnionGenericAlias", remove=(3, 15)) + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) return isinstance(inst, Union) def __eq__(self, other): - warnings._deprecated("_UnionGenericAlias", remove=(3, 15)) + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) if other is _UnionGenericAlias or other is Union: return True return NotImplemented @@ -1669,7 +1669,7 @@ class _UnionGenericAlias(metaclass=_UnionGenericAliasMeta): """ def __new__(cls, self_cls, parameters, /, *, name=None): - warnings._deprecated("_UnionGenericAlias", remove=(3, 15)) + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) return Union[parameters] From f500f5fe2bac17a219ad8abac3a36753400e5433 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 28 Sep 2024 14:41:18 -0700 Subject: [PATCH 31/35] Make union support unhashable objects --- Lib/test/test_typing.py | 10 +- Objects/unionobject.c | 475 ++++++++++++++++++++-------------------- 2 files changed, 235 insertions(+), 250 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index d61e3ab5974cbe..b99b8badf7b1f7 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2062,16 +2062,10 @@ class B(metaclass=UnhashableMeta): ... self.assertEqual(Union[A, B].__args__, (A, B)) union1 = Union[A, B] - with self.assertRaises(TypeError): - hash(union1) - union2 = Union[int, B] - with self.assertRaises(TypeError): - hash(union2) - union3 = Union[A, int] - with self.assertRaises(TypeError): - hash(union3) + + self.assertEqual(len({union1, union2, union3}), 3) def test_repr(self): u = Union[Employee, int] diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 3cf735f53153b8..d2ed67f987daf1 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -5,12 +5,11 @@ #include "pycore_unionobject.h" -static PyObject *make_union(PyObject *); - - typedef struct { PyObject_HEAD - PyObject *args; + PyObject *args; // all args (tuple) + PyObject *hashable_args; // frozenset or NULL + PyObject *unhashable_args; // tuple or NULL PyObject *parameters; PyObject *weakreflist; } unionobject; @@ -26,6 +25,8 @@ unionobject_dealloc(PyObject *self) } Py_XDECREF(alias->args); + Py_XDECREF(alias->hashable_args); + Py_XDECREF(alias->unhashable_args); Py_XDECREF(alias->parameters); Py_TYPE(self)->tp_free(self); } @@ -35,6 +36,8 @@ union_traverse(PyObject *self, visitproc visit, void *arg) { unionobject *alias = (unionobject *)self; Py_VISIT(alias->args); + Py_VISIT(alias->hashable_args); + Py_VISIT(alias->unhashable_args); Py_VISIT(alias->parameters); return 0; } @@ -43,15 +46,70 @@ static Py_hash_t union_hash(PyObject *self) { unionobject *alias = (unionobject *)self; - PyObject *args = PyFrozenSet_New(alias->args); - if (args == NULL) { - return (Py_hash_t)-1; + Py_hash_t hash; + if (alias->hashable_args) { + hash = PyObject_Hash(alias->hashable_args); + if (hash == -1) { + return -1; + } + } + else { + hash = 604; + } + // Mix in the ids of all the unhashable args. + if (alias->unhashable_args) { + assert(PyTuple_CheckExact(alias->unhashable_args)); + Py_ssize_t n = PyTuple_GET_SIZE(alias->unhashable_args); + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *arg = PyTuple_GET_ITEM(alias->unhashable_args, i); + hash ^= (Py_hash_t)arg; + } } - Py_hash_t hash = PyObject_Hash(args); - Py_DECREF(args); return hash; } +static int +unions_equal(unionobject *a, unionobject *b) +{ + int result = PyObject_RichCompareBool(a->hashable_args, b->hashable_args, Py_EQ); + if (result == -1) { + return -1; + } + if (result == 0) { + return 0; + } + if (a->unhashable_args && b->unhashable_args) { + Py_ssize_t n = PyTuple_GET_SIZE(a->unhashable_args); + if (n != PyTuple_GET_SIZE(b->unhashable_args)) { + return 0; + } + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *arg_a = PyTuple_GET_ITEM(a->unhashable_args, i); + int result = PySequence_Contains(b->unhashable_args, arg_a); + if (result == -1) { + return -1; + } + if (!result) { + return 0; + } + } + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *arg_b = PyTuple_GET_ITEM(b->unhashable_args, i); + int result = PySequence_Contains(a->unhashable_args, arg_b); + if (result == -1) { + return -1; + } + if (!result) { + return 0; + } + } + } + else if (a->unhashable_args || b->unhashable_args) { + return 0; + } + return 1; +} + static PyObject * union_richcompare(PyObject *a, PyObject *b, int op) { @@ -59,95 +117,130 @@ union_richcompare(PyObject *a, PyObject *b, int op) Py_RETURN_NOTIMPLEMENTED; } - PyObject *a_set = PySet_New(((unionobject*)a)->args); - if (a_set == NULL) { + int equal = unions_equal((unionobject*)a, (unionobject*)b); + if (equal == -1) { return NULL; } - PyObject *b_set = PySet_New(((unionobject*)b)->args); - if (b_set == NULL) { - Py_DECREF(a_set); - return NULL; + if (op == Py_EQ) { + return PyBool_FromLong(equal); + } + else { + return PyBool_FromLong(!equal); } - PyObject *result = PyObject_RichCompare(a_set, b_set, op); - Py_DECREF(b_set); - Py_DECREF(a_set); - return result; } -static int -is_same(PyObject *left, PyObject *right) +typedef struct { + PyObject *args; // list + PyObject *hashable_args; // set + PyObject *unhashable_args; // list or NULL + bool is_checked; // whether to call type_check() +} unionbuilder; + +static bool unionbuilder_add_tuple(unionbuilder *, PyObject *); +static PyObject *make_union(unionbuilder *); +static PyObject *type_check(PyObject *, const char *); + +static bool +unionbuilder_init(unionbuilder *ub, bool is_checked) { - int is_ga = _PyGenericAlias_Check(left) && _PyGenericAlias_Check(right); - return is_ga ? PyObject_RichCompareBool(left, right, Py_EQ) : left == right; + ub->args = PyList_New(0); + if (ub->args == NULL) { + return false; + } + ub->hashable_args = PySet_New(NULL); + if (ub->hashable_args == NULL) { + Py_DECREF(ub->args); + return false; + } + ub->unhashable_args = NULL; + ub->is_checked = is_checked; + return true; } -static int -contains(PyObject **items, Py_ssize_t size, PyObject *obj) +static void +unionbuilder_finalize(unionbuilder *ub) { - for (Py_ssize_t i = 0; i < size; i++) { - int is_duplicate = is_same(items[i], obj); - if (is_duplicate) { // -1 or 1 - return is_duplicate; - } - } - return 0; + Py_DECREF(ub->args); + Py_DECREF(ub->hashable_args); + Py_XDECREF(ub->unhashable_args); } -static PyObject * -merge(PyObject **items1, Py_ssize_t size1, - PyObject **items2, Py_ssize_t size2) +static bool +unionbuilder_add_single_unchecked(unionbuilder *ub, PyObject *arg) { - PyObject *tuple = NULL; - Py_ssize_t pos = 0; - - for (Py_ssize_t i = 0; i < size2; i++) { - PyObject *arg = items2[i]; - int is_duplicate = contains(items1, size1, arg); - if (is_duplicate < 0) { - Py_XDECREF(tuple); - return NULL; - } - if (is_duplicate) { - continue; + Py_hash_t hash = PyObject_Hash(arg); + if (hash == -1) { + PyErr_Clear(); + if (ub->unhashable_args == NULL) { + ub->unhashable_args = PyList_New(0); + if (ub->unhashable_args == NULL) { + return false; + } } - - if (tuple == NULL) { - tuple = PyTuple_New(size1 + size2 - i); - if (tuple == NULL) { - return NULL; + else { + int contains = PySequence_Contains(ub->unhashable_args, arg); + if (contains < 0) { + return false; } - for (; pos < size1; pos++) { - PyObject *a = items1[pos]; - PyTuple_SET_ITEM(tuple, pos, Py_NewRef(a)); + if (contains == 1) { + return true; } } - PyTuple_SET_ITEM(tuple, pos, Py_NewRef(arg)); - pos++; + if (PyList_Append(ub->unhashable_args, arg) < 0) { + return false; + } } - - if (tuple) { - (void) _PyTuple_Resize(&tuple, pos); + else { + int contains = PySet_Contains(ub->hashable_args, arg); + if (contains < 0) { + return false; + } + if (contains == 1) { + return true; + } + if (PySet_Add(ub->hashable_args, arg) < 0) { + return false; + } } - return tuple; + return PyList_Append(ub->args, arg) == 0; } -static PyObject ** -get_types(PyObject **obj, Py_ssize_t *size) +static bool +unionbuilder_add_single(unionbuilder *ub, PyObject *arg) { - if (*obj == Py_None) { - *obj = (PyObject *)&_PyNone_Type; + if (Py_IsNone(arg)) { + arg = (PyObject *)&_PyNone_Type; // immortal, so no refcounting needed + } + else if (_PyUnion_Check(arg)) { + PyObject *args = ((unionobject *)arg)->args; + return unionbuilder_add_tuple(ub, args); } - if (_PyUnion_Check(*obj)) { - PyObject *args = ((unionobject *) *obj)->args; - *size = PyTuple_GET_SIZE(args); - return &PyTuple_GET_ITEM(args, 0); + if (ub->is_checked) { + PyObject *type = type_check(arg, "Union[arg, ...]: each arg must be a type."); + if (type == NULL) { + return false; + } + bool result = unionbuilder_add_single_unchecked(ub, type); + Py_DECREF(type); + return result; } else { - *size = 1; - return obj; + return unionbuilder_add_single_unchecked(ub, arg); } } +static bool +unionbuilder_add_tuple(unionbuilder *ub, PyObject *tuple) +{ + Py_ssize_t n = PyTuple_GET_SIZE(tuple); + for (Py_ssize_t i = 0; i < n; i++) { + if (!unionbuilder_add_single(ub, PyTuple_GET_ITEM(tuple, i))) { + return false; + } + } + return true; +} + static int is_unionable(PyObject *obj) { @@ -168,19 +261,18 @@ _Py_union_type_or(PyObject* self, PyObject* other) Py_RETURN_NOTIMPLEMENTED; } - Py_ssize_t size1, size2; - PyObject **items1 = get_types(&self, &size1); - PyObject **items2 = get_types(&other, &size2); - PyObject *tuple = merge(items1, size1, items2, size2); - if (tuple == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(self); + unionbuilder ub; + // unchecked because we already checked is_unionable() + if (!unionbuilder_init(&ub, false)) { + return NULL; + } + if (!unionbuilder_add_single(&ub, self) || + !unionbuilder_add_single(&ub, other)) { + unionbuilder_finalize(&ub); + return NULL; } - PyObject *new_union = make_union(tuple); - Py_DECREF(tuple); + PyObject *new_union = make_union(&ub); return new_union; } @@ -206,6 +298,18 @@ union_repr(PyObject *self) goto error; } } + +#if 0 + PyUnicodeWriter_WriteUTF8(writer, "|args=", 6); + PyUnicodeWriter_WriteRepr(writer, alias->args); + PyUnicodeWriter_WriteUTF8(writer, "|h=", 3); + PyUnicodeWriter_WriteRepr(writer, alias->hashable_args); + if (alias->unhashable_args) { + PyUnicodeWriter_WriteUTF8(writer, "|u=", 3); + PyUnicodeWriter_WriteRepr(writer, alias->unhashable_args); + } +#endif + return PyUnicodeWriter_Finish(writer); error: @@ -235,21 +339,7 @@ union_getitem(PyObject *self, PyObject *item) return NULL; } - PyObject *res; - Py_ssize_t nargs = PyTuple_GET_SIZE(newargs); - if (nargs == 0) { - res = make_union(newargs); - } - else { - res = Py_NewRef(PyTuple_GET_ITEM(newargs, 0)); - for (Py_ssize_t iarg = 1; iarg < nargs; iarg++) { - PyObject *arg = PyTuple_GET_ITEM(newargs, iarg); - Py_SETREF(res, PyNumber_Or(res, arg)); - if (res == NULL) { - break; - } - } - } + PyObject *res = _Py_union_from_tuple(newargs); Py_DECREF(newargs); return res; } @@ -367,159 +457,24 @@ type_check(PyObject *arg, const char *msg) return result; } -static int -add_object_to_union_args(PyObject *args_list, PyObject *args_set, - PyObject *unhashable_args, PyObject *obj) -{ - if (Py_IS_TYPE(obj, &_PyUnion_Type)) { - PyObject *args = ((unionobject *) obj)->args; - Py_ssize_t size = PyTuple_GET_SIZE(args); - for (Py_ssize_t i = 0; i < size; i++) { - PyObject *arg = PyTuple_GET_ITEM(args, i); - if (add_object_to_union_args(args_list, args_set, unhashable_args, arg) < 0) { - return -1; - } - } - return 0; - } - PyObject *type = type_check(obj, "Union[arg, ...]: each arg must be a type."); - if (type == NULL) { - return -1; - } - int contains = PySet_Contains(args_set, type); - if (contains < 0) { - if (!PyErr_ExceptionMatches(PyExc_TypeError)) { - Py_DECREF(type); - return -1; - } - PyErr_Clear(); - if (PyList_Append(unhashable_args, obj) < 0) { - Py_DECREF(type); - return -1; - } - Py_DECREF(type); - return 0; - } - else if (contains == 1) { - Py_DECREF(type); - return 0; - } - if (PyList_Append(args_list, type) < 0) { - Py_DECREF(type); - return -1; - } - if (PySet_Add(args_set, type) < 0) { - Py_DECREF(type); - return -1; - } - Py_DECREF(type); - return 0; -} - PyObject * _Py_union_from_tuple(PyObject *args) { - PyObject *args_list = PyList_New(0); - if (args_list == NULL) { - return NULL; - } - PyObject *args_set = PySet_New(NULL); - if (args_set == NULL) { - Py_DECREF(args_list); - return NULL; - } - PyObject *unhashable_args = PyList_New(0); - if (unhashable_args == NULL) { - Py_DECREF(args_list); - Py_DECREF(args_set); + unionbuilder ub; + if (!unionbuilder_init(&ub, true)) { return NULL; } - if (!PyTuple_CheckExact(args)) { - if (add_object_to_union_args(args_list, args_set, unhashable_args, args) < 0) { - Py_DECREF(args_list); - Py_DECREF(args_set); - Py_DECREF(unhashable_args); + if (PyTuple_CheckExact(args)) { + if (!unionbuilder_add_tuple(&ub, args)) { return NULL; } } else { - Py_ssize_t size = PyTuple_GET_SIZE(args); - for (Py_ssize_t i = 0; i < size; i++) { - PyObject *arg = PyTuple_GET_ITEM(args, i); - if (add_object_to_union_args(args_list, args_set, unhashable_args, arg) < 0) { - Py_DECREF(args_list); - Py_DECREF(args_set); - Py_DECREF(unhashable_args); - return NULL; - } - } - } - Py_DECREF(args_set); - Py_ssize_t num_unhashable = PyList_Size(unhashable_args); - if (num_unhashable < 0) { - Py_DECREF(args_list); - Py_DECREF(unhashable_args); - return NULL; - } - if (num_unhashable > 0) { - PyObject *new_unhashable = PyList_New(0); - if (new_unhashable == NULL) { - Py_DECREF(args_list); - Py_DECREF(unhashable_args); - Py_DECREF(new_unhashable); + if (!unionbuilder_add_single(&ub, args)) { return NULL; } - for (Py_ssize_t i = 0; i < num_unhashable; i++) { - PyObject *obj = PyList_GetItemRef(unhashable_args, i); - if (obj == NULL) { - Py_DECREF(args_list); - Py_DECREF(unhashable_args); - Py_DECREF(new_unhashable); - return NULL; - } - int contains = PySequence_Contains(new_unhashable, obj); - if (contains < 0) { - Py_DECREF(args_list); - Py_DECREF(unhashable_args); - Py_DECREF(new_unhashable); - Py_DECREF(obj); - return NULL; - } - if (contains == 1) { - Py_DECREF(obj); - continue; - } - if (PyList_Append(args_list, obj) < 0) { - Py_DECREF(args_list); - Py_DECREF(unhashable_args); - Py_DECREF(new_unhashable); - Py_DECREF(obj); - return NULL; - } - Py_DECREF(obj); - } - Py_DECREF(new_unhashable); - } - Py_DECREF(unhashable_args); - if (PyList_GET_SIZE(args_list) == 0) { - Py_DECREF(args_list); - PyErr_SetString(PyExc_TypeError, "Cannot take a Union of no types."); - return NULL; - } - else if (PyList_GET_SIZE(args_list) == 1) { - PyObject *result = PyList_GET_ITEM(args_list, 0); - Py_INCREF(result); - Py_DECREF(args_list); - return result; } - PyObject *args_tuple = PyList_AsTuple(args_list); - Py_DECREF(args_list); - if (args_tuple == NULL) { - return NULL; - } - PyObject *u = make_union(args_tuple); - Py_DECREF(args_tuple); - return u; + return make_union(&ub); } static PyObject * @@ -566,18 +521,54 @@ PyTypeObject _PyUnion_Type = { }; static PyObject * -make_union(PyObject *args) +make_union(unionbuilder *ub) { - assert(PyTuple_CheckExact(args)); + Py_ssize_t n = PyList_GET_SIZE(ub->args); + if (n == 0) { + PyErr_SetString(PyExc_TypeError, "Cannot take a Union of no types."); + unionbuilder_finalize(ub); + return NULL; + } + if (n == 1) { + PyObject *result = PyList_GET_ITEM(ub->args, 0); + Py_INCREF(result); + unionbuilder_finalize(ub); + return result; + } + + PyObject *args = NULL, *hashable_args = NULL, *unhashable_args = NULL; + args = PyList_AsTuple(ub->args); + if (args == NULL) { + goto error; + } + hashable_args = PyFrozenSet_New(ub->hashable_args); + if (hashable_args == NULL) { + goto error; + } + if (ub->unhashable_args != NULL) { + unhashable_args = PyList_AsTuple(ub->unhashable_args); + if (unhashable_args == NULL) { + goto error; + } + } unionobject *result = PyObject_GC_New(unionobject, &_PyUnion_Type); if (result == NULL) { - return NULL; + goto error; } + unionbuilder_finalize(ub); result->parameters = NULL; - result->args = Py_NewRef(args); + result->args = args; + result->hashable_args = hashable_args; + result->unhashable_args = unhashable_args; result->weakreflist = NULL; _PyObject_GC_TRACK(result); return (PyObject*)result; +error: + Py_XDECREF(args); + Py_XDECREF(hashable_args); + Py_XDECREF(unhashable_args); + unionbuilder_finalize(ub); + return NULL; } From 5f9e599226a81045703d37088436159aa8c4f365 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 28 Sep 2024 14:53:07 -0700 Subject: [PATCH 32/35] simplify, extend docs --- Doc/library/typing.rst | 4 +++- Lib/functools.py | 9 +++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c349307618b274..1ba38575d98f27 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1125,7 +1125,9 @@ These can be used as types in annotations. They all support subscription using :class:`types.UnionType` is now an alias for :class:`Union`, and both ``Union[int, str]`` and ``int | str`` create instances of the same class. To check whether an object is a ``Union`` at runtime, use - ``isinstance(obj, Union)``. + ``isinstance(obj, Union)``. For compatibility with earlier versions of + Python, use + ``get_origin(obj) is typing.Union or get_origin(obj) is types.UnionType``. .. data:: Optional diff --git a/Lib/functools.py b/Lib/functools.py index 5265587e9493d1..97b98f663b5d4d 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -930,13 +930,10 @@ def dispatch(cls): dispatch_cache[cls] = impl return impl - def _is_union_type(cls): - return isinstance(cls, UnionType) - def _is_valid_dispatch_type(cls): if isinstance(cls, type): return True - return (_is_union_type(cls) and + return (isinstance(cls, UnionType) and all(isinstance(arg, type) for arg in cls.__args__)) def register(cls, func=None): @@ -969,7 +966,7 @@ def register(cls, func=None): from annotationlib import Format, ForwardRef argname, cls = next(iter(get_type_hints(func, format=Format.FORWARDREF).items())) if not _is_valid_dispatch_type(cls): - if _is_union_type(cls): + if isinstance(cls, UnionType): raise TypeError( f"Invalid annotation for {argname!r}. " f"{cls!r} not all arguments are classes." @@ -985,7 +982,7 @@ def register(cls, func=None): f"{cls!r} is not a class." ) - if _is_union_type(cls): + if isinstance(cls, UnionType): for arg in cls.__args__: registry[arg] = func else: From 9927b38272904e32355c1afd7f36933a4bca73f4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 28 Sep 2024 15:06:42 -0700 Subject: [PATCH 33/35] fix more tests --- Lib/test/test_types.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 1e7ac3abefa56a..1bd218e8237b76 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -723,16 +723,10 @@ class B(metaclass=UnhashableMeta): ... self.assertEqual((A | B).__args__, (A, B)) union1 = A | B - with self.assertRaises(TypeError): - hash(union1) - union2 = int | B - with self.assertRaises(TypeError): - hash(union2) - union3 = A | int - with self.assertRaises(TypeError): - hash(union3) + + self.assertEqual(len({union1, union2, union3}), 3) def test_instancecheck_and_subclasscheck(self): for x in (int | str, typing.Union[int, str]): @@ -1011,9 +1005,12 @@ def __eq__(self, other): return 1 / 0 bt = BadType('bt', (), {}) + bt2 = BadType('bt2', (), {}) # Comparison should fail and errors should propagate out for bad types. + union1 = int | bt + union2 = int | bt2 with self.assertRaises(ZeroDivisionError): - list[int] | list[bt] + union1 == union2 union_ga = (list[str] | int, collections.abc.Callable[..., str] | int, d | int) From eea6ecaaf2c1a6f66d76b896611865567d8f09b3 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 28 Sep 2024 15:08:16 -0700 Subject: [PATCH 34/35] another --- Lib/test/test_types.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 1bd218e8237b76..1cf67726c853f7 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1011,6 +1011,8 @@ def __eq__(self, other): union2 = int | bt2 with self.assertRaises(ZeroDivisionError): union1 == union2 + with self.assertRaises(ZeroDivisionError): + bt | bt2 union_ga = (list[str] | int, collections.abc.Callable[..., str] | int, d | int) From 07859518929ec5a0ed686bc2d4e00597711086a1 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 29 Sep 2024 06:26:20 -0700 Subject: [PATCH 35/35] change hash --- Lib/test/test_types.py | 31 ++++++++++++++++++++++++++++++- Lib/test/test_typing.py | 10 ++++++++-- Objects/unionobject.c | 25 ++++++++++++------------- 3 files changed, 50 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 1cf67726c853f7..786cc6852a8458 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -723,10 +723,39 @@ class B(metaclass=UnhashableMeta): ... self.assertEqual((A | B).__args__, (A, B)) union1 = A | B + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union1) + union2 = int | B + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union2) + union3 = A | int + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union3) + + def test_unhashable_becomes_hashable(self): + is_hashable = False + class UnhashableMeta(type): + def __hash__(self): + if is_hashable: + return 1 + else: + raise TypeError("not hashable") + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + union = A | B + self.assertEqual(union.__args__, (A, B)) + + with self.assertRaisesRegex(TypeError, "not hashable"): + hash(union) + + is_hashable = True - self.assertEqual(len({union1, union2, union3}), 3) + with self.assertRaisesRegex(TypeError, "union contains 2 unhashable elements"): + hash(union) def test_instancecheck_and_subclasscheck(self): for x in (int | str, typing.Union[int, str]): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b99b8badf7b1f7..561c901b5a24ec 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2062,10 +2062,16 @@ class B(metaclass=UnhashableMeta): ... self.assertEqual(Union[A, B].__args__, (A, B)) union1 = Union[A, B] + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union1) + union2 = Union[int, B] - union3 = Union[A, int] + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union2) - self.assertEqual(len({union1, union2, union3}), 3) + union3 = Union[A, int] + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union3) def test_repr(self): u = Union[Employee, int] diff --git a/Objects/unionobject.c b/Objects/unionobject.c index d2ed67f987daf1..065b0b8539775c 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -46,26 +46,25 @@ static Py_hash_t union_hash(PyObject *self) { unionobject *alias = (unionobject *)self; - Py_hash_t hash; - if (alias->hashable_args) { - hash = PyObject_Hash(alias->hashable_args); - if (hash == -1) { - return -1; - } - } - else { - hash = 604; - } - // Mix in the ids of all the unhashable args. + // If there are any unhashable args, treat this union as unhashable. + // Otherwise, two unions might compare equal but have different hashes. if (alias->unhashable_args) { + // Attempt to get an error from one of the values. assert(PyTuple_CheckExact(alias->unhashable_args)); Py_ssize_t n = PyTuple_GET_SIZE(alias->unhashable_args); for (Py_ssize_t i = 0; i < n; i++) { PyObject *arg = PyTuple_GET_ITEM(alias->unhashable_args, i); - hash ^= (Py_hash_t)arg; + Py_hash_t hash = PyObject_Hash(arg); + if (hash == -1) { + return -1; + } } + // The unhashable values somehow became hashable again. Still raise + // an error. + PyErr_Format(PyExc_TypeError, "union contains %d unhashable elements", n); + return -1; } - return hash; + return PyObject_Hash(alias->hashable_args); } static int