From c3bbd1cdeca02e63e1102a3274415f056e8d1e43 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 31 May 2024 11:29:55 -0700 Subject: [PATCH 001/247] Use Never in more messages, use ambiguous in join (#17304) Switches the logic from https://github.com/python/mypy/pull/16994 to use ambiguous (since is_noreturn was only meant for error messages) See also https://github.com/python/mypy/pull/15996 --- mypy/copytype.py | 2 +- mypy/join.py | 4 ++-- mypy/messages.py | 7 ++----- mypy/typeanal.py | 2 +- mypy/types.py | 13 ++++--------- test-data/unit/check-dataclasses.test | 4 ++-- test-data/unit/check-flags.test | 4 ++-- test-data/unit/check-literal.test | 6 ++---- test-data/unit/check-plugin-attrs.test | 4 ++-- test-data/unit/check-python310.test | 2 +- test-data/unit/check-type-aliases.test | 2 +- test-data/unit/check-typeddict.test | 6 +++--- 12 files changed, 23 insertions(+), 33 deletions(-) diff --git a/mypy/copytype.py b/mypy/copytype.py index 4ca381c4a8c4..465f06566f54 100644 --- a/mypy/copytype.py +++ b/mypy/copytype.py @@ -53,7 +53,7 @@ def visit_none_type(self, t: NoneType) -> ProperType: return self.copy_common(t, NoneType()) def visit_uninhabited_type(self, t: UninhabitedType) -> ProperType: - dup = UninhabitedType(t.is_noreturn) + dup = UninhabitedType() dup.ambiguous = t.ambiguous return self.copy_common(t, dup) diff --git a/mypy/join.py b/mypy/join.py index 782b4fbebd7b..c711697ec46d 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -108,9 +108,9 @@ def join_instances(self, t: Instance, s: Instance) -> ProperType: # TODO: contravariant case should use meet but pass seen instances as # an argument to keep track of recursive checks. elif type_var.variance in (INVARIANT, CONTRAVARIANT): - if isinstance(ta_proper, UninhabitedType) and not ta_proper.is_noreturn: + if isinstance(ta_proper, UninhabitedType) and ta_proper.ambiguous: new_type = sa - elif isinstance(sa_proper, UninhabitedType) and not sa_proper.is_noreturn: + elif isinstance(sa_proper, UninhabitedType) and sa_proper.ambiguous: new_type = ta elif not is_equivalent(ta, sa): self.seen_instances.pop() diff --git a/mypy/messages.py b/mypy/messages.py index 8f923462c789..53a7f7d97774 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2430,7 +2430,7 @@ def quote_type_string(type_string: str) -> str: """Quotes a type representation for use in messages.""" no_quote_regex = r"^<(tuple|union): \d+ items>$" if ( - type_string in ["Module", "overloaded function", "Never", ""] + type_string in ["Module", "overloaded function", ""] or type_string.startswith("Module ") or re.match(no_quote_regex, type_string) is not None or type_string.endswith("?") @@ -2633,10 +2633,7 @@ def format_literal_value(typ: LiteralType) -> str: elif isinstance(typ, DeletedType): return "" elif isinstance(typ, UninhabitedType): - if typ.is_noreturn: - return "NoReturn" - else: - return "Never" + return "Never" elif isinstance(typ, TypeType): type_name = "type" if options.use_lowercase_names() else "Type" return f"{type_name}[{format(typ.item)}]" diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 31d451b0831a..8f138ab5698f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -645,7 +645,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ return AnyType(TypeOfAny.from_error) return self.anal_type(t.args[0]) elif fullname in NEVER_NAMES: - return UninhabitedType(is_noreturn=True) + return UninhabitedType() elif fullname in LITERAL_TYPE_NAMES: return self.analyze_literal_type(t) elif fullname in ANNOTATED_TYPE_NAMES: diff --git a/mypy/types.py b/mypy/types.py index 0ef3803c5687..2cacc3e44085 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1169,17 +1169,12 @@ class UninhabitedType(ProperType): is_subtype(UninhabitedType, T) = True """ - __slots__ = ("ambiguous", "is_noreturn") + __slots__ = ("ambiguous",) - is_noreturn: bool # Does this come from a NoReturn? Purely for error messages. - # It is important to track whether this is an actual NoReturn type, or just a result - # of ambiguous type inference, in the latter case we don't want to mark a branch as - # unreachable in binder. ambiguous: bool # Is this a result of inference for a variable without constraints? - def __init__(self, is_noreturn: bool = False, line: int = -1, column: int = -1) -> None: + def __init__(self, line: int = -1, column: int = -1) -> None: super().__init__(line, column) - self.is_noreturn = is_noreturn self.ambiguous = False def can_be_true_default(self) -> bool: @@ -1198,12 +1193,12 @@ def __eq__(self, other: object) -> bool: return isinstance(other, UninhabitedType) def serialize(self) -> JsonDict: - return {".class": "UninhabitedType", "is_noreturn": self.is_noreturn} + return {".class": "UninhabitedType"} @classmethod def deserialize(cls, data: JsonDict) -> UninhabitedType: assert data[".class"] == "UninhabitedType" - return UninhabitedType(is_noreturn=data["is_noreturn"]) + return UninhabitedType() class NoneType(ProperType): diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index a055507cdd78..924f9c7bb5be 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -2080,8 +2080,8 @@ class B: a_or_b: Union[A[int], B] _ = replace(a_or_b, x=42, y=True, init_var=42) _ = replace(a_or_b, x=42, y=True) # E: Missing named argument "init_var" for "replace" of "Union[A[int], B]" -_ = replace(a_or_b, x=42, y=True, z='42', init_var=42) # E: Argument "z" to "replace" of "Union[A[int], B]" has incompatible type "str"; expected Never -_ = replace(a_or_b, x=42, y=True, w={}, init_var=42) # E: Argument "w" to "replace" of "Union[A[int], B]" has incompatible type "Dict[Never, Never]"; expected Never +_ = replace(a_or_b, x=42, y=True, z='42', init_var=42) # E: Argument "z" to "replace" of "Union[A[int], B]" has incompatible type "str"; expected "Never" +_ = replace(a_or_b, x=42, y=True, w={}, init_var=42) # E: Argument "w" to "replace" of "Union[A[int], B]" has incompatible type "Dict[Never, Never]"; expected "Never" _ = replace(a_or_b, y=42, init_var=42) # E: Argument "y" to "replace" of "Union[A[int], B]" has incompatible type "int"; expected "bool" [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index c90c773e320f..62711d5f0071 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -408,7 +408,7 @@ reveal_type(f() or no_return()) # N: Revealed type is "builtins.int" # flags: --warn-no-return from mypy_extensions import NoReturn -x = 0 # type: NoReturn # E: Incompatible types in assignment (expression has type "int", variable has type "NoReturn") +x = 0 # type: NoReturn # E: Incompatible types in assignment (expression has type "int", variable has type "Never") [builtins fixtures/dict.pyi] [case testNoReturnAsync] @@ -477,7 +477,7 @@ def no_return() -> NoReturn: pass def f() -> NoReturn: no_return() -x: NoReturn = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "NoReturn") +x: NoReturn = 0 # E: Incompatible types in assignment (expression has type "int", variable has type "Never") [builtins fixtures/dict.pyi] [case testShowErrorContextFunction] diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index 423ba74eba72..8f8aaf6a3982 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -839,14 +839,13 @@ b: NoReturn c: None fa(lit) -fb(lit) # E: Argument 1 to "fb" has incompatible type "Literal[1]"; expected "NoReturn" +fb(lit) # E: Argument 1 to "fb" has incompatible type "Literal[1]"; expected "Never" fc(lit) # E: Argument 1 to "fc" has incompatible type "Literal[1]"; expected "None" f_lit(a) f_lit(b) f_lit(c) # E: Argument 1 to "f_lit" has incompatible type "None"; expected "Literal[1]" [builtins fixtures/tuple.pyi] -[out] [case testLiteralCheckSubtypingNoStrictOptional] # flags: --no-strict-optional @@ -865,14 +864,13 @@ b: NoReturn c: None fa(lit) -fb(lit) # E: Argument 1 to "fb" has incompatible type "Literal[1]"; expected "NoReturn" +fb(lit) # E: Argument 1 to "fb" has incompatible type "Literal[1]"; expected "Never" fc(lit) # E: Argument 1 to "fc" has incompatible type "Literal[1]"; expected "None" f_lit(a) f_lit(b) f_lit(c) [builtins fixtures/tuple.pyi] -[out] [case testLiteralCallingOverloadedFunction] from typing import overload, Generic, TypeVar, Any diff --git a/test-data/unit/check-plugin-attrs.test b/test-data/unit/check-plugin-attrs.test index 39b266dba50e..b96c00730a74 100644 --- a/test-data/unit/check-plugin-attrs.test +++ b/test-data/unit/check-plugin-attrs.test @@ -2170,8 +2170,8 @@ class B: a_or_b: A[int] | B a2 = attrs.evolve(a_or_b, x=42, y=True) -a2 = attrs.evolve(a_or_b, x=42, y=True, z='42') # E: Argument "z" to "evolve" of "Union[A[int], B]" has incompatible type "str"; expected Never -a2 = attrs.evolve(a_or_b, x=42, y=True, w={}) # E: Argument "w" to "evolve" of "Union[A[int], B]" has incompatible type "Dict[Never, Never]"; expected Never +a2 = attrs.evolve(a_or_b, x=42, y=True, z='42') # E: Argument "z" to "evolve" of "Union[A[int], B]" has incompatible type "str"; expected "Never" +a2 = attrs.evolve(a_or_b, x=42, y=True, w={}) # E: Argument "w" to "evolve" of "Union[A[int], B]" has incompatible type "Dict[Never, Never]"; expected "Never" [builtins fixtures/plugin_attrs.pyi] diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 8991b65f67b5..5ecc69dc7c32 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -1406,7 +1406,7 @@ def f(value: int) -> int: # E: Missing return statement case 2: return 1 case o: - assert_never(o) # E: Argument 1 to "assert_never" has incompatible type "int"; expected "NoReturn" + assert_never(o) # E: Argument 1 to "assert_never" has incompatible type "int"; expected "Never" [case testMatchExhaustiveNoError] from typing import NoReturn, Union, Literal diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index aebb0381d962..f77c3c1c34e2 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -57,7 +57,7 @@ Never = NoReturn a: Never # Used to be an error here def f(a: Never): ... -f(5) # E: Argument 1 to "f" has incompatible type "int"; expected "NoReturn" +f(5) # E: Argument 1 to "f" has incompatible type "int"; expected "Never" [case testImportUnionAlias] import typing from _m import U diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index bd1fbe3f2667..09b86e4afd2d 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1648,7 +1648,7 @@ a.setdefault('y', '') # E: Argument 2 to "setdefault" of "TypedDict" has incompa x = '' a.setdefault(x, 1) # E: Expected TypedDict key to be string literal alias = a.setdefault -alias(x, 1) # E: Argument 1 has incompatible type "str"; expected "NoReturn" +alias(x, 1) # E: Argument 1 has incompatible type "str"; expected "Never" a.update({}) a.update({'x': 1}) @@ -1680,8 +1680,8 @@ b.pop('x') # E: Key "x" of TypedDict "B" cannot be deleted x = '' b.pop(x) # E: Expected TypedDict key to be string literal pop = b.pop -pop('x') # E: Argument 1 has incompatible type "str"; expected "NoReturn" -pop('invalid') # E: Argument 1 has incompatible type "str"; expected "NoReturn" +pop('x') # E: Argument 1 has incompatible type "str"; expected "Never" +pop('invalid') # E: Argument 1 has incompatible type "str"; expected "Never" [builtins fixtures/dict.pyi] [case testTypedDictDel] From 2116386c7752a5c78425419df8e28f654c893045 Mon Sep 17 00:00:00 2001 From: Ben Brown Date: Sun, 2 Jun 2024 14:51:56 +0100 Subject: [PATCH 002/247] Update 'typing_extensions' to >=4.6.0 to fix python 3.12 error (#17312) With earlier versions of typing_extensions, the following traceback is seen: ``` Traceback (most recent call last): File ".../bin/mypy", line 5, in from mypy.__main__ import console_entry File ".../lib/python3.12/site-packages/mypy/__main__.py", line 9, in from mypy.main import main, process_options File ".../lib/python3.12/site-packages/mypy/main.py", line 12, in from typing_extensions import Final File ".../lib/python3.12/site-packages/typing_extensions.py", line 1174, in class TypeVar(typing.TypeVar, _DefaultMixin, _root=True): TypeError: type 'typing.TypeVar' is not an acceptable base type ``` The error is addressed in typing_extensions in https://github.com/python/typing_extensions/pull/162, which is included in the 4.6.0 release. --- mypy-requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index f81412be761e..341052822f25 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ # NOTE: this needs to be kept in sync with the "requires" list in pyproject.toml -typing_extensions>=4.1.0 +typing_extensions>=4.6.0 mypy_extensions>=1.0.0 tomli>=1.1.0; python_version<'3.11' diff --git a/pyproject.toml b/pyproject.toml index 35f1592ca83c..33d4ec094f50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "setuptools >= 40.6.2", "wheel >= 0.30.0", # the following is from mypy-requirements.txt - "typing_extensions>=4.1.0", + "typing_extensions>=4.6.0", "mypy_extensions>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", # the following is from build-requirements.txt From b207550318fc5d58372abe3dac0d34f895d3ead9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 2 Jun 2024 15:23:26 +0100 Subject: [PATCH 003/247] Sync typing_extensions pin in setup.py with the pin in the other two places (#17313) Followup to #17312. (Can't say I fully understand why we have to have this pin in three places :) --- mypy-requirements.txt | 1 + pyproject.toml | 2 +- setup.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index 341052822f25..8d41a3fc7003 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,5 @@ # NOTE: this needs to be kept in sync with the "requires" list in pyproject.toml +# and the pins in setup.py typing_extensions>=4.6.0 mypy_extensions>=1.0.0 tomli>=1.1.0; python_version<'3.11' diff --git a/pyproject.toml b/pyproject.toml index 33d4ec094f50..12a0dc109cd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ # self-typechecking :/ "setuptools >= 40.6.2", "wheel >= 0.30.0", - # the following is from mypy-requirements.txt + # the following is from mypy-requirements.txt/setup.py "typing_extensions>=4.6.0", "mypy_extensions>=1.0.0", "tomli>=1.1.0; python_version<'3.11'", diff --git a/setup.py b/setup.py index a17ee562eb39..160e2b054b0e 100644 --- a/setup.py +++ b/setup.py @@ -218,9 +218,9 @@ def run(self): }, classifiers=classifiers, cmdclass=cmdclass, - # When changing this, also update mypy-requirements.txt. + # When changing this, also update mypy-requirements.txt and pyproject.toml install_requires=[ - "typing_extensions>=4.1.0", + "typing_extensions>=4.6.0", "mypy_extensions >= 1.0.0", "tomli>=1.1.0; python_version<'3.11'", ], From 6c24ea66e20166964aa5d42e28fda5b4b69f44b1 Mon Sep 17 00:00:00 2001 From: urnest Date: Mon, 3 Jun 2024 14:38:44 +1000 Subject: [PATCH 004/247] fix #16935 fix type of tuple[X,Y] expression (#17235) implement the mypy/checkexpr.py TODO: Specialize the callable for the type arguments ... so e.g. reveal_type(tuple[int, int]) gives expected def (p0: tuple[builtins.int, builtins.int]) -> tuple[builtins.int, builtins.int] ... rather than def [_T_co] (typing.Iterable[_T_co`1] =) -> builtins.tuple[_T_co`1, ...] Fixes #16935 --- mypy/checkexpr.py | 17 +++++++- .../check-type-object-type-inference.test | 41 +++++++++++++++++++ test-data/unit/pythoneval.test | 3 +- 3 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 test-data/unit/check-type-object-type-inference.test diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 479ef228b038..0a4af069ea17 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4836,8 +4836,21 @@ def apply_type_arguments_to_callable( len(args) < min_arg_count or len(args) > len(tp.variables) ) and not has_type_var_tuple: if tp.is_type_obj() and tp.type_object().fullname == "builtins.tuple": - # TODO: Specialize the callable for the type arguments - return tp + # e.g. expression tuple[X, Y] + # - want the type of the expression i.e. a function with that as its return type + # - tp is type of tuple (note it won't have params as we are only called + # with generic callable type) + # - tuple[X, Y]() takes a single arg that is a tuple containing an X and a Y + return CallableType( + [TupleType(list(args), self.chk.named_type("tuple"))], + [ARG_POS], + [None], + TupleType(list(args), self.chk.named_type("tuple")), + tp.fallback, + name="tuple", + definition=tp.definition, + bound_args=tp.bound_args, + ) self.msg.incompatible_type_application( min_arg_count, len(tp.variables), len(args), ctx ) diff --git a/test-data/unit/check-type-object-type-inference.test b/test-data/unit/check-type-object-type-inference.test new file mode 100644 index 000000000000..baeca1e22ac7 --- /dev/null +++ b/test-data/unit/check-type-object-type-inference.test @@ -0,0 +1,41 @@ +[case testInferTupleType] +# flags: --python-version 3.9 +from typing import TypeVar, Generic, Type +from abc import abstractmethod + +T = TypeVar('T') +class E(Generic[T]): + @abstractmethod + def e(self, t: T) -> str: + ... + +class F: + @abstractmethod + def f(self, tp: Type[T]) -> E[T]: + ... + +def g(f: F): + f.f(int).e(7) + f.f(tuple[int,str]) + f.f(tuple[int,str]).e('x') # E: Argument 1 to "e" of "E" has incompatible type "str"; expected "Tuple[int, str]" + f.f(tuple[int,str]).e( (7,8) ) # E: Argument 1 to "e" of "E" has incompatible type "Tuple[int, int]"; expected "Tuple[int, str]" + f.f(tuple[int,str]).e( (7,'x') ) # OK + reveal_type(f.f(tuple[int,str]).e) # N: Revealed type is "def (t: Tuple[builtins.int, builtins.str]) -> builtins.str" + +def h(f: F): + f.f(int).e(7) + f.f(tuple) + f.f(tuple).e('y') # E: Argument 1 to "e" of "E" has incompatible type "str"; expected "Tuple[Any, ...]" + f.f(tuple).e( (8,'y') ) # OK + reveal_type(f.f(tuple).e) # N: Revealed type is "def (t: builtins.tuple[Any, ...]) -> builtins.str" + +def i(f: F): + f.f(tuple[int,tuple[int,str]]) + f.f(tuple[int,tuple[int,str]]).e('z') # E: Argument 1 to "e" of "E" has incompatible type "str"; expected "Tuple[int, Tuple[int, str]]" + f.f(tuple[int,tuple[int,str]]).e( (8,9) ) # E: Argument 1 to "e" of "E" has incompatible type "Tuple[int, int]"; expected "Tuple[int, Tuple[int, str]]" + f.f(tuple[int,tuple[int,str]]).e( (17, (28, 29)) ) # E: Argument 1 to "e" of "E" has incompatible type "Tuple[int, Tuple[int, int]]"; expected "Tuple[int, Tuple[int, str]]" + f.f(tuple[int,tuple[int,str]]).e( (27,(28,'z')) ) # OK + reveal_type(f.f(tuple[int,tuple[int,str]]).e) # N: Revealed type is "def (t: Tuple[builtins.int, Tuple[builtins.int, builtins.str]]) -> builtins.str" + +x = tuple[int,str][str] # E: The type "Type[Tuple[Any, ...]]" is not generic and not indexable +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 0cf6d6b5aa38..a76d3abd7114 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1834,7 +1834,6 @@ RHSAlias3: type = tuple[int, ...] WrongTypeElement = str | tuple[float, 1] # Error WrongEllipsis = tuple[float, float, ...] | str # Error -# TODO: This should produce a fixed-length tuple reveal_type(tuple[int, str]((1, "x"))) [out] _testTupleWithDifferentArgsPy310.py:15: note: Revealed type is "Union[builtins.str, Tuple[builtins.float, builtins.float, builtins.str]]" @@ -1845,7 +1844,7 @@ _testTupleWithDifferentArgsPy310.py:19: note: Revealed type is "builtins.tuple[b _testTupleWithDifferentArgsPy310.py:20: note: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str]]" _testTupleWithDifferentArgsPy310.py:26: error: Invalid type: try using Literal[1] instead? _testTupleWithDifferentArgsPy310.py:27: error: Unexpected "..." -_testTupleWithDifferentArgsPy310.py:30: note: Revealed type is "builtins.tuple[builtins.object, ...]" +_testTupleWithDifferentArgsPy310.py:29: note: Revealed type is "Tuple[builtins.int, builtins.str]" [case testEnumIterMetaInference] import socket From aa4410ff7806425f143c3c4a21324d8f10b3f76d Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 3 Jun 2024 08:49:12 -0700 Subject: [PATCH 005/247] [mypyc] Fix ParamSpec (#17309) Fixes https://github.com/mypyc/mypyc/issues/1051 --- mypyc/irbuild/mapper.py | 4 +-- mypyc/test-data/irbuild-generics.test | 41 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index a3abbb1f84fb..90ce0e16c741 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -15,7 +15,7 @@ Type, TypedDictType, TypeType, - TypeVarType, + TypeVarLikeType, UnboundType, UninhabitedType, UnionType, @@ -131,7 +131,7 @@ def type_to_rtype(self, typ: Type | None) -> RType: return object_rprimitive elif isinstance(typ, TypeType): return object_rprimitive - elif isinstance(typ, TypeVarType): + elif isinstance(typ, TypeVarLikeType): # Erase type variable to upper bound. # TODO: Erase to union if object has value restriction? return self.type_to_rtype(typ.upper_bound) diff --git a/mypyc/test-data/irbuild-generics.test b/mypyc/test-data/irbuild-generics.test index 4f9d0ab83a16..910148f80dda 100644 --- a/mypyc/test-data/irbuild-generics.test +++ b/mypyc/test-data/irbuild-generics.test @@ -148,3 +148,44 @@ L2: r4 = x L3: return r4 + + +[case testParamSpec] +from typing import Callable, ParamSpec, TypeVar + +P = ParamSpec("P") + +def execute(func: Callable[P, int], *args: P.args, **kwargs: P.kwargs) -> int: + return func(*args, **kwargs) + +def f(x: int) -> int: + return x + +execute(f, 1) +[out] +def execute(func, args, kwargs): + func :: object + args :: tuple + kwargs :: dict + r0 :: list + r1 :: object + r2 :: dict + r3 :: i32 + r4 :: bit + r5 :: tuple + r6 :: object + r7 :: int +L0: + r0 = PyList_New(0) + r1 = CPyList_Extend(r0, args) + r2 = PyDict_New() + r3 = CPyDict_UpdateInDisplay(r2, kwargs) + r4 = r3 >= 0 :: signed + r5 = PyList_AsTuple(r0) + r6 = PyObject_Call(func, r5, r2) + r7 = unbox(int, r6) + return r7 +def f(x): + x :: int +L0: + return x From 93dac05cc8461f13c2031dff48711eecbe2595af Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 3 Jun 2024 17:10:38 +0100 Subject: [PATCH 006/247] [PEP 695] Fix a few issues and add tests (#17318) Fix badly formed types that could be created when using aliases like `type A = list`. Improve some error messages when using PEP 695 syntax. Add a few PEP 695 tests. Work on #15238. --- mypy/message_registry.py | 3 ++ mypy/messages.py | 6 ++- mypy/semanal.py | 5 +++ mypy/typeanal.py | 30 ++++++++++--- test-data/unit/check-python312.test | 69 +++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 8 deletions(-) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index ccc1443dacf0..3852431f2290 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -194,6 +194,9 @@ def with_additional_msg(self, info: str) -> ErrorMessage: "A function returning TypeVar should receive at least " "one argument containing the same TypeVar" ) +TYPE_PARAMETERS_SHOULD_BE_DECLARED: Final = ( + "All type parameters should be declared ({} not declared)" +) # Super TOO_MANY_ARGS_FOR_SUPER: Final = ErrorMessage('Too many arguments for "super"') diff --git a/mypy/messages.py b/mypy/messages.py index 53a7f7d97774..de079feda048 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2423,7 +2423,11 @@ def annotation_in_unchecked_function(self, context: Context) -> None: def type_parameters_should_be_declared(self, undeclared: list[str], context: Context) -> None: names = ", ".join('"' + n + '"' for n in undeclared) - self.fail(f"All type parameters should be declared ({names} not declared)", context) + self.fail( + message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format(names), + context, + code=codes.VALID_TYPE, + ) def quote_type_string(type_string: str) -> str: diff --git a/mypy/semanal.py b/mypy/semanal.py index 0689d5416efe..44db7ddf5618 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3694,6 +3694,7 @@ def analyze_alias( allow_placeholder: bool = False, declared_type_vars: TypeVarLikeList | None = None, all_declared_type_params_names: list[str] | None = None, + python_3_12_type_alias: bool = False, ) -> tuple[Type | None, list[TypeVarLikeType], set[str], list[str], bool]: """Check if 'rvalue' is a valid type allowed for aliasing (e.g. not a type variable). @@ -3747,6 +3748,7 @@ def analyze_alias( global_scope=global_scope, allowed_alias_tvars=tvar_defs, alias_type_params_names=all_declared_type_params_names, + python_3_12_type_alias=python_3_12_type_alias, ) # There can be only one variadic variable at most, the error is reported elsewhere. @@ -5321,6 +5323,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: allow_placeholder=True, declared_type_vars=type_params, all_declared_type_params_names=all_type_params_names, + python_3_12_type_alias=True, ) if not res: res = AnyType(TypeOfAny.from_error) @@ -5355,6 +5358,8 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: # so we need to replace it with non-explicit Anys. res = make_any_non_explicit(res) eager = self.is_func_scope() + if isinstance(res, ProperType) and isinstance(res, Instance) and not res.args: + fix_instance(res, self.fail, self.note, disallow_any=False, options=self.options) alias_node = TypeAlias( res, self.qualified_name(s.name.name), diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 8f138ab5698f..bf53204ffce9 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -142,6 +142,7 @@ def analyze_type_alias( global_scope: bool = True, allowed_alias_tvars: list[TypeVarLikeType] | None = None, alias_type_params_names: list[str] | None = None, + python_3_12_type_alias: bool = False, ) -> tuple[Type, set[str]]: """Analyze r.h.s. of a (potential) type alias definition. @@ -160,6 +161,7 @@ def analyze_type_alias( prohibit_self_type="type alias target", allowed_alias_tvars=allowed_alias_tvars, alias_type_params_names=alias_type_params_names, + python_3_12_type_alias=python_3_12_type_alias, ) analyzer.in_dynamic_func = in_dynamic_func analyzer.global_scope = global_scope @@ -202,6 +204,7 @@ def __init__( is_typeshed_stub: bool, *, defining_alias: bool = False, + python_3_12_type_alias: bool = False, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, allow_placeholder: bool = False, @@ -220,6 +223,7 @@ def __init__( self.tvar_scope = tvar_scope # Are we analysing a type alias definition rvalue? self.defining_alias = defining_alias + self.python_3_12_type_alias = python_3_12_type_alias self.allow_tuple_literal = allow_tuple_literal # Positive if we are analyzing arguments of another (outer) type self.nesting_level = 0 @@ -364,7 +368,12 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) and (tvar_def is None or tvar_def not in self.allowed_alias_tvars) ): if self.not_declared_in_type_params(t.name): - msg = f'Type variable "{t.name}" is not included in type_params' + if self.python_3_12_type_alias: + msg = message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format( + f'"{t.name}"' + ) + else: + msg = f'Type variable "{t.name}" is not included in type_params' else: msg = f'Can\'t use bound type variable "{t.name}" to define generic alias' self.fail(msg, t, code=codes.VALID_TYPE) @@ -393,7 +402,12 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) if self.allow_unbound_tvars: return t if self.defining_alias and self.not_declared_in_type_params(t.name): - msg = f'TypeVarTuple "{t.name}" is not included in type_params' + if self.python_3_12_type_alias: + msg = message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format( + f'"{t.name}"' + ) + else: + msg = f'TypeVarTuple "{t.name}" is not included in type_params' else: msg = f'TypeVarTuple "{t.name}" is unbound' self.fail(msg, t, code=codes.VALID_TYPE) @@ -1309,11 +1323,13 @@ def analyze_callable_args_for_paramspec( and self.not_declared_in_type_params(tvar_def.name) and tvar_def not in self.allowed_alias_tvars ): - self.fail( - f'ParamSpec "{tvar_def.name}" is not included in type_params', - callable_args, - code=codes.VALID_TYPE, - ) + if self.python_3_12_type_alias: + msg = message_registry.TYPE_PARAMETERS_SHOULD_BE_DECLARED.format( + f'"{tvar_def.name}"' + ) + else: + msg = f'ParamSpec "{tvar_def.name}" is not included in type_params' + self.fail(msg, callable_args, code=codes.VALID_TYPE) return callable_with_ellipsis( AnyType(TypeOfAny.special_form), ret_type=ret_type, fallback=fallback ) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 6dd61351d7a8..8443aadb6905 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1244,3 +1244,72 @@ class C[T]: class D[T](C[S]): # E: All type parameters should be declared ("S" not declared) pass + +[case testPEP695MixNewAndOldStyleTypeVarTupleAndParamSpec] +# mypy: enable-incomplete-feature=NewGenericSyntax +from typing import TypeVarTuple, ParamSpec, Callable +Ts = TypeVarTuple("Ts") +P = ParamSpec("P") + +def f[T](x: T, f: Callable[P, None] # E: All type parameters should be declared ("P" not declared) + ) -> Callable[P, T]: ... +def g[T](x: T, f: tuple[*Ts] # E: All type parameters should be declared ("Ts" not declared) + ) -> tuple[T, *Ts]: ... +[builtins fixtures/tuple.pyi] + +[case testPEP695MixNewAndOldStyleGenericsInTypeAlias] +# mypy: enable-incomplete-feature=NewGenericSyntax +from typing import TypeVar, ParamSpec, TypeVarTuple, Callable + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +P = ParamSpec("P") + +type A = list[T] # E: All type parameters should be declared ("T" not declared) +a: A[int] # E: Bad number of arguments for type alias, expected 0, given 1 +reveal_type(a) # N: Revealed type is "builtins.list[Any]" + +type B = tuple[*Ts] # E: All type parameters should be declared ("Ts" not declared) +type C = Callable[P, None] # E: All type parameters should be declared ("P" not declared) +[builtins fixtures/tuple.pyi] + +[case testPEP695NonGenericAliasToGenericClass] +# mypy: enable-incomplete-feature=NewGenericSyntax +class C[T]: pass +type A = C +x: C +y: A +reveal_type(x) # N: Revealed type is "__main__.C[Any]" +reveal_type(y) # N: Revealed type is "__main__.C[Any]" +z: A[int] # E: Bad number of arguments for type alias, expected 0, given 1 + +[case testPEP695SelfType] +# mypy: enable-incomplete-feature=NewGenericSyntax +from typing import Self + +class C: + @classmethod + def m[T](cls, x: T) -> tuple[Self, T]: + return cls(), x + +class D(C): + pass + +reveal_type(C.m(1)) # N: Revealed type is "Tuple[__main__.C, builtins.int]" +reveal_type(D.m(1)) # N: Revealed type is "Tuple[__main__.D, builtins.int]" + +class E[T]: + def m(self) -> Self: + return self + + def mm[S](self, x: S) -> tuple[Self, S]: + return self, x + +class F[T](E[T]): + pass + +reveal_type(E[int]().m()) # N: Revealed type is "__main__.E[builtins.int]" +reveal_type(E[int]().mm(b'x')) # N: Revealed type is "Tuple[__main__.E[builtins.int], builtins.bytes]" +reveal_type(F[str]().m()) # N: Revealed type is "__main__.F[builtins.str]" +reveal_type(F[str]().mm(b'x')) # N: Revealed type is "Tuple[__main__.F[builtins.str], builtins.bytes]" +[builtins fixtures/tuple.pyi] From c7621912e2451f7135169f7d458f91c6c947ddba Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 4 Jun 2024 13:32:15 +0100 Subject: [PATCH 007/247] [PEP 695] Generate error if 3.12 type alias is called (#17320) PEP 695 type aliases raise an exception at runtime if called. Work on #15238. --- mypy/checkexpr.py | 4 +++ mypy/nodes.py | 6 ++++ mypy/semanal.py | 2 ++ test-data/unit/check-python312.test | 47 +++++++++++++++++++++++++ test-data/unit/check-type-aliases.test | 7 ++++ test-data/unit/fixtures/typing-full.pyi | 1 + 6 files changed, 67 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 0a4af069ea17..f826d4c11dd3 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4659,6 +4659,8 @@ def visit_type_application(self, tapp: TypeApplication) -> Type: is due to slight differences in how type arguments are applied and checked. """ if isinstance(tapp.expr, RefExpr) and isinstance(tapp.expr.node, TypeAlias): + if tapp.expr.node.python_3_12_type_alias: + return self.named_type("typing.TypeAliasType") # Subscription of a (generic) alias in runtime context, expand the alias. item = instantiate_type_alias( tapp.expr.node, @@ -4721,6 +4723,8 @@ class LongName(Generic[T]): ... x = A() y = cast(A, ...) """ + if alias.python_3_12_type_alias: + return self.named_type("typing.TypeAliasType") if isinstance(alias.target, Instance) and alias.target.invalid: # type: ignore[misc] # An invalid alias, error already has been reported return AnyType(TypeOfAny.from_error) diff --git a/mypy/nodes.py b/mypy/nodes.py index dbde3ddf4f1b..90561779051d 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3578,6 +3578,7 @@ def f(x: B[T]) -> T: ... # without T, Any would be used here "_is_recursive", "eager", "tvar_tuple_index", + "python_3_12_type_alias", ) __match_args__ = ("name", "target", "alias_tvars", "no_args") @@ -3593,6 +3594,7 @@ def __init__( no_args: bool = False, normalized: bool = False, eager: bool = False, + python_3_12_type_alias: bool = False, ) -> None: self._fullname = fullname self.target = target @@ -3605,6 +3607,7 @@ def __init__( # it is the cached value. self._is_recursive: bool | None = None self.eager = eager + self.python_3_12_type_alias = python_3_12_type_alias self.tvar_tuple_index = None for i, t in enumerate(alias_tvars): if isinstance(t, mypy.types.TypeVarTupleType): @@ -3675,6 +3678,7 @@ def serialize(self) -> JsonDict: "normalized": self.normalized, "line": self.line, "column": self.column, + "python_3_12_type_alias": self.python_3_12_type_alias, } return data @@ -3692,6 +3696,7 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: normalized = data["normalized"] line = data["line"] column = data["column"] + python_3_12_type_alias = data["python_3_12_type_alias"] return cls( target, fullname, @@ -3700,6 +3705,7 @@ def deserialize(cls, data: JsonDict) -> TypeAlias: alias_tvars=cast(List[mypy.types.TypeVarLikeType], alias_tvars), no_args=no_args, normalized=normalized, + python_3_12_type_alias=python_3_12_type_alias, ) diff --git a/mypy/semanal.py b/mypy/semanal.py index 44db7ddf5618..e4f123cbfc20 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3922,6 +3922,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: alias_tvars=alias_tvars, no_args=no_args, eager=eager, + python_3_12_type_alias=pep_695, ) if isinstance(s.rvalue, (IndexExpr, CallExpr, OpExpr)) and ( not isinstance(rvalue, OpExpr) @@ -5368,6 +5369,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: alias_tvars=alias_tvars, no_args=False, eager=eager, + python_3_12_type_alias=True, ) existing = self.current_symbol_table().get(s.name.name) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 8443aadb6905..77276d9ee079 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1313,3 +1313,50 @@ reveal_type(E[int]().mm(b'x')) # N: Revealed type is "Tuple[__main__.E[builtins reveal_type(F[str]().m()) # N: Revealed type is "__main__.F[builtins.str]" reveal_type(F[str]().mm(b'x')) # N: Revealed type is "Tuple[__main__.F[builtins.str], builtins.bytes]" [builtins fixtures/tuple.pyi] + +[case testPEP695CallAlias] +# mypy: enable-incomplete-feature=NewGenericSyntax + +class C: + def __init__(self, x: str) -> None: ... +type A = C + +class D[T]: pass +type B[T] = D[T] + +reveal_type(A) # N: Revealed type is "typing.TypeAliasType" +reveal_type(B) # N: Revealed type is "typing.TypeAliasType" +reveal_type(B[int]) # N: Revealed type is "typing.TypeAliasType" + +A(1) # E: "TypeAliasType" not callable +B[int]() # E: "TypeAliasType" not callable + +A2 = C +B2 = D +A2(1) # E: Argument 1 to "C" has incompatible type "int"; expected "str" +B2[int]() +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695IncrementalTypeAliasKinds] +# flags: --enable-incomplete-feature=NewGenericSyntax +import a + +[file a.py] +from b import A + +[file a.py.2] +from b import A, B, C +A() +B() +C() + +[file b.py] +from typing_extensions import TypeAlias +type A = int +B = int +C: TypeAlias = int +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] +[out2] +tmp/a.py:2: error: "TypeAliasType" not callable diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index f77c3c1c34e2..c13331e0a61b 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -1075,11 +1075,15 @@ x: TestType = 42 y: TestType = 'a' z: TestType = object() # E: Incompatible types in assignment (expression has type "object", variable has type "Union[int, str]") +reveal_type(TestType) # N: Revealed type is "typing.TypeAliasType" +TestType() # E: "TypeAliasType" not callable + class A: ClassAlias = TypeAliasType("ClassAlias", int) xc: A.ClassAlias = 1 yc: A.ClassAlias = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testTypeAliasTypeInvalid] from typing_extensions import TypeAliasType @@ -1094,6 +1098,7 @@ T3 = TypeAliasType("T3", -1) # E: Invalid type: try using Literal[-1] instead? t3: T3 reveal_type(t3) # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testTypeAliasTypeGeneric] from typing import Callable, Dict, Generic, TypeVar, Tuple @@ -1140,6 +1145,7 @@ ParamAlias2 = TypeAliasType("ParamAlias2", G[P, T], type_params=(P, T)) xp: ParamAlias2[[int], str] reveal_type(xp) # N: Revealed type is "__main__.G[[builtins.int], builtins.str]" [builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] [case testTypeAliasTypeInvalidGeneric] from typing_extensions import TypeAliasType, TypeVarTuple, ParamSpec @@ -1200,6 +1206,7 @@ class A(Generic[T]): x: A.Ta11 = {"a": 1} reveal_type(x) # N: Revealed type is "builtins.dict[builtins.str, Any]" [builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] [case testTypeAliasTypeNoUnpackInTypeParams311] # flags: --python-version 3.11 diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index f7da75fa4cd0..71d4dcb58853 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -35,6 +35,7 @@ TypedDict = 0 NoReturn = 0 NewType = 0 Self = 0 +Unpack = 0 T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) From 16d5aaf7f7f1e7f267c0a51924f6ca53ac5abe99 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 4 Jun 2024 15:31:39 +0100 Subject: [PATCH 008/247] [PEP 695] Add daemon tests that use new type parameter syntax (#17327) Add some basic mypy daemon test coverage, and make it possible to write daemon tests that only work on recent Python versions. Everything tested seems to work already. Work on #15238. --- mypy/test/testdeps.py | 5 ++ mypy/test/testdiff.py | 7 +- mypy/test/testfinegrained.py | 4 + mypy/traverser.py | 1 - test-data/unit/deps.test | 16 ++++ test-data/unit/diff.test | 99 ++++++++++++++++++++++ test-data/unit/fine-grained-python312.test | 81 ++++++++++++++++++ 7 files changed, 210 insertions(+), 3 deletions(-) create mode 100644 test-data/unit/fine-grained-python312.test diff --git a/mypy/test/testdeps.py b/mypy/test/testdeps.py index f9a059672de8..7c845eab8b57 100644 --- a/mypy/test/testdeps.py +++ b/mypy/test/testdeps.py @@ -3,8 +3,11 @@ from __future__ import annotations import os +import sys from collections import defaultdict +import pytest + from mypy import build from mypy.errors import CompileError from mypy.modulefinder import BuildSource @@ -28,6 +31,8 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: src = "\n".join(testcase.input) dump_all = "# __dump_all__" in src options = parse_options(src, testcase, incremental_step=1) + if options.python_version > sys.version_info: + pytest.skip("Test case requires a newer Python version") options.use_builtins_fixtures = True options.show_traceback = True options.cache_dir = os.devnull diff --git a/mypy/test/testdiff.py b/mypy/test/testdiff.py index 5e2e0bc2ca5a..0559b33c33e2 100644 --- a/mypy/test/testdiff.py +++ b/mypy/test/testdiff.py @@ -3,9 +3,11 @@ from __future__ import annotations import os +import sys + +import pytest from mypy import build -from mypy.defaults import PYTHON3_VERSION from mypy.errors import CompileError from mypy.modulefinder import BuildSource from mypy.nodes import MypyFile @@ -24,6 +26,8 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: files_dict = dict(testcase.files) second_src = files_dict["tmp/next.py"] options = parse_options(first_src, testcase, 1) + if options.python_version > sys.version_info: + pytest.skip("Test case requires a newer Python version") messages1, files1 = self.build(first_src, options) messages2, files2 = self.build(second_src, options) @@ -53,7 +57,6 @@ def build(self, source: str, options: Options) -> tuple[list[str], dict[str, Myp options.use_builtins_fixtures = True options.show_traceback = True options.cache_dir = os.devnull - options.python_version = PYTHON3_VERSION options.allow_empty_bodies = True try: result = build.build( diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index f61a58c425fc..800ba2dff087 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -16,6 +16,7 @@ import os import re +import sys import unittest from typing import Any @@ -82,6 +83,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: f.write(main_src) options = self.get_options(main_src, testcase, build_cache=False) + if options.python_version > sys.version_info: + pytest.skip("Test case requires a newer Python version") + build_options = self.get_options(main_src, testcase, build_cache=True) server = Server(options, DEFAULT_STATUS_FILE) diff --git a/mypy/traverser.py b/mypy/traverser.py index 225de27e7002..6f162c9ec576 100644 --- a/mypy/traverser.py +++ b/mypy/traverser.py @@ -246,7 +246,6 @@ def visit_match_stmt(self, o: MatchStmt) -> None: def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: o.name.accept(self) - # TODO: params o.value.accept(self) def visit_member_expr(self, o: MemberExpr) -> None: diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index d18b4aae963b..f46cfebb113f 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -1431,3 +1431,19 @@ class B(A): -> m -> m -> m + +[case testPEP695TypeAliasDeps] +# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +from a import C, E +type A = C +type A2 = A +type A3 = E +[file a.py] +class C: pass +class D: pass +type E = D +[out] + -> m + -> m + -> m + -> m diff --git a/test-data/unit/diff.test b/test-data/unit/diff.test index 8fc74868123e..9212d902e8b2 100644 --- a/test-data/unit/diff.test +++ b/test-data/unit/diff.test @@ -1530,3 +1530,102 @@ class C: [out] __main__.C.get_by_team_and_id __main__.Optional + +[case testPEP695TypeAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +from typing_extensions import TypeAlias, TypeAliasType +type A = int +type B = str +type C = int +D = int +E: TypeAlias = int +F = TypeAliasType("F", int) +G = TypeAliasType("G", int) +type H = int + +[file next.py] +# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +from typing_extensions import TypeAlias, TypeAliasType +type A = str +type B = str +type C[T] = int +type D = int +type E = int +type F = int +type G = str +type H[T] = int + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] +[out] +__main__.A +__main__.C +__main__.D +__main__.E +__main__.G +__main__.H + +[case testPEP695TypeAlias2] +# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +type A[T: int] = list[T] +type B[T: int] = list[T] +type C[T: (int, str)] = list[T] +type D[T: (int, str)] = list[T] +type E[T: int] = list[T] +type F[T: (int, str)] = list[T] + +[file next.py] +# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +type A[T] = list[T] +type B[T: str] = list[T] +type C[T: (int, None)] = list[T] +type D[T] = list[T] +type E[T: int] = list[T] +type F[T: (int, str)] = list[T] + +[out] +__main__.A +__main__.B +__main__.C +__main__.D + +[case testPEP695GenericFunction] +# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +def f[T](x: T) -> T: + return x +def g[T](x: T, y: T) -> T: + return x +[file next.py] +# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +def f[T](x: T) -> T: + return x +def g[T, S](x: T, y: S) -> S: + return y +[out] +__main__.g + +[case testPEP695GenericClass] +# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +class C[T]: + pass +class D[T]: + pass +class E[T]: + pass +class F[T]: + def f(self, x: object) -> T: ... +[file next.py] +# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +class C[T]: + pass +class D[T: int]: + pass +class E: + pass +class F[T]: + def f(self, x: T) -> T: ... +[out] +__main__.D +__main__.E +__main__.F +__main__.F.f diff --git a/test-data/unit/fine-grained-python312.test b/test-data/unit/fine-grained-python312.test new file mode 100644 index 000000000000..70cf427d6798 --- /dev/null +++ b/test-data/unit/fine-grained-python312.test @@ -0,0 +1,81 @@ +[case testPEP695TypeAliasDep] +# flags: --enable-incomplete-feature=NewGenericSyntax +import m +def g() -> m.C: + return m.f() +[file m.py] +type C = int + +def f() -> int: + pass +[file m.py.2] +type C = str + +def f() -> int: + pass +[out] +== +main:4: error: Incompatible return value type (got "int", expected "str") + +[case testPEP695ChangeOldStyleToNewStyleTypeAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax +from m import A +A() + +[file m.py] +A = int + +[file m.py.2] +type A = int +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] +[out] +== +main:3: error: "TypeAliasType" not callable + +[case testPEP695VarianceChangesDueToDependency] +# flags: --enable-incomplete-feature=NewGenericSyntax +from a import C + +x: C[object] = C[int]() + +[file a.py] +from b import A + +class C[T]: + def f(self) -> A[T]: ... + +[file b.py] +class A[T]: + def f(self) -> T: ... + +[file b.py.2] +class A[T]: + def f(self) -> list[T]: ... + +[out] +== +main:4: error: Incompatible types in assignment (expression has type "C[int]", variable has type "C[object]") + +[case testPEP695TypeAliasChangesDueToDependency] +# flags: --enable-incomplete-feature=NewGenericSyntax +from a import A +x: A +x = 0 +x = '' + +[file a.py] +from b import B +type A = B[int, str] + +[file b.py] +from typing import Union as B + +[file b.py.2] +from builtins import tuple as B + +[builtins fixtures/tuple.pyi] +[out] +== +main:4: error: Incompatible types in assignment (expression has type "int", variable has type "tuple[int, str]") +main:5: error: Incompatible types in assignment (expression has type "str", variable has type "tuple[int, str]") From ad0e180659f8bb1ef0c270045c655888b3b1223a Mon Sep 17 00:00:00 2001 From: GiorgosPapoutsakis <116210016+GiorgosPapoutsakis@users.noreply.github.com> Date: Tue, 4 Jun 2024 22:38:17 +0300 Subject: [PATCH 009/247] Fix false positive for Final local scope variable in Protocol (#17308) This PR fixes and closes #17281 ,which reported a false positive when using Final within the local function scope of a protocol method. With these changes: - Local variables within protocol methods can be marked as Final. - Protocol members still cannot be marked as Final Modified ``semanal.py`` file and added a unit test in ``test-data/unit/check.final.test`` --------- Co-authored-by: Jelle Zijlstra Co-authored-by: Stanislav Terliakov --- mypy/semanal.py | 3 ++- test-data/unit/check-final.test | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index e4f123cbfc20..2448ea8485f7 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3505,7 +3505,8 @@ def unwrap_final(self, s: AssignmentStmt) -> bool: if self.loop_depth[-1] > 0: self.fail("Cannot use Final inside a loop", s) if self.type and self.type.is_protocol: - self.msg.protocol_members_cant_be_final(s) + if self.is_class_scope(): + self.msg.protocol_members_cant_be_final(s) if ( isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs diff --git a/test-data/unit/check-final.test b/test-data/unit/check-final.test index 26a0d0782503..dadf76a283b0 100644 --- a/test-data/unit/check-final.test +++ b/test-data/unit/check-final.test @@ -301,6 +301,50 @@ class P(Protocol): pass [out] +[case testFinalInProtocol] +from typing import Final, Protocol, final + +class P(Protocol): + var1 : Final[int] = 0 # E: Protocol member cannot be final + + @final # E: Protocol member cannot be final + def meth1(self) -> None: + var2: Final = 0 + + def meth2(self) -> None: + var3: Final = 0 + + def meth3(self) -> None: + class Inner: + var3: Final = 0 # OK + + @final + def inner(self) -> None: ... + + class Inner: + var3: Final = 0 # OK + + @final + def inner(self) -> None: ... + +[out] + +[case testFinalWithClassVarInProtocol] +from typing import Protocol, Final, final, ClassVar + +class P(Protocol): + var1 : Final[ClassVar[int]] = 0 # E: Variable should not be annotated with both ClassVar and Final + var2: ClassVar[int] = 1 + + @final # E: Protocol member cannot be final + def meth1(self) -> None: + ... + + def meth2(self) -> None: + var3: Final[ClassVar[int]] = 0 # E: Variable should not be annotated with both ClassVar and Final # E: ClassVar can only be used for assignments in class body + +[out] + [case testFinalNotInLoops] from typing import Final From fbaa7e0121a180051c28d72bf1ed73f5c3c3b947 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 5 Jun 2024 10:49:13 +0100 Subject: [PATCH 010/247] [PEP 695] Add tests for type aliases with bounds and value restrictions (#17330) The functionality already works, but there was missing test coverage. Work on #15238. --- test-data/unit/check-python312.test | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 77276d9ee079..905012d9099c 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1360,3 +1360,42 @@ C: TypeAlias = int [typing fixtures/typing-full.pyi] [out2] tmp/a.py:2: error: "TypeAliasType" not callable + +[case testPEP695TypeAliasBoundAndValueChecking] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import Any, cast + +class C: pass +class D(C): pass + +type A[T: C] = list[T] +a1: A +reveal_type(a1) # N: Revealed type is "builtins.list[Any]" +a2: A[Any] +a3: A[C] +a4: A[D] +a5: A[object] # E: Type argument "object" of "A" must be a subtype of "C" +a6: A[int] # E: Type argument "int" of "A" must be a subtype of "C" + +x1 = cast(A[C], a1) +x2 = cast(A[None], a1) # E: Type argument "None" of "A" must be a subtype of "C" + +type A2[T: (int, C)] = list[T] +b1: A2 +reveal_type(b1) # N: Revealed type is "builtins.list[Any]" +b2: A2[Any] +b3: A2[int] +b4: A2[C] +b5: A2[D] # E: Value of type variable "T" of "A2" cannot be "D" +b6: A2[object] # E: Value of type variable "T" of "A2" cannot be "object" + +list[A2[int]]() +list[A2[None]]() # E: Invalid type argument value for "A2" + +class N(int): pass + +type A3[T: C, S: (int, str)] = T | S +c1: A3[C, int] +c2: A3[D, str] +c3: A3[C, N] # E: Value of type variable "S" of "A3" cannot be "N" +c4: A3[int, str] # E: Type argument "int" of "A3" must be a subtype of "C" From ddcf90d1c43b078b8acc5a341074a1c9ab260569 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 5 Jun 2024 16:16:19 +0100 Subject: [PATCH 011/247] [PEP 695] Add tests for type alias in class body or function (#17334) Work on #15238. --- test-data/unit/check-python312.test | 54 +++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 905012d9099c..2b67f56e679c 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1399,3 +1399,57 @@ c1: A3[C, int] c2: A3[D, str] c3: A3[C, N] # E: Value of type variable "S" of "A3" cannot be "N" c4: A3[int, str] # E: Type argument "int" of "A3" must be a subtype of "C" + +[case testPEP695TypeAliasInClassBodyOrFunction] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class C: + type A = int + type B[T] = list[T] | None + a: A + b: B[str] + + def method(self) -> None: + v: C.A + reveal_type(v) # N: Revealed type is "builtins.int" + +reveal_type(C.a) # N: Revealed type is "builtins.int" +reveal_type(C.b) # N: Revealed type is "Union[builtins.list[builtins.str], None]" + +C.A = str # E: Incompatible types in assignment (expression has type "Type[str]", variable has type "TypeAliasType") + +x: C.A +y: C.B[int] +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "Union[builtins.list[builtins.int], None]" + +def f() -> None: + type A = int + type B[T] = list[T] | None + a: A + reveal_type(a) # N: Revealed type is "builtins.int" + + def g() -> None: + b: B[int] + reveal_type(b) # N: Revealed type is "Union[builtins.list[builtins.int], None]" + +class D: + def __init__(self) -> None: + type A = int + self.a: A = 0 + type B[T] = list[T] + self.b: B[int] = [1] + +reveal_type(D().a) # N: Revealed type is "builtins.int" +reveal_type(D().b) # N: Revealed type is "builtins.list[builtins.int]" + +class E[T]: + type X = list[T] # E: All type parameters should be declared ("T" not declared) + + def __init__(self) -> None: + type A = list[T] # E: All type parameters should be declared ("T" not declared) + self.a: A + +reveal_type(E[str]().a) # N: Revealed type is "builtins.list[Any]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] From 6bdd854083604e4416123edf87a8f549356f783f Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Wed, 5 Jun 2024 23:32:42 +0200 Subject: [PATCH 012/247] Do not forget that a `TypedDict` was wrapped in `Unpack` after a `name-defined` error occurred. (#17226) Do not set the `unpacked_kwargs` attribute of `CallableType` to False when visiting a callable of which the `Unpack` wrapper of a `TypedDict` has already been removed. Fixes #17225 --- mypy/typeanal.py | 2 +- test-data/unit/check-typeddict.test | 38 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index bf53204ffce9..ded8b8412a9a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1030,7 +1030,7 @@ def visit_parameters(self, t: Parameters) -> Type: def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: # Every Callable can bind its own type variables, if they're not in the outer scope with self.tvar_scope_frame(): - unpacked_kwargs = False + unpacked_kwargs = t.unpack_kwargs if self.defining_alias: variables = t.variables else: diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 09b86e4afd2d..5fb74f66dd89 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -3487,3 +3487,41 @@ class A(Generic[T]): return self.a(x=1) [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] + +[case testNameUndefinedErrorDoesNotLoseUnpackedKWArgsInformation] +from typing import overload +from typing_extensions import TypedDict, Unpack + +class TD(TypedDict, total=False): + x: int + y: str + +@overload +def f(self, *, x: int) -> None: ... +@overload +def f(self, *, y: str) -> None: ... +def f(self, **kwargs: Unpack[TD]) -> None: + z # E: Name "z" is not defined + +@overload +def g(self, *, x: float) -> None: ... +@overload +def g(self, *, y: str) -> None: ... +def g(self, **kwargs: Unpack[TD]) -> None: # E: Overloaded function implementation does not accept all possible arguments of signature 1 + z # E: Name "z" is not defined + +class A: + def f(self, *, x: int) -> None: ... + def g(self, *, x: float) -> None: ... +class B(A): + def f(self, **kwargs: Unpack[TD]) -> None: + z # E: Name "z" is not defined + def g(self, **kwargs: Unpack[TD]) -> None: # E: Signature of "g" incompatible with supertype "A" \ + # N: Superclass: \ + # N: def g(self, *, x: float) -> None \ + # N: Subclass: \ + # N: def g(*, x: int = ..., y: str = ...) -> None + z # E: Name "z" is not defined +reveal_type(B.f) # N: Revealed type is "def (self: __main__.B, **kwargs: Unpack[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: builtins.str})])" +B().f(x=1.0) # E: Argument "x" to "f" of "B" has incompatible type "float"; expected "int" +[builtins fixtures/primitives.pyi] From 668256364bc320a371e0a705436f1a700a724edc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 5 Jun 2024 23:11:41 +0100 Subject: [PATCH 013/247] Use namespaces for function type variables (#17311) Fixes https://github.com/python/mypy/issues/16582 IMO this is long overdue. Currently, type variable IDs are 99% unique, but when they accidentally clash, it causes hard to debug issues. The implementation is generally straightforward, but it uncovered a whole bunch of unrelated bugs. Few notes: * This still doesn't fix the type variables in nested generic callable types (those that appear in return types of another generic callable). It is non-trivial to put namespace there, and luckily this situation is already special-cased in `checkexpr.py` to avoid ID clashes. * This uncovered a bug in overloaded dunder overrides handling, fix is simple. * This also uncovered a deeper problem in unsafe overload overlap logic (w.r.t. partial parameters overlap). Here proper fix would be hard, so instead I tweak current logic so it will not cause false positives, at a cost of possible false negatives. * This makes explicit that we use a somewhat ad-hoc logic for join/meet of generic callables. FWIW I decided to keep it, since it seems to work reasonably well. * This accidentally highlighted two bugs in error message locations. One very old one related to type aliases, I fixed newly discovered cases by extending a previous partial fix. Second, the error locations generated by `partial` plugin were completely off (you can see examples in `mypy_primer` where there were errors on empty lines etc). * This PR (naturally) causes a significant amount of new valid errors (fixed false negatives). To improve the error messages, I extend the name disambiguation logic to include type variables (and also type aliases, while I am at it), previously it only applied to `Instance`s. Note that I use a notation `TypeVar@namespace`, which is a semantic equivalent of qualified name for type variables. For now, I shorten the namespace to only the last component, to make errors less verbose. We can reconsider this if it causes confusion. * Finally, this PR will hopefully allow a more principled implementation of https://github.com/python/mypy/issues/15907 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/checker.py | 4 +- mypy/checkexpr.py | 13 +++-- mypy/expandtype.py | 2 +- mypy/join.py | 31 ++++++++++ mypy/meet.py | 3 +- mypy/messages.py | 76 ++++++++++++++++++------- mypy/plugins/attrs.py | 17 +++--- mypy/plugins/dataclasses.py | 5 +- mypy/plugins/functools.py | 14 ++++- mypy/semanal.py | 19 ++++--- mypy/semanal_namedtuple.py | 31 ++++++---- mypy/semanal_shared.py | 4 +- mypy/subtypes.py | 16 ++++-- mypy/test/testtypes.py | 33 ++++++++--- mypy/test/typefixture.py | 9 +-- mypy/tvar_scope.py | 24 ++++---- mypy/typeanal.py | 15 +++-- mypy/types.py | 32 ++++++----- mypyc/test-data/fixtures/testutil.py | 4 +- test-data/unit/check-functions.test | 62 ++++++++++++++++++++ test-data/unit/check-functools.test | 4 +- test-data/unit/check-generics.test | 29 +++++----- test-data/unit/check-inference.test | 2 +- test-data/unit/check-python311.test | 2 +- test-data/unit/check-type-aliases.test | 11 ++++ test-data/unit/check-typevar-tuple.test | 3 +- 26 files changed, 332 insertions(+), 133 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 179ff6e0b4b6..38976d4ce15e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2171,7 +2171,9 @@ def bind_and_map_method( def get_op_other_domain(self, tp: FunctionLike) -> Type | None: if isinstance(tp, CallableType): if tp.arg_kinds and tp.arg_kinds[0] == ARG_POS: - return tp.arg_types[0] + # For generic methods, domain comparison is tricky, as a first + # approximation erase all remaining type variables to bounds. + return erase_typevars(tp.arg_types[0], {v.id for v in tp.variables}) return None elif isinstance(tp, Overloaded): raw_items = [self.get_op_other_domain(it) for it in tp.items] diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index f826d4c11dd3..8e6af0218c32 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -167,6 +167,7 @@ TypedDictType, TypeOfAny, TypeType, + TypeVarId, TypeVarLikeType, TypeVarTupleType, TypeVarType, @@ -4933,7 +4934,7 @@ def check_lst_expr(self, e: ListExpr | SetExpr | TupleExpr, fullname: str, tag: tv = TypeVarType( "T", "T", - id=-1, + id=TypeVarId(-1, namespace=""), values=[], upper_bound=self.object_type(), default=AnyType(TypeOfAny.from_omitted_generics), @@ -5164,7 +5165,7 @@ def visit_dict_expr(self, e: DictExpr) -> Type: kt = TypeVarType( "KT", "KT", - id=-1, + id=TypeVarId(-1, namespace=""), values=[], upper_bound=self.object_type(), default=AnyType(TypeOfAny.from_omitted_generics), @@ -5172,7 +5173,7 @@ def visit_dict_expr(self, e: DictExpr) -> Type: vt = TypeVarType( "VT", "VT", - id=-2, + id=TypeVarId(-2, namespace=""), values=[], upper_bound=self.object_type(), default=AnyType(TypeOfAny.from_omitted_generics), @@ -5564,7 +5565,7 @@ def check_generator_or_comprehension( tv = TypeVarType( "T", "T", - id=-1, + id=TypeVarId(-1, namespace=""), values=[], upper_bound=self.object_type(), default=AnyType(TypeOfAny.from_omitted_generics), @@ -5591,7 +5592,7 @@ def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: ktdef = TypeVarType( "KT", "KT", - id=-1, + id=TypeVarId(-1, namespace=""), values=[], upper_bound=self.object_type(), default=AnyType(TypeOfAny.from_omitted_generics), @@ -5599,7 +5600,7 @@ def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> Type: vtdef = TypeVarType( "VT", "VT", - id=-2, + id=TypeVarId(-2, namespace=""), values=[], upper_bound=self.object_type(), default=AnyType(TypeOfAny.from_omitted_generics), diff --git a/mypy/expandtype.py b/mypy/expandtype.py index f7fa0258f588..86875bc6079a 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -316,7 +316,7 @@ def interpolate_args_for_unpack(self, t: CallableType, var_arg: UnpackType) -> l new_unpack: Type if isinstance(var_arg_type, Instance): # we have something like Unpack[Tuple[Any, ...]] - new_unpack = var_arg + new_unpack = UnpackType(var_arg.type.accept(self)) elif isinstance(var_arg_type, TupleType): # We have something like Unpack[Tuple[Unpack[Ts], X1, X2]] expanded_tuple = var_arg_type.accept(self) diff --git a/mypy/join.py b/mypy/join.py index c711697ec46d..5284be7dd2a1 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -5,6 +5,7 @@ from typing import Sequence, overload import mypy.typeops +from mypy.expandtype import expand_type from mypy.maptype import map_instance_to_supertype from mypy.nodes import CONTRAVARIANT, COVARIANT, INVARIANT, VARIANCE_NOT_READY from mypy.state import state @@ -36,6 +37,7 @@ TypedDictType, TypeOfAny, TypeType, + TypeVarId, TypeVarLikeType, TypeVarTupleType, TypeVarType, @@ -718,7 +720,35 @@ def is_similar_callables(t: CallableType, s: CallableType) -> bool: ) +def update_callable_ids(c: CallableType, ids: list[TypeVarId]) -> CallableType: + tv_map = {} + tvs = [] + for tv, new_id in zip(c.variables, ids): + new_tv = tv.copy_modified(id=new_id) + tvs.append(new_tv) + tv_map[tv.id] = new_tv + return expand_type(c, tv_map).copy_modified(variables=tvs) + + +def match_generic_callables(t: CallableType, s: CallableType) -> tuple[CallableType, CallableType]: + # The case where we combine/join/meet similar callables, situation where both are generic + # requires special care. A more principled solution may involve unify_generic_callable(), + # but it would have two problems: + # * This adds risk of infinite recursion: e.g. join -> unification -> solver -> join + # * Using unification is an incorrect thing for meets, as it "widens" the types + # Finally, this effectively falls back to an old behaviour before namespaces were added to + # type variables, and it worked relatively well. + max_len = max(len(t.variables), len(s.variables)) + min_len = min(len(t.variables), len(s.variables)) + if min_len == 0: + return t, s + new_ids = [TypeVarId.new(meta_level=0) for _ in range(max_len)] + # Note: this relies on variables being in order they appear in function definition. + return update_callable_ids(t, new_ids), update_callable_ids(s, new_ids) + + def join_similar_callables(t: CallableType, s: CallableType) -> CallableType: + t, s = match_generic_callables(t, s) arg_types: list[Type] = [] for i in range(len(t.arg_types)): arg_types.append(safe_meet(t.arg_types[i], s.arg_types[i])) @@ -771,6 +801,7 @@ def safe_meet(t: Type, s: Type) -> Type: def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType: + t, s = match_generic_callables(t, s) arg_types: list[Type] = [] for i in range(len(t.arg_types)): arg_types.append(safe_join(t.arg_types[i], s.arg_types[i])) diff --git a/mypy/meet.py b/mypy/meet.py index df8b960cdf3f..2d44cafb23b3 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -1024,8 +1024,9 @@ def default(self, typ: Type) -> ProperType: def meet_similar_callables(t: CallableType, s: CallableType) -> CallableType: - from mypy.join import safe_join + from mypy.join import match_generic_callables, safe_join + t, s = match_generic_callables(t, s) arg_types: list[Type] = [] for i in range(len(t.arg_types)): arg_types.append(safe_join(t.arg_types[i], s.arg_types[i])) diff --git a/mypy/messages.py b/mypy/messages.py index de079feda048..f01b0a726584 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -83,6 +83,7 @@ TypeOfAny, TypeStrVisitor, TypeType, + TypeVarLikeType, TypeVarTupleType, TypeVarType, UnboundType, @@ -2502,14 +2503,16 @@ def format_literal_value(typ: LiteralType) -> str: return typ.value_repr() if isinstance(typ, TypeAliasType) and typ.is_recursive: - # TODO: find balance here, str(typ) doesn't support custom verbosity, and may be - # too verbose for user messages, OTOH it nicely shows structure of recursive types. - if verbosity < 2: - type_str = typ.alias.name if typ.alias else "" + if typ.alias is None: + type_str = "" + else: + if verbosity >= 2 or (fullnames and typ.alias.fullname in fullnames): + type_str = typ.alias.fullname + else: + type_str = typ.alias.name if typ.args: type_str += f"[{format_list(typ.args)}]" - return type_str - return str(typ) + return type_str # TODO: always mention type alias names in errors. typ = get_proper_type(typ) @@ -2550,9 +2553,15 @@ def format_literal_value(typ: LiteralType) -> str: return f"Unpack[{format(typ.type)}]" elif isinstance(typ, TypeVarType): # This is similar to non-generic instance types. + fullname = scoped_type_var_name(typ) + if verbosity >= 2 or (fullnames and fullname in fullnames): + return fullname return typ.name elif isinstance(typ, TypeVarTupleType): # This is similar to non-generic instance types. + fullname = scoped_type_var_name(typ) + if verbosity >= 2 or (fullnames and fullname in fullnames): + return fullname return typ.name elif isinstance(typ, ParamSpecType): # Concatenate[..., P] @@ -2563,6 +2572,7 @@ def format_literal_value(typ: LiteralType) -> str: return f"[{args}, **{typ.name_with_suffix()}]" else: + # TODO: better disambiguate ParamSpec name clashes. return typ.name_with_suffix() elif isinstance(typ, TupleType): # Prefer the name of the fallback class (if not tuple), as it's more informative. @@ -2680,29 +2690,51 @@ def format_literal_value(typ: LiteralType) -> str: return "object" -def collect_all_instances(t: Type) -> list[Instance]: - """Return all instances that `t` contains (including `t`). +def collect_all_named_types(t: Type) -> list[Type]: + """Return all instances/aliases/type variables that `t` contains (including `t`). This is similar to collect_all_inner_types from typeanal but only returns instances and will recurse into fallbacks. """ - visitor = CollectAllInstancesQuery() + visitor = CollectAllNamedTypesQuery() t.accept(visitor) - return visitor.instances + return visitor.types -class CollectAllInstancesQuery(TypeTraverserVisitor): +class CollectAllNamedTypesQuery(TypeTraverserVisitor): def __init__(self) -> None: - self.instances: list[Instance] = [] + self.types: list[Type] = [] def visit_instance(self, t: Instance) -> None: - self.instances.append(t) + self.types.append(t) super().visit_instance(t) def visit_type_alias_type(self, t: TypeAliasType) -> None: if t.alias and not t.is_recursive: - t.alias.target.accept(self) - super().visit_type_alias_type(t) + get_proper_type(t).accept(self) + else: + self.types.append(t) + super().visit_type_alias_type(t) + + def visit_type_var(self, t: TypeVarType) -> None: + self.types.append(t) + super().visit_type_var(t) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: + self.types.append(t) + super().visit_type_var_tuple(t) + + def visit_param_spec(self, t: ParamSpecType) -> None: + self.types.append(t) + super().visit_param_spec(t) + + +def scoped_type_var_name(t: TypeVarLikeType) -> str: + if not t.id.namespace: + return t.name + # TODO: support rare cases when both TypeVar name and namespace suffix coincide. + *_, suffix = t.id.namespace.split(".") + return f"{t.name}@{suffix}" def find_type_overlaps(*types: Type) -> set[str]: @@ -2713,8 +2745,14 @@ def find_type_overlaps(*types: Type) -> set[str]: """ d: dict[str, set[str]] = {} for type in types: - for inst in collect_all_instances(type): - d.setdefault(inst.type.name, set()).add(inst.type.fullname) + for t in collect_all_named_types(type): + if isinstance(t, ProperType) and isinstance(t, Instance): + d.setdefault(t.type.name, set()).add(t.type.fullname) + elif isinstance(t, TypeAliasType) and t.alias: + d.setdefault(t.alias.name, set()).add(t.alias.fullname) + else: + assert isinstance(t, TypeVarLikeType) + d.setdefault(t.name, set()).add(scoped_type_var_name(t)) for shortname in d.keys(): if f"typing.{shortname}" in TYPES_FOR_UNIMPORTED_HINTS: d[shortname].add(f"typing.{shortname}") @@ -2732,7 +2770,7 @@ def format_type( """ Convert a type to a relatively short string suitable for error messages. - `verbosity` is a coarse grained control on the verbosity of the type + `verbosity` is a coarse-grained control on the verbosity of the type This function returns a string appropriate for unmodified use in error messages; this means that it will be quoted in most cases. If @@ -2748,7 +2786,7 @@ def format_type_bare( """ Convert a type to a relatively short string suitable for error messages. - `verbosity` is a coarse grained control on the verbosity of the type + `verbosity` is a coarse-grained control on the verbosity of the type `fullnames` specifies a set of names that should be printed in full This function will return an unquoted string. If a caller doesn't need to diff --git a/mypy/plugins/attrs.py b/mypy/plugins/attrs.py index 83f685f57a16..db976385ee56 100644 --- a/mypy/plugins/attrs.py +++ b/mypy/plugins/attrs.py @@ -69,6 +69,7 @@ Type, TypeOfAny, TypeType, + TypeVarId, TypeVarType, UninhabitedType, UnionType, @@ -807,25 +808,25 @@ def _add_order(ctx: mypy.plugin.ClassDefContext, adder: MethodAdder) -> None: # AT = TypeVar('AT') # def __lt__(self: AT, other: AT) -> bool # This way comparisons with subclasses will work correctly. + fullname = f"{ctx.cls.info.fullname}.{SELF_TVAR_NAME}" tvd = TypeVarType( SELF_TVAR_NAME, - ctx.cls.info.fullname + "." + SELF_TVAR_NAME, - id=-1, + fullname, + # Namespace is patched per-method below. + id=TypeVarId(-1, namespace=""), values=[], upper_bound=object_type, default=AnyType(TypeOfAny.from_omitted_generics), ) self_tvar_expr = TypeVarExpr( - SELF_TVAR_NAME, - ctx.cls.info.fullname + "." + SELF_TVAR_NAME, - [], - object_type, - AnyType(TypeOfAny.from_omitted_generics), + SELF_TVAR_NAME, fullname, [], object_type, AnyType(TypeOfAny.from_omitted_generics) ) ctx.cls.info.names[SELF_TVAR_NAME] = SymbolTableNode(MDEF, self_tvar_expr) - args = [Argument(Var("other", tvd), tvd, None, ARG_POS)] for method in ["__lt__", "__le__", "__gt__", "__ge__"]: + namespace = f"{ctx.cls.info.fullname}.{method}" + tvd = tvd.copy_modified(id=TypeVarId(tvd.id.raw_id, namespace=namespace)) + args = [Argument(Var("other", tvd), tvd, None, ARG_POS)] adder.add_method(method, args, bool_type, self_type=tvd, tvd=tvd) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index dead512a2202..dd2eceab217f 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -65,6 +65,7 @@ TupleType, Type, TypeOfAny, + TypeVarId, TypeVarType, UninhabitedType, UnionType, @@ -314,8 +315,8 @@ def transform(self) -> bool: obj_type = self._api.named_type("builtins.object") order_tvar_def = TypeVarType( SELF_TVAR_NAME, - info.fullname + "." + SELF_TVAR_NAME, - id=-1, + f"{info.fullname}.{SELF_TVAR_NAME}", + id=TypeVarId(-1, namespace=f"{info.fullname}.{method_name}"), values=[], upper_bound=obj_type, default=AnyType(TypeOfAny.from_omitted_generics), diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index 81a3b4d96ef3..335123a4a108 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -7,7 +7,7 @@ import mypy.checker import mypy.plugin from mypy.argmap import map_actuals_to_formals -from mypy.nodes import ARG_POS, ARG_STAR2, ArgKind, Argument, FuncItem, Var +from mypy.nodes import ARG_POS, ARG_STAR2, ArgKind, Argument, CallExpr, FuncItem, Var from mypy.plugins.common import add_method_to_class from mypy.types import ( AnyType, @@ -151,12 +151,22 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: actual_arg_names = [a for param in ctx.arg_names[1:] for a in param] actual_types = [a for param in ctx.arg_types[1:] for a in param] + # Create a valid context for various ad-hoc inspections in check_call(). + call_expr = CallExpr( + callee=ctx.args[0][0], + args=actual_args, + arg_kinds=actual_arg_kinds, + arg_names=actual_arg_names, + analyzed=ctx.context.analyzed if isinstance(ctx.context, CallExpr) else None, + ) + call_expr.set_line(ctx.context) + _, bound = ctx.api.expr_checker.check_call( callee=defaulted, args=actual_args, arg_kinds=actual_arg_kinds, arg_names=actual_arg_names, - context=defaulted, + context=call_expr, ) bound = get_proper_type(bound) if not isinstance(bound, CallableType): diff --git a/mypy/semanal.py b/mypy/semanal.py index 2448ea8485f7..8505b3a9ccac 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -279,6 +279,7 @@ TypedDictType, TypeOfAny, TypeType, + TypeVarId, TypeVarLikeType, TypeVarTupleType, TypeVarType, @@ -894,7 +895,7 @@ def analyze_func_def(self, defn: FuncDef) -> None: self.prepare_method_signature(defn, self.type, has_self_type) # Analyze function signature - with self.tvar_scope_frame(self.tvar_scope.method_frame()): + with self.tvar_scope_frame(self.tvar_scope.method_frame(defn.fullname)): if defn.type: self.check_classvar_in_signature(defn.type) assert isinstance(defn.type, CallableType) @@ -902,7 +903,9 @@ def analyze_func_def(self, defn: FuncDef) -> None: # class-level imported names and type variables are in scope. analyzer = self.type_analyzer() tag = self.track_incomplete_refs() - result = analyzer.visit_callable_type(defn.type, nested=False) + result = analyzer.visit_callable_type( + defn.type, nested=False, namespace=defn.fullname + ) # Don't store not ready types (including placeholders). if self.found_incomplete_ref(tag) or has_placeholder(result): self.defer(defn) @@ -1114,7 +1117,7 @@ def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) if defn is generic. Return True, if the signature contains typing.Self type, or False otherwise. """ - with self.tvar_scope_frame(self.tvar_scope.method_frame()): + with self.tvar_scope_frame(self.tvar_scope.method_frame(defn.fullname)): a = self.type_analyzer() fun_type.variables, has_self_type = a.bind_function_type_variables(fun_type, defn) if has_self_type and self.type is not None: @@ -1152,7 +1155,7 @@ def setup_self_type(self) -> None: info.self_type = TypeVarType( "Self", f"{info.fullname}.Self", - id=0, + id=TypeVarId(0), # 0 is a special value for self-types. values=[], upper_bound=fill_typevars(info), default=AnyType(TypeOfAny.from_omitted_generics), @@ -1441,7 +1444,7 @@ def add_function_to_symbol_table(self, func: FuncDef | OverloadedFuncDef) -> Non self.add_symbol(func.name, func, func) def analyze_arg_initializers(self, defn: FuncItem) -> None: - with self.tvar_scope_frame(self.tvar_scope.method_frame()): + with self.tvar_scope_frame(self.tvar_scope.method_frame(defn.fullname)): # Analyze default arguments for arg in defn.arguments: if arg.initializer: @@ -1449,7 +1452,7 @@ def analyze_arg_initializers(self, defn: FuncItem) -> None: def analyze_function_body(self, defn: FuncItem) -> None: is_method = self.is_class_scope() - with self.tvar_scope_frame(self.tvar_scope.method_frame()): + with self.tvar_scope_frame(self.tvar_scope.method_frame(defn.fullname)): # Bind the type variables again to visit the body. if defn.type: a = self.type_analyzer() @@ -3930,7 +3933,9 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: or (self.options.python_version >= (3, 10) or self.is_stub_file) ): # Note: CallExpr is for "void = type(None)" and OpExpr is for "X | Y" union syntax. - s.rvalue.analyzed = TypeAliasExpr(alias_node) + if not isinstance(s.rvalue.analyzed, TypeAliasExpr): + # Any existing node will be updated in-place below. + s.rvalue.analyzed = TypeAliasExpr(alias_node) s.rvalue.analyzed.line = s.line # we use the column from resulting target, to get better location for errors s.rvalue.analyzed.column = res.column diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 753deafe103b..768dd265b338 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -62,6 +62,7 @@ Type, TypeOfAny, TypeType, + TypeVarId, TypeVarLikeType, TypeVarType, UnboundType, @@ -569,27 +570,33 @@ def add_field( add_field(Var("__match_args__", match_args_type), is_initialized_in_class=True) assert info.tuple_type is not None # Set by update_tuple_type() above. - tvd = TypeVarType( + shared_self_type = TypeVarType( name=SELF_TVAR_NAME, - fullname=info.fullname + "." + SELF_TVAR_NAME, + fullname=f"{info.fullname}.{SELF_TVAR_NAME}", + # Namespace is patched per-method below. id=self.api.tvar_scope.new_unique_func_id(), values=[], upper_bound=info.tuple_type, default=AnyType(TypeOfAny.from_omitted_generics), ) - selftype = tvd def add_method( funcname: str, - ret: Type, + ret: Type | None, # None means use (patched) self-type args: list[Argument], is_classmethod: bool = False, is_new: bool = False, ) -> None: + fullname = f"{info.fullname}.{funcname}" + self_type = shared_self_type.copy_modified( + id=TypeVarId(shared_self_type.id.raw_id, namespace=fullname) + ) + if ret is None: + ret = self_type if is_classmethod or is_new: - first = [Argument(Var("_cls"), TypeType.make_normalized(selftype), None, ARG_POS)] + first = [Argument(Var("_cls"), TypeType.make_normalized(self_type), None, ARG_POS)] else: - first = [Argument(Var("_self"), selftype, None, ARG_POS)] + first = [Argument(Var("_self"), self_type, None, ARG_POS)] args = first + args types = [arg.type_annotation for arg in args] @@ -597,12 +604,12 @@ def add_method( arg_kinds = [arg.kind for arg in args] assert None not in types signature = CallableType(cast(List[Type], types), arg_kinds, items, ret, function_type) - signature.variables = [tvd] + signature.variables = [self_type] func = FuncDef(funcname, args, Block([])) func.info = info func.is_class = is_classmethod func.type = set_callable_name(signature, func) - func._fullname = info.fullname + "." + funcname + func._fullname = fullname func.line = line if is_classmethod: v = Var(funcname, func.type) @@ -620,13 +627,13 @@ def add_method( add_method( "_replace", - ret=selftype, + ret=None, args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], ) if self.options.python_version >= (3, 13): add_method( "__replace__", - ret=selftype, + ret=None, args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], ) @@ -635,11 +642,11 @@ def make_init_arg(var: Var) -> Argument: kind = ARG_POS if default is None else ARG_OPT return Argument(var, var.type, default, kind) - add_method("__new__", ret=selftype, args=[make_init_arg(var) for var in vars], is_new=True) + add_method("__new__", ret=None, args=[make_init_arg(var) for var in vars], is_new=True) add_method("_asdict", args=[], ret=ordereddictype) add_method( "_make", - ret=selftype, + ret=None, is_classmethod=True, args=[Argument(Var("iterable", iterable_type), iterable_type, None, ARG_POS)], ) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index b5ec2bb52a0d..01d8e9aafffb 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -314,7 +314,7 @@ def __call__(self, fully_qualified_name: str, args: list[Type] | None = None) -> def paramspec_args( name: str, fullname: str, - id: TypeVarId | int, + id: TypeVarId, *, named_type_func: _NamedTypeCallback, line: int = -1, @@ -337,7 +337,7 @@ def paramspec_args( def paramspec_kwargs( name: str, fullname: str, - id: TypeVarId | int, + id: TypeVarId, *, named_type_func: _NamedTypeCallback, line: int = -1, diff --git a/mypy/subtypes.py b/mypy/subtypes.py index a5523fbe0d45..971caa3991ae 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -944,7 +944,7 @@ def visit_overloaded(self, left: Overloaded) -> bool: # When it is the same overload, then the types are equal. return True - # Ensure each overload in the right side (the supertype) is accounted for. + # Ensure each overload on the right side (the supertype) is accounted for. previous_match_left_index = -1 matched_overloads = set() @@ -1792,7 +1792,9 @@ def are_args_compatible( # If both arguments are required allow_partial_overlap has no effect. allow_partial_overlap = False - def is_different(left_item: object | None, right_item: object | None) -> bool: + def is_different( + left_item: object | None, right_item: object | None, allow_overlap: bool + ) -> bool: """Checks if the left and right items are different. If the right item is unspecified (e.g. if the right callable doesn't care @@ -1802,19 +1804,21 @@ def is_different(left_item: object | None, right_item: object | None) -> bool: if the left callable also doesn't care.""" if right_item is None: return False - if allow_partial_overlap and left_item is None: + if allow_overlap and left_item is None: return False return left_item != right_item # If right has a specific name it wants this argument to be, left must # have the same. - if is_different(left.name, right.name): + if is_different(left.name, right.name, allow_partial_overlap): # But pay attention to whether we're ignoring positional arg names if not ignore_pos_arg_names or right.pos is None: return False - # If right is at a specific position, left must have the same: - if is_different(left.pos, right.pos) and not allow_imprecise_kinds: + # If right is at a specific position, left must have the same. + # TODO: partial overlap logic is flawed for positions. + # We disable it to avoid false positives at a cost of few false negatives. + if is_different(left.pos, right.pos, allow_overlap=False) and not allow_imprecise_kinds: return False # If right's argument is optional, left's must also be diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index b3f84905c47e..0218d33cc124 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -144,7 +144,11 @@ def test_tuple_type_upper(self) -> None: def test_type_variable_binding(self) -> None: assert_equal( - str(TypeVarType("X", "X", 1, [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics))), + str( + TypeVarType( + "X", "X", TypeVarId(1), [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics) + ) + ), "X`1", ) assert_equal( @@ -152,7 +156,7 @@ def test_type_variable_binding(self) -> None: TypeVarType( "X", "X", - 1, + TypeVarId(1), [self.x, self.y], self.fx.o, AnyType(TypeOfAny.from_omitted_generics), @@ -170,14 +174,25 @@ def test_generic_function_type(self) -> None: self.function, name=None, variables=[ - TypeVarType("X", "X", -1, [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics)) + TypeVarType( + "X", + "X", + TypeVarId(-1), + [], + self.fx.o, + AnyType(TypeOfAny.from_omitted_generics), + ) ], ) assert_equal(str(c), "def [X] (X?, Y?) -> Y?") v = [ - TypeVarType("Y", "Y", -1, [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics)), - TypeVarType("X", "X", -2, [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics)), + TypeVarType( + "Y", "Y", TypeVarId(-1), [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics) + ), + TypeVarType( + "X", "X", TypeVarId(-2), [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics) + ), ] c2 = CallableType([], [], [], NoneType(), self.function, name=None, variables=v) assert_equal(str(c2), "def [Y, X] ()") @@ -205,7 +220,9 @@ def test_type_alias_expand_all(self) -> None: def test_recursive_nested_in_non_recursive(self) -> None: A, _ = self.fx.def_alias_1(self.fx.a) - T = TypeVarType("T", "T", -1, [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics)) + T = TypeVarType( + "T", "T", TypeVarId(-1), [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics) + ) NA = self.fx.non_rec_alias(Instance(self.fx.gi, [T]), [T], [A]) assert not NA.is_recursive assert has_recursive_types(NA) @@ -657,7 +674,9 @@ def callable(self, vars: list[str], *a: Type) -> CallableType: n = -1 for v in vars: tv.append( - TypeVarType(v, v, n, [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics)) + TypeVarType( + v, v, TypeVarId(n), [], self.fx.o, AnyType(TypeOfAny.from_omitted_generics) + ) ) n -= 1 return CallableType( diff --git a/mypy/test/typefixture.py b/mypy/test/typefixture.py index b7bde16e6be2..5a813f70117c 100644 --- a/mypy/test/typefixture.py +++ b/mypy/test/typefixture.py @@ -30,6 +30,7 @@ TypeAliasType, TypeOfAny, TypeType, + TypeVarId, TypeVarLikeType, TypeVarTupleType, TypeVarType, @@ -57,7 +58,7 @@ def make_type_var( return TypeVarType( name, name, - id, + TypeVarId(id), values, upper_bound, AnyType(TypeOfAny.from_omitted_generics), @@ -227,7 +228,7 @@ def make_type_var_tuple(name: str, id: int, upper_bound: Type) -> TypeVarTupleTy return TypeVarTupleType( name, name, - id, + TypeVarId(id), upper_bound, self.std_tuple, AnyType(TypeOfAny.from_omitted_generics), @@ -325,7 +326,7 @@ def make_type_info( TypeVarTupleType( n, n, - id, + TypeVarId(id), self.std_tuple.copy_modified(args=[self.o]), self.std_tuple.copy_modified(args=[self.o]), AnyType(TypeOfAny.from_omitted_generics), @@ -340,7 +341,7 @@ def make_type_info( TypeVarType( n, n, - id, + TypeVarId(id), [], self.o, AnyType(TypeOfAny.from_omitted_generics), diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index 4dc663df0399..fe97a8359287 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -85,29 +85,27 @@ def allow_binding(self, fullname: str) -> bool: return False return True - def method_frame(self) -> TypeVarLikeScope: + def method_frame(self, namespace: str) -> TypeVarLikeScope: """A new scope frame for binding a method""" - return TypeVarLikeScope(self, False, None) + return TypeVarLikeScope(self, False, None, namespace=namespace) def class_frame(self, namespace: str) -> TypeVarLikeScope: """A new scope frame for binding a class. Prohibits *this* class's tvars""" return TypeVarLikeScope(self.get_function_scope(), True, self, namespace=namespace) - def new_unique_func_id(self) -> int: + def new_unique_func_id(self) -> TypeVarId: """Used by plugin-like code that needs to make synthetic generic functions.""" self.func_id -= 1 - return self.func_id + return TypeVarId(self.func_id) def bind_new(self, name: str, tvar_expr: TypeVarLikeExpr) -> TypeVarLikeType: if self.is_class_scope: self.class_id += 1 i = self.class_id - namespace = self.namespace else: self.func_id -= 1 i = self.func_id - # TODO: Consider also using namespaces for functions - namespace = "" + namespace = self.namespace tvar_expr.default.accept(TypeVarLikeNamespaceSetter(namespace)) if isinstance(tvar_expr, TypeVarExpr): @@ -124,9 +122,9 @@ def bind_new(self, name: str, tvar_expr: TypeVarLikeExpr) -> TypeVarLikeType: ) elif isinstance(tvar_expr, ParamSpecExpr): tvar_def = ParamSpecType( - name, - tvar_expr.fullname, - i, + name=name, + fullname=tvar_expr.fullname, + id=TypeVarId(i, namespace=namespace), flavor=ParamSpecFlavor.BARE, upper_bound=tvar_expr.upper_bound, default=tvar_expr.default, @@ -135,9 +133,9 @@ def bind_new(self, name: str, tvar_expr: TypeVarLikeExpr) -> TypeVarLikeType: ) elif isinstance(tvar_expr, TypeVarTupleExpr): tvar_def = TypeVarTupleType( - name, - tvar_expr.fullname, - i, + name=name, + fullname=tvar_expr.fullname, + id=TypeVarId(i, namespace=namespace), upper_bound=tvar_expr.upper_bound, tuple_fallback=tvar_expr.tuple_fallback, default=tvar_expr.default, diff --git a/mypy/typeanal.py b/mypy/typeanal.py index ded8b8412a9a..a513b0716a01 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1027,9 +1027,12 @@ def visit_unpack_type(self, t: UnpackType) -> Type: def visit_parameters(self, t: Parameters) -> Type: raise NotImplementedError("ParamSpec literals cannot have unbound TypeVars") - def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: + def visit_callable_type( + self, t: CallableType, nested: bool = True, namespace: str = "" + ) -> Type: # Every Callable can bind its own type variables, if they're not in the outer scope - with self.tvar_scope_frame(): + # TODO: attach namespace for nested free type variables (these appear in return type only). + with self.tvar_scope_frame(namespace=namespace): unpacked_kwargs = t.unpack_kwargs if self.defining_alias: variables = t.variables @@ -1432,7 +1435,7 @@ def analyze_callable_type(self, t: UnboundType) -> Type: ) else: # Callable[P, RET] (where P is ParamSpec) - with self.tvar_scope_frame(): + with self.tvar_scope_frame(namespace=""): # Temporarily bind ParamSpecs to allow code like this: # my_fun: Callable[Q, Foo[Q]] # We usually do this later in visit_callable_type(), but the analysis @@ -1648,9 +1651,9 @@ def note(self, msg: str, ctx: Context, *, code: ErrorCode | None = None) -> None self.note_func(msg, ctx, code=code) @contextmanager - def tvar_scope_frame(self) -> Iterator[None]: + def tvar_scope_frame(self, namespace: str) -> Iterator[None]: old_scope = self.tvar_scope - self.tvar_scope = self.tvar_scope.method_frame() + self.tvar_scope = self.tvar_scope.method_frame(namespace) yield self.tvar_scope = old_scope @@ -1795,7 +1798,7 @@ def anal_var_def(self, var_def: TypeVarLikeType) -> TypeVarLikeType: return TypeVarType( name=var_def.name, fullname=var_def.fullname, - id=var_def.id.raw_id, + id=var_def.id, values=self.anal_array(var_def.values), upper_bound=var_def.upper_bound.accept(self), default=var_def.default.accept(self), diff --git a/mypy/types.py b/mypy/types.py index 2cacc3e44085..cdcb26f435b8 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -510,9 +510,8 @@ class TypeVarId: # Class variable used for allocating fresh ids for metavariables. next_raw_id: ClassVar[int] = 1 - # Fullname of class (or potentially function in the future) which - # declares this type variable (not the fullname of the TypeVar - # definition!), or '' + # Fullname of class or function/method which declares this type + # variable (not the fullname of the TypeVar definition!), or '' namespace: str def __init__(self, raw_id: int, meta_level: int = 0, *, namespace: str = "") -> None: @@ -560,7 +559,7 @@ def __init__( self, name: str, fullname: str, - id: TypeVarId | int, + id: TypeVarId, upper_bound: Type, default: Type, line: int = -1, @@ -569,8 +568,6 @@ def __init__( super().__init__(line, column) self.name = name self.fullname = fullname - if isinstance(id, int): - id = TypeVarId(id) self.id = id self.upper_bound = upper_bound self.default = default @@ -607,7 +604,7 @@ def __init__( self, name: str, fullname: str, - id: TypeVarId | int, + id: TypeVarId, values: list[Type], upper_bound: Type, default: Type, @@ -626,7 +623,7 @@ def copy_modified( values: Bogus[list[Type]] = _dummy, upper_bound: Bogus[Type] = _dummy, default: Bogus[Type] = _dummy, - id: Bogus[TypeVarId | int] = _dummy, + id: Bogus[TypeVarId] = _dummy, line: int = _dummy_int, column: int = _dummy_int, **kwargs: Any, @@ -722,7 +719,7 @@ def __init__( self, name: str, fullname: str, - id: TypeVarId | int, + id: TypeVarId, flavor: int, upper_bound: Type, default: Type, @@ -749,7 +746,7 @@ def with_flavor(self, flavor: int) -> ParamSpecType: def copy_modified( self, *, - id: Bogus[TypeVarId | int] = _dummy, + id: Bogus[TypeVarId] = _dummy, flavor: int = _dummy_int, prefix: Bogus[Parameters] = _dummy, default: Bogus[Type] = _dummy, @@ -794,6 +791,7 @@ def serialize(self) -> JsonDict: "name": self.name, "fullname": self.fullname, "id": self.id.raw_id, + "namespace": self.id.namespace, "flavor": self.flavor, "upper_bound": self.upper_bound.serialize(), "default": self.default.serialize(), @@ -806,7 +804,7 @@ def deserialize(cls, data: JsonDict) -> ParamSpecType: return ParamSpecType( data["name"], data["fullname"], - data["id"], + TypeVarId(data["id"], namespace=data["namespace"]), data["flavor"], deserialize_type(data["upper_bound"]), deserialize_type(data["default"]), @@ -826,7 +824,7 @@ def __init__( self, name: str, fullname: str, - id: TypeVarId | int, + id: TypeVarId, upper_bound: Type, tuple_fallback: Instance, default: Type, @@ -848,6 +846,7 @@ def serialize(self) -> JsonDict: "name": self.name, "fullname": self.fullname, "id": self.id.raw_id, + "namespace": self.id.namespace, "upper_bound": self.upper_bound.serialize(), "tuple_fallback": self.tuple_fallback.serialize(), "default": self.default.serialize(), @@ -860,7 +859,7 @@ def deserialize(cls, data: JsonDict) -> TypeVarTupleType: return TypeVarTupleType( data["name"], data["fullname"], - data["id"], + TypeVarId(data["id"], namespace=data["namespace"]), deserialize_type(data["upper_bound"]), Instance.deserialize(data["tuple_fallback"]), deserialize_type(data["default"]), @@ -881,7 +880,7 @@ def __eq__(self, other: object) -> bool: def copy_modified( self, *, - id: Bogus[TypeVarId | int] = _dummy, + id: Bogus[TypeVarId] = _dummy, upper_bound: Bogus[Type] = _dummy, default: Bogus[Type] = _dummy, min_len: Bogus[int] = _dummy, @@ -3499,6 +3498,11 @@ def visit_instance(self, typ: Instance) -> None: typ.column = self.column super().visit_instance(typ) + def visit_type_alias_type(self, typ: TypeAliasType) -> None: + typ.line = self.line + typ.column = self.column + super().visit_type_alias_type(typ) + class HasTypeVars(BoolTypeQuery): def __init__(self) -> None: diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index 7f00ee5aea00..f210faf71109 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -5,7 +5,7 @@ import math from typing import ( Any, Iterator, TypeVar, Generator, Optional, List, Tuple, Sequence, - Union, Callable, Awaitable, + Union, Callable, Awaitable, Generic ) from typing import Final @@ -86,7 +86,7 @@ def run_generator(gen: Generator[T, V, U], F = TypeVar('F', bound=Callable) -class async_val(Awaitable[V]): +class async_val(Awaitable[V], Generic[T, V]): def __init__(self, val: T) -> None: self.val = val diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index fe01590c6c71..917b74fd2147 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -3240,3 +3240,65 @@ class Base: class Derived(Base): def foo(self): # E: Cannot override final attribute "foo" (previously declared in base class "Base") pass + +[case testTypeVarIdClashPolymorphic] +from typing import Callable, Generic, TypeVar + +A = TypeVar("A") +B = TypeVar("B") + +class Gen(Generic[A]): ... + +def id_(x: A) -> A: ... +def f(x: Gen[A], y: A) -> Gen[Gen[A]]: ... +def g(x: Gen[A], id_: Callable[[B], B], f: Callable[[A, B], Gen[A]]) -> A: ... + +def test(x: Gen[Gen[A]]) -> Gen[A]: + return g(x, id_, f) # Technically OK + +x: Gen[Gen[int]] +reveal_type(g(x, id_, f)) # N: Revealed type is "__main__.Gen[builtins.int]" + +def h(x: A, y: A) -> A: ... +def gn(id_: Callable[[B], B], step: Callable[[A, B], A]) -> A: ... + +def fn(x: A) -> A: + return gn(id_, h) # Technically OK + +[case testTypeVarIdsNested] +from typing import Callable, TypeVar + +A = TypeVar("A") +B = TypeVar("B") + +def f(x: Callable[[A], A]) -> Callable[[B], B]: + def g(x: B) -> B: ... + return g + +reveal_type(f(f)) # N: Revealed type is "def [B] (B`1) -> B`1" +reveal_type(f(f)(f)) # N: Revealed type is "def [A] (x: def (A`-1) -> A`-1) -> def [B] (B`-2) -> B`-2" + +[case testGenericUnionFunctionJoin] +from typing import TypeVar, Union + +T = TypeVar("T") +S = TypeVar("S") + +def f(x: T, y: S) -> Union[T, S]: ... +def g(x: T, y: S) -> Union[T, S]: ... + +x = [f, g] +reveal_type(x) # N: Revealed type is "builtins.list[def [T, S] (x: T`4, y: S`5) -> Union[T`4, S`5]]" +[builtins fixtures/list.pyi] + +[case testTypeVariableClashErrorMessage] +from typing import TypeVar + +T = TypeVar("T") + +class C: # Note: Generic[T] missing + def bad_idea(self, x: T) -> None: + self.x = x + + def nope(self, x: T) -> None: + self.x = x # E: Incompatible types in assignment (expression has type "T@nope", variable has type "T@bad_idea") diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index 38083ad98f21..283500f25a7d 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -183,8 +183,8 @@ p3(1, 3) # E: Too many positional arguments for "foo" \ # E: Argument 2 to "foo" has incompatible type "int"; expected "str" functools.partial(foo, "a") # E: Argument 1 to "foo" has incompatible type "str"; expected "int" -functools.partial(foo, b=1) # E: Argument 1 to "foo" has incompatible type "int"; expected "str" -functools.partial(foo, a=1, b=2, c=3) # E: Argument 2 to "foo" has incompatible type "int"; expected "str" +functools.partial(foo, b=1) # E: Argument "b" to "foo" has incompatible type "int"; expected "str" +functools.partial(foo, a=1, b=2, c=3) # E: Argument "b" to "foo" has incompatible type "int"; expected "str" functools.partial(1) # E: "int" not callable \ # E: Argument 1 to "partial" has incompatible type "int"; expected "Callable[..., Never]" [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index b1d1ff3f46a1..bd327745e2ed 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -1608,17 +1608,17 @@ if int(): if int(): y1 = f3 if int(): - y1 = f4 # E: Incompatible types in assignment (expression has type "Callable[[int], A]", variable has type "Callable[[A], A]") + y1 = f4 # E: Incompatible types in assignment (expression has type "Callable[[int], A@f4]", variable has type "Callable[[A@f1], A@f1]") y2 = f2 if int(): y2 = f2 if int(): - y2 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A], A]", variable has type "Callable[[A], B]") + y2 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A@f1], A@f1]", variable has type "Callable[[A@f2], B]") if int(): - y2 = f3 # E: Incompatible types in assignment (expression has type "Callable[[B], B]", variable has type "Callable[[A], B]") + y2 = f3 # E: Incompatible types in assignment (expression has type "Callable[[B@f3], B@f3]", variable has type "Callable[[A], B@f2]") if int(): - y2 = f4 # E: Incompatible types in assignment (expression has type "Callable[[int], A]", variable has type "Callable[[A], B]") + y2 = f4 # E: Incompatible types in assignment (expression has type "Callable[[int], A@f4]", variable has type "Callable[[A@f2], B]") y3 = f3 if int(): @@ -1634,7 +1634,7 @@ y4 = f4 if int(): y4 = f4 if int(): - y4 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A], A]", variable has type "Callable[[int], A]") + y4 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A@f1], A@f1]", variable has type "Callable[[int], A@f4]") if int(): y4 = f2 if int(): @@ -1655,26 +1655,26 @@ def outer(t: T) -> None: y1 = f1 if int(): y1 = f2 - y1 = f3 # E: Incompatible types in assignment (expression has type "Callable[[T], A]", variable has type "Callable[[A], A]") - y1 = f4 # E: Incompatible types in assignment (expression has type "Callable[[A], T]", variable has type "Callable[[A], A]") + y1 = f3 # E: Incompatible types in assignment (expression has type "Callable[[T], A@f3]", variable has type "Callable[[A@f1], A@f1]") + y1 = f4 # E: Incompatible types in assignment (expression has type "Callable[[A@f4], T]", variable has type "Callable[[A@f1], A@f1]") y1 = f5 # E: Incompatible types in assignment (expression has type "Callable[[T], T]", variable has type "Callable[[A], A]") y2 = f2 if int(): - y2 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A], A]", variable has type "Callable[[A], B]") + y2 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A@f1], A@f1]", variable has type "Callable[[A@f2], B]") y3 = f3 if int(): - y3 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A], A]", variable has type "Callable[[T], A]") + y3 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A@f1], A@f1]", variable has type "Callable[[T], A@f3]") y3 = f2 - y3 = f4 # E: Incompatible types in assignment (expression has type "Callable[[A], T]", variable has type "Callable[[T], A]") + y3 = f4 # E: Incompatible types in assignment (expression has type "Callable[[A@f4], T]", variable has type "Callable[[T], A@f3]") y3 = f5 # E: Incompatible types in assignment (expression has type "Callable[[T], T]", variable has type "Callable[[T], A]") y4 = f4 if int(): - y4 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A], A]", variable has type "Callable[[A], T]") + y4 = f1 # E: Incompatible types in assignment (expression has type "Callable[[A@f1], A@f1]", variable has type "Callable[[A@f4], T]") y4 = f2 - y4 = f3 # E: Incompatible types in assignment (expression has type "Callable[[T], A]", variable has type "Callable[[A], T]") + y4 = f3 # E: Incompatible types in assignment (expression has type "Callable[[T], A@f3]", variable has type "Callable[[A@f4], T]") y4 = f5 # E: Incompatible types in assignment (expression has type "Callable[[T], T]", variable has type "Callable[[A], T]") y5 = f5 @@ -1683,7 +1683,6 @@ def outer(t: T) -> None: y5 = f2 y5 = f3 y5 = f4 -[out] [case testSubtypingWithGenericFunctionUsingTypevarWithValues] from typing import TypeVar, Callable @@ -2928,8 +2927,8 @@ def mix(fs: List[Callable[[S], T]]) -> Callable[[S], List[T]]: def id(__x: U) -> U: ... fs = [id, id, id] -reveal_type(mix(fs)) # N: Revealed type is "def [S] (S`3) -> builtins.list[S`3]" -reveal_type(mix([id, id, id])) # N: Revealed type is "def [S] (S`5) -> builtins.list[S`5]" +reveal_type(mix(fs)) # N: Revealed type is "def [S] (S`7) -> builtins.list[S`7]" +reveal_type(mix([id, id, id])) # N: Revealed type is "def [S] (S`9) -> builtins.list[S`9]" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCurry] diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 08b53ab16972..fcd03f8efe01 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -988,7 +988,7 @@ a = k2 if int(): a = k2 if int(): - a = k1 # E: Incompatible types in assignment (expression has type "Callable[[int, List[T]], List[Union[T, int]]]", variable has type "Callable[[S, List[T]], List[Union[T, int]]]") + a = k1 # E: Incompatible types in assignment (expression has type "Callable[[int, List[T@k1]], List[Union[T@k1, int]]]", variable has type "Callable[[S, List[T@k2]], List[Union[T@k2, int]]]") b = k1 if int(): b = k1 diff --git a/test-data/unit/check-python311.test b/test-data/unit/check-python311.test index 2d1a09ef3336..28951824999f 100644 --- a/test-data/unit/check-python311.test +++ b/test-data/unit/check-python311.test @@ -171,5 +171,5 @@ reveal_type(x3) # N: Revealed type is "def (*Any) -> builtins.int" IntList = List[int] Alias4 = Callable[[*IntList], int] # E: "List[int]" cannot be unpacked (must be tuple or TypeVarTuple) x4: Alias4[int] # E: Bad number of arguments for type alias, expected 0, given 1 -reveal_type(x4) # N: Revealed type is "def (*Unpack[builtins.tuple[Any, ...]]) -> builtins.int" +reveal_type(x4) # N: Revealed type is "def (*Any) -> builtins.int" [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index c13331e0a61b..86bd4422003b 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -1220,3 +1220,14 @@ Ta2 = TypeAliasType("Ta2", None, type_params=(Unpack[Ts],)) # E: Free type vari # N: Don't Unpack type variables in type_params [builtins fixtures/tuple.pyi] + +[case testAliasInstanceNameClash] +from lib import func +class A: ... +func(A()) # E: Argument 1 to "func" has incompatible type "__main__.A"; expected "lib.A" +[file lib.py] +from typing import List, Union + +A = Union[int, List[A]] +def func(x: A) -> int: ... +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index f704e3c5c713..21415abb9c28 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -881,7 +881,8 @@ y: B z: C reveal_type(x) # N: Revealed type is "Any" reveal_type(y) # N: Revealed type is "Any" -reveal_type(z) # N: Revealed type is "Tuple[builtins.int, Unpack[Any]]" +reveal_type(z) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[Any, ...]]]" + [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericVariadicWithBadType] From f989414d9db8c94268e593200ab94b4aec2cc3d3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Jun 2024 10:34:06 +0100 Subject: [PATCH 014/247] [PEP 695] Don't crash when redefining something as a type alias (#17335) Generate an error instead. Work on #15238. --- mypy/semanal.py | 9 +++++++- test-data/unit/check-python312.test | 36 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 8505b3a9ccac..8592a6f05e1f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5323,6 +5323,14 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: all_type_params_names = [p.name for p in s.type_args] try: + existing = self.current_symbol_table().get(s.name.name) + if existing and not ( + isinstance(existing.node, TypeAlias) + or (isinstance(existing.node, PlaceholderNode) and existing.node.line == s.line) + ): + self.already_defined(s.name.name, s, existing, "Name") + return + tag = self.track_incomplete_refs() res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias( s.name.name, @@ -5378,7 +5386,6 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: python_3_12_type_alias=True, ) - existing = self.current_symbol_table().get(s.name.name) if ( existing and isinstance(existing.node, (PlaceholderNode, TypeAlias)) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 2b67f56e679c..52f77243fd0a 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1453,3 +1453,39 @@ class E[T]: reveal_type(E[str]().a) # N: Revealed type is "builtins.list[Any]" [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] + +[case testPEP695RedefineAsTypeAlias1] +# flags: --enable-incomplete-feature=NewGenericSyntax +class C: pass +type C = int # E: Name "C" already defined on line 2 + +A = 0 +type A = str # E: Name "A" already defined on line 5 +reveal_type(A) # N: Revealed type is "builtins.int" + +[case testPEP695RedefineAsTypeAlias2] +# flags: --enable-incomplete-feature=NewGenericSyntax +from m import D +type D = int # E: Name "D" already defined (possibly by an import) +a: D +reveal_type(a) # N: Revealed type is "m.D" +[file m.py] +class D: pass + +[case testPEP695RedefineAsTypeAlias3] +# flags: --enable-incomplete-feature=NewGenericSyntax +D = list["Forward"] +type D = int # E: Name "D" already defined on line 2 +Forward = str +x: D +reveal_type(x) # N: Revealed type is "builtins.list[builtins.str]" + +[case testPEP695MultiDefinitionsForTypeAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax +if int(): + type A[T] = list[T] +else: + type A[T] = str # E: Name "A" already defined on line 3 +x: T # E: Name "T" is not defined +a: A[int] +reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" From 09c48a4d9da2b0d8bf5ce116ca35d51a46c415f2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Jun 2024 10:35:37 +0100 Subject: [PATCH 015/247] [mypyc] Inline tagged integer arithmetic and bitwise operations (#17265) Inline the fast path of various tagged integer operations by using C inline functions. Most of these operations are very quick, so getting rid of the overhead of a C call improves performance significantly. This also enables the C compiler to optimize things more, if there are constant operands, for example. This speeds up an older version of the richards benchmark, which didn't use native integers, by 10% (on CPython 3.12). Even bigger improvements are possible in some microbenchmarks. We didn't do this in the past because of worries about compilation time. However, I couldn't measure an impact to self-compilation speed, and the binary size is only increased by about 0.1%. Work on mypyc/mypyc#757. --- mypyc/lib-rt/CPy.h | 162 +++++++++++++++++++++++++++++++++++++--- mypyc/lib-rt/int_ops.c | 165 +++++++++-------------------------------- 2 files changed, 185 insertions(+), 142 deletions(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 1a03f049ecb0..9e85647226fe 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -129,20 +129,20 @@ Py_ssize_t CPyTagged_AsSsize_t(CPyTagged x); void CPyTagged_IncRef(CPyTagged x); void CPyTagged_DecRef(CPyTagged x); void CPyTagged_XDecRef(CPyTagged x); -CPyTagged CPyTagged_Negate(CPyTagged num); -CPyTagged CPyTagged_Invert(CPyTagged num); -CPyTagged CPyTagged_Add(CPyTagged left, CPyTagged right); -CPyTagged CPyTagged_Subtract(CPyTagged left, CPyTagged right); -CPyTagged CPyTagged_Multiply(CPyTagged left, CPyTagged right); -CPyTagged CPyTagged_FloorDivide(CPyTagged left, CPyTagged right); -CPyTagged CPyTagged_Remainder(CPyTagged left, CPyTagged right); -CPyTagged CPyTagged_And(CPyTagged left, CPyTagged right); -CPyTagged CPyTagged_Or(CPyTagged left, CPyTagged right); -CPyTagged CPyTagged_Xor(CPyTagged left, CPyTagged right); -CPyTagged CPyTagged_Rshift(CPyTagged left, CPyTagged right); -CPyTagged CPyTagged_Lshift(CPyTagged left, CPyTagged right); + bool CPyTagged_IsEq_(CPyTagged left, CPyTagged right); bool CPyTagged_IsLt_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_Negate_(CPyTagged num); +CPyTagged CPyTagged_Invert_(CPyTagged num); +CPyTagged CPyTagged_Add_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_Subtract_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_Multiply_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_FloorDivide_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_Remainder_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_BitwiseLongOp_(CPyTagged a, CPyTagged b, char op); +CPyTagged CPyTagged_Rshift_(CPyTagged left, CPyTagged right); +CPyTagged CPyTagged_Lshift_(CPyTagged left, CPyTagged right); + PyObject *CPyTagged_Str(CPyTagged n); CPyTagged CPyTagged_FromFloat(double f); PyObject *CPyLong_FromStrWithBase(PyObject *o, CPyTagged base); @@ -286,6 +286,144 @@ static inline bool CPyTagged_IsLe(CPyTagged left, CPyTagged right) { } } +static inline CPyTagged CPyTagged_Negate(CPyTagged num) { + if (likely(CPyTagged_CheckShort(num) + && num != (CPyTagged) ((Py_ssize_t)1 << (CPY_INT_BITS - 1)))) { + // The only possibility of an overflow error happening when negating a short is if we + // attempt to negate the most negative number. + return -num; + } + return CPyTagged_Negate_(num); +} + +static inline CPyTagged CPyTagged_Add(CPyTagged left, CPyTagged right) { + // TODO: Use clang/gcc extension __builtin_saddll_overflow instead. + if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { + CPyTagged sum = left + right; + if (likely(!CPyTagged_IsAddOverflow(sum, left, right))) { + return sum; + } + } + return CPyTagged_Add_(left, right); +} + +static inline CPyTagged CPyTagged_Subtract(CPyTagged left, CPyTagged right) { + // TODO: Use clang/gcc extension __builtin_saddll_overflow instead. + if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { + CPyTagged diff = left - right; + if (likely(!CPyTagged_IsSubtractOverflow(diff, left, right))) { + return diff; + } + } + return CPyTagged_Subtract_(left, right); +} + +static inline CPyTagged CPyTagged_Multiply(CPyTagged left, CPyTagged right) { + // TODO: Consider using some clang/gcc extension to check for overflow + if (CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right)) { + if (!CPyTagged_IsMultiplyOverflow(left, right)) { + return left * CPyTagged_ShortAsSsize_t(right); + } + } + return CPyTagged_Multiply_(left, right); +} + +static inline CPyTagged CPyTagged_FloorDivide(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(left) + && CPyTagged_CheckShort(right) + && !CPyTagged_MaybeFloorDivideFault(left, right)) { + Py_ssize_t result = CPyTagged_ShortAsSsize_t(left) / CPyTagged_ShortAsSsize_t(right); + if (((Py_ssize_t)left < 0) != (((Py_ssize_t)right) < 0)) { + if (result * right != left) { + // Round down + result--; + } + } + return result << 1; + } + return CPyTagged_FloorDivide_(left, right); +} + +static inline CPyTagged CPyTagged_Remainder(CPyTagged left, CPyTagged right) { + if (CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right) + && !CPyTagged_MaybeRemainderFault(left, right)) { + Py_ssize_t result = (Py_ssize_t)left % (Py_ssize_t)right; + if (((Py_ssize_t)right < 0) != ((Py_ssize_t)left < 0) && result != 0) { + result += right; + } + return result; + } + return CPyTagged_Remainder_(left, right); +} + +// Bitwise '~' +static inline CPyTagged CPyTagged_Invert(CPyTagged num) { + if (likely(CPyTagged_CheckShort(num) && num != CPY_TAGGED_ABS_MIN)) { + return ~num & ~CPY_INT_TAG; + } + return CPyTagged_Invert_(num); +} + +// Bitwise '&' +static inline CPyTagged CPyTagged_And(CPyTagged left, CPyTagged right) { + if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { + return left & right; + } + return CPyTagged_BitwiseLongOp_(left, right, '&'); +} + +// Bitwise '|' +static inline CPyTagged CPyTagged_Or(CPyTagged left, CPyTagged right) { + if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { + return left | right; + } + return CPyTagged_BitwiseLongOp_(left, right, '|'); +} + +// Bitwise '^' +static inline CPyTagged CPyTagged_Xor(CPyTagged left, CPyTagged right) { + if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { + return left ^ right; + } + return CPyTagged_BitwiseLongOp_(left, right, '^'); +} + +// Bitwise '>>' +static inline CPyTagged CPyTagged_Rshift(CPyTagged left, CPyTagged right) { + if (likely(CPyTagged_CheckShort(left) + && CPyTagged_CheckShort(right) + && (Py_ssize_t)right >= 0)) { + CPyTagged count = CPyTagged_ShortAsSsize_t(right); + if (unlikely(count >= CPY_INT_BITS)) { + if ((Py_ssize_t)left >= 0) { + return 0; + } else { + return CPyTagged_ShortFromInt(-1); + } + } + return ((Py_ssize_t)left >> count) & ~CPY_INT_TAG; + } + return CPyTagged_Rshift_(left, right); +} + +static inline bool IsShortLshiftOverflow(Py_ssize_t short_int, Py_ssize_t shift) { + return ((Py_ssize_t)(short_int << shift) >> shift) != short_int; +} + +// Bitwise '<<' +static inline CPyTagged CPyTagged_Lshift(CPyTagged left, CPyTagged right) { + if (likely(CPyTagged_CheckShort(left) + && CPyTagged_CheckShort(right) + && (Py_ssize_t)right >= 0 + && right < CPY_INT_BITS * 2)) { + CPyTagged shift = CPyTagged_ShortAsSsize_t(right); + if (!IsShortLshiftOverflow(left, shift)) + // Short integers, no overflow + return left << shift; + } + return CPyTagged_Lshift_(left, right); +} + // Float operations diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index b57d88c6ac93..b1b3d6e125f3 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -135,13 +135,8 @@ void CPyTagged_XDecRef(CPyTagged x) { } } -CPyTagged CPyTagged_Negate(CPyTagged num) { - if (CPyTagged_CheckShort(num) - && num != (CPyTagged) ((Py_ssize_t)1 << (CPY_INT_BITS - 1))) { - // The only possibility of an overflow error happening when negating a short is if we - // attempt to negate the most negative number. - return -num; - } +// Tagged int negation slow path, where the result may be a long integer +CPyTagged CPyTagged_Negate_(CPyTagged num) { PyObject *num_obj = CPyTagged_AsObject(num); PyObject *result = PyNumber_Negative(num_obj); if (result == NULL) { @@ -151,14 +146,8 @@ CPyTagged CPyTagged_Negate(CPyTagged num) { return CPyTagged_StealFromObject(result); } -CPyTagged CPyTagged_Add(CPyTagged left, CPyTagged right) { - // TODO: Use clang/gcc extension __builtin_saddll_overflow instead. - if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { - CPyTagged sum = left + right; - if (likely(!CPyTagged_IsAddOverflow(sum, left, right))) { - return sum; - } - } +// Tagged int addition slow path, where the result may be a long integer +CPyTagged CPyTagged_Add_(CPyTagged left, CPyTagged right) { PyObject *left_obj = CPyTagged_AsObject(left); PyObject *right_obj = CPyTagged_AsObject(right); PyObject *result = PyNumber_Add(left_obj, right_obj); @@ -170,14 +159,8 @@ CPyTagged CPyTagged_Add(CPyTagged left, CPyTagged right) { return CPyTagged_StealFromObject(result); } -CPyTagged CPyTagged_Subtract(CPyTagged left, CPyTagged right) { - // TODO: Use clang/gcc extension __builtin_saddll_overflow instead. - if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { - CPyTagged diff = left - right; - if (likely(!CPyTagged_IsSubtractOverflow(diff, left, right))) { - return diff; - } - } +// Tagged int subraction slow path, where the result may be a long integer +CPyTagged CPyTagged_Subtract_(CPyTagged left, CPyTagged right) { PyObject *left_obj = CPyTagged_AsObject(left); PyObject *right_obj = CPyTagged_AsObject(right); PyObject *result = PyNumber_Subtract(left_obj, right_obj); @@ -189,13 +172,8 @@ CPyTagged CPyTagged_Subtract(CPyTagged left, CPyTagged right) { return CPyTagged_StealFromObject(result); } -CPyTagged CPyTagged_Multiply(CPyTagged left, CPyTagged right) { - // TODO: Consider using some clang/gcc extension - if (CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right)) { - if (!CPyTagged_IsMultiplyOverflow(left, right)) { - return left * CPyTagged_ShortAsSsize_t(right); - } - } +// Tagged int multiplication slow path, where the result may be a long integer +CPyTagged CPyTagged_Multiply_(CPyTagged left, CPyTagged right) { PyObject *left_obj = CPyTagged_AsObject(left); PyObject *right_obj = CPyTagged_AsObject(right); PyObject *result = PyNumber_Multiply(left_obj, right_obj); @@ -207,19 +185,8 @@ CPyTagged CPyTagged_Multiply(CPyTagged left, CPyTagged right) { return CPyTagged_StealFromObject(result); } -CPyTagged CPyTagged_FloorDivide(CPyTagged left, CPyTagged right) { - if (CPyTagged_CheckShort(left) - && CPyTagged_CheckShort(right) - && !CPyTagged_MaybeFloorDivideFault(left, right)) { - Py_ssize_t result = CPyTagged_ShortAsSsize_t(left) / CPyTagged_ShortAsSsize_t(right); - if (((Py_ssize_t)left < 0) != (((Py_ssize_t)right) < 0)) { - if (result * right != left) { - // Round down - result--; - } - } - return result << 1; - } +// Tagged int // slow path, where the result may be a long integer (or raise) +CPyTagged CPyTagged_FloorDivide_(CPyTagged left, CPyTagged right) { PyObject *left_obj = CPyTagged_AsObject(left); PyObject *right_obj = CPyTagged_AsObject(right); PyObject *result = PyNumber_FloorDivide(left_obj, right_obj); @@ -233,15 +200,8 @@ CPyTagged CPyTagged_FloorDivide(CPyTagged left, CPyTagged right) { } } -CPyTagged CPyTagged_Remainder(CPyTagged left, CPyTagged right) { - if (CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right) - && !CPyTagged_MaybeRemainderFault(left, right)) { - Py_ssize_t result = (Py_ssize_t)left % (Py_ssize_t)right; - if (((Py_ssize_t)right < 0) != ((Py_ssize_t)left < 0) && result != 0) { - result += right; - } - return result; - } +// Tagged int % slow path, where the result may be a long integer (or raise) +CPyTagged CPyTagged_Remainder_(CPyTagged left, CPyTagged right) { PyObject *left_obj = CPyTagged_AsObject(left); PyObject *right_obj = CPyTagged_AsObject(right); PyObject *result = PyNumber_Remainder(left_obj, right_obj); @@ -368,7 +328,7 @@ static digit *GetIntDigits(CPyTagged n, Py_ssize_t *size, digit *buf) { // Shared implementation of bitwise '&', '|' and '^' (specified by op) for at least // one long operand. This is somewhat optimized for performance. -static CPyTagged BitwiseLongOp(CPyTagged a, CPyTagged b, char op) { +CPyTagged CPyTagged_BitwiseLongOp_(CPyTagged a, CPyTagged b, char op) { // Directly access the digits, as there is no fast C API function for this. digit abuf[3]; digit bbuf[3]; @@ -419,89 +379,34 @@ static CPyTagged BitwiseLongOp(CPyTagged a, CPyTagged b, char op) { return CPyTagged_StealFromObject((PyObject *)r); } -// Bitwise '&' -CPyTagged CPyTagged_And(CPyTagged left, CPyTagged right) { - if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { - return left & right; - } - return BitwiseLongOp(left, right, '&'); -} - -// Bitwise '|' -CPyTagged CPyTagged_Or(CPyTagged left, CPyTagged right) { - if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { - return left | right; - } - return BitwiseLongOp(left, right, '|'); -} - -// Bitwise '^' -CPyTagged CPyTagged_Xor(CPyTagged left, CPyTagged right) { - if (likely(CPyTagged_CheckShort(left) && CPyTagged_CheckShort(right))) { - return left ^ right; - } - return BitwiseLongOp(left, right, '^'); -} - -// Bitwise '~' -CPyTagged CPyTagged_Invert(CPyTagged num) { - if (likely(CPyTagged_CheckShort(num) && num != CPY_TAGGED_ABS_MIN)) { - return ~num & ~CPY_INT_TAG; - } else { - PyObject *obj = CPyTagged_AsObject(num); - PyObject *result = PyNumber_Invert(obj); - if (unlikely(result == NULL)) { - CPyError_OutOfMemory(); - } - Py_DECREF(obj); - return CPyTagged_StealFromObject(result); +// Bitwise '~' slow path +CPyTagged CPyTagged_Invert_(CPyTagged num) { + PyObject *obj = CPyTagged_AsObject(num); + PyObject *result = PyNumber_Invert(obj); + if (unlikely(result == NULL)) { + CPyError_OutOfMemory(); } + Py_DECREF(obj); + return CPyTagged_StealFromObject(result); } -// Bitwise '>>' -CPyTagged CPyTagged_Rshift(CPyTagged left, CPyTagged right) { - if (likely(CPyTagged_CheckShort(left) - && CPyTagged_CheckShort(right) - && (Py_ssize_t)right >= 0)) { - CPyTagged count = CPyTagged_ShortAsSsize_t(right); - if (unlikely(count >= CPY_INT_BITS)) { - if ((Py_ssize_t)left >= 0) { - return 0; - } else { - return CPyTagged_ShortFromInt(-1); - } - } - return ((Py_ssize_t)left >> count) & ~CPY_INT_TAG; - } else { - // Long integer or negative shift -- use generic op - PyObject *lobj = CPyTagged_AsObject(left); - PyObject *robj = CPyTagged_AsObject(right); - PyObject *result = PyNumber_Rshift(lobj, robj); - Py_DECREF(lobj); - Py_DECREF(robj); - if (result == NULL) { - // Propagate error (could be negative shift count) - return CPY_INT_TAG; - } - return CPyTagged_StealFromObject(result); +// Bitwise '>>' slow path +CPyTagged CPyTagged_Rshift_(CPyTagged left, CPyTagged right) { + // Long integer or negative shift -- use generic op + PyObject *lobj = CPyTagged_AsObject(left); + PyObject *robj = CPyTagged_AsObject(right); + PyObject *result = PyNumber_Rshift(lobj, robj); + Py_DECREF(lobj); + Py_DECREF(robj); + if (result == NULL) { + // Propagate error (could be negative shift count) + return CPY_INT_TAG; } + return CPyTagged_StealFromObject(result); } -static inline bool IsShortLshiftOverflow(Py_ssize_t short_int, Py_ssize_t shift) { - return ((Py_ssize_t)(short_int << shift) >> shift) != short_int; -} - -// Bitwise '<<' -CPyTagged CPyTagged_Lshift(CPyTagged left, CPyTagged right) { - if (likely(CPyTagged_CheckShort(left) - && CPyTagged_CheckShort(right) - && (Py_ssize_t)right >= 0 - && right < CPY_INT_BITS * 2)) { - CPyTagged shift = CPyTagged_ShortAsSsize_t(right); - if (!IsShortLshiftOverflow(left, shift)) - // Short integers, no overflow - return left << shift; - } +// Bitwise '<<' slow path +CPyTagged CPyTagged_Lshift_(CPyTagged left, CPyTagged right) { // Long integer or out of range shift -- use generic op PyObject *lobj = CPyTagged_AsObject(left); PyObject *robj = CPyTagged_AsObject(right); From 8dd268ffd84ccf549b3aa9105dd35766a899b2bd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 6 Jun 2024 11:43:11 +0100 Subject: [PATCH 016/247] [PEP 695] Fix handling of undefined name in generic function annotation (#17338) This was generating a false positive. Work on #15238. --- mypy/semanal.py | 2 +- test-data/unit/check-python312.test | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 8592a6f05e1f..98184ab41dd7 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -909,7 +909,7 @@ def analyze_func_def(self, defn: FuncDef) -> None: # Don't store not ready types (including placeholders). if self.found_incomplete_ref(tag) or has_placeholder(result): self.defer(defn) - # TODO: pop type args + self.pop_type_args(defn.type_args) return assert isinstance(result, ProperType) if isinstance(result, CallableType): diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 52f77243fd0a..a1c819667087 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1489,3 +1489,8 @@ else: x: T # E: Name "T" is not defined a: A[int] reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + +[case testPEP695UndefinedNameInAnnotation] +# flags: --enable-incomplete-feature=NewGenericSyntax +def f[T](x: foobar, y: T) -> T: ... # E: Name "foobar" is not defined +reveal_type(f) # N: Revealed type is "def [T] (x: Any, y: T`-1) -> T`-1" From 3518f2499f5677792888bc97484cc53404472fca Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 8 Jun 2024 20:01:27 +0100 Subject: [PATCH 017/247] Move apply_type() to applytype.py (#17346) Moving towards https://github.com/python/mypy/issues/15907 This is a pure refactoring. It was surprisingly easy, this didn't add new import cycles, because there is already (somewhat fundamental) cycle `applytype.py` <-> `subtypes.py`. --- mypy/applytype.py | 135 +++++++++++++++++++++++++++++++++++++++++++++- mypy/checkexpr.py | 128 +------------------------------------------ 2 files changed, 134 insertions(+), 129 deletions(-) diff --git a/mypy/applytype.py b/mypy/applytype.py index eecd555bf90d..4847570b1712 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -1,17 +1,24 @@ from __future__ import annotations -from typing import Callable, Sequence +from typing import Callable, Iterable, Sequence import mypy.subtypes from mypy.erasetype import erase_typevars from mypy.expandtype import expand_type -from mypy.nodes import Context +from mypy.nodes import Context, TypeInfo +from mypy.type_visitor import TypeTranslator +from mypy.typeops import get_all_type_vars from mypy.types import ( AnyType, CallableType, + Instance, + Parameters, + ParamSpecFlavor, ParamSpecType, PartialType, + ProperType, Type, + TypeAliasType, TypeVarId, TypeVarLikeType, TypeVarTupleType, @@ -19,6 +26,7 @@ UninhabitedType, UnpackType, get_proper_type, + remove_dups, ) @@ -170,3 +178,126 @@ def apply_generic_arguments( type_guard=type_guard, type_is=type_is, ) + + +def apply_poly(tp: CallableType, poly_tvars: Sequence[TypeVarLikeType]) -> CallableType | None: + """Make free type variables generic in the type if possible. + + This will translate the type `tp` while trying to create valid bindings for + type variables `poly_tvars` while traversing the type. This follows the same rules + as we do during semantic analysis phase, examples: + * Callable[Callable[[T], T], T] -> def [T] (def (T) -> T) -> T + * Callable[[], Callable[[T], T]] -> def () -> def [T] (T -> T) + * List[T] -> None (not possible) + """ + try: + return tp.copy_modified( + arg_types=[t.accept(PolyTranslator(poly_tvars)) for t in tp.arg_types], + ret_type=tp.ret_type.accept(PolyTranslator(poly_tvars)), + variables=[], + ) + except PolyTranslationError: + return None + + +class PolyTranslationError(Exception): + pass + + +class PolyTranslator(TypeTranslator): + """Make free type variables generic in the type if possible. + + See docstring for apply_poly() for details. + """ + + def __init__( + self, + poly_tvars: Iterable[TypeVarLikeType], + bound_tvars: frozenset[TypeVarLikeType] = frozenset(), + seen_aliases: frozenset[TypeInfo] = frozenset(), + ) -> None: + self.poly_tvars = set(poly_tvars) + # This is a simplified version of TypeVarScope used during semantic analysis. + self.bound_tvars = bound_tvars + self.seen_aliases = seen_aliases + + def collect_vars(self, t: CallableType | Parameters) -> list[TypeVarLikeType]: + found_vars = [] + for arg in t.arg_types: + for tv in get_all_type_vars(arg): + if isinstance(tv, ParamSpecType): + normalized: TypeVarLikeType = tv.copy_modified( + flavor=ParamSpecFlavor.BARE, prefix=Parameters([], [], []) + ) + else: + normalized = tv + if normalized in self.poly_tvars and normalized not in self.bound_tvars: + found_vars.append(normalized) + return remove_dups(found_vars) + + def visit_callable_type(self, t: CallableType) -> Type: + found_vars = self.collect_vars(t) + self.bound_tvars |= set(found_vars) + result = super().visit_callable_type(t) + self.bound_tvars -= set(found_vars) + + assert isinstance(result, ProperType) and isinstance(result, CallableType) + result.variables = list(result.variables) + found_vars + return result + + def visit_type_var(self, t: TypeVarType) -> Type: + if t in self.poly_tvars and t not in self.bound_tvars: + raise PolyTranslationError() + return super().visit_type_var(t) + + def visit_param_spec(self, t: ParamSpecType) -> Type: + if t in self.poly_tvars and t not in self.bound_tvars: + raise PolyTranslationError() + return super().visit_param_spec(t) + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: + if t in self.poly_tvars and t not in self.bound_tvars: + raise PolyTranslationError() + return super().visit_type_var_tuple(t) + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + if not t.args: + return t.copy_modified() + if not t.is_recursive: + return get_proper_type(t).accept(self) + # We can't handle polymorphic application for recursive generic aliases + # without risking an infinite recursion, just give up for now. + raise PolyTranslationError() + + def visit_instance(self, t: Instance) -> Type: + if t.type.has_param_spec_type: + # We need this special-casing to preserve the possibility to store a + # generic function in an instance type. Things like + # forall T . Foo[[x: T], T] + # are not really expressible in current type system, but this looks like + # a useful feature, so let's keep it. + param_spec_index = next( + i for (i, tv) in enumerate(t.type.defn.type_vars) if isinstance(tv, ParamSpecType) + ) + p = get_proper_type(t.args[param_spec_index]) + if isinstance(p, Parameters): + found_vars = self.collect_vars(p) + self.bound_tvars |= set(found_vars) + new_args = [a.accept(self) for a in t.args] + self.bound_tvars -= set(found_vars) + + repl = new_args[param_spec_index] + assert isinstance(repl, ProperType) and isinstance(repl, Parameters) + repl.variables = list(repl.variables) + list(found_vars) + return t.copy_modified(args=new_args) + # There is the same problem with callback protocols as with aliases + # (callback protocols are essentially more flexible aliases to callables). + if t.args and t.type.is_protocol and t.type.protocol_members == ["__call__"]: + if t.type in self.seen_aliases: + raise PolyTranslationError() + call = mypy.subtypes.find_member("__call__", t, t, is_operator=True) + assert call is not None + return call.accept( + PolyTranslator(self.poly_tvars, self.bound_tvars, self.seen_aliases | {t.type}) + ) + return super().visit_instance(t) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 8e6af0218c32..779d63c8d385 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -115,7 +115,6 @@ non_method_protocol_members, ) from mypy.traverser import has_await_expression -from mypy.type_visitor import TypeTranslator from mypy.typeanal import ( check_for_explicit_any, fix_instance, @@ -168,7 +167,6 @@ TypeOfAny, TypeType, TypeVarId, - TypeVarLikeType, TypeVarTupleType, TypeVarType, UnboundType, @@ -182,7 +180,6 @@ get_proper_types, has_recursive_types, is_named_instance, - remove_dups, split_with_prefix_and_suffix, ) from mypy.types_utils import ( @@ -2136,7 +2133,7 @@ def infer_function_type_arguments( ) # Try applying inferred polymorphic type if possible, e.g. Callable[[T], T] can # be interpreted as def [T] (T) -> T, but dict[T, T] cannot be expressed. - applied = apply_poly(poly_callee_type, free_vars) + applied = applytype.apply_poly(poly_callee_type, free_vars) if applied is not None and all( a is not None and not isinstance(get_proper_type(a), UninhabitedType) for a in poly_inferred_args @@ -6220,129 +6217,6 @@ def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> Callabl return c.copy_modified(ret_type=new_ret_type) -def apply_poly(tp: CallableType, poly_tvars: Sequence[TypeVarLikeType]) -> CallableType | None: - """Make free type variables generic in the type if possible. - - This will translate the type `tp` while trying to create valid bindings for - type variables `poly_tvars` while traversing the type. This follows the same rules - as we do during semantic analysis phase, examples: - * Callable[Callable[[T], T], T] -> def [T] (def (T) -> T) -> T - * Callable[[], Callable[[T], T]] -> def () -> def [T] (T -> T) - * List[T] -> None (not possible) - """ - try: - return tp.copy_modified( - arg_types=[t.accept(PolyTranslator(poly_tvars)) for t in tp.arg_types], - ret_type=tp.ret_type.accept(PolyTranslator(poly_tvars)), - variables=[], - ) - except PolyTranslationError: - return None - - -class PolyTranslationError(Exception): - pass - - -class PolyTranslator(TypeTranslator): - """Make free type variables generic in the type if possible. - - See docstring for apply_poly() for details. - """ - - def __init__( - self, - poly_tvars: Iterable[TypeVarLikeType], - bound_tvars: frozenset[TypeVarLikeType] = frozenset(), - seen_aliases: frozenset[TypeInfo] = frozenset(), - ) -> None: - self.poly_tvars = set(poly_tvars) - # This is a simplified version of TypeVarScope used during semantic analysis. - self.bound_tvars = bound_tvars - self.seen_aliases = seen_aliases - - def collect_vars(self, t: CallableType | Parameters) -> list[TypeVarLikeType]: - found_vars = [] - for arg in t.arg_types: - for tv in get_all_type_vars(arg): - if isinstance(tv, ParamSpecType): - normalized: TypeVarLikeType = tv.copy_modified( - flavor=ParamSpecFlavor.BARE, prefix=Parameters([], [], []) - ) - else: - normalized = tv - if normalized in self.poly_tvars and normalized not in self.bound_tvars: - found_vars.append(normalized) - return remove_dups(found_vars) - - def visit_callable_type(self, t: CallableType) -> Type: - found_vars = self.collect_vars(t) - self.bound_tvars |= set(found_vars) - result = super().visit_callable_type(t) - self.bound_tvars -= set(found_vars) - - assert isinstance(result, ProperType) and isinstance(result, CallableType) - result.variables = list(result.variables) + found_vars - return result - - def visit_type_var(self, t: TypeVarType) -> Type: - if t in self.poly_tvars and t not in self.bound_tvars: - raise PolyTranslationError() - return super().visit_type_var(t) - - def visit_param_spec(self, t: ParamSpecType) -> Type: - if t in self.poly_tvars and t not in self.bound_tvars: - raise PolyTranslationError() - return super().visit_param_spec(t) - - def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: - if t in self.poly_tvars and t not in self.bound_tvars: - raise PolyTranslationError() - return super().visit_type_var_tuple(t) - - def visit_type_alias_type(self, t: TypeAliasType) -> Type: - if not t.args: - return t.copy_modified() - if not t.is_recursive: - return get_proper_type(t).accept(self) - # We can't handle polymorphic application for recursive generic aliases - # without risking an infinite recursion, just give up for now. - raise PolyTranslationError() - - def visit_instance(self, t: Instance) -> Type: - if t.type.has_param_spec_type: - # We need this special-casing to preserve the possibility to store a - # generic function in an instance type. Things like - # forall T . Foo[[x: T], T] - # are not really expressible in current type system, but this looks like - # a useful feature, so let's keep it. - param_spec_index = next( - i for (i, tv) in enumerate(t.type.defn.type_vars) if isinstance(tv, ParamSpecType) - ) - p = get_proper_type(t.args[param_spec_index]) - if isinstance(p, Parameters): - found_vars = self.collect_vars(p) - self.bound_tvars |= set(found_vars) - new_args = [a.accept(self) for a in t.args] - self.bound_tvars -= set(found_vars) - - repl = new_args[param_spec_index] - assert isinstance(repl, ProperType) and isinstance(repl, Parameters) - repl.variables = list(repl.variables) + list(found_vars) - return t.copy_modified(args=new_args) - # There is the same problem with callback protocols as with aliases - # (callback protocols are essentially more flexible aliases to callables). - if t.args and t.type.is_protocol and t.type.protocol_members == ["__call__"]: - if t.type in self.seen_aliases: - raise PolyTranslationError() - call = find_member("__call__", t, t, is_operator=True) - assert call is not None - return call.accept( - PolyTranslator(self.poly_tvars, self.bound_tvars, self.seen_aliases | {t.type}) - ) - return super().visit_instance(t) - - class ArgInferSecondPassQuery(types.BoolTypeQuery): """Query whether an argument type should be inferred in the second pass. From 428a0354867911ef9666266bb060ce6d1d203e5a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 9 Jun 2024 20:50:43 +0100 Subject: [PATCH 018/247] Fix crash on recursive alias with an optional type (#17350) Fixes https://github.com/python/mypy/issues/17132 Fix is trivial, we don't need that extra `get_proper_type()`. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/typeanal.py | 12 +++--------- test-data/unit/check-recursive-types.test | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index a513b0716a01..a9b4576c8f42 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -93,7 +93,6 @@ callable_with_ellipsis, find_unpack_in_list, flatten_nested_tuples, - flatten_nested_unions, get_proper_type, has_type_vars, ) @@ -2337,16 +2336,11 @@ def make_optional_type(t: Type) -> Type: is called during semantic analysis and simplification only works during type checking. """ - p_t = get_proper_type(t) - if isinstance(p_t, NoneType): + if isinstance(t, ProperType) and isinstance(t, NoneType): return t - elif isinstance(p_t, UnionType): + elif isinstance(t, ProperType) and isinstance(t, UnionType): # Eagerly expanding aliases is not safe during semantic analysis. - items = [ - item - for item in flatten_nested_unions(p_t.items, handle_type_alias_type=False) - if not isinstance(get_proper_type(item), NoneType) - ] + items = [item for item in t.items if not isinstance(get_proper_type(item), NoneType)] return UnionType(items + [NoneType()], t.line, t.column) else: return UnionType([t, NoneType()], t.line, t.column) diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 84593933a2de..b67818e169b1 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -942,3 +942,19 @@ NotFilter = Tuple[Literal["not"], "NotFilter"] n: NotFilter reveal_type(n[1][1][0]) # N: Revealed type is "Literal['not']" [builtins fixtures/tuple.pyi] + +[case testNoCrashOnRecursiveAliasWithNone] +# flags: --strict-optional +from typing import Union, Generic, TypeVar, Optional + +T = TypeVar("T") +class A(Generic[T]): ... +class B(Generic[T]): ... + +Z = Union[A[Z], B[Optional[Z]]] +X = Union[A[Optional[X]], B[Optional[X]]] + +z: Z +x: X +reveal_type(z) # N: Revealed type is "Union[__main__.A[...], __main__.B[Union[..., None]]]" +reveal_type(x) # N: Revealed type is "Union[__main__.A[Union[..., None]], __main__.B[Union[..., None]]]" From 09e6a2bea4cb4a523791220b37b5c664db895760 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 9 Jun 2024 21:04:42 +0100 Subject: [PATCH 019/247] Fix crash on unpacking self in NamedTuple (#17351) Fixes https://github.com/python/mypy/issues/17010 Fix is trivial: replicate the `TypeVar` handling logic in the caller. --- mypy/checker.py | 2 ++ test-data/unit/check-namedtuple.test | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 38976d4ce15e..42fcc05c5976 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3839,6 +3839,8 @@ def check_multi_assignment_from_tuple( self.expr_checker.accept(rvalue, lvalue_type) ) + if isinstance(reinferred_rvalue_type, TypeVarLikeType): + reinferred_rvalue_type = get_proper_type(reinferred_rvalue_type.upper_bound) if isinstance(reinferred_rvalue_type, UnionType): # If this is an Optional type in non-strict Optional code, unwrap it. relevant_items = reinferred_rvalue_type.relevant_items() diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index a0d984b30279..2007d574f922 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1412,3 +1412,14 @@ A(x=0).__replace__(x="asdf") # E: Argument "x" to "__replace__" of "A" has inco A(x=0).__replace__(y=1) # E: Unexpected keyword argument "y" for "__replace__" of "A" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] + +[case testUnpackSelfNamedTuple] +import typing + +class Foo(typing.NamedTuple): + bar: int + def baz(self: typing.Self) -> None: + x, = self + reveal_type(x) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] From 7c391ddb2f72833822309e5baef1ab533b149e1b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 9 Jun 2024 21:33:13 +0100 Subject: [PATCH 020/247] Fix crash on invalid callable property override (#17352) Fixes https://github.com/python/mypy/issues/16896 Fix is simple, do not assume that an error context given by the caller of the override check for callable type is a method defining such type, because it may be a property. --- mypy/checker.py | 2 +- test-data/unit/check-functions.test | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 42fcc05c5976..38f6f5f44816 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2279,7 +2279,7 @@ def erase_override(t: Type) -> Type: ): arg_type_in_super = original.arg_types[i] - if isinstance(node, FuncDef): + if isinstance(node, FuncDef) and not node.is_property: context: Context = node.arguments[i + len(override.bound_args)] else: context = node diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 917b74fd2147..4b04a3b96ae4 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -3302,3 +3302,25 @@ class C: # Note: Generic[T] missing def nope(self, x: T) -> None: self.x = x # E: Incompatible types in assignment (expression has type "T@nope", variable has type "T@bad_idea") + +[case testNoCrashOnBadCallablePropertyOverride] +from typing import Callable, Union + +class C: ... +class D: ... + +A = Callable[[C], None] +B = Callable[[D], None] + +class Foo: + @property + def method(self) -> Callable[[int, Union[A, B]], None]: + ... + +class Bar(Foo): + @property + def method(self) -> Callable[[int, A], None]: # E: Argument 2 of "method" is incompatible with supertype "Foo"; supertype defines the argument type as "Union[Callable[[C], None], Callable[[D], None]]" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + ... +[builtins fixtures/property.pyi] From 5ae9e69480985d5eba423b718c675ea8714ac66c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 9 Jun 2024 23:32:04 +0100 Subject: [PATCH 021/247] Fix crash involving recursive union of tuples (#17353) Fixes https://github.com/python/mypy/issues/17236 It turns out we were calculating tuple fallbacks where we don't really need to. We can rely on the fact that tuple fallback is trivial for non-trivial partial fallbacks to simplify the logic and avoid the infinite recursion. --- mypy/subtypes.py | 11 ++++--- test-data/unit/check-recursive-types.test | 36 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 971caa3991ae..63f5137ef8ae 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -794,15 +794,18 @@ def visit_tuple_type(self, left: TupleType) -> bool: return False if any(not self._is_subtype(l, r) for l, r in zip(left.items, right.items)): return False - rfallback = mypy.typeops.tuple_fallback(right) - if is_named_instance(rfallback, "builtins.tuple"): + if is_named_instance(right.partial_fallback, "builtins.tuple"): # No need to verify fallback. This is useful since the calculated fallback # may be inconsistent due to how we calculate joins between unions vs. # non-unions. For example, join(int, str) == object, whereas # join(Union[int, C], Union[str, C]) == Union[int, str, C]. return True - lfallback = mypy.typeops.tuple_fallback(left) - return self._is_subtype(lfallback, rfallback) + if is_named_instance(left.partial_fallback, "builtins.tuple"): + # Again, no need to verify. At this point we know the right fallback + # is a subclass of tuple, so if left is plain tuple, it cannot be a subtype. + return False + # At this point we know both fallbacks are non-tuple. + return self._is_subtype(left.partial_fallback, right.partial_fallback) else: return False diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index b67818e169b1..33cb9ccad9af 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -958,3 +958,39 @@ z: Z x: X reveal_type(z) # N: Revealed type is "Union[__main__.A[...], __main__.B[Union[..., None]]]" reveal_type(x) # N: Revealed type is "Union[__main__.A[Union[..., None]], __main__.B[Union[..., None]]]" + +[case testRecursiveTupleFallback1] +from typing import NewType, Tuple, Union + +T1 = NewType("T1", str) +T2 = Tuple[T1, "T4", "T4"] +T3 = Tuple[str, "T4", "T4"] +T4 = Union[T2, T3] +[builtins fixtures/tuple.pyi] + +[case testRecursiveTupleFallback2] +from typing import NewType, Tuple, Union + +T1 = NewType("T1", str) +class T2(Tuple[T1, "T4", "T4"]): ... +T3 = Tuple[str, "T4", "T4"] +T4 = Union[T2, T3] +[builtins fixtures/tuple.pyi] + +[case testRecursiveTupleFallback3] +from typing import NewType, Tuple, Union + +T1 = NewType("T1", str) +T2 = Tuple[T1, "T4", "T4"] +class T3(Tuple[str, "T4", "T4"]): ... +T4 = Union[T2, T3] +[builtins fixtures/tuple.pyi] + +[case testRecursiveTupleFallback4] +from typing import NewType, Tuple, Union + +T1 = NewType("T1", str) +class T2(Tuple[T1, "T4", "T4"]): ... +class T3(Tuple[str, "T4", "T4"]): ... +T4 = Union[T2, T3] +[builtins fixtures/tuple.pyi] From 83d54ffb01af9cba76a36d2ec0938acc7dfa2197 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 10 Jun 2024 20:11:38 +0100 Subject: [PATCH 022/247] Use polymorphic inference in unification (#17348) Moving towards https://github.com/python/mypy/issues/15907 Fixes https://github.com/python/mypy/issues/17206 This PR enables polymorphic inference during unification. This will allow us to handle even more tricky situations involving generic higher-order functions (see a random example I added in tests). Implementation is mostly straightforward, few notes: * This uncovered another issue with unions in solver, unfortunately current constraint inference algorithm can sometimes infer weird constraints like `T <: Union[T, int]`, that later confuse the solver. * This uncovered another possible type variable clash scenario that was not handled properly. In overloaded generic function, each overload should have a different namespace for type variables (currently they all just get function name). I use `module.some_func#0` etc. for overloads namespaces instead. * Another thing with overloads is that the switch caused unsafe overlap check to change: after some back and forth I am keeping it mostly the same to avoid possible regressions (unfortunately this requires some extra refreshing of type variables). * This makes another `ParamSpec` crash to happen more often so I fix it in this same PR. * Finally this uncovered a bug in handling of overloaded `__init__()` that I am fixing here as well. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/checker.py | 16 ++- mypy/constraints.py | 9 +- mypy/message_registry.py | 4 + mypy/semanal.py | 38 ++++-- mypy/semanal_typeargs.py | 22 +++- mypy/solve.py | 9 +- mypy/subtypes.py | 16 ++- mypy/typeanal.py | 17 ++- mypy/typeops.py | 33 +++-- test-data/unit/check-generics.test | 120 +++++++++++++++--- .../unit/check-parameter-specification.test | 28 ++-- test-data/unit/check-selftype.test | 6 +- 12 files changed, 253 insertions(+), 65 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 38f6f5f44816..04e90c3e94cd 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -791,9 +791,21 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: if impl_type is not None: assert defn.impl is not None + # This is what we want from implementation, it should accept all arguments + # of an overload, but the return types should go the opposite way. + if is_callable_compatible( + impl_type, + sig1, + is_compat=is_subtype, + is_proper_subtype=False, + is_compat_return=lambda l, r: is_subtype(r, l), + ): + continue + # If the above check didn't work, we repeat some key steps in + # is_callable_compatible() to give a better error message. + # We perform a unification step that's very similar to what - # 'is_callable_compatible' would have done if we had set - # 'unify_generics' to True -- the only difference is that + # 'is_callable_compatible' does -- the only difference is that # we check and see if the impl_type's return value is a # *supertype* of the overload alternative, not a *subtype*. # diff --git a/mypy/constraints.py b/mypy/constraints.py index cdfa39ac45f3..46221bd82628 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -688,14 +688,19 @@ def visit_unpack_type(self, template: UnpackType) -> list[Constraint]: def visit_parameters(self, template: Parameters) -> list[Constraint]: # Constraining Any against C[P] turns into infer_against_any([P], Any) - # ... which seems like the only case this can happen. Better to fail loudly otherwise. if isinstance(self.actual, AnyType): return self.infer_against_any(template.arg_types, self.actual) if type_state.infer_polymorphic and isinstance(self.actual, Parameters): # For polymorphic inference we need to be able to infer secondary constraints # in situations like [x: T] <: P <: [x: int]. return infer_callable_arguments_constraints(template, self.actual, self.direction) - raise RuntimeError("Parameters cannot be constrained to") + if type_state.infer_polymorphic and isinstance(self.actual, ParamSpecType): + # Similar for [x: T] <: Q <: Concatenate[int, P]. + return infer_callable_arguments_constraints( + template, self.actual.prefix, self.direction + ) + # There also may be unpatched types after a user error, simply ignore them. + return [] # Non-leaf types diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 3852431f2290..52bd9a1ce00c 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -180,6 +180,10 @@ def with_additional_msg(self, info: str) -> ErrorMessage: ) INVALID_UNPACK: Final = "{} cannot be unpacked (must be tuple or TypeVarTuple)" INVALID_UNPACK_POSITION: Final = "Unpack is only valid in a variadic position" +INVALID_PARAM_SPEC_LOCATION: Final = "Invalid location for ParamSpec {}" +INVALID_PARAM_SPEC_LOCATION_NOTE: Final = ( + 'You can use ParamSpec as the first argument to Callable, e.g., "Callable[{}, int]"' +) # TypeVar INCOMPATIBLE_TYPEVAR_VALUE: Final = 'Value of type variable "{}" of {} cannot be {}' diff --git a/mypy/semanal.py b/mypy/semanal.py index 98184ab41dd7..903af80fe404 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -479,6 +479,9 @@ def __init__( # new uses of this, as this may cause leaking `UnboundType`s to type checking. self.allow_unbound_tvars = False + # Used to pass information about current overload index to visit_func_def(). + self.current_overload_item: int | None = None + # mypyc doesn't properly handle implementing an abstractproperty # with a regular attribute so we make them properties @property @@ -869,6 +872,11 @@ def visit_func_def(self, defn: FuncDef) -> None: with self.scope.function_scope(defn): self.analyze_func_def(defn) + def function_fullname(self, fullname: str) -> str: + if self.current_overload_item is None: + return fullname + return f"{fullname}#{self.current_overload_item}" + def analyze_func_def(self, defn: FuncDef) -> None: if self.push_type_args(defn.type_args, defn) is None: self.defer(defn) @@ -895,7 +903,8 @@ def analyze_func_def(self, defn: FuncDef) -> None: self.prepare_method_signature(defn, self.type, has_self_type) # Analyze function signature - with self.tvar_scope_frame(self.tvar_scope.method_frame(defn.fullname)): + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): if defn.type: self.check_classvar_in_signature(defn.type) assert isinstance(defn.type, CallableType) @@ -903,9 +912,7 @@ def analyze_func_def(self, defn: FuncDef) -> None: # class-level imported names and type variables are in scope. analyzer = self.type_analyzer() tag = self.track_incomplete_refs() - result = analyzer.visit_callable_type( - defn.type, nested=False, namespace=defn.fullname - ) + result = analyzer.visit_callable_type(defn.type, nested=False, namespace=fullname) # Don't store not ready types (including placeholders). if self.found_incomplete_ref(tag) or has_placeholder(result): self.defer(defn) @@ -1117,7 +1124,8 @@ def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) if defn is generic. Return True, if the signature contains typing.Self type, or False otherwise. """ - with self.tvar_scope_frame(self.tvar_scope.method_frame(defn.fullname)): + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): a = self.type_analyzer() fun_type.variables, has_self_type = a.bind_function_type_variables(fun_type, defn) if has_self_type and self.type is not None: @@ -1175,6 +1183,14 @@ def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: with self.scope.function_scope(defn): self.analyze_overloaded_func_def(defn) + @contextmanager + def overload_item_set(self, item: int | None) -> Iterator[None]: + self.current_overload_item = item + try: + yield + finally: + self.current_overload_item = None + def analyze_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: # OverloadedFuncDef refers to any legitimate situation where you have # more than one declaration for the same function in a row. This occurs @@ -1187,7 +1203,8 @@ def analyze_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: first_item = defn.items[0] first_item.is_overload = True - first_item.accept(self) + with self.overload_item_set(0): + first_item.accept(self) if isinstance(first_item, Decorator) and first_item.func.is_property: # This is a property. @@ -1272,7 +1289,8 @@ def analyze_overload_sigs_and_impl( if i != 0: # Assume that the first item was already visited item.is_overload = True - item.accept(self) + with self.overload_item_set(i if i < len(defn.items) - 1 else None): + item.accept(self) # TODO: support decorated overloaded functions properly if isinstance(item, Decorator): callable = function_type(item.func, self.named_type("builtins.function")) @@ -1444,7 +1462,8 @@ def add_function_to_symbol_table(self, func: FuncDef | OverloadedFuncDef) -> Non self.add_symbol(func.name, func, func) def analyze_arg_initializers(self, defn: FuncItem) -> None: - with self.tvar_scope_frame(self.tvar_scope.method_frame(defn.fullname)): + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): # Analyze default arguments for arg in defn.arguments: if arg.initializer: @@ -1452,7 +1471,8 @@ def analyze_arg_initializers(self, defn: FuncItem) -> None: def analyze_function_body(self, defn: FuncItem) -> None: is_method = self.is_class_scope() - with self.tvar_scope_frame(self.tvar_scope.method_frame(defn.fullname)): + fullname = self.function_fullname(defn.fullname) + with self.tvar_scope_frame(self.tvar_scope.method_frame(fullname)): # Bind the type variables again to visit the body. if defn.type: a = self.type_analyzer() diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index 15ea15d612c0..02cb1b1f6128 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -12,6 +12,7 @@ from mypy import errorcodes as codes, message_registry from mypy.errorcodes import ErrorCode from mypy.errors import Errors +from mypy.message_registry import INVALID_PARAM_SPEC_LOCATION, INVALID_PARAM_SPEC_LOCATION_NOTE from mypy.messages import format_type from mypy.mixedtraverser import MixedTraverserVisitor from mypy.nodes import ARG_STAR, Block, ClassDef, Context, FakeInfo, FuncItem, MypyFile @@ -146,13 +147,25 @@ def validate_args( for (i, arg), tvar in zip(enumerate(args), type_vars): if isinstance(tvar, TypeVarType): if isinstance(arg, ParamSpecType): - # TODO: Better message is_error = True - self.fail(f'Invalid location for ParamSpec "{arg.name}"', ctx) + self.fail( + INVALID_PARAM_SPEC_LOCATION.format(format_type(arg, self.options)), + ctx, + code=codes.VALID_TYPE, + ) self.note( - "You can use ParamSpec as the first argument to Callable, e.g., " - "'Callable[{}, int]'".format(arg.name), + INVALID_PARAM_SPEC_LOCATION_NOTE.format(arg.name), + ctx, + code=codes.VALID_TYPE, + ) + continue + if isinstance(arg, Parameters): + is_error = True + self.fail( + f"Cannot use {format_type(arg, self.options)} for regular type variable," + " only for ParamSpec", ctx, + code=codes.VALID_TYPE, ) continue if tvar.values: @@ -204,6 +217,7 @@ def validate_args( "Can only replace ParamSpec with a parameter types list or" f" another ParamSpec, got {format_type(arg, self.options)}", ctx, + code=codes.VALID_TYPE, ) return is_error diff --git a/mypy/solve.py b/mypy/solve.py index 9770364bf892..bb87b6576ada 100644 --- a/mypy/solve.py +++ b/mypy/solve.py @@ -514,7 +514,8 @@ def skip_reverse_union_constraints(cs: list[Constraint]) -> list[Constraint]: is a linear constraint. This is however not true in presence of union types, for example T :> Union[S, int] vs S <: T. Trying to solve such constraints would be detected ambiguous as (T, S) form a non-linear SCC. However, simply removing the linear part results in a valid - solution T = Union[S, int], S = . + solution T = Union[S, int], S = . A similar scenario is when we get T <: Union[T, int], + such constraints carry no information, and will equally confuse linearity check. TODO: a cleaner solution may be to avoid inferring such constraints in first place, but this would require passing around a flag through all infer_constraints() calls. @@ -525,7 +526,13 @@ def skip_reverse_union_constraints(cs: list[Constraint]) -> list[Constraint]: if isinstance(p_target, UnionType): for item in p_target.items: if isinstance(item, TypeVarType): + if item == c.origin_type_var and c.op == SUBTYPE_OF: + reverse_union_cs.add(c) + continue + # These two forms are semantically identical, but are different from + # the point of view of Constraint.__eq__(). reverse_union_cs.add(Constraint(item, neg_op(c.op), c.origin_type_var)) + reverse_union_cs.add(Constraint(c.origin_type_var, c.op, item)) return [c for c in cs if c not in reverse_union_cs] diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 63f5137ef8ae..a5d1d5d8194a 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -8,7 +8,12 @@ import mypy.constraints import mypy.typeops from mypy.erasetype import erase_type -from mypy.expandtype import expand_self_type, expand_type, expand_type_by_instance +from mypy.expandtype import ( + expand_self_type, + expand_type, + expand_type_by_instance, + freshen_function_type_vars, +) from mypy.maptype import map_instance_to_supertype # Circular import; done in the function instead. @@ -1860,6 +1865,11 @@ def unify_generic_callable( """ import mypy.solve + if set(type.type_var_ids()) & {v.id for v in mypy.typeops.get_all_type_vars(target)}: + # Overload overlap check does nasty things like unifying in opposite direction. + # This can easily create type variable clashes, so we need to refresh. + type = freshen_function_type_vars(type) + if return_constraint_direction is None: return_constraint_direction = mypy.constraints.SUBTYPE_OF @@ -1882,7 +1892,9 @@ def unify_generic_callable( constraints = [ c for c in constraints if not isinstance(get_proper_type(c.target), NoneType) ] - inferred_vars, _ = mypy.solve.solve_constraints(type.variables, constraints) + inferred_vars, _ = mypy.solve.solve_constraints( + type.variables, constraints, allow_polymorphic=True + ) if None in inferred_vars: return None non_none_inferred_vars = cast(List[Type], inferred_vars) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index a9b4576c8f42..28abd24149e6 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -10,7 +10,14 @@ from mypy import errorcodes as codes, message_registry, nodes from mypy.errorcodes import ErrorCode from mypy.expandtype import expand_type -from mypy.messages import MessageBuilder, format_type_bare, quote_type_string, wrong_type_arg_count +from mypy.message_registry import INVALID_PARAM_SPEC_LOCATION, INVALID_PARAM_SPEC_LOCATION_NOTE +from mypy.messages import ( + MessageBuilder, + format_type, + format_type_bare, + quote_type_string, + wrong_type_arg_count, +) from mypy.nodes import ( ARG_NAMED, ARG_NAMED_OPT, @@ -1782,12 +1789,14 @@ def anal_type( analyzed = AnyType(TypeOfAny.from_error) else: self.fail( - f'Invalid location for ParamSpec "{analyzed.name}"', t, code=codes.VALID_TYPE + INVALID_PARAM_SPEC_LOCATION.format(format_type(analyzed, self.options)), + t, + code=codes.VALID_TYPE, ) self.note( - "You can use ParamSpec as the first argument to Callable, e.g., " - "'Callable[{}, int]'".format(analyzed.name), + INVALID_PARAM_SPEC_LOCATION_NOTE.format(analyzed.name), t, + code=codes.VALID_TYPE, ) analyzed = AnyType(TypeOfAny.from_error) return analyzed diff --git a/mypy/typeops.py b/mypy/typeops.py index a59bd3739562..62c850452516 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -152,7 +152,14 @@ def type_object_type_from_function( # ... # # We need to map B's __init__ to the type (List[T]) -> None. - signature = bind_self(signature, original_type=default_self, is_classmethod=is_new) + signature = bind_self( + signature, + original_type=default_self, + is_classmethod=is_new, + # Explicit instance self annotations have special handling in class_callable(), + # we don't need to bind any type variables in them if they are generic. + ignore_instances=True, + ) signature = cast(FunctionLike, map_type_from_supertype(signature, info, def_info)) special_sig: str | None = None @@ -244,7 +251,9 @@ class C(D[E[T]], Generic[T]): ... return expand_type_by_instance(typ, inst_type) -def supported_self_type(typ: ProperType, allow_callable: bool = True) -> bool: +def supported_self_type( + typ: ProperType, allow_callable: bool = True, allow_instances: bool = True +) -> bool: """Is this a supported kind of explicit self-types? Currently, this means an X or Type[X], where X is an instance or @@ -257,14 +266,19 @@ def supported_self_type(typ: ProperType, allow_callable: bool = True) -> bool: # as well as callable self for callback protocols. return True return isinstance(typ, TypeVarType) or ( - isinstance(typ, Instance) and typ != fill_typevars(typ.type) + allow_instances and isinstance(typ, Instance) and typ != fill_typevars(typ.type) ) F = TypeVar("F", bound=FunctionLike) -def bind_self(method: F, original_type: Type | None = None, is_classmethod: bool = False) -> F: +def bind_self( + method: F, + original_type: Type | None = None, + is_classmethod: bool = False, + ignore_instances: bool = False, +) -> F: """Return a copy of `method`, with the type of its first parameter (usually self or cls) bound to original_type. @@ -288,9 +302,10 @@ class B(A): pass """ if isinstance(method, Overloaded): - return cast( - F, Overloaded([bind_self(c, original_type, is_classmethod) for c in method.items]) - ) + items = [ + bind_self(c, original_type, is_classmethod, ignore_instances) for c in method.items + ] + return cast(F, Overloaded(items)) assert isinstance(method, CallableType) func = method if not func.arg_types: @@ -310,7 +325,9 @@ class B(A): pass # this special-casing looks not very principled, there is nothing meaningful we can infer # from such definition, since it is inherently indefinitely recursive. allow_callable = func.name is None or not func.name.startswith("__call__ of") - if func.variables and supported_self_type(self_param_type, allow_callable=allow_callable): + if func.variables and supported_self_type( + self_param_type, allow_callable=allow_callable, allow_instances=not ignore_instances + ): from mypy.infer import infer_type_arguments if original_type is None: diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index bd327745e2ed..b4b075694bb4 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -2927,8 +2927,8 @@ def mix(fs: List[Callable[[S], T]]) -> Callable[[S], List[T]]: def id(__x: U) -> U: ... fs = [id, id, id] -reveal_type(mix(fs)) # N: Revealed type is "def [S] (S`7) -> builtins.list[S`7]" -reveal_type(mix([id, id, id])) # N: Revealed type is "def [S] (S`9) -> builtins.list[S`9]" +reveal_type(mix(fs)) # N: Revealed type is "def [S] (S`11) -> builtins.list[S`11]" +reveal_type(mix([id, id, id])) # N: Revealed type is "def [S] (S`13) -> builtins.list[S`13]" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericCurry] @@ -3027,7 +3027,7 @@ def dec(f: Callable[[T], S], g: Callable[[T], U]) -> Callable[[T], Tuple[S, U]]: def id(x: V) -> V: ... -reveal_type(dec(id, id)) # N: Revealed type is "def [T] (T`1) -> Tuple[T`1, T`1]" +reveal_type(dec(id, id)) # N: Revealed type is "def [T] (T`7) -> Tuple[T`7, T`7]" [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericEllipsisSelfSpecialCase] @@ -3099,13 +3099,13 @@ def dec4_bound(f: Callable[[I], List[T]]) -> Callable[[I], T]: ... reveal_type(dec1(lambda x: x)) # N: Revealed type is "def [T] (T`3) -> builtins.list[T`3]" -reveal_type(dec2(lambda x: x)) # N: Revealed type is "def [S] (S`4) -> builtins.list[S`4]" -reveal_type(dec3(lambda x: x[0])) # N: Revealed type is "def [S] (S`6) -> S`6" -reveal_type(dec4(lambda x: [x])) # N: Revealed type is "def [S] (S`9) -> S`9" +reveal_type(dec2(lambda x: x)) # N: Revealed type is "def [S] (S`5) -> builtins.list[S`5]" +reveal_type(dec3(lambda x: x[0])) # N: Revealed type is "def [S] (S`8) -> S`8" +reveal_type(dec4(lambda x: [x])) # N: Revealed type is "def [S] (S`12) -> S`12" reveal_type(dec1(lambda x: 1)) # N: Revealed type is "def (builtins.int) -> builtins.list[builtins.int]" reveal_type(dec5(lambda x: x)) # N: Revealed type is "def (builtins.int) -> builtins.list[builtins.int]" -reveal_type(dec3(lambda x: x)) # N: Revealed type is "def [S] (S`16) -> builtins.list[S`16]" -reveal_type(dec4(lambda x: x)) # N: Revealed type is "def [T] (builtins.list[T`19]) -> T`19" +reveal_type(dec3(lambda x: x)) # N: Revealed type is "def [S] (S`20) -> builtins.list[S`20]" +reveal_type(dec4(lambda x: x)) # N: Revealed type is "def [T] (builtins.list[T`24]) -> T`24" dec4_bound(lambda x: x) # E: Value of type variable "I" of "dec4_bound" cannot be "List[T]" [builtins fixtures/list.pyi] @@ -3185,7 +3185,7 @@ reveal_type(dec(id)) # N: Revealed type is "def [T] (T`3) -> T`3" reveal_type(dec(either)) # N: Revealed type is "def [T] (T`6, x: T`6) -> T`6" reveal_type(dec(pair)) # N: Revealed type is "def [T, U] (T`9, x: U`-1) -> Tuple[T`9, U`-1]" # This is counter-intuitive but looks correct, dec matches itself only if P can be empty -reveal_type(dec(dec)) # N: Revealed type is "def [T, S] (T`12, f: def () -> def (T`12) -> S`13) -> S`13" +reveal_type(dec(dec)) # N: Revealed type is "def [T, S] (T`13, f: def () -> def (T`13) -> S`14) -> S`14" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericParamSpecVsParamSpec] @@ -3263,8 +3263,8 @@ def transform( def dec(f: Callable[W, U]) -> Callable[W, U]: ... def dec2(f: Callable[Concatenate[str, W], U]) -> Callable[Concatenate[bytes, W], U]: ... -reveal_type(transform(dec)) # N: Revealed type is "def [P, T] (def (builtins.int, *P.args, **P.kwargs) -> T`3) -> def (builtins.int, *P.args, **P.kwargs) -> T`3" -reveal_type(transform(dec2)) # N: Revealed type is "def [W, T] (def (builtins.int, builtins.str, *W.args, **W.kwargs) -> T`7) -> def (builtins.int, builtins.bytes, *W.args, **W.kwargs) -> T`7" +reveal_type(transform(dec)) # N: Revealed type is "def [P, T] (def (builtins.int, *P.args, **P.kwargs) -> T`9) -> def (builtins.int, *P.args, **P.kwargs) -> T`9" +reveal_type(transform(dec2)) # N: Revealed type is "def [W, T] (def (builtins.int, builtins.str, *W.args, **W.kwargs) -> T`13) -> def (builtins.int, builtins.bytes, *W.args, **W.kwargs) -> T`13" [builtins fixtures/tuple.pyi] [case testNoAccidentalVariableClashInNestedGeneric] @@ -3318,8 +3318,8 @@ def id(x: U) -> U: ... def either(x: U, y: U) -> U: ... def pair(x: U, y: V) -> Tuple[U, V]: ... -reveal_type(dec(id)) # N: Revealed type is "def [T] (T`3) -> builtins.list[T`3]" -reveal_type(dec(either)) # N: Revealed type is "def [T] (T`5, T`5) -> builtins.list[T`5]" +reveal_type(dec(id)) # N: Revealed type is "def [T] (T`9) -> builtins.list[T`9]" +reveal_type(dec(either)) # N: Revealed type is "def [T] (T`11, T`11) -> builtins.list[T`11]" reveal_type(dec(pair)) # N: Revealed type is "def [U, V] (U`-1, V`-2) -> builtins.list[Tuple[U`-1, V`-2]]" [builtins fixtures/tuple.pyi] @@ -3337,8 +3337,8 @@ V = TypeVar("V") def id(x: U) -> U: ... def either(x: U, y: U) -> U: ... -reveal_type(dec(id)) # N: Revealed type is "def [T] (builtins.list[T`3]) -> T`3" -reveal_type(dec(either)) # N: Revealed type is "def [T] (builtins.list[T`5], builtins.list[T`5]) -> T`5" +reveal_type(dec(id)) # N: Revealed type is "def [T] (builtins.list[T`9]) -> T`9" +reveal_type(dec(either)) # N: Revealed type is "def [T] (builtins.list[T`11], builtins.list[T`11]) -> T`11" [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericVariadicPopOff] @@ -3383,7 +3383,7 @@ reveal_type(dec(id)) # N: Revealed type is "def [T] (T`3) -> T`3" reveal_type(dec(either)) # N: Revealed type is "def [T] (T`6, T`6) -> T`6" reveal_type(dec(pair)) # N: Revealed type is "def [T, U] (T`9, U`-1) -> Tuple[T`9, U`-1]" # This is counter-intuitive but looks correct, dec matches itself only if Ts is empty -reveal_type(dec(dec)) # N: Revealed type is "def [T, S] (T`12, def () -> def (T`12) -> S`13) -> S`13" +reveal_type(dec(dec)) # N: Revealed type is "def [T, S] (T`13, def () -> def (T`13) -> S`14) -> S`14" [builtins fixtures/list.pyi] [case testInferenceAgainstGenericVariadicVsVariadic] @@ -3442,3 +3442,91 @@ reveal_type(dec(g)) # N: Revealed type is "def (builtins.int) -> __main__.Foo[b h: Callable[[Unpack[Us]], Foo[int]] reveal_type(dec(h)) # N: Revealed type is "def (builtins.int) -> __main__.Foo[builtins.int]" [builtins fixtures/list.pyi] + +[case testHigherOrderGenericPartial] +from typing import TypeVar, Callable + +T = TypeVar("T") +S = TypeVar("S") +U = TypeVar("U") +def apply(f: Callable[[T], S], x: T) -> S: ... +def id(x: U) -> U: ... + +A1 = TypeVar("A1") +A2 = TypeVar("A2") +R = TypeVar("R") +def fake_partial(fun: Callable[[A1, A2], R], arg: A1) -> Callable[[A2], R]: ... + +f_pid = fake_partial(apply, id) +reveal_type(f_pid) # N: Revealed type is "def [A2] (A2`2) -> A2`2" +reveal_type(f_pid(1)) # N: Revealed type is "builtins.int" + +[case testInvalidTypeVarParametersConcrete] +from typing import Callable, Generic, ParamSpec, Protocol, TypeVar, overload + +P = ParamSpec('P') +P2 = ParamSpec('P2') +R = TypeVar('R') +R2 = TypeVar('R2') + +class C(Generic[P, R, P2, R2]): ... + +class Proto(Protocol[P, R]): + @overload + def __call__(self, f: Callable[P2, R2]) -> C[P2, R2, ..., R]: ... + @overload + def __call__(self, **kwargs) -> C[P, R, ..., [int, str]]: ... # E: Cannot use "[int, str]" for regular type variable, only for ParamSpec +[builtins fixtures/tuple.pyi] + +[case testInvalidTypeVarParametersArbitrary] +from typing import Callable, Generic, ParamSpec, Protocol, TypeVar, overload + +P = ParamSpec('P') +P2 = ParamSpec('P2') +R = TypeVar('R') +R2 = TypeVar('R2') + +class C(Generic[P, R, P2, R2]): ... + +class Proto(Protocol[P, R]): + @overload + def __call__(self, f: Callable[P2, R2]) -> C[P2, R2, ..., R]: ... + @overload + def __call__(self, **kwargs) -> C[P, R, ..., ...]: ... # E: Cannot use "[VarArg(Any), KwArg(Any)]" for regular type variable, only for ParamSpec +[builtins fixtures/tuple.pyi] + +[case testGenericOverloadOverlapUnion] +from typing import TypeVar, overload, Union, Generic + +K = TypeVar("K") +V = TypeVar("V") +T = TypeVar("T") + +class C(Generic[K, V]): + @overload + def pop(self, key: K) -> V: ... + @overload + def pop(self, key: K, default: Union[V, T] = ...) -> Union[V, T]: ... + def pop(self, key: K, default: Union[V, T] = ...) -> Union[V, T]: + ... + +[case testOverloadedGenericInit] +from typing import TypeVar, overload, Union, Generic + +T = TypeVar("T") +S = TypeVar("S") + +class Int(Generic[T]): ... +class Str(Generic[T]): ... + +class C(Generic[T]): + @overload + def __init__(self: C[Int[S]], x: int, y: S) -> None: ... + @overload + def __init__(self: C[Str[S]], x: str, y: S) -> None: ... + def __init__(self, x, y) -> None: ... + +def foo(x: T): + reveal_type(C) # N: Revealed type is "Overload(def [T, S] (x: builtins.int, y: S`-1) -> __main__.C[__main__.Int[S`-1]], def [T, S] (x: builtins.str, y: S`-1) -> __main__.C[__main__.Str[S`-1]])" + reveal_type(C(0, x)) # N: Revealed type is "__main__.C[__main__.Int[T`-1]]" + reveal_type(C("yes", x)) # N: Revealed type is "__main__.C[__main__.Str[T`-1]]" diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index cab7d2bf6819..37916c2155fe 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -23,19 +23,19 @@ x: P # E: ParamSpec "P" is unbound def foo1(x: Callable[P, int]) -> Callable[P, str]: ... def foo2(x: P) -> P: ... # E: Invalid location for ParamSpec "P" \ - # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" def foo3(x: Concatenate[int, P]) -> int: ... # E: Invalid location for Concatenate \ # N: You can use Concatenate as the first argument to Callable def foo4(x: List[P]) -> None: ... # E: Invalid location for ParamSpec "P" \ - # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" def foo5(x: Callable[[int, str], P]) -> None: ... # E: Invalid location for ParamSpec "P" \ - # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" def foo6(x: Callable[[P], int]) -> None: ... # E: Invalid location for ParamSpec "P" \ - # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" [builtins fixtures/paramspec.pyi] [case testParamSpecImports] @@ -901,8 +901,8 @@ class A: def func(self, action: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs) -> _R: ... -reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, action: def (*_P.args, **_P.kwargs) -> _R`4, *_P.args, **_P.kwargs) -> _R`4" -reveal_type(A().func) # N: Revealed type is "def [_P, _R] (action: def (*_P.args, **_P.kwargs) -> _R`8, *_P.args, **_P.kwargs) -> _R`8" +reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, action: def (*_P.args, **_P.kwargs) -> _R`15, *_P.args, **_P.kwargs) -> _R`15" +reveal_type(A().func) # N: Revealed type is "def [_P, _R] (action: def (*_P.args, **_P.kwargs) -> _R`19, *_P.args, **_P.kwargs) -> _R`19" def f(x: int) -> int: ... @@ -933,8 +933,8 @@ class A: def func(self, action: Job[_P, None]) -> Job[_P, None]: ... -reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`3, None]) -> __main__.Job[_P`3, None]" -reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`5, None]) -> __main__.Job[_P`5, None]" +reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`13, None]) -> __main__.Job[_P`13, None]" +reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`15, None]) -> __main__.Job[_P`15, None]" reveal_type(A().func(Job(lambda x: x))) # N: Revealed type is "__main__.Job[[x: Any], None]" def f(x: int, y: int) -> None: ... @@ -1096,7 +1096,7 @@ j = Job(generic_f) reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`-1]]" jf = j.into_callable() -reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`3)" +reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`13)" reveal_type(jf(1)) # N: Revealed type is "None" [builtins fixtures/paramspec.pyi] @@ -1115,10 +1115,10 @@ class Job(Generic[_P, _T]): def generic_f(x: _T) -> _T: ... j = Job(generic_f) -reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`3], _T`3]" +reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`12], _T`12]" jf = j.into_callable() -reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`4) -> _T`4" +reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`13) -> _T`13" reveal_type(jf(1)) # N: Revealed type is "builtins.int" [builtins fixtures/paramspec.pyi] @@ -1520,7 +1520,7 @@ T = TypeVar("T") A = List[T] def f(x: A[[int, str]]) -> None: ... # E: Bracketed expression "[...]" is not valid as a type def g(x: A[P]) -> None: ... # E: Invalid location for ParamSpec "P" \ - # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' + # N: You can use ParamSpec as the first argument to Callable, e.g., "Callable[P, int]" C = Callable[P, T] x: C[int] # E: Bad number of arguments for type alias, expected 2, given 1 @@ -1640,13 +1640,13 @@ U = TypeVar("U") def dec(f: Callable[P, T]) -> Callable[P, List[T]]: ... def test(x: U) -> U: ... reveal_type(dec) # N: Revealed type is "def [P, T] (f: def (*P.args, **P.kwargs) -> T`-2) -> def (*P.args, **P.kwargs) -> builtins.list[T`-2]" -reveal_type(dec(test)) # N: Revealed type is "def [T] (x: T`3) -> builtins.list[T`3]" +reveal_type(dec(test)) # N: Revealed type is "def [T] (x: T`12) -> builtins.list[T`12]" class A: ... TA = TypeVar("TA", bound=A) def test_with_bound(x: TA) -> TA: ... -reveal_type(dec(test_with_bound)) # N: Revealed type is "def [T <: __main__.A] (x: T`5) -> builtins.list[T`5]" +reveal_type(dec(test_with_bound)) # N: Revealed type is "def [T <: __main__.A] (x: T`14) -> builtins.list[T`14]" dec(test_with_bound)(0) # E: Value of type variable "T" of function cannot be "int" dec(test_with_bound)(A()) # OK [builtins fixtures/paramspec.pyi] diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index e49a7a0e2e2f..e99b859bbcd0 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1793,7 +1793,7 @@ class C: def bar(self) -> Self: ... def foo(self, x: S) -> Tuple[Self, S]: ... -reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`1, x: S`2) -> Tuple[Self`1, S`2]" +reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`7, x: S`8) -> Tuple[Self`7, S`8]" reveal_type(C().foo(42)) # N: Revealed type is "Tuple[__main__.C, builtins.int]" [builtins fixtures/tuple.pyi] @@ -1807,7 +1807,7 @@ class C: def bar(self) -> Self: ... foo: Callable[[S, Self], Tuple[Self, S]] -reveal_type(C().foo) # N: Revealed type is "def [S] (S`1, __main__.C) -> Tuple[__main__.C, S`1]" +reveal_type(C().foo) # N: Revealed type is "def [S] (S`7, __main__.C) -> Tuple[__main__.C, S`7]" reveal_type(C().foo(42, C())) # N: Revealed type is "Tuple[__main__.C, builtins.int]" class This: ... [builtins fixtures/tuple.pyi] @@ -2032,7 +2032,7 @@ class Ben(Object): } @classmethod def doit(cls) -> Foo: - reveal_type(cls.MY_MAP) # N: Revealed type is "builtins.dict[builtins.str, def [Self <: __main__.Foo] (self: Self`4) -> Self`4]" + reveal_type(cls.MY_MAP) # N: Revealed type is "builtins.dict[builtins.str, def [Self <: __main__.Foo] (self: Self`10) -> Self`10]" foo_method = cls.MY_MAP["foo"] return foo_method(Foo()) [builtins fixtures/isinstancelist.pyi] From 6427da62add428d452f690aabbb6c272ff713710 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 10 Jun 2024 20:12:15 +0100 Subject: [PATCH 023/247] Properly handle unpacks in overlap checks (#17356) Fixes https://github.com/python/mypy/issues/17319 This is still not 100% robust, but at least it should not crash, and should cover correctly vast majority of cases. --- mypy/meet.py | 34 +++++++++++++++ test-data/unit/check-typevar-tuple.test | 57 +++++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/mypy/meet.py b/mypy/meet.py index 2d44cafb23b3..48e5dfaa18ee 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -611,6 +611,19 @@ def are_tuples_overlapping( right = adjust_tuple(right, left) or right assert isinstance(left, TupleType), f"Type {left} is not a tuple" assert isinstance(right, TupleType), f"Type {right} is not a tuple" + + # This algorithm works well if only one tuple is variadic, if both are + # variadic we may get rare false negatives for overlapping prefix/suffix. + # Also, this ignores empty unpack case, but it is probably consistent with + # how we handle e.g. empty lists in overload overlaps. + # TODO: write a more robust algorithm for cases where both types are variadic. + left_unpack = find_unpack_in_list(left.items) + right_unpack = find_unpack_in_list(right.items) + if left_unpack is not None: + left = expand_tuple_if_possible(left, len(right.items)) + if right_unpack is not None: + right = expand_tuple_if_possible(right, len(left.items)) + if len(left.items) != len(right.items): return False return all( @@ -624,6 +637,27 @@ def are_tuples_overlapping( ) +def expand_tuple_if_possible(tup: TupleType, target: int) -> TupleType: + if len(tup.items) > target + 1: + return tup + extra = target + 1 - len(tup.items) + new_items = [] + for it in tup.items: + if not isinstance(it, UnpackType): + new_items.append(it) + continue + unpacked = get_proper_type(it.type) + if isinstance(unpacked, TypeVarTupleType): + instance = unpacked.tuple_fallback + else: + # Nested non-variadic tuples should be normalized at this point. + assert isinstance(unpacked, Instance) + instance = unpacked + assert instance.type.fullname == "builtins.tuple" + new_items.extend([instance.args[0]] * extra) + return tup.copy_modified(items=new_items) + + def adjust_tuple(left: ProperType, r: ProperType) -> TupleType | None: """Find out if `left` is a Tuple[A, ...], and adjust its length to `right`""" if isinstance(left, Instance) and left.type.fullname == "builtins.tuple": diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 21415abb9c28..2751e01aa21a 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -1808,6 +1808,63 @@ def test(a: Tuple[int, str], b: Tuple[bool], c: Tuple[bool, ...]): reveal_type(add(b, c)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" [builtins fixtures/tuple.pyi] +[case testTypeVarTupleOverloadOverlap] +from typing import Union, overload, Tuple +from typing_extensions import Unpack + +class Int(int): ... + +A = Tuple[int, Unpack[Tuple[int, ...]]] +B = Tuple[int, Unpack[Tuple[str, ...]]] + +@overload +def f(arg: A) -> int: ... +@overload +def f(arg: B) -> str: ... +def f(arg: Union[A, B]) -> Union[int, str]: + ... + +A1 = Tuple[int, Unpack[Tuple[Int, ...]]] +B1 = Tuple[Unpack[Tuple[Int, ...]], int] + +@overload +def f1(arg: A1) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f1(arg: B1) -> str: ... +def f1(arg: Union[A1, B1]) -> Union[int, str]: + ... + +A2 = Tuple[int, int, int] +B2 = Tuple[int, Unpack[Tuple[int, ...]]] + +@overload +def f2(arg: A2) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def f2(arg: B2) -> str: ... +def f2(arg: Union[A2, B2]) -> Union[int, str]: + ... + +A3 = Tuple[int, int, int] +B3 = Tuple[int, Unpack[Tuple[str, ...]]] + +@overload +def f3(arg: A3) -> int: ... +@overload +def f3(arg: B3) -> str: ... +def f3(arg: Union[A3, B3]) -> Union[int, str]: + ... + +A4 = Tuple[int, int, Unpack[Tuple[int, ...]]] +B4 = Tuple[int] + +@overload +def f4(arg: A4) -> int: ... +@overload +def f4(arg: B4) -> str: ... +def f4(arg: Union[A4, B4]) -> Union[int, str]: + ... +[builtins fixtures/tuple.pyi] + [case testTypeVarTupleIndexOldStyleNonNormalizedAndNonLiteral] from typing import Any, Tuple from typing_extensions import Unpack From 4fa4657d7d9e41b5b3b7ae093ccdc360ce6e1b95 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 10 Jun 2024 20:40:06 +0100 Subject: [PATCH 024/247] Fix type application for classes with generic constructors (#17354) Fixes https://github.com/python/mypy/issues/17212 Note I removed the problematic asset after all. It is hard to maintain it, since this function may be called from both explicit application, and from type inference code paths. And these two cases may have different min/max type argument count (see tests and comments for examples). --- mypy/applytype.py | 5 ++- mypy/checkexpr.py | 41 +++++++++++++++++++------ mypy/typeanal.py | 6 ++++ test-data/unit/check-generics.test | 13 ++++++++ test-data/unit/check-typevar-tuple.test | 27 ++++++++++++++++ 5 files changed, 80 insertions(+), 12 deletions(-) diff --git a/mypy/applytype.py b/mypy/applytype.py index 4847570b1712..783748cd8a5e 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -101,8 +101,7 @@ def apply_generic_arguments( bound or constraints, instead of giving an error. """ tvars = callable.variables - min_arg_count = sum(not tv.has_default() for tv in tvars) - assert min_arg_count <= len(orig_types) <= len(tvars) + assert len(orig_types) <= len(tvars) # Check that inferred type variable values are compatible with allowed # values and bounds. Also, promote subtype values to allowed values. # Create a map from type variable id to target type. @@ -156,7 +155,7 @@ def apply_generic_arguments( type_is = None # The callable may retain some type vars if only some were applied. - # TODO: move apply_poly() logic from checkexpr.py here when new inference + # TODO: move apply_poly() logic here when new inference # becomes universally used (i.e. in all passes + in unification). # With this new logic we can actually *add* some new free variables. remaining_tvars: list[TypeVarLikeType] = [] diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 779d63c8d385..c34952b084f9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4781,7 +4781,11 @@ class C(Generic[T, Unpack[Ts]]): ... We simply group the arguments that need to go into Ts variable into a TupleType, similar to how it is done in other places using split_with_prefix_and_suffix(). """ - vars = t.variables + if t.is_type_obj(): + # Type arguments must map to class type variables, ignoring constructor vars. + vars = t.type_object().defn.type_vars + else: + vars = list(t.variables) args = flatten_nested_tuples(args) # TODO: this logic is duplicated with semanal_typeargs. @@ -4799,6 +4803,7 @@ class C(Generic[T, Unpack[Ts]]): ... if not vars or not any(isinstance(v, TypeVarTupleType) for v in vars): return list(args) + # TODO: in future we may want to support type application to variadic functions. assert t.is_type_obj() info = t.type_object() # We reuse the logic from semanal phase to reduce code duplication. @@ -4832,10 +4837,23 @@ def apply_type_arguments_to_callable( tp = get_proper_type(tp) if isinstance(tp, CallableType): - min_arg_count = sum(not v.has_default() for v in tp.variables) - has_type_var_tuple = any(isinstance(v, TypeVarTupleType) for v in tp.variables) + if tp.is_type_obj(): + # If we have a class object in runtime context, then the available type + # variables are those of the class, we don't include additional variables + # of the constructor. So that with + # class C(Generic[T]): + # def __init__(self, f: Callable[[S], T], x: S) -> None + # C[int] is valid + # C[int, str] is invalid (although C as a callable has 2 type variables) + # Note: various logic below and in applytype.py relies on the fact that + # class type variables appear *before* constructor variables. + type_vars = tp.type_object().defn.type_vars + else: + type_vars = list(tp.variables) + min_arg_count = sum(not v.has_default() for v in type_vars) + has_type_var_tuple = any(isinstance(v, TypeVarTupleType) for v in type_vars) if ( - len(args) < min_arg_count or len(args) > len(tp.variables) + len(args) < min_arg_count or len(args) > len(type_vars) ) and not has_type_var_tuple: if tp.is_type_obj() and tp.type_object().fullname == "builtins.tuple": # e.g. expression tuple[X, Y] @@ -4854,19 +4872,24 @@ def apply_type_arguments_to_callable( bound_args=tp.bound_args, ) self.msg.incompatible_type_application( - min_arg_count, len(tp.variables), len(args), ctx + min_arg_count, len(type_vars), len(args), ctx ) return AnyType(TypeOfAny.from_error) return self.apply_generic_arguments(tp, self.split_for_callable(tp, args, ctx), ctx) if isinstance(tp, Overloaded): for it in tp.items: - min_arg_count = sum(not v.has_default() for v in it.variables) - has_type_var_tuple = any(isinstance(v, TypeVarTupleType) for v in it.variables) + if tp.is_type_obj(): + # Same as above. + type_vars = tp.type_object().defn.type_vars + else: + type_vars = list(it.variables) + min_arg_count = sum(not v.has_default() for v in type_vars) + has_type_var_tuple = any(isinstance(v, TypeVarTupleType) for v in type_vars) if ( - len(args) < min_arg_count or len(args) > len(it.variables) + len(args) < min_arg_count or len(args) > len(type_vars) ) and not has_type_var_tuple: self.msg.incompatible_type_application( - min_arg_count, len(it.variables), len(args), ctx + min_arg_count, len(type_vars), len(args), ctx ) return AnyType(TypeOfAny.from_error) return Overloaded( diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 28abd24149e6..82c90272d6c2 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -2376,6 +2376,12 @@ def validate_instance(t: Instance, fail: MsgCallback, empty_tuple_index: bool) - if not t.args: if not (empty_tuple_index and len(t.type.type_vars) == 1): # The Any arguments should be set by the caller. + if empty_tuple_index and min_tv_count: + fail( + f"At least {min_tv_count} type argument(s) expected, none given", + t, + code=codes.TYPE_ARG, + ) return False elif not correct: fail( diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index b4b075694bb4..ea3f501fd949 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -3443,6 +3443,19 @@ h: Callable[[Unpack[Us]], Foo[int]] reveal_type(dec(h)) # N: Revealed type is "def (builtins.int) -> __main__.Foo[builtins.int]" [builtins fixtures/list.pyi] +[case testTypeApplicationGenericConstructor] +from typing import Generic, TypeVar, Callable + +T = TypeVar("T") +S = TypeVar("S") +class C(Generic[T]): + def __init__(self, f: Callable[[S], T], x: S) -> None: + self.x = f(x) + +reveal_type(C[int]) # N: Revealed type is "def [S] (f: def (S`-1) -> builtins.int, x: S`-1) -> __main__.C[builtins.int]" +Alias = C[int] +C[int, str] # E: Type application has too many types (1 expected) + [case testHigherOrderGenericPartial] from typing import TypeVar, Callable diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 2751e01aa21a..0aff702e1b22 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2378,3 +2378,30 @@ def a2(x: Array[int, str]) -> None: reveal_type(func(x, 2, "Hello", True)) # E: Cannot infer type argument 1 of "func" \ # N: Revealed type is "builtins.tuple[Any, ...]" [builtins fixtures/tuple.pyi] + +[case testTypeVarTupleTypeApplicationOverload] +from typing import Generic, TypeVar, TypeVarTuple, Unpack, overload, Callable + +T = TypeVar("T") +T1 = TypeVar("T1") +T2 = TypeVar("T2") +T3 = TypeVar("T3") +Ts = TypeVarTuple("Ts") + +class C(Generic[T, Unpack[Ts]]): + @overload + def __init__(self, f: Callable[[Unpack[Ts]], T]) -> None: ... + @overload + def __init__(self, f: Callable[[T1, T2, T3, Unpack[Ts]], T], a: T1, b: T2, c: T3) -> None: ... + def __init__(self, f, *args, **kwargs) -> None: + ... + +reveal_type(C[int, str]) # N: Revealed type is "Overload(def (f: def (builtins.str) -> builtins.int) -> __main__.C[builtins.int, builtins.str], def [T1, T2, T3] (f: def (T1`-1, T2`-2, T3`-3, builtins.str) -> builtins.int, a: T1`-1, b: T2`-2, c: T3`-3) -> __main__.C[builtins.int, builtins.str])" +Alias = C[int, str] + +def f(x: int, y: int, z: int, t: int) -> str: ... +x = C(f, 0, 0, "hm") # E: Argument 1 to "C" has incompatible type "Callable[[int, int, int, int], str]"; expected "Callable[[int, int, str, int], str]" +reveal_type(x) # N: Revealed type is "__main__.C[builtins.str, builtins.int]" +reveal_type(C(f)) # N: Revealed type is "__main__.C[builtins.str, builtins.int, builtins.int, builtins.int, builtins.int]" +C[()] # E: At least 1 type argument(s) expected, none given +[builtins fixtures/tuple.pyi] From 317533c5589c5778bad4dbf3b0205974491debac Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 10 Jun 2024 23:09:51 +0100 Subject: [PATCH 025/247] Fix crash on TypedDict unpacking for ParamSpec (#17358) Fixes https://github.com/python/mypy/issues/17345 Fixes https://github.com/python/mypy/issues/17112 Fixes https://github.com/python/mypy/issues/16616 Oh well, I clearly remember I have put those lines before `if` only because otherwise the line would be 101 chars long, and I didn't want to wrap arguments. Now I see it was a bad idea, LOL. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/constraints.py | 9 ++++++--- test-data/unit/check-typeddict.test | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 46221bd82628..56ca51d19486 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -223,9 +223,6 @@ def infer_constraints_for_callable( if actual_arg_type is None: continue - actual_type = mapper.expand_actual_type( - actual_arg_type, arg_kinds[actual], callee.arg_names[i], callee.arg_kinds[i] - ) if param_spec and callee.arg_kinds[i] in (ARG_STAR, ARG_STAR2): # If actual arguments are mapped to ParamSpec type, we can't infer individual # constraints, instead store them and infer single constraint at the end. @@ -243,6 +240,12 @@ def infer_constraints_for_callable( ) param_spec_arg_names.append(arg_names[actual] if arg_names else None) else: + actual_type = mapper.expand_actual_type( + actual_arg_type, + arg_kinds[actual], + callee.arg_names[i], + callee.arg_kinds[i], + ) c = infer_constraints(callee.arg_types[i], actual_type, SUPERTYPE_OF) constraints.extend(c) if ( diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 5fb74f66dd89..fa77d98e4a34 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -3525,3 +3525,28 @@ class B(A): reveal_type(B.f) # N: Revealed type is "def (self: __main__.B, **kwargs: Unpack[TypedDict('__main__.TD', {'x'?: builtins.int, 'y'?: builtins.str})])" B().f(x=1.0) # E: Argument "x" to "f" of "B" has incompatible type "float"; expected "int" [builtins fixtures/primitives.pyi] + +[case testTypedDictUnpackWithParamSpecInference] +from typing import TypeVar, ParamSpec, Callable +from typing_extensions import TypedDict, Unpack + +P = ParamSpec("P") +R = TypeVar("R") + +def run(func: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R: ... + +class Params(TypedDict): + temperature: float + +def test(temperature: int) -> None: ... +def test2(temperature: float, other: str) -> None: ... + +class Test: + def f(self, c: Callable[..., None], **params: Unpack[Params]) -> None: + run(c, **params) + def g(self, **params: Unpack[Params]) -> None: + run(test, **params) # E: Argument "temperature" to "run" has incompatible type "float"; expected "int" + def h(self, **params: Unpack[Params]) -> None: + run(test2, other="yes", **params) + run(test2, other=0, **params) # E: Argument "other" to "run" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] From b8a026017de10969d35de9d1ea7951428b95dfbc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 11 Jun 2024 09:35:08 +0100 Subject: [PATCH 026/247] Fix crash when overriding with unpacked TypedDict (#17359) Fixes https://github.com/python/mypy/issues/17208 While writing the fix (that is trivial), I could not notice that the relevant code simply assumes functions can have nothing but positional parameters. This could lead really misleading error messages, so I decided to fix this as well. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/checker.py | 26 ++++++++++---- test-data/unit/check-functions.test | 53 ++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 04e90c3e94cd..70db31c9a94f 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2255,6 +2255,7 @@ def check_override( if fail: emitted_msg = False + offset_arguments = isinstance(override, CallableType) and override.unpack_kwargs # Normalize signatures, so we get better diagnostics. if isinstance(override, (CallableType, Overloaded)): override = override.with_unpacked_kwargs() @@ -2285,12 +2286,23 @@ def check_override( def erase_override(t: Type) -> Type: return erase_typevars(t, ids_to_erase=override_ids) - for i in range(len(override.arg_types)): - if not is_subtype( - original.arg_types[i], erase_override(override.arg_types[i]) - ): - arg_type_in_super = original.arg_types[i] - + for i, (sub_kind, super_kind) in enumerate( + zip(override.arg_kinds, original.arg_kinds) + ): + if sub_kind.is_positional() and super_kind.is_positional(): + override_arg_type = override.arg_types[i] + original_arg_type = original.arg_types[i] + elif sub_kind.is_named() and super_kind.is_named() and not offset_arguments: + arg_name = override.arg_names[i] + if arg_name in original.arg_names: + override_arg_type = override.arg_types[i] + original_i = original.arg_names.index(arg_name) + original_arg_type = original.arg_types[original_i] + else: + continue + else: + continue + if not is_subtype(original_arg_type, erase_override(override_arg_type)): if isinstance(node, FuncDef) and not node.is_property: context: Context = node.arguments[i + len(override.bound_args)] else: @@ -2300,7 +2312,7 @@ def erase_override(t: Type) -> Type: name, type_name, name_in_super, - arg_type_in_super, + original_arg_type, supertype, context, secondary_context=node, diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 4b04a3b96ae4..ef6ca9f3b285 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -40,11 +40,10 @@ class B(A): class C(A): def f(self, *, b: int, a: str) -> None: pass # Fail [out] -main:10: error: Signature of "f" incompatible with supertype "A" -main:10: note: Superclass: -main:10: note: def f(self, *, a: int, b: str) -> None -main:10: note: Subclass: -main:10: note: def f(self, *, b: int, a: str) -> None +main:10: error: Argument 1 of "f" is incompatible with supertype "A"; supertype defines the argument type as "str" +main:10: note: This violates the Liskov substitution principle +main:10: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +main:10: error: Argument 2 of "f" is incompatible with supertype "A"; supertype defines the argument type as "int" [case testPositionalOverridingArgumentNameInsensitivity] import typing @@ -3324,3 +3323,47 @@ class Bar(Foo): # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides ... [builtins fixtures/property.pyi] + +[case testNoCrashOnUnpackOverride] +from typing import Unpack +from typing_extensions import TypedDict + +class Params(TypedDict): + x: int + y: str + +class Other(TypedDict): + x: int + y: int + +class B: + def meth(self, **kwargs: Unpack[Params]) -> None: + ... +class C(B): + def meth(self, **kwargs: Unpack[Other]) -> None: # E: Signature of "meth" incompatible with supertype "B" \ + # N: Superclass: \ + # N: def meth(*, x: int, y: str) -> None \ + # N: Subclass: \ + # N: def meth(*, x: int, y: int) -> None + + ... +[builtins fixtures/tuple.pyi] + +[case testOverrideErrorLocationNamed] +class B: + def meth( + self, *, + x: int, + y: str, + ) -> None: + ... +class C(B): + def meth( + self, *, + y: int, # E: Argument 1 of "meth" is incompatible with supertype "B"; supertype defines the argument type as "str" \ + # N: This violates the Liskov substitution principle \ + # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides + x: int, + ) -> None: + ... +[builtins fixtures/tuple.pyi] From 415d49f25b6315cf1b7a04046a942246a033498d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 11 Jun 2024 17:13:46 +0100 Subject: [PATCH 027/247] [mypyc] Support new syntax for generic functions and classes (PEP 695) (#17357) Generate an implicit `Generic` base class for new-style generic classes. For this to work, also create C statics that can be used to access type variable objects (e.g. `T` or `Ts`) at runtime. These are needed when evaluating base classes. Import `TypeVar` and friends from the `_typing` C extension instead of `typing`, since the latter is pretty slow to import, and we don't want to add a hidden new runtime dependency in case the full `typing` module isn't needed. Generic functions don't need any changes, since they don't support indexing with a type, and type variable types aren't valid in runtime contexts. Type erasure seems sufficient, especially considering that mypyc doesn't support classes nested within functions. (I'm not 100% sure about this though, and we might need to put function type variables into statics eventually.) Update builtins test fixtures used in mypyc tests to not defined type variables such as `T`, since these leak into tests and can produce unexpected or unrealistic results. Ignore upper bounds and value restrictions. These are only used for type checking. This should only affect introspection of type variables, which isn't properly supported in compiled code anyway. New type alias syntax is not supported in this PR. --- mypy/nodes.py | 9 +- mypy/semanal.py | 9 +- mypyc/codegen/emitfunc.py | 11 +- mypyc/codegen/emitmodule.py | 11 ++ mypyc/common.py | 1 + mypyc/ir/module_ir.py | 6 + mypyc/ir/ops.py | 3 + mypyc/irbuild/builder.py | 17 +++ mypyc/irbuild/classdef.py | 62 +++++++- mypyc/irbuild/expression.py | 5 + mypyc/irbuild/main.py | 1 + mypyc/primitives/generic_ops.py | 2 +- mypyc/test-data/fixtures/ir.py | 132 ++++++++--------- mypyc/test-data/fixtures/typing-full.pyi | 3 + mypyc/test-data/irbuild-set.test | 5 +- mypyc/test-data/run-loops.test | 11 +- mypyc/test-data/run-python312.test | 172 +++++++++++++++++++++++ mypyc/test/test_run.py | 3 + 18 files changed, 382 insertions(+), 81 deletions(-) create mode 100644 mypyc/test-data/run-python312.test diff --git a/mypy/nodes.py b/mypy/nodes.py index 90561779051d..850b1db87556 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2535,8 +2535,9 @@ def __init__( default: mypy.types.Type, variance: int = INVARIANT, is_new_style: bool = False, + line: int = -1, ) -> None: - super().__init__() + super().__init__(line=line) self._name = name self._fullname = fullname self.upper_bound = upper_bound @@ -2582,8 +2583,9 @@ def __init__( default: mypy.types.Type, variance: int = INVARIANT, is_new_style: bool = False, + line: int = -1, ) -> None: - super().__init__(name, fullname, upper_bound, default, variance, is_new_style) + super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line) self.values = values def accept(self, visitor: ExpressionVisitor[T]) -> T: @@ -2661,8 +2663,9 @@ def __init__( default: mypy.types.Type, variance: int = INVARIANT, is_new_style: bool = False, + line: int = -1, ) -> None: - super().__init__(name, fullname, upper_bound, default, variance, is_new_style) + super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line) self.tuple_fallback = tuple_fallback def accept(self, visitor: ExpressionVisitor[T]) -> T: diff --git a/mypy/semanal.py b/mypy/semanal.py index 903af80fe404..8da5c68d562d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1709,7 +1709,7 @@ def push_type_args( self.scope_stack.append(SCOPE_ANNOTATION) tvs: list[tuple[str, TypeVarLikeExpr]] = [] for p in type_args: - tv = self.analyze_type_param(p) + tv = self.analyze_type_param(p, context) if tv is None: return None tvs.append((p.name, tv)) @@ -1732,7 +1732,9 @@ def is_defined_type_param(self, name: str) -> bool: return True return False - def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None: + def analyze_type_param( + self, type_param: TypeParam, context: Context + ) -> TypeVarLikeExpr | None: fullname = self.qualified_name(type_param.name) if type_param.upper_bound: upper_bound = self.anal_type(type_param.upper_bound) @@ -1757,6 +1759,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None: default=default, variance=VARIANCE_NOT_READY, is_new_style=True, + line=context.line, ) elif type_param.kind == PARAM_SPEC_KIND: return ParamSpecExpr( @@ -1765,6 +1768,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None: upper_bound=upper_bound, default=default, is_new_style=True, + line=context.line, ) else: assert type_param.kind == TYPE_VAR_TUPLE_KIND @@ -1777,6 +1781,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None: tuple_fallback=tuple_fallback, default=default, is_new_style=True, + line=context.line, ) def pop_type_args(self, type_args: list[TypeParam] | None) -> None: diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 12f57b9cee6f..d945a28d8481 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -6,7 +6,14 @@ from mypyc.analysis.blockfreq import frequently_executed_blocks from mypyc.codegen.emit import DEBUG_ERRORS, Emitter, TracebackAndGotoHandler, c_array_initializer -from mypyc.common import MODULE_PREFIX, NATIVE_PREFIX, REG_PREFIX, STATIC_PREFIX, TYPE_PREFIX +from mypyc.common import ( + MODULE_PREFIX, + NATIVE_PREFIX, + REG_PREFIX, + STATIC_PREFIX, + TYPE_PREFIX, + TYPE_VAR_PREFIX, +) from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values from mypyc.ir.ops import ( @@ -14,6 +21,7 @@ NAMESPACE_MODULE, NAMESPACE_STATIC, NAMESPACE_TYPE, + NAMESPACE_TYPE_VAR, Assign, AssignMulti, BasicBlock, @@ -477,6 +485,7 @@ def visit_set_attr(self, op: SetAttr) -> None: NAMESPACE_STATIC: STATIC_PREFIX, NAMESPACE_TYPE: TYPE_PREFIX, NAMESPACE_MODULE: MODULE_PREFIX, + NAMESPACE_TYPE_VAR: TYPE_VAR_PREFIX, } def visit_load_static(self, op: LoadStatic) -> None: diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 6c8f5ac91335..1d8708912de5 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -41,6 +41,7 @@ PREFIX, RUNTIME_C_FILES, TOP_LEVEL_NAME, + TYPE_VAR_PREFIX, shared_lib_name, short_id_from_name, use_vectorcall, @@ -590,6 +591,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: self.declare_finals(module_name, module.final_names, declarations) for cl in module.classes: generate_class_type_decl(cl, emitter, ext_declarations, declarations) + self.declare_type_vars(module_name, module.type_var_names, declarations) for fn in module.functions: generate_function_declaration(fn, declarations) @@ -1063,6 +1065,15 @@ def declare_static_pyobject(self, identifier: str, emitter: Emitter) -> None: symbol = emitter.static_name(identifier, None) self.declare_global("PyObject *", symbol) + def declare_type_vars(self, module: str, type_var_names: list[str], emitter: Emitter) -> None: + for name in type_var_names: + static_name = emitter.static_name(name, module, prefix=TYPE_VAR_PREFIX) + emitter.context.declarations[static_name] = HeaderDeclaration( + f"PyObject *{static_name};", + [f"PyObject *{static_name} = NULL;"], + needs_export=False, + ) + def sort_classes(classes: list[tuple[str, ClassIR]]) -> list[tuple[str, ClassIR]]: mod_name = {ir: name for name, ir in classes} diff --git a/mypyc/common.py b/mypyc/common.py index 3d07f6c3d0d3..d7610fe15c41 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -13,6 +13,7 @@ STATIC_PREFIX: Final = "CPyStatic_" # Static variables (for literals etc.) TYPE_PREFIX: Final = "CPyType_" # Type object struct MODULE_PREFIX: Final = "CPyModule_" # Cached modules +TYPE_VAR_PREFIX: Final = "CPyTypeVar_" # Type variables when using new-style Python 3.12 syntax ATTR_PREFIX: Final = "_" # Attributes ENV_ATTR_NAME: Final = "__mypyc_env__" diff --git a/mypyc/ir/module_ir.py b/mypyc/ir/module_ir.py index dcf6f8768547..e3b240629eda 100644 --- a/mypyc/ir/module_ir.py +++ b/mypyc/ir/module_ir.py @@ -21,12 +21,17 @@ def __init__( functions: list[FuncIR], classes: list[ClassIR], final_names: list[tuple[str, RType]], + type_var_names: list[str], ) -> None: self.fullname = fullname self.imports = imports.copy() self.functions = functions self.classes = classes self.final_names = final_names + # Names of C statics used for Python 3.12 type variable objects. + # These are only visible in the module that defined them, so no need + # to serialize. + self.type_var_names = type_var_names def serialize(self) -> JsonDict: return { @@ -45,6 +50,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ModuleIR: [ctx.functions[FuncDecl.get_id_from_json(f)] for f in data["functions"]], [ClassIR.deserialize(c, ctx) for c in data["classes"]], [(k, deserialize_type(t, ctx)) for k, t in data["final_names"]], + [], ) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 377266e797d9..896ba3ac091c 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -789,6 +789,9 @@ def accept(self, visitor: OpVisitor[T]) -> T: # Namespace for modules NAMESPACE_MODULE: Final = "module" +# Namespace for Python 3.12 type variable objects (implicitly created TypeVar instances, etc.) +NAMESPACE_TYPE_VAR: Final = "typevar" + class LoadStatic(RegisterOp): """Load a static name (name :: static). diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index cca771e82c83..1b4f551d4a2a 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -69,6 +69,7 @@ from mypyc.ir.func_ir import INVALID_FUNC_DEF, FuncDecl, FuncIR, FuncSignature, RuntimeArg from mypyc.ir.ops import ( NAMESPACE_MODULE, + NAMESPACE_TYPE_VAR, Assign, BasicBlock, Branch, @@ -179,6 +180,7 @@ def __init__( self.function_names: set[tuple[str | None, str]] = set() self.classes: list[ClassIR] = [] self.final_names: list[tuple[str, RType]] = [] + self.type_var_names: list[str] = [] self.callable_class_names: set[str] = set() self.options = options @@ -541,6 +543,21 @@ def load_final_static( error_msg=f'value for final name "{error_name}" was not set', ) + def init_type_var(self, value: Value, name: str, line: int) -> None: + unique_name = name + "___" + str(line) + self.type_var_names.append(unique_name) + self.add(InitStatic(value, unique_name, self.module_name, namespace=NAMESPACE_TYPE_VAR)) + + def load_type_var(self, name: str, line: int) -> Value: + return self.add( + LoadStatic( + object_rprimitive, + name + "___" + str(line), + self.module_name, + namespace=NAMESPACE_TYPE_VAR, + ) + ) + def load_literal_value(self, val: int | str | bytes | float | complex | bool) -> Value: """Load value of a final name, class-level attribute, or constant folded expression.""" if isinstance(val, bool): diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 3f6ec0f33822..303ee8849244 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -7,6 +7,9 @@ from typing import Callable, Final from mypy.nodes import ( + PARAM_SPEC_KIND, + TYPE_VAR_KIND, + TYPE_VAR_TUPLE_KIND, AssignmentStmt, CallExpr, ClassDef, @@ -22,6 +25,7 @@ StrExpr, TempNode, TypeInfo, + TypeParam, is_class_var, ) from mypy.types import ENUM_REMOVED_PROPS, Instance, RawExpressionType, get_proper_type @@ -63,9 +67,16 @@ ) from mypyc.irbuild.util import dataclass_type, get_func_def, is_constant, is_dataclass_decorator from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op -from mypyc.primitives.generic_ops import py_hasattr_op, py_setattr_op +from mypyc.primitives.generic_ops import ( + iter_op, + next_op, + py_get_item_op, + py_hasattr_op, + py_setattr_op, +) from mypyc.primitives.misc_ops import ( dataclass_sleight_of_hand, + import_op, not_implemented_op, py_calc_meta_op, pytype_from_template_op, @@ -405,8 +416,14 @@ def get_type_annotation(self, stmt: AssignmentStmt) -> TypeInfo | None: def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value: # OK AND NOW THE FUN PART base_exprs = cdef.base_type_exprs + cdef.removed_base_type_exprs - if base_exprs: - bases = [builder.accept(x) for x in base_exprs] + new_style_type_args = cdef.type_args + if new_style_type_args: + bases = [make_generic_base_class(builder, cdef.fullname, new_style_type_args, cdef.line)] + else: + bases = [] + + if base_exprs or new_style_type_args: + bases.extend([builder.accept(x) for x in base_exprs]) tp_bases = builder.new_tuple(bases, cdef.line) else: tp_bases = builder.add(LoadErrorValue(object_rprimitive, is_borrowed=True)) @@ -453,6 +470,45 @@ def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value: return tp +def make_generic_base_class( + builder: IRBuilder, fullname: str, type_args: list[TypeParam], line: int +) -> Value: + """Construct Generic[...] base class object for a new-style generic class (Python 3.12).""" + mod = builder.call_c(import_op, [builder.load_str("_typing")], line) + tvs = [] + type_var_imported: Value | None = None + for type_param in type_args: + unpack = False + if type_param.kind == TYPE_VAR_KIND: + if type_var_imported: + # Reuse previously imported value as a minor optimization + tvt = type_var_imported + else: + tvt = builder.py_get_attr(mod, "TypeVar", line) + type_var_imported = tvt + elif type_param.kind == TYPE_VAR_TUPLE_KIND: + tvt = builder.py_get_attr(mod, "TypeVarTuple", line) + unpack = True + else: + assert type_param.kind == PARAM_SPEC_KIND + tvt = builder.py_get_attr(mod, "ParamSpec", line) + tv = builder.py_call(tvt, [builder.load_str(type_param.name)], line) + builder.init_type_var(tv, type_param.name, line) + if unpack: + # Evaluate *Ts for a TypeVarTuple + it = builder.call_c(iter_op, [tv], line) + tv = builder.call_c(next_op, [it], line) + tvs.append(tv) + gent = builder.py_get_attr(mod, "Generic", line) + if len(tvs) == 1: + arg = tvs[0] + else: + arg = builder.new_tuple(tvs, line) + + base = builder.call_c(py_get_item_op, [gent, arg], line) + return base + + # Mypy uses these internally as base classes of TypedDict classes. These are # lies and don't have any runtime equivalent. MAGIC_TYPED_DICT_CLASSES: Final[tuple[str, ...]] = ( diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index a16faf6cd7d7..8d7c089e20cd 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -44,6 +44,7 @@ TupleExpr, TypeApplication, TypeInfo, + TypeVarLikeExpr, UnaryExpr, Var, ) @@ -106,6 +107,10 @@ def transform_name_expr(builder: IRBuilder, expr: NameExpr) -> Value: + if isinstance(expr.node, TypeVarLikeExpr) and expr.node.is_new_style: + # Reference to Python 3.12 implicit TypeVar/TupleVarTuple/... object. + # These are stored in C statics and not visible in Python namespaces. + return builder.load_type_var(expr.node.name, expr.node.line) if expr.node is None: builder.add( RaiseStandardError( diff --git a/mypyc/irbuild/main.py b/mypyc/irbuild/main.py index 85b905393af1..15928d939cbf 100644 --- a/mypyc/irbuild/main.py +++ b/mypyc/irbuild/main.py @@ -99,6 +99,7 @@ def build_ir( builder.functions, builder.classes, builder.final_names, + builder.type_var_names, ) result[module.fullname] = module_ir class_irs.extend(builder.classes) diff --git a/mypyc/primitives/generic_ops.py b/mypyc/primitives/generic_ops.py index 3caec0a9875e..fe42767db11e 100644 --- a/mypyc/primitives/generic_ops.py +++ b/mypyc/primitives/generic_ops.py @@ -178,7 +178,7 @@ ) # obj1[obj2] -method_op( +py_get_item_op = method_op( name="__getitem__", arg_types=[object_rprimitive, object_rprimitive], return_type=object_rprimitive, diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index bf06613ad2a8..6f0d8da90d57 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -7,12 +7,12 @@ overload, Mapping, Union, Callable, Sequence, FrozenSet, Protocol ) -T = TypeVar('T') +_T = TypeVar('_T') T_co = TypeVar('T_co', covariant=True) T_contra = TypeVar('T_contra', contravariant=True) -S = TypeVar('S') -K = TypeVar('K') # for keys in mapping -V = TypeVar('V') # for values in mapping +_S = TypeVar('_S') +_K = TypeVar('_K') # for keys in mapping +_V = TypeVar('_V') # for values in mapping class __SupportsAbs(Protocol[T_co]): def __abs__(self) -> T_co: pass @@ -199,76 +199,76 @@ def __contains__(self, item: object) -> int: ... class function: pass -class list(Generic[T], Sequence[T], Iterable[T]): - def __init__(self, i: Optional[Iterable[T]] = None) -> None: pass +class list(Generic[_T], Sequence[_T], Iterable[_T]): + def __init__(self, i: Optional[Iterable[_T]] = None) -> None: pass @overload - def __getitem__(self, i: int) -> T: ... + def __getitem__(self, i: int) -> _T: ... @overload - def __getitem__(self, s: slice) -> List[T]: ... - def __setitem__(self, i: int, o: T) -> None: pass + def __getitem__(self, s: slice) -> List[_T]: ... + def __setitem__(self, i: int, o: _T) -> None: pass def __delitem__(self, i: int) -> None: pass - def __mul__(self, i: int) -> List[T]: pass - def __rmul__(self, i: int) -> List[T]: pass - def __iter__(self) -> Iterator[T]: pass + def __mul__(self, i: int) -> List[_T]: pass + def __rmul__(self, i: int) -> List[_T]: pass + def __iter__(self) -> Iterator[_T]: pass def __len__(self) -> int: pass def __contains__(self, item: object) -> int: ... - def __add__(self, x: List[T]) -> List[T]: ... - def append(self, x: T) -> None: pass - def pop(self, i: int = -1) -> T: pass - def count(self, T) -> int: pass - def extend(self, l: Iterable[T]) -> None: pass - def insert(self, i: int, x: T) -> None: pass + def __add__(self, x: List[_T]) -> List[_T]: ... + def append(self, x: _T) -> None: pass + def pop(self, i: int = -1) -> _T: pass + def count(self, _T) -> int: pass + def extend(self, l: Iterable[_T]) -> None: pass + def insert(self, i: int, x: _T) -> None: pass def sort(self) -> None: pass def reverse(self) -> None: pass - def remove(self, o: T) -> None: pass - def index(self, o: T) -> int: pass + def remove(self, o: _T) -> None: pass + def index(self, o: _T) -> int: pass -class dict(Mapping[K, V]): +class dict(Mapping[_K, _V]): @overload - def __init__(self, **kwargs: K) -> None: ... + def __init__(self, **kwargs: _K) -> None: ... @overload - def __init__(self, map: Mapping[K, V], **kwargs: V) -> None: ... + def __init__(self, map: Mapping[_K, _V], **kwargs: _V) -> None: ... @overload - def __init__(self, iterable: Iterable[Tuple[K, V]], **kwargs: V) -> None: ... - def __getitem__(self, key: K) -> V: pass - def __setitem__(self, k: K, v: V) -> None: pass - def __delitem__(self, k: K) -> None: pass + def __init__(self, iterable: Iterable[Tuple[_K, _V]], **kwargs: _V) -> None: ... + def __getitem__(self, key: _K) -> _V: pass + def __setitem__(self, k: _K, v: _V) -> None: pass + def __delitem__(self, k: _K) -> None: pass def __contains__(self, item: object) -> int: pass - def __iter__(self) -> Iterator[K]: pass + def __iter__(self) -> Iterator[_K]: pass def __len__(self) -> int: pass @overload - def update(self, __m: Mapping[K, V], **kwargs: V) -> None: pass + def update(self, __m: Mapping[_K, _V], **kwargs: _V) -> None: pass @overload - def update(self, __m: Iterable[Tuple[K, V]], **kwargs: V) -> None: ... + def update(self, __m: Iterable[Tuple[_K, _V]], **kwargs: _V) -> None: ... @overload - def update(self, **kwargs: V) -> None: ... - def pop(self, x: int) -> K: pass - def keys(self) -> Iterable[K]: pass - def values(self) -> Iterable[V]: pass - def items(self) -> Iterable[Tuple[K, V]]: pass + def update(self, **kwargs: _V) -> None: ... + def pop(self, x: int) -> _K: pass + def keys(self) -> Iterable[_K]: pass + def values(self) -> Iterable[_V]: pass + def items(self) -> Iterable[Tuple[_K, _V]]: pass def clear(self) -> None: pass - def copy(self) -> Dict[K, V]: pass - def setdefault(self, key: K, val: V = ...) -> V: pass + def copy(self) -> Dict[_K, _V]: pass + def setdefault(self, key: _K, val: _V = ...) -> _V: pass -class set(Generic[T]): - def __init__(self, i: Optional[Iterable[T]] = None) -> None: pass - def __iter__(self) -> Iterator[T]: pass +class set(Generic[_T]): + def __init__(self, i: Optional[Iterable[_T]] = None) -> None: pass + def __iter__(self) -> Iterator[_T]: pass def __len__(self) -> int: pass - def add(self, x: T) -> None: pass - def remove(self, x: T) -> None: pass - def discard(self, x: T) -> None: pass + def add(self, x: _T) -> None: pass + def remove(self, x: _T) -> None: pass + def discard(self, x: _T) -> None: pass def clear(self) -> None: pass - def pop(self) -> T: pass - def update(self, x: Iterable[S]) -> None: pass - def __or__(self, s: Union[Set[S], FrozenSet[S]]) -> Set[Union[T, S]]: ... - def __xor__(self, s: Union[Set[S], FrozenSet[S]]) -> Set[Union[T, S]]: ... - -class frozenset(Generic[T]): - def __init__(self, i: Optional[Iterable[T]] = None) -> None: pass - def __iter__(self) -> Iterator[T]: pass + def pop(self) -> _T: pass + def update(self, x: Iterable[_S]) -> None: pass + def __or__(self, s: Union[Set[_S], FrozenSet[_S]]) -> Set[Union[_T, _S]]: ... + def __xor__(self, s: Union[Set[_S], FrozenSet[_S]]) -> Set[Union[_T, _S]]: ... + +class frozenset(Generic[_T]): + def __init__(self, i: Optional[Iterable[_T]] = None) -> None: pass + def __iter__(self) -> Iterator[_T]: pass def __len__(self) -> int: pass - def __or__(self, s: Union[Set[S], FrozenSet[S]]) -> FrozenSet[Union[T, S]]: ... - def __xor__(self, s: Union[Set[S], FrozenSet[S]]) -> FrozenSet[Union[T, S]]: ... + def __or__(self, s: Union[Set[_S], FrozenSet[_S]]) -> FrozenSet[Union[_T, _S]]: ... + def __xor__(self, s: Union[Set[_S], FrozenSet[_S]]) -> FrozenSet[Union[_T, _S]]: ... class slice: pass @@ -323,31 +323,31 @@ class OverflowError(ArithmeticError): pass class GeneratorExit(BaseException): pass -def any(i: Iterable[T]) -> bool: pass -def all(i: Iterable[T]) -> bool: pass -def sum(i: Iterable[T]) -> int: pass -def reversed(object: Sequence[T]) -> Iterator[T]: ... +def any(i: Iterable[_T]) -> bool: pass +def all(i: Iterable[_T]) -> bool: pass +def sum(i: Iterable[_T]) -> int: pass +def reversed(object: Sequence[_T]) -> Iterator[_T]: ... def id(o: object) -> int: pass # This type is obviously wrong but the test stubs don't have Sized anymore def len(o: object) -> int: pass def print(*object) -> None: pass def isinstance(x: object, t: object) -> bool: pass -def iter(i: Iterable[T]) -> Iterator[T]: pass +def iter(i: Iterable[_T]) -> Iterator[_T]: pass @overload -def next(i: Iterator[T]) -> T: pass +def next(i: Iterator[_T]) -> _T: pass @overload -def next(i: Iterator[T], default: T) -> T: pass +def next(i: Iterator[_T], default: _T) -> _T: pass def hash(o: object) -> int: ... def globals() -> Dict[str, Any]: ... def getattr(obj: object, name: str, default: Any = None) -> Any: ... def setattr(obj: object, name: str, value: Any) -> None: ... -def enumerate(x: Iterable[T]) -> Iterator[Tuple[int, T]]: ... +def enumerate(x: Iterable[_T]) -> Iterator[Tuple[int, _T]]: ... @overload -def zip(x: Iterable[T], y: Iterable[S]) -> Iterator[Tuple[T, S]]: ... +def zip(x: Iterable[_T], y: Iterable[_S]) -> Iterator[Tuple[_T, _S]]: ... @overload -def zip(x: Iterable[T], y: Iterable[S], z: Iterable[V]) -> Iterator[Tuple[T, S, V]]: ... +def zip(x: Iterable[_T], y: Iterable[_S], z: Iterable[_V]) -> Iterator[Tuple[_T, _S, _V]]: ... def eval(e: str) -> Any: ... -def abs(x: __SupportsAbs[T]) -> T: ... +def abs(x: __SupportsAbs[_T]) -> _T: ... @overload def divmod(x: __SupportsDivMod[T_contra, T_co], y: T_contra) -> T_co: ... @overload @@ -359,8 +359,8 @@ def pow(base: __SupportsPow3NoneOnly[T_contra, T_co], exp: T_contra, mod: None = @overload def pow(base: __SupportsPow3[T_contra, _M, T_co], exp: T_contra, mod: _M) -> T_co: ... def exit() -> None: ... -def min(x: T, y: T) -> T: ... -def max(x: T, y: T) -> T: ... +def min(x: _T, y: _T) -> _T: ... +def max(x: _T, y: _T) -> _T: ... def repr(o: object) -> str: ... def ascii(o: object) -> str: ... def ord(o: object) -> int: ... diff --git a/mypyc/test-data/fixtures/typing-full.pyi b/mypyc/test-data/fixtures/typing-full.pyi index 52bca09a1dec..3ddc1f1bba08 100644 --- a/mypyc/test-data/fixtures/typing-full.pyi +++ b/mypyc/test-data/fixtures/typing-full.pyi @@ -167,3 +167,6 @@ class _TypedDict(Mapping[str, object]): def pop(self, k: NoReturn, default: T = ...) -> object: ... def update(self: T, __m: T) -> None: ... def __delitem__(self, k: NoReturn) -> None: ... + +class TypeAliasType: + pass diff --git a/mypyc/test-data/irbuild-set.test b/mypyc/test-data/irbuild-set.test index 1ac638754a8b..110801b78a66 100644 --- a/mypyc/test-data/irbuild-set.test +++ b/mypyc/test-data/irbuild-set.test @@ -39,7 +39,10 @@ L0: return r0 [case testNewSetFromIterable] -from typing import Set, List +from typing import Set, List, TypeVar + +T = TypeVar("T") + def f(l: List[T]) -> Set[T]: return set(l) [out] diff --git a/mypyc/test-data/run-loops.test b/mypyc/test-data/run-loops.test index 994b30b42347..6f7d79059a6d 100644 --- a/mypyc/test-data/run-loops.test +++ b/mypyc/test-data/run-loops.test @@ -276,7 +276,10 @@ for k in range(12): [out] [case testForIterable] -from typing import Iterable, Dict, Any, Tuple +from typing import Iterable, Dict, Any, Tuple, TypeVar + +T = TypeVar("T") + def iterate_over_any(a: Any) -> None: for element in a: print(element) @@ -350,13 +353,13 @@ iterate_over_tuple((1, 2, 3)) Traceback (most recent call last): File "driver.py", line 16, in iterate_over_any(5) - File "native.py", line 3, in iterate_over_any + File "native.py", line 6, in iterate_over_any for element in a: TypeError: 'int' object is not iterable Traceback (most recent call last): File "driver.py", line 20, in iterate_over_iterable(broken_generator(5)) - File "native.py", line 7, in iterate_over_iterable + File "native.py", line 10, in iterate_over_iterable for element in iterable: File "driver.py", line 8, in broken_generator raise Exception('Exception Manually Raised') @@ -364,7 +367,7 @@ Exception: Exception Manually Raised Traceback (most recent call last): File "driver.py", line 24, in iterate_and_delete(d) - File "native.py", line 11, in iterate_and_delete + File "native.py", line 14, in iterate_and_delete for key in d: RuntimeError: dictionary changed size during iteration 15 diff --git a/mypyc/test-data/run-python312.test b/mypyc/test-data/run-python312.test new file mode 100644 index 000000000000..fbafeaf3e65f --- /dev/null +++ b/mypyc/test-data/run-python312.test @@ -0,0 +1,172 @@ +[case testPEP695Basics] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import Any, TypeAliasType, cast + +from testutil import assertRaises + +def id[T](x: T) -> T: + return x + +def test_call_generic_function() -> None: + assert id(2) == 2 + assert id('x') == 'x' + +class C[T]: + x: T + + def __init__(self, x: T) -> None: + self.x = x + +class D[T, S]: + x: T + y: S + + def __init__(self, x: T, y: S) -> None: + self.x = x + self.y = y + + def set(self, x: object, y: object) -> None: + self.x = cast(T, x) + self.y = cast(S, y) + +def test_generic_class() -> None: + c = C(5) + assert c.x == 5 + c2 = C[str]('x') + assert c2.x == 'x' + d = D[str, int]('a', 5) + assert d.x == 'a' + assert d.y == 5 + d.set('b', 6) + assert d.x == 'b' + assert d.y == 6 + +def test_generic_class_via_any() -> None: + c_any: Any = C + c = c_any(2) + assert c.x == 2 + c2 = c_any[str]('y') + assert c2.x == 'y' + assert str(c_any[str]) == 'native.C[str]' + + d_any: Any = D + d = d_any(1, 'x') + assert d.x == 1 + assert d.y == 'x' + d2 = d_any[int, str](2, 'y') + assert d2.x == 2 + assert d2.y == 'y' + + with assertRaises(TypeError): + c_any[int, str] + with assertRaises(TypeError): + d_any[int] + +class E[*Ts]: pass + +def test_type_var_tuple() -> None: + e: E[int, str] = E() + e_any: Any = E + assert isinstance(e_any(), E) + assert isinstance(e_any[int](), E) + assert isinstance(e_any[int, str](), E) + +class F[**P]: pass + +def test_param_spec() -> None: + f: F[[int, str]] = F() + f_any: Any = F + assert isinstance(f_any(), F) + assert isinstance(f_any[[int, str]](), F) + +class SubC[S](C[S]): + def __init__(self, x: S) -> None: + super().__init__(x) + +def test_generic_subclass() -> None: + s = SubC(1) + assert s.x == 1 + s2 = SubC[str]('y') + assert s2.x == 'y' + sub_any: Any = SubC + assert sub_any(1).x == 1 + assert sub_any[str]('x').x == 'x' + assert isinstance(s, SubC) + assert isinstance(s, C) + +class SubD[ + T, # Put everything on separate lines + S]( + D[T, + S]): pass + +def test_generic_subclass_two_params() -> None: + s = SubD(3, 'y') + assert s.x == 3 + assert s.y == 'y' + s2 = SubD[str, int]('z', 4) + assert s2.x == 'z' + assert s2.y == 4 + sub_any: Any = SubD + assert sub_any(3, 'y').y == 'y' + assert sub_any[int, str](3, 'y').y == 'y' + assert isinstance(s, SubD) + assert isinstance(s, D) + +class SubE[*Ts](E[*Ts]): pass + +def test_type_var_tuple_subclass() -> None: + sub_any: Any = SubE + assert isinstance(sub_any(), SubE) + assert isinstance(sub_any(), E) + assert isinstance(sub_any[int](), SubE) + assert isinstance(sub_any[int, str](), SubE) + + +class SubF[**P](F[P]): pass + +def test_param_spec_subclass() -> None: + sub_any: Any = SubF + assert isinstance(sub_any(), SubF) + assert isinstance(sub_any(), F) + assert isinstance(sub_any[[int]](), SubF) + assert isinstance(sub_any[[int, str]](), SubF) + +# We test that upper bounds and restricted values can be used, but not that +# they are introspectable + +def bound[T: C](x: T) -> T: + return x + +def test_function_with_upper_bound() -> None: + c = C(1) + assert bound(c) is c + +def restriction[T: (int, str)](x: T) -> T: + return x + +def test_function_with_value_restriction() -> None: + assert restriction(1) == 1 + assert restriction('x') == 'x' + +class Bound[T: C]: + def __init__(self, x: T) -> None: + self.x = x + +def test_class_with_upper_bound() -> None: + c = C(1) + b = Bound(c) + assert b.x is c + b2 = Bound[C](c) + assert b2.x is c + +class Restriction[T: (int, str)]: + def __init__(self, x: T) -> None: + self.x = x + +def test_class_with_value_restriction() -> None: + r = Restriction(1) + assert r.x == 1 + r2 = Restriction[str]('a') + assert r2.x == 'a' +[typing fixtures/typing-full.pyi] diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 467ef8b87a92..37de192a9291 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -71,6 +71,8 @@ if sys.version_info >= (3, 10): files.append("run-match.test") +if sys.version_info >= (3, 12): + files.append("run-python312.test") setup_format = """\ from setuptools import setup @@ -194,6 +196,7 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> options.preserve_asts = True options.allow_empty_bodies = True options.incremental = self.separate + options.enable_incomplete_feature.append("NewGenericSyntax") # Avoid checking modules/packages named 'unchecked', to provide a way # to test interacting with code we don't have types for. From 8fb99695a01848f47a12a01c63ddd82cd354762d Mon Sep 17 00:00:00 2001 From: Tadeu Manoel Date: Wed, 12 Jun 2024 08:22:08 -0300 Subject: [PATCH 028/247] Write stubs for C module with utf-8 encoding (#17367) Avoid encoding errors if functions contain non-ASCII characters. --- mypy/stubgenc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 7e3ef49c6e9a..9acd3f171a41 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -178,7 +178,7 @@ def generate_stub_for_c_module( gen.generate_module() output = gen.output() - with open(target, "w") as file: + with open(target, "w", encoding="utf-8") as file: file.write(output) From 98a22c44c26ff436f1c343ad4727258ffd72e055 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 13 Jun 2024 14:17:05 -0700 Subject: [PATCH 029/247] Fix isinstance with type aliases to PEP 604 unions (#17371) Fixes #12155, fixes #11673, seems pretty commonly reported issue --- mypy/checker.py | 2 ++ mypy/checkexpr.py | 10 ++++++++++ mypy/exprtotype.py | 3 ++- mypy/type_visitor.py | 7 ++++++- mypy/typeanal.py | 2 +- mypy/types.py | 1 + test-data/unit/check-type-aliases.test | 1 + test-data/unit/check-union-or-syntax.test | 19 +++++++++++++++++++ test-data/unit/fine-grained.test | 1 + test-data/unit/fixtures/type.pyi | 7 +++++++ test-data/unit/lib-stub/types.pyi | 3 +++ 11 files changed, 53 insertions(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 70db31c9a94f..119aa9f3cea2 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -7323,6 +7323,8 @@ def get_isinstance_type(self, expr: Expression) -> list[TypeRange] | None: elif isinstance(typ, Instance) and typ.type.fullname == "builtins.type": object_type = Instance(typ.type.mro[-1], []) types.append(TypeRange(object_type, is_upper_bound=True)) + elif isinstance(typ, Instance) and typ.type.fullname == "types.UnionType" and typ.args: + types.append(TypeRange(UnionType(typ.args), is_upper_bound=False)) elif isinstance(typ, AnyType): types.append(TypeRange(typ, is_upper_bound=False)) else: # we didn't see an actual type, but rather a variable with unknown value diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index c34952b084f9..861c28e5b54c 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -527,6 +527,10 @@ def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) -> and node and isinstance(node.node, TypeAlias) and not node.node.no_args + and not ( + isinstance(union_target := get_proper_type(node.node.target), UnionType) + and union_target.uses_pep604_syntax + ) ): self.msg.type_arguments_not_allowed(e) if isinstance(typ, RefExpr) and isinstance(typ.node, TypeInfo): @@ -4762,6 +4766,12 @@ class LongName(Generic[T]): ... return TypeType(item, line=item.line, column=item.column) elif isinstance(item, AnyType): return AnyType(TypeOfAny.from_another_any, source_any=item) + elif ( + isinstance(item, UnionType) + and item.uses_pep604_syntax + and self.chk.options.python_version >= (3, 10) + ): + return self.chk.named_generic_type("types.UnionType", item.items) else: if alias_definition: return AnyType(TypeOfAny.special_form) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 2218a950788c..d9bdf2e2b20b 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -122,7 +122,8 @@ def expr_to_unanalyzed_type( [ expr_to_unanalyzed_type(expr.left, options, allow_new_syntax), expr_to_unanalyzed_type(expr.right, options, allow_new_syntax), - ] + ], + uses_pep604_syntax=True, ) elif isinstance(expr, CallExpr) and isinstance(_parent, ListExpr): c = expr.callee diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index a6ae77832ceb..d0876629fc08 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -266,7 +266,12 @@ def visit_literal_type(self, t: LiteralType) -> Type: return LiteralType(value=t.value, fallback=fallback, line=t.line, column=t.column) def visit_union_type(self, t: UnionType) -> Type: - return UnionType(self.translate_types(t.items), t.line, t.column) + return UnionType( + self.translate_types(t.items), + t.line, + t.column, + uses_pep604_syntax=t.uses_pep604_syntax, + ) def translate_types(self, types: Iterable[Type]) -> list[Type]: return [t.accept(self) for t in types] diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 82c90272d6c2..6651af7dad4f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1271,7 +1271,7 @@ def visit_union_type(self, t: UnionType) -> Type: and not self.options.python_version >= (3, 10) ): self.fail("X | Y syntax for unions requires Python 3.10", t, code=codes.SYNTAX) - return UnionType(self.anal_array(t.items), t.line) + return UnionType(self.anal_array(t.items), t.line, uses_pep604_syntax=t.uses_pep604_syntax) def visit_partial_type(self, t: PartialType) -> Type: assert False, "Internal error: Unexpected partial type" diff --git a/mypy/types.py b/mypy/types.py index cdcb26f435b8..0f8c48c8cb7d 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2821,6 +2821,7 @@ def __init__( items: Sequence[Type], line: int = -1, column: int = -1, + *, is_evaluated: bool = True, uses_pep604_syntax: bool = False, ) -> None: diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 86bd4422003b..5eea1fb2b53e 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -967,6 +967,7 @@ a: A b: B reveal_type(a) # N: Revealed type is "Union[builtins.list[Any], builtins.int]" reveal_type(b) # N: Revealed type is "Union[builtins.int, builtins.list[Any]]" +[builtins fixtures/type.pyi] [case testValidTypeAliasValues] from typing import TypeVar, Generic, List diff --git a/test-data/unit/check-union-or-syntax.test b/test-data/unit/check-union-or-syntax.test index 85e268f348f0..b5fd85cb7ed8 100644 --- a/test-data/unit/check-union-or-syntax.test +++ b/test-data/unit/check-union-or-syntax.test @@ -207,6 +207,25 @@ foo: ReadableBuffer [file was_mmap.pyi] from was_builtins import * class mmap: ... +[builtins fixtures/type.pyi] + +[case testTypeAliasWithNewUnionIsInstance] +# flags: --python-version 3.10 +SimpleAlias = int | str + +def foo(x: int | str | tuple): + if isinstance(x, SimpleAlias): + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + else: + reveal_type(x) # N: Revealed type is "builtins.tuple[Any, ...]" + +ParameterizedAlias = str | list[str] + +# these are false negatives: +isinstance(5, str | list[str]) +isinstance(5, ParameterizedAlias) +[builtins fixtures/type.pyi] + # TODO: Get this test to pass [case testImplicit604TypeAliasWithCyclicImportNotInStub-xfail] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 9c379d8f60da..a87f8ceca15c 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10380,6 +10380,7 @@ from b import C, D A = C | D a: A reveal_type(a) +[builtins fixtures/type.pyi] [file b.py] C = int diff --git a/test-data/unit/fixtures/type.pyi b/test-data/unit/fixtures/type.pyi index 39357a693638..084b7f8388d8 100644 --- a/test-data/unit/fixtures/type.pyi +++ b/test-data/unit/fixtures/type.pyi @@ -1,6 +1,8 @@ # builtins stub used in type-related test cases. from typing import Any, Generic, TypeVar, List, Union +import sys +import types T = TypeVar("T") S = TypeVar("S") @@ -25,3 +27,8 @@ class bool: pass class int: pass class str: pass class ellipsis: pass + +if sys.version_info >= (3, 10): # type: ignore + def isinstance(obj: object, class_or_tuple: type | types.UnionType, /) -> bool: ... +else: + def isinstance(obj: object, class_or_tuple: type, /) -> bool: ... diff --git a/test-data/unit/lib-stub/types.pyi b/test-data/unit/lib-stub/types.pyi index 012fd8503377..e4869dbc3093 100644 --- a/test-data/unit/lib-stub/types.pyi +++ b/test-data/unit/lib-stub/types.pyi @@ -15,3 +15,6 @@ if sys.version_info >= (3, 10): class NoneType: ... + + class UnionType: + ... From 3d9256b3c508b69426dea9d672f517459f372fa8 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 14 Jun 2024 09:10:37 +0300 Subject: [PATCH 030/247] Support `enum.nonmember` for python3.11+ (#17376) This PR adds support for https://docs.python.org/3.11/library/enum.html#enum.nonmember Refs https://github.com/python/mypy/issues/12841 --- mypy/checkmember.py | 11 +++++++++++ mypy/plugins/enums.py | 15 ++++++++++++--- test-data/unit/check-enum.test | 28 ++++++++++++++++++++++++++++ test-data/unit/lib-stub/enum.pyi | 5 +++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index fa847de2e4a0..7525db25d9cd 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -1143,6 +1143,17 @@ def analyze_enum_class_attribute_access( if name.startswith("__") and name.replace("_", "") != "": return None + node = itype.type.get(name) + if node and node.type: + proper = get_proper_type(node.type) + # Support `A = nonmember(1)` function call and decorator. + if ( + isinstance(proper, Instance) + and proper.type.fullname == "enum.nonmember" + and proper.args + ): + return proper.args[0] + enum_literal = LiteralType(name, fallback=itype) return itype.copy_modified(last_known_value=enum_literal) diff --git a/mypy/plugins/enums.py b/mypy/plugins/enums.py index 83350fe2fe11..167b330f9b09 100644 --- a/mypy/plugins/enums.py +++ b/mypy/plugins/enums.py @@ -20,7 +20,15 @@ from mypy.semanal_enum import ENUM_BASES from mypy.subtypes import is_equivalent from mypy.typeops import fixup_partial_type, make_simplified_union -from mypy.types import CallableType, Instance, LiteralType, ProperType, Type, get_proper_type +from mypy.types import ( + CallableType, + Instance, + LiteralType, + ProperType, + Type, + get_proper_type, + is_named_instance, +) ENUM_NAME_ACCESS: Final = {f"{prefix}.name" for prefix in ENUM_BASES} | { f"{prefix}._name_" for prefix in ENUM_BASES @@ -159,7 +167,7 @@ class SomeEnum: stnodes = (info.get(name) for name in info.names) - # Enums _can_ have methods and instance attributes. + # Enums _can_ have methods, instance attributes, and `nonmember`s. # Omit methods and attributes created by assigning to self.* # for our value inference. node_types = ( @@ -170,7 +178,8 @@ class SomeEnum: proper_types = [ _infer_value_type_with_auto_fallback(ctx, t) for t in node_types - if t is None or not isinstance(t, CallableType) + if t is None + or (not isinstance(t, CallableType) and not is_named_instance(t, "enum.nonmember")) ] underlying_type = _first(proper_types) if underlying_type is None: diff --git a/test-data/unit/check-enum.test b/test-data/unit/check-enum.test index e8e65f464eaf..183901416604 100644 --- a/test-data/unit/check-enum.test +++ b/test-data/unit/check-enum.test @@ -2138,3 +2138,31 @@ elif e == MyEnum.B: else: reveal_type(e) # E: Statement is unreachable [builtins fixtures/dict.pyi] + + +[case testEnumNonMemberSupport] +# flags: --python-version 3.11 +# This was added in 3.11 +from enum import Enum, nonmember + +class My(Enum): + a = 1 + b = 2 + c = nonmember(3) + +reveal_type(My.a) # N: Revealed type is "Literal[__main__.My.a]?" +reveal_type(My.b) # N: Revealed type is "Literal[__main__.My.b]?" +reveal_type(My.c) # N: Revealed type is "builtins.int" + +def accepts_my(my: My): + reveal_type(my.value) # N: Revealed type is "Union[Literal[1]?, Literal[2]?]" + +class Other(Enum): + a = 1 + @nonmember + class Support: + b = 2 + +reveal_type(Other.a) # N: Revealed type is "Literal[__main__.Other.a]?" +reveal_type(Other.Support.b) # N: Revealed type is "builtins.int" +[builtins fixtures/dict.pyi] diff --git a/test-data/unit/lib-stub/enum.pyi b/test-data/unit/lib-stub/enum.pyi index 11adfc597955..32dd7c38d251 100644 --- a/test-data/unit/lib-stub/enum.pyi +++ b/test-data/unit/lib-stub/enum.pyi @@ -48,3 +48,8 @@ class auto(IntFlag): # It is python-3.11+ only: class StrEnum(str, Enum): def __new__(cls: Type[_T], value: str | _T) -> _T: ... + +# It is python-3.11+ only: +class nonmember(Generic[_T]): + value: _T + def __init__(self, value: _T) -> None: ... From 5dd062a14b2d2c9b5c73fd266f166f4031e746a1 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 14 Jun 2024 02:35:22 -0700 Subject: [PATCH 031/247] Use inline config in the optional error codes docs (#17374) This page already says: > The examples in this section use inline configuration But some of these examples predate support for inline configuration of error codes that was added in #13502 --- docs/source/error_code_list2.rst | 20 ++++++++++---------- docs/source/error_codes.rst | 10 +++++++--- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst index 465d1c7a6583..2b765e412913 100644 --- a/docs/source/error_code_list2.rst +++ b/docs/source/error_code_list2.rst @@ -5,8 +5,8 @@ Error codes for optional checks This section documents various errors codes that mypy generates only if you enable certain options. See :ref:`error-codes` for general -documentation about error codes. :ref:`error-code-list` documents -error codes that are enabled by default. +documentation about error codes and their configuration. +:ref:`error-code-list` documents error codes that are enabled by default. .. note:: @@ -241,7 +241,7 @@ mypy generates an error if it thinks that an expression is redundant. .. code-block:: python - # Use "mypy --enable-error-code redundant-expr ..." + # mypy: enable-error-code="redundant-expr" def example(x: int) -> None: # Error: Left operand of "and" is always true [redundant-expr] @@ -268,7 +268,7 @@ example: .. code-block:: python - # Use "mypy --enable-error-code possibly-undefined ..." + # mypy: enable-error-code="possibly-undefined" from typing import Iterable @@ -297,7 +297,7 @@ Using an iterable value in a boolean context has a separate error code .. code-block:: python - # Use "mypy --enable-error-code truthy-bool ..." + # mypy: enable-error-code="truthy-bool" class Foo: pass @@ -347,7 +347,7 @@ Example: .. code-block:: python - # Use "mypy --enable-error-code ignore-without-code ..." + # mypy: enable-error-code="ignore-without-code" class Foo: def __init__(self, name: str) -> None: @@ -378,7 +378,7 @@ Example: .. code-block:: python - # Use "mypy --enable-error-code unused-awaitable ..." + # mypy: enable-error-code="unused-awaitable" import asyncio @@ -462,7 +462,7 @@ Example: .. code-block:: python - # Use "mypy --enable-error-code explicit-override ..." + # mypy: enable-error-code="explicit-override" from typing import override @@ -536,7 +536,7 @@ Now users can actually import ``reveal_type`` to make the runtime code safe. .. code-block:: python - # Use "mypy --enable-error-code unimported-reveal" + # mypy: enable-error-code="unimported-reveal" x = 1 reveal_type(x) # Note: Revealed type is "builtins.int" \ @@ -546,7 +546,7 @@ Correct usage: .. code-block:: python - # Use "mypy --enable-error-code unimported-reveal" + # mypy: enable-error-code="unimported-reveal" from typing import reveal_type # or `typing_extensions` x = 1 diff --git a/docs/source/error_codes.rst b/docs/source/error_codes.rst index 35fad161f8a2..485d70cb59bc 100644 --- a/docs/source/error_codes.rst +++ b/docs/source/error_codes.rst @@ -87,9 +87,13 @@ still keep the other two error codes enabled. The overall logic is following: * Individual config sections *adjust* them per glob/module -* Inline ``# mypy: disable-error-code="..."`` comments can further - *adjust* them for a specific module. - For example: ``# mypy: disable-error-code="truthy-bool, ignore-without-code"`` +* Inline ``# mypy: disable-error-code="..."`` and ``# mypy: enable-error-code="..."`` + comments can further *adjust* them for a specific file. + For example: + +.. code-block:: python + + # mypy: enable-error-code="truthy-bool, ignore-without-code" So one can e.g. enable some code globally, disable it for all tests in the corresponding config section, and then re-enable it with an inline From dac88f346cada58ae022599fffbbd961643b5d5b Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 14 Jun 2024 17:35:19 +0300 Subject: [PATCH 032/247] Support `enum.member` for python3.11+ (#17382) There are no tests for `@enum.member` used as a decorator, because I can only decorate classes and functions, which are not supported right now: https://mypy-play.net/?mypy=latest&python=3.12&gist=449ee8c12eba9f807cfc7832f1ea2c49 ```python import enum class A(enum.Enum): class x: ... reveal_type(A.x) # Revealed type is "def () -> __main__.A.x" ``` This issue is separate and rather complex, so I would prefer to solve it independently. Refs https://github.com/python/mypy/pull/17376 --------- Co-authored-by: Alex Waygood --- mypy/plugins/default.py | 4 +++- mypy/plugins/enums.py | 18 ++++++++++++++++++ test-data/unit/check-enum.test | 19 +++++++++++++++++++ test-data/unit/lib-stub/enum.pyi | 5 +++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 3ad301a15f6c..5139b9b82289 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -41,7 +41,7 @@ class DefaultPlugin(Plugin): """Type checker plugin that is enabled by default.""" def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None: - from mypy.plugins import ctypes, singledispatch + from mypy.plugins import ctypes, enums, singledispatch if fullname == "_ctypes.Array": return ctypes.array_constructor_callback @@ -51,6 +51,8 @@ def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] import mypy.plugins.functools return mypy.plugins.functools.partial_new_callback + elif fullname == "enum.member": + return enums.enum_member_callback return None diff --git a/mypy/plugins/enums.py b/mypy/plugins/enums.py index 167b330f9b09..816241fa6e9a 100644 --- a/mypy/plugins/enums.py +++ b/mypy/plugins/enums.py @@ -87,6 +87,8 @@ def _infer_value_type_with_auto_fallback( return None proper_type = get_proper_type(fixup_partial_type(proper_type)) if not (isinstance(proper_type, Instance) and proper_type.type.fullname == "enum.auto"): + if is_named_instance(proper_type, "enum.member") and proper_type.args: + return proper_type.args[0] return proper_type assert isinstance(ctx.type, Instance), "An incorrect ctx.type was passed." info = ctx.type.type @@ -126,6 +128,22 @@ def _implements_new(info: TypeInfo) -> bool: return type_with_new.fullname not in ("enum.Enum", "enum.IntEnum", "enum.StrEnum") +def enum_member_callback(ctx: mypy.plugin.FunctionContext) -> Type: + """By default `member(1)` will be infered as `member[int]`, + we want to improve the inference to be `Literal[1]` here.""" + if ctx.arg_types or ctx.arg_types[0]: + arg = get_proper_type(ctx.arg_types[0][0]) + proper_return = get_proper_type(ctx.default_return_type) + if ( + isinstance(arg, Instance) + and arg.last_known_value + and isinstance(proper_return, Instance) + and len(proper_return.args) == 1 + ): + return proper_return.copy_modified(args=[arg]) + return ctx.default_return_type + + def enum_value_callback(ctx: mypy.plugin.AttributeContext) -> Type: """This plugin refines the 'value' attribute in enums to refer to the original underlying value. For example, suppose we have the diff --git a/test-data/unit/check-enum.test b/test-data/unit/check-enum.test index 183901416604..d53935085325 100644 --- a/test-data/unit/check-enum.test +++ b/test-data/unit/check-enum.test @@ -2166,3 +2166,22 @@ class Other(Enum): reveal_type(Other.a) # N: Revealed type is "Literal[__main__.Other.a]?" reveal_type(Other.Support.b) # N: Revealed type is "builtins.int" [builtins fixtures/dict.pyi] + + +[case testEnumMemberSupport] +# flags: --python-version 3.11 +# This was added in 3.11 +from enum import Enum, member + +class A(Enum): + x = member(1) + y = 2 + +reveal_type(A.x) # N: Revealed type is "Literal[__main__.A.x]?" +reveal_type(A.x.value) # N: Revealed type is "Literal[1]?" +reveal_type(A.y) # N: Revealed type is "Literal[__main__.A.y]?" +reveal_type(A.y.value) # N: Revealed type is "Literal[2]?" + +def some_a(a: A): + reveal_type(a.value) # N: Revealed type is "Union[Literal[1]?, Literal[2]?]" +[builtins fixtures/dict.pyi] diff --git a/test-data/unit/lib-stub/enum.pyi b/test-data/unit/lib-stub/enum.pyi index 32dd7c38d251..0e0b8e025d9f 100644 --- a/test-data/unit/lib-stub/enum.pyi +++ b/test-data/unit/lib-stub/enum.pyi @@ -53,3 +53,8 @@ class StrEnum(str, Enum): class nonmember(Generic[_T]): value: _T def __init__(self, value: _T) -> None: ... + +# It is python-3.11+ only: +class member(Generic[_T]): + value: _T + def __init__(self, value: _T) -> None: ... From 740d39ecc31c64675439e6796dcf46e6a9110896 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 14 Jun 2024 16:43:56 +0100 Subject: [PATCH 033/247] [PEP 695] Add more error checks and tests for error conditions (#17339) Detect invalid number of constrained types. At least two are required, according do PEP 695. Add tests for other simple errors. Work on #15238. --- mypy/errorcodes.py | 2 +- mypy/fastparse.py | 12 +++++++++-- mypy/message_registry.py | 3 +++ mypy/semanal.py | 9 +++++++-- test-data/unit/check-python312.test | 31 +++++++++++++++++++++++++++++ test-data/unit/semanal-errors.test | 2 +- 6 files changed, 53 insertions(+), 6 deletions(-) diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index 7de796a70c8d..6e8763264ddd 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -271,7 +271,7 @@ def __hash__(self) -> int: del error_codes[FILE.code] # This is a catch-all for remaining uncategorized errors. -MISC: Final = ErrorCode("misc", "Miscellaneous other checks", "General") +MISC: Final[ErrorCode] = ErrorCode("misc", "Miscellaneous other checks", "General") OVERLOAD_OVERLAP: Final[ErrorCode] = ErrorCode( "overload-overlap", diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 49f0a938b750..70afe9010583 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1185,8 +1185,16 @@ def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: explicit_type_params.append(TypeParam(p.name, TYPE_VAR_TUPLE_KIND, None, [])) else: if isinstance(p.bound, ast3.Tuple): - conv = TypeConverter(self.errors, line=p.lineno) - values = [conv.visit(t) for t in p.bound.elts] + if len(p.bound.elts) < 2: + self.fail( + message_registry.TYPE_VAR_TOO_FEW_CONSTRAINED_TYPES, + p.lineno, + p.col_offset, + blocker=False, + ) + else: + conv = TypeConverter(self.errors, line=p.lineno) + values = [conv.visit(t) for t in p.bound.elts] elif p.bound is not None: bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) explicit_type_params.append(TypeParam(p.name, TYPE_VAR_KIND, bound, values)) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 52bd9a1ce00c..befacc9e6182 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -334,3 +334,6 @@ def with_additional_msg(self, info: str) -> ErrorMessage: NARROWED_TYPE_NOT_SUBTYPE: Final = ErrorMessage( "Narrowed type {} is not a subtype of input type {}", codes.NARROWED_TYPE_NOT_SUBTYPE ) +TYPE_VAR_TOO_FEW_CONSTRAINED_TYPES: Final = ErrorMessage( + "Type variable must have at least two constrained types", codes.MISC +) diff --git a/mypy/semanal.py b/mypy/semanal.py index 8da5c68d562d..d2f02d4835e2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -59,6 +59,7 @@ from mypy.errorcodes import PROPERTY_DECORATOR, ErrorCode from mypy.errors import Errors, report_internal_error from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type +from mypy.message_registry import ErrorMessage from mypy.messages import ( SUGGESTED_TEST_FIXTURES, TYPES_FOR_UNIMPORTED_HINTS, @@ -4618,7 +4619,7 @@ def process_typevar_parameters( self.fail("TypeVar cannot be both covariant and contravariant", context) return None elif num_values == 1: - self.fail("TypeVar cannot have only a single constraint", context) + self.fail(message_registry.TYPE_VAR_TOO_FEW_CONSTRAINED_TYPES, context) return None elif covariant: variance = COVARIANT @@ -7034,7 +7035,7 @@ def in_checked_function(self) -> bool: def fail( self, - msg: str, + msg: str | ErrorMessage, ctx: Context, serious: bool = False, *, @@ -7045,6 +7046,10 @@ def fail( return # In case it's a bug and we don't really have context assert ctx is not None, msg + if isinstance(msg, ErrorMessage): + if code is None: + code = msg.code + msg = msg.value self.errors.report(ctx.line, ctx.column, msg, blocker=blocker, code=code) def note(self, msg: str, ctx: Context, code: ErrorCode | None = None) -> None: diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index a1c819667087..06c5bada1e92 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1494,3 +1494,34 @@ reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" # flags: --enable-incomplete-feature=NewGenericSyntax def f[T](x: foobar, y: T) -> T: ... # E: Name "foobar" is not defined reveal_type(f) # N: Revealed type is "def [T] (x: Any, y: T`-1) -> T`-1" + +[case testPEP695WrongNumberOfConstrainedTypes] +# flags: --enable-incomplete-feature=NewGenericSyntax +type A[T: ()] = list[T] # E: Type variable must have at least two constrained types +a: A[int] +reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" + +type B[T: (int,)] = list[T] # E: Type variable must have at least two constrained types +b: B[str] +reveal_type(b) # N: Revealed type is "builtins.list[builtins.str]" + +[case testPEP695UsingTypeVariableInOwnBoundOrConstraint] +# flags: --enable-incomplete-feature=NewGenericSyntax +type A[T: list[T]] = str # E: Name "T" is not defined +type B[S: (list[S], str)] = str # E: Name "S" is not defined +type C[T, S: list[T]] = str # E: Name "T" is not defined + +def f[T: T](x: T) -> T: ... # E: Name "T" is not defined +class D[T: T]: # E: Name "T" is not defined + pass + +[case testPEP695InvalidType] +# flags: --enable-incomplete-feature=NewGenericSyntax +def f[T: 1](x: T) -> T: ... # E: Invalid type: try using Literal[1] instead? +class C[T: (int, (1 + 2))]: pass # E: Invalid type comment or annotation +type A = list[1] # E: Invalid type: try using Literal[1] instead? +type B = (1 + 2) # E: Invalid type alias: expression is not a valid type +a: A +reveal_type(a) # N: Revealed type is "builtins.list[Any]" +b: B +reveal_type(b) # N: Revealed type is "Any" diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 269536f868a4..33c8f9b80aa0 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -1046,7 +1046,7 @@ T = TypeVar(b'T') # E: TypeVar() expects a string literal as first argument d = TypeVar('D') # E: String argument 1 "D" to TypeVar(...) does not match variable name "d" e = TypeVar('e', int, str, x=1) # E: Unexpected argument to "TypeVar()": "x" f = TypeVar('f', (int, str), int) # E: Type expected -g = TypeVar('g', int) # E: TypeVar cannot have only a single constraint +g = TypeVar('g', int) # E: Type variable must have at least two constrained types h = TypeVar('h', x=(int, str)) # E: Unexpected argument to "TypeVar()": "x" i = TypeVar('i', bound=1) # E: TypeVar "bound" must be a type j = TypeVar('j', covariant=None) # E: TypeVar "covariant" may only be a literal bool From 693e1d8d74ba331c2bd36ff4774c9a4eefe03ab6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 15 Jun 2024 15:28:25 +0100 Subject: [PATCH 034/247] Add a test case for no overload overlap with self-types (#17388) Fixes https://github.com/python/mypy/issues/14641 I don't think we have a test case for this, and although it is a bit in a grey area (because of `x: Any = None`), I think we should allow this. --- test-data/unit/check-overloading.test | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 7bca5cc7b508..bcb775ba5dac 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -4542,6 +4542,23 @@ reveal_type(Child().foo("...")) # N: Revealed type is "builtins.st reveal_type(Child().foo(x)) # N: Revealed type is "Union[__main__.Child, builtins.str]" reveal_type(Child().foo(3).child_only()) # N: Revealed type is "builtins.int" +[case testOverloadAndSelfTypesGenericNoOverlap] +from typing import Generic, TypeVar, Any, overload, Self, Union + +T = TypeVar("T", bound=Any) +class C(Generic[T]): + @overload + def get(self, obj: None) -> Self: ... + @overload + def get(self, obj: Any) -> T: ... + def get(self, obj: Union[Any, None]) -> Union[T, Self]: + return self + +class D(C[int]): ... +d: D +reveal_type(d.get(None)) # N: Revealed type is "__main__.D" +reveal_type(d.get("whatever")) # N: Revealed type is "builtins.int" + [case testOverloadAndClassTypes] from typing import overload, Union, TypeVar, Type From b81b9e0a230f135c9b8abdafba674922e37e1a88 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Jun 2024 12:25:18 +0100 Subject: [PATCH 035/247] [mypyc] Sync pythoncapi_compat.h (#17390) This helps with Python 3.13 support. --- mypyc/lib-rt/pythoncapi_compat.h | 1071 +++++++++++++++++++++++++++--- 1 file changed, 967 insertions(+), 104 deletions(-) diff --git a/mypyc/lib-rt/pythoncapi_compat.h b/mypyc/lib-rt/pythoncapi_compat.h index f22e92f7358f..1b59f93de7ec 100644 --- a/mypyc/lib-rt/pythoncapi_compat.h +++ b/mypyc/lib-rt/pythoncapi_compat.h @@ -19,34 +19,25 @@ extern "C" { #endif #include -#include "frameobject.h" // PyFrameObject, PyFrame_GetBack() - -// Compatibility with Visual Studio 2013 and older which don't support -// the inline keyword in C (only in C++): use __inline instead. -#if (defined(_MSC_VER) && _MSC_VER < 1900 \ - && !defined(__cplusplus) && !defined(inline)) -# define PYCAPI_COMPAT_STATIC_INLINE(TYPE) static __inline TYPE -#else -# define PYCAPI_COMPAT_STATIC_INLINE(TYPE) static inline TYPE +// Python 3.11.0b4 added PyFrame_Back() to Python.h +#if PY_VERSION_HEX < 0x030b00B4 && !defined(PYPY_VERSION) +# include "frameobject.h" // PyFrameObject, PyFrame_GetBack() #endif -// C++ compatibility: _Py_CAST() and _Py_NULL #ifndef _Py_CAST -# ifdef __cplusplus -# define _Py_CAST(type, expr) \ - const_cast(reinterpret_cast(expr)) -# else -# define _Py_CAST(type, expr) ((type)(expr)) -# endif +# define _Py_CAST(type, expr) ((type)(expr)) #endif -#ifndef _Py_NULL -# ifdef __cplusplus -# define _Py_NULL nullptr -# else -# define _Py_NULL NULL -# endif + +// Static inline functions should use _Py_NULL rather than using directly NULL +// to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, +// _Py_NULL is defined as nullptr. +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + || (defined(__cplusplus) && __cplusplus >= 201103) +# define _Py_NULL nullptr +#else +# define _Py_NULL NULL #endif // Cast argument to PyObject* type. @@ -57,8 +48,7 @@ extern "C" { // bpo-42262 added Py_NewRef() to Python 3.10.0a3 #if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_NewRef) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -_Py_NewRef(PyObject *obj) +static inline PyObject* _Py_NewRef(PyObject *obj) { Py_INCREF(obj); return obj; @@ -69,8 +59,7 @@ _Py_NewRef(PyObject *obj) // bpo-42262 added Py_XNewRef() to Python 3.10.0a3 #if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_XNewRef) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -_Py_XNewRef(PyObject *obj) +static inline PyObject* _Py_XNewRef(PyObject *obj) { Py_XINCREF(obj); return obj; @@ -81,8 +70,7 @@ _Py_XNewRef(PyObject *obj) // bpo-39573 added Py_SET_REFCNT() to Python 3.9.0a4 #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_REFCNT) -PYCAPI_COMPAT_STATIC_INLINE(void) -_Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) +static inline void _Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { ob->ob_refcnt = refcnt; } @@ -93,18 +81,20 @@ _Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) // Py_SETREF() and Py_XSETREF() were added to Python 3.5.2. // It is excluded from the limited C API. #if (PY_VERSION_HEX < 0x03050200 && !defined(Py_SETREF)) && !defined(Py_LIMITED_API) -#define Py_SETREF(op, op2) \ - do { \ - PyObject *_py_tmp = _PyObject_CAST(op); \ - (op) = (op2); \ - Py_DECREF(_py_tmp); \ +#define Py_SETREF(dst, src) \ + do { \ + PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ + PyObject *_tmp_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = _PyObject_CAST(src); \ + Py_DECREF(_tmp_dst); \ } while (0) -#define Py_XSETREF(op, op2) \ - do { \ - PyObject *_py_tmp = _PyObject_CAST(op); \ - (op) = (op2); \ - Py_XDECREF(_py_tmp); \ +#define Py_XSETREF(dst, src) \ + do { \ + PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \ + PyObject *_tmp_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = _PyObject_CAST(src); \ + Py_XDECREF(_tmp_dst); \ } while (0) #endif @@ -117,18 +107,17 @@ _Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) #if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsNone) # define Py_IsNone(x) Py_Is(x, Py_None) #endif -#if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsTrue) +#if (PY_VERSION_HEX < 0x030A00B1 || defined(PYPY_VERSION)) && !defined(Py_IsTrue) # define Py_IsTrue(x) Py_Is(x, Py_True) #endif -#if PY_VERSION_HEX < 0x030A00B1 && !defined(Py_IsFalse) +#if (PY_VERSION_HEX < 0x030A00B1 || defined(PYPY_VERSION)) && !defined(Py_IsFalse) # define Py_IsFalse(x) Py_Is(x, Py_False) #endif // bpo-39573 added Py_SET_TYPE() to Python 3.9.0a4 #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_TYPE) -PYCAPI_COMPAT_STATIC_INLINE(void) -_Py_SET_TYPE(PyObject *ob, PyTypeObject *type) +static inline void _Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } @@ -138,8 +127,7 @@ _Py_SET_TYPE(PyObject *ob, PyTypeObject *type) // bpo-39573 added Py_SET_SIZE() to Python 3.9.0a4 #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_SIZE) -PYCAPI_COMPAT_STATIC_INLINE(void) -_Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) +static inline void _Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { ob->ob_size = size; } @@ -148,9 +136,8 @@ _Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) // bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1 -#if PY_VERSION_HEX < 0x030900B1 -PYCAPI_COMPAT_STATIC_INLINE(PyCodeObject*) -PyFrame_GetCode(PyFrameObject *frame) +#if PY_VERSION_HEX < 0x030900B1 || defined(PYPY_VERSION) +static inline PyCodeObject* PyFrame_GetCode(PyFrameObject *frame) { assert(frame != _Py_NULL); assert(frame->f_code != _Py_NULL); @@ -158,8 +145,7 @@ PyFrame_GetCode(PyFrameObject *frame) } #endif -PYCAPI_COMPAT_STATIC_INLINE(PyCodeObject*) -_PyFrame_GetCodeBorrow(PyFrameObject *frame) +static inline PyCodeObject* _PyFrame_GetCodeBorrow(PyFrameObject *frame) { PyCodeObject *code = PyFrame_GetCode(frame); Py_DECREF(code); @@ -169,8 +155,7 @@ _PyFrame_GetCodeBorrow(PyFrameObject *frame) // bpo-40421 added PyFrame_GetBack() to Python 3.9.0b1 #if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) -PyFrame_GetBack(PyFrameObject *frame) +static inline PyFrameObject* PyFrame_GetBack(PyFrameObject *frame) { assert(frame != _Py_NULL); return _Py_CAST(PyFrameObject*, Py_XNewRef(frame->f_back)); @@ -178,8 +163,7 @@ PyFrame_GetBack(PyFrameObject *frame) #endif #if !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) -_PyFrame_GetBackBorrow(PyFrameObject *frame) +static inline PyFrameObject* _PyFrame_GetBackBorrow(PyFrameObject *frame) { PyFrameObject *back = PyFrame_GetBack(frame); Py_XDECREF(back); @@ -190,8 +174,7 @@ _PyFrame_GetBackBorrow(PyFrameObject *frame) // bpo-40421 added PyFrame_GetLocals() to Python 3.11.0a7 #if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyFrame_GetLocals(PyFrameObject *frame) +static inline PyObject* PyFrame_GetLocals(PyFrameObject *frame) { #if PY_VERSION_HEX >= 0x030400B1 if (PyFrame_FastToLocalsWithError(frame) < 0) { @@ -207,8 +190,7 @@ PyFrame_GetLocals(PyFrameObject *frame) // bpo-40421 added PyFrame_GetGlobals() to Python 3.11.0a7 #if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyFrame_GetGlobals(PyFrameObject *frame) +static inline PyObject* PyFrame_GetGlobals(PyFrameObject *frame) { return Py_NewRef(frame->f_globals); } @@ -217,8 +199,7 @@ PyFrame_GetGlobals(PyFrameObject *frame) // bpo-40421 added PyFrame_GetBuiltins() to Python 3.11.0a7 #if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyFrame_GetBuiltins(PyFrameObject *frame) +static inline PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) { return Py_NewRef(frame->f_builtins); } @@ -227,8 +208,7 @@ PyFrame_GetBuiltins(PyFrameObject *frame) // bpo-40421 added PyFrame_GetLasti() to Python 3.11.0b1 #if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(int) -PyFrame_GetLasti(PyFrameObject *frame) +static inline int PyFrame_GetLasti(PyFrameObject *frame) { #if PY_VERSION_HEX >= 0x030A00A7 // bpo-27129: Since Python 3.10.0a7, f_lasti is an instruction offset, @@ -245,9 +225,63 @@ PyFrame_GetLasti(PyFrameObject *frame) #endif +// gh-91248 added PyFrame_GetVar() to Python 3.12.0a2 +#if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) +static inline PyObject* PyFrame_GetVar(PyFrameObject *frame, PyObject *name) +{ + PyObject *locals, *value; + + locals = PyFrame_GetLocals(frame); + if (locals == NULL) { + return NULL; + } +#if PY_VERSION_HEX >= 0x03000000 + value = PyDict_GetItemWithError(locals, name); +#else + value = _PyDict_GetItemWithError(locals, name); +#endif + Py_DECREF(locals); + + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } +#if PY_VERSION_HEX >= 0x03000000 + PyErr_Format(PyExc_NameError, "variable %R does not exist", name); +#else + PyErr_SetString(PyExc_NameError, "variable does not exist"); +#endif + return NULL; + } + return Py_NewRef(value); +} +#endif + + +// gh-91248 added PyFrame_GetVarString() to Python 3.12.0a2 +#if PY_VERSION_HEX < 0x030C00A2 && !defined(PYPY_VERSION) +static inline PyObject* +PyFrame_GetVarString(PyFrameObject *frame, const char *name) +{ + PyObject *name_obj, *value; +#if PY_VERSION_HEX >= 0x03000000 + name_obj = PyUnicode_FromString(name); +#else + name_obj = PyString_FromString(name); +#endif + if (name_obj == NULL) { + return NULL; + } + value = PyFrame_GetVar(frame, name_obj); + Py_DECREF(name_obj); + return value; +} +#endif + + // bpo-39947 added PyThreadState_GetInterpreter() to Python 3.9.0a5 -#if PY_VERSION_HEX < 0x030900A5 -PYCAPI_COMPAT_STATIC_INLINE(PyInterpreterState *) +#if PY_VERSION_HEX < 0x030900A5 || defined(PYPY_VERSION) +static inline PyInterpreterState * PyThreadState_GetInterpreter(PyThreadState *tstate) { assert(tstate != _Py_NULL); @@ -258,8 +292,7 @@ PyThreadState_GetInterpreter(PyThreadState *tstate) // bpo-40429 added PyThreadState_GetFrame() to Python 3.9.0b1 #if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) -PyThreadState_GetFrame(PyThreadState *tstate) +static inline PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) { assert(tstate != _Py_NULL); return _Py_CAST(PyFrameObject *, Py_XNewRef(tstate->frame)); @@ -267,7 +300,7 @@ PyThreadState_GetFrame(PyThreadState *tstate) #endif #if !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyFrameObject*) +static inline PyFrameObject* _PyThreadState_GetFrameBorrow(PyThreadState *tstate) { PyFrameObject *frame = PyThreadState_GetFrame(tstate); @@ -278,9 +311,8 @@ _PyThreadState_GetFrameBorrow(PyThreadState *tstate) // bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a5 -#if PY_VERSION_HEX < 0x030900A5 -PYCAPI_COMPAT_STATIC_INLINE(PyInterpreterState*) -PyInterpreterState_Get(void) +#if PY_VERSION_HEX < 0x030900A5 || defined(PYPY_VERSION) +static inline PyInterpreterState* PyInterpreterState_Get(void) { PyThreadState *tstate; PyInterpreterState *interp; @@ -300,8 +332,7 @@ PyInterpreterState_Get(void) // bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a6 #if 0x030700A1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030900A6 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(uint64_t) -PyThreadState_GetID(PyThreadState *tstate) +static inline uint64_t PyThreadState_GetID(PyThreadState *tstate) { assert(tstate != _Py_NULL); return tstate->id; @@ -310,8 +341,7 @@ PyThreadState_GetID(PyThreadState *tstate) // bpo-43760 added PyThreadState_EnterTracing() to Python 3.11.0a2 #if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(void) -PyThreadState_EnterTracing(PyThreadState *tstate) +static inline void PyThreadState_EnterTracing(PyThreadState *tstate) { tstate->tracing++; #if PY_VERSION_HEX >= 0x030A00A1 @@ -324,8 +354,7 @@ PyThreadState_EnterTracing(PyThreadState *tstate) // bpo-43760 added PyThreadState_LeaveTracing() to Python 3.11.0a2 #if PY_VERSION_HEX < 0x030B00A2 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(void) -PyThreadState_LeaveTracing(PyThreadState *tstate) +static inline void PyThreadState_LeaveTracing(PyThreadState *tstate) { int use_tracing = (tstate->c_tracefunc != _Py_NULL || tstate->c_profilefunc != _Py_NULL); @@ -340,9 +369,9 @@ PyThreadState_LeaveTracing(PyThreadState *tstate) // bpo-37194 added PyObject_CallNoArgs() to Python 3.9.0a1 -#if PY_VERSION_HEX < 0x030900A1 -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyObject_CallNoArgs(PyObject *func) +// PyObject_CallNoArgs() added to PyPy 3.9.16-v7.3.11 +#if !defined(PyObject_CallNoArgs) && PY_VERSION_HEX < 0x030900A1 +static inline PyObject* PyObject_CallNoArgs(PyObject *func) { return PyObject_CallFunctionObjArgs(func, NULL); } @@ -351,9 +380,9 @@ PyObject_CallNoArgs(PyObject *func) // bpo-39245 made PyObject_CallOneArg() public (previously called // _PyObject_CallOneArg) in Python 3.9.0a4 -#if PY_VERSION_HEX < 0x030900A4 -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyObject_CallOneArg(PyObject *func, PyObject *arg) +// PyObject_CallOneArg() added to PyPy 3.9.16-v7.3.11 +#if !defined(PyObject_CallOneArg) && PY_VERSION_HEX < 0x030900A4 +static inline PyObject* PyObject_CallOneArg(PyObject *func, PyObject *arg) { return PyObject_CallFunctionObjArgs(func, arg, NULL); } @@ -362,10 +391,19 @@ PyObject_CallOneArg(PyObject *func, PyObject *arg) // bpo-1635741 added PyModule_AddObjectRef() to Python 3.10.0a3 #if PY_VERSION_HEX < 0x030A00A3 -PYCAPI_COMPAT_STATIC_INLINE(int) +static inline int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) { int res; + + if (!value && !PyErr_Occurred()) { + // PyModule_AddObject() raises TypeError in this case + PyErr_SetString(PyExc_SystemError, + "PyModule_AddObjectRef() must be called " + "with an exception raised if value is NULL"); + return -1; + } + Py_XINCREF(value); res = PyModule_AddObject(module, name, value); if (res < 0) { @@ -378,8 +416,7 @@ PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) // bpo-40024 added PyModule_AddType() to Python 3.9.0a5 #if PY_VERSION_HEX < 0x030900A5 -PYCAPI_COMPAT_STATIC_INLINE(int) -PyModule_AddType(PyObject *module, PyTypeObject *type) +static inline int PyModule_AddType(PyObject *module, PyTypeObject *type) { const char *name, *dot; @@ -403,8 +440,7 @@ PyModule_AddType(PyObject *module, PyTypeObject *type) // bpo-40241 added PyObject_GC_IsTracked() to Python 3.9.0a6. // bpo-4688 added _PyObject_GC_IS_TRACKED() to Python 2.7.0a2. #if PY_VERSION_HEX < 0x030900A6 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(int) -PyObject_GC_IsTracked(PyObject* obj) +static inline int PyObject_GC_IsTracked(PyObject* obj) { return (PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)); } @@ -413,8 +449,7 @@ PyObject_GC_IsTracked(PyObject* obj) // bpo-40241 added PyObject_GC_IsFinalized() to Python 3.9.0a6. // bpo-18112 added _PyGCHead_FINALIZED() to Python 3.4.0 final. #if PY_VERSION_HEX < 0x030900A6 && PY_VERSION_HEX >= 0x030400F0 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(int) -PyObject_GC_IsFinalized(PyObject *obj) +static inline int PyObject_GC_IsFinalized(PyObject *obj) { PyGC_Head *gc = _Py_CAST(PyGC_Head*, obj) - 1; return (PyObject_IS_GC(obj) && _PyGCHead_FINALIZED(gc)); @@ -424,8 +459,7 @@ PyObject_GC_IsFinalized(PyObject *obj) // bpo-39573 added Py_IS_TYPE() to Python 3.9.0a4 #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_IS_TYPE) -PYCAPI_COMPAT_STATIC_INLINE(int) -_Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { +static inline int _Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; } #define Py_IS_TYPE(ob, type) _Py_IS_TYPE(_PyObject_CAST(ob), type) @@ -437,12 +471,10 @@ _Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { // Python 3.11a2 moved _PyFloat_Pack2() and _PyFloat_Unpack2() to the internal // C API: Python 3.11a2-3.11a6 versions are not supported. #if 0x030600B1 <= PY_VERSION_HEX && PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(int) -PyFloat_Pack2(double x, char *p, int le) +static inline int PyFloat_Pack2(double x, char *p, int le) { return _PyFloat_Pack2(x, (unsigned char*)p, le); } -PYCAPI_COMPAT_STATIC_INLINE(double) -PyFloat_Unpack2(const char *p, int le) +static inline double PyFloat_Unpack2(const char *p, int le) { return _PyFloat_Unpack2((const unsigned char *)p, le); } #endif @@ -453,34 +485,54 @@ PyFloat_Unpack2(const char *p, int le) // and _PyFloat_Unpack8() to the internal C API: Python 3.11a2-3.11a6 versions // are not supported. #if PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(int) -PyFloat_Pack4(double x, char *p, int le) +static inline int PyFloat_Pack4(double x, char *p, int le) { return _PyFloat_Pack4(x, (unsigned char*)p, le); } -PYCAPI_COMPAT_STATIC_INLINE(int) -PyFloat_Pack8(double x, char *p, int le) +static inline int PyFloat_Pack8(double x, char *p, int le) { return _PyFloat_Pack8(x, (unsigned char*)p, le); } -PYCAPI_COMPAT_STATIC_INLINE(double) -PyFloat_Unpack4(const char *p, int le) +static inline double PyFloat_Unpack4(const char *p, int le) { return _PyFloat_Unpack4((const unsigned char *)p, le); } -PYCAPI_COMPAT_STATIC_INLINE(double) -PyFloat_Unpack8(const char *p, int le) +static inline double PyFloat_Unpack8(const char *p, int le) { return _PyFloat_Unpack8((const unsigned char *)p, le); } #endif // gh-92154 added PyCode_GetCode() to Python 3.11.0b1 #if PY_VERSION_HEX < 0x030B00B1 && !defined(PYPY_VERSION) -PYCAPI_COMPAT_STATIC_INLINE(PyObject*) -PyCode_GetCode(PyCodeObject *code) +static inline PyObject* PyCode_GetCode(PyCodeObject *code) { return Py_NewRef(code->co_code); } #endif +// gh-95008 added PyCode_GetVarnames() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetVarnames(PyCodeObject *code) +{ + return Py_NewRef(code->co_varnames); +} +#endif + +// gh-95008 added PyCode_GetFreevars() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetFreevars(PyCodeObject *code) +{ + return Py_NewRef(code->co_freevars); +} +#endif + +// gh-95008 added PyCode_GetCellvars() to Python 3.11.0rc1 +#if PY_VERSION_HEX < 0x030B00C1 && !defined(PYPY_VERSION) +static inline PyObject* PyCode_GetCellvars(PyCodeObject *code) +{ + return Py_NewRef(code->co_cellvars); +} +#endif + + // Py_UNUSED() was added to Python 3.4.0b2. #if PY_VERSION_HEX < 0x030400B2 && !defined(Py_UNUSED) # if defined(__GNUC__) || defined(__clang__) @@ -491,6 +543,817 @@ PyCode_GetCode(PyCodeObject *code) #endif +// gh-105922 added PyImport_AddModuleRef() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A0 +static inline PyObject* PyImport_AddModuleRef(const char *name) +{ + return Py_XNewRef(PyImport_AddModule(name)); +} +#endif + + +// gh-105927 added PyWeakref_GetRef() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D0000 +static inline int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) +{ + PyObject *obj; + if (ref != NULL && !PyWeakref_Check(ref)) { + *pobj = NULL; + PyErr_SetString(PyExc_TypeError, "expected a weakref"); + return -1; + } + obj = PyWeakref_GetObject(ref); + if (obj == NULL) { + // SystemError if ref is NULL + *pobj = NULL; + return -1; + } + if (obj == Py_None) { + *pobj = NULL; + return 0; + } + *pobj = Py_NewRef(obj); + return (*pobj != NULL); +} +#endif + + +// bpo-36974 added PY_VECTORCALL_ARGUMENTS_OFFSET to Python 3.8b1 +#ifndef PY_VECTORCALL_ARGUMENTS_OFFSET +# define PY_VECTORCALL_ARGUMENTS_OFFSET (_Py_CAST(size_t, 1) << (8 * sizeof(size_t) - 1)) +#endif + +// bpo-36974 added PyVectorcall_NARGS() to Python 3.8b1 +#if PY_VERSION_HEX < 0x030800B1 +static inline Py_ssize_t PyVectorcall_NARGS(size_t n) +{ + return n & ~PY_VECTORCALL_ARGUMENTS_OFFSET; +} +#endif + + +// gh-105922 added PyObject_Vectorcall() to Python 3.9.0a4 +#if PY_VERSION_HEX < 0x030900A4 +static inline PyObject* +PyObject_Vectorcall(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ +#if PY_VERSION_HEX >= 0x030800B1 && !defined(PYPY_VERSION) + // bpo-36974 added _PyObject_Vectorcall() to Python 3.8.0b1 + return _PyObject_Vectorcall(callable, args, nargsf, kwnames); +#else + PyObject *posargs = NULL, *kwargs = NULL; + PyObject *res; + Py_ssize_t nposargs, nkwargs, i; + + if (nargsf != 0 && args == NULL) { + PyErr_BadInternalCall(); + goto error; + } + if (kwnames != NULL && !PyTuple_Check(kwnames)) { + PyErr_BadInternalCall(); + goto error; + } + + nposargs = (Py_ssize_t)PyVectorcall_NARGS(nargsf); + if (kwnames) { + nkwargs = PyTuple_GET_SIZE(kwnames); + } + else { + nkwargs = 0; + } + + posargs = PyTuple_New(nposargs); + if (posargs == NULL) { + goto error; + } + if (nposargs) { + for (i=0; i < nposargs; i++) { + PyTuple_SET_ITEM(posargs, i, Py_NewRef(*args)); + args++; + } + } + + if (nkwargs) { + kwargs = PyDict_New(); + if (kwargs == NULL) { + goto error; + } + + for (i = 0; i < nkwargs; i++) { + PyObject *key = PyTuple_GET_ITEM(kwnames, i); + PyObject *value = *args; + args++; + if (PyDict_SetItem(kwargs, key, value) < 0) { + goto error; + } + } + } + else { + kwargs = NULL; + } + + res = PyObject_Call(callable, posargs, kwargs); + Py_DECREF(posargs); + Py_XDECREF(kwargs); + return res; + +error: + Py_DECREF(posargs); + Py_XDECREF(kwargs); + return NULL; +#endif +} +#endif + + +// gh-106521 added PyObject_GetOptionalAttr() and +// PyObject_GetOptionalAttrString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_GetOptionalAttr(PyObject *obj, PyObject *attr_name, PyObject **result) +{ + // bpo-32571 added _PyObject_LookupAttr() to Python 3.7.0b1 +#if PY_VERSION_HEX >= 0x030700B1 && !defined(PYPY_VERSION) + return _PyObject_LookupAttr(obj, attr_name, result); +#else + *result = PyObject_GetAttr(obj, attr_name); + if (*result != NULL) { + return 1; + } + if (!PyErr_Occurred()) { + return 0; + } + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + return 0; + } + return -1; +#endif +} + +static inline int +PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result) +{ + PyObject *name_obj; + int rc; +#if PY_VERSION_HEX >= 0x03000000 + name_obj = PyUnicode_FromString(attr_name); +#else + name_obj = PyString_FromString(attr_name); +#endif + if (name_obj == NULL) { + *result = NULL; + return -1; + } + rc = PyObject_GetOptionalAttr(obj, name_obj, result); + Py_DECREF(name_obj); + return rc; +} +#endif + + +// gh-106307 added PyObject_GetOptionalAttr() and +// PyMapping_GetOptionalItemString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) +{ + *result = PyObject_GetItem(obj, key); + if (*result) { + return 1; + } + if (!PyErr_ExceptionMatches(PyExc_KeyError)) { + return -1; + } + PyErr_Clear(); + return 0; +} + +static inline int +PyMapping_GetOptionalItemString(PyObject *obj, const char *key, PyObject **result) +{ + PyObject *key_obj; + int rc; +#if PY_VERSION_HEX >= 0x03000000 + key_obj = PyUnicode_FromString(key); +#else + key_obj = PyString_FromString(key); +#endif + if (key_obj == NULL) { + *result = NULL; + return -1; + } + rc = PyMapping_GetOptionalItem(obj, key_obj, result); + Py_DECREF(key_obj); + return rc; +} +#endif + +// gh-108511 added PyMapping_HasKeyWithError() and +// PyMapping_HasKeyStringWithError() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyMapping_HasKeyWithError(PyObject *obj, PyObject *key) +{ + PyObject *res; + int rc = PyMapping_GetOptionalItem(obj, key, &res); + Py_XDECREF(res); + return rc; +} + +static inline int +PyMapping_HasKeyStringWithError(PyObject *obj, const char *key) +{ + PyObject *res; + int rc = PyMapping_GetOptionalItemString(obj, key, &res); + Py_XDECREF(res); + return rc; +} +#endif + + +// gh-108511 added PyObject_HasAttrWithError() and +// PyObject_HasAttrStringWithError() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_HasAttrWithError(PyObject *obj, PyObject *attr) +{ + PyObject *res; + int rc = PyObject_GetOptionalAttr(obj, attr, &res); + Py_XDECREF(res); + return rc; +} + +static inline int +PyObject_HasAttrStringWithError(PyObject *obj, const char *attr) +{ + PyObject *res; + int rc = PyObject_GetOptionalAttrString(obj, attr, &res); + Py_XDECREF(res); + return rc; +} +#endif + + +// gh-106004 added PyDict_GetItemRef() and PyDict_GetItemStringRef() +// to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result) +{ +#if PY_VERSION_HEX >= 0x03000000 + PyObject *item = PyDict_GetItemWithError(mp, key); +#else + PyObject *item = _PyDict_GetItemWithError(mp, key); +#endif + if (item != NULL) { + *result = Py_NewRef(item); + return 1; // found + } + if (!PyErr_Occurred()) { + *result = NULL; + return 0; // not found + } + *result = NULL; + return -1; +} + +static inline int +PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result) +{ + int res; +#if PY_VERSION_HEX >= 0x03000000 + PyObject *key_obj = PyUnicode_FromString(key); +#else + PyObject *key_obj = PyString_FromString(key); +#endif + if (key_obj == NULL) { + *result = NULL; + return -1; + } + res = PyDict_GetItemRef(mp, key_obj, result); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-106307 added PyModule_Add() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyModule_Add(PyObject *mod, const char *name, PyObject *value) +{ + int res = PyModule_AddObjectRef(mod, name, value); + Py_XDECREF(value); + return res; +} +#endif + + +// gh-108014 added Py_IsFinalizing() to Python 3.13.0a1 +// bpo-1856 added _Py_Finalizing to Python 3.2.1b1. +// _Py_IsFinalizing() was added to PyPy 7.3.0. +#if (0x030201B1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030D00A1) \ + && (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM >= 0x7030000) +static inline int Py_IsFinalizing(void) +{ +#if PY_VERSION_HEX >= 0x030700A1 + // _Py_IsFinalizing() was added to Python 3.7.0a1. + return _Py_IsFinalizing(); +#else + return (_Py_Finalizing != NULL); +#endif +} +#endif + + +// gh-108323 added PyDict_ContainsString() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int PyDict_ContainsString(PyObject *op, const char *key) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + return -1; + } + int res = PyDict_Contains(op, key_obj); + Py_DECREF(key_obj); + return res; +} +#endif + + +// gh-108445 added PyLong_AsInt() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int PyLong_AsInt(PyObject *obj) +{ +#ifdef PYPY_VERSION + long value = PyLong_AsLong(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + if (value < (long)INT_MIN || (long)INT_MAX < value) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)value; +#else + return _PyLong_AsInt(obj); +#endif +} +#endif + + +// gh-107073 added PyObject_VisitManagedDict() to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) +{ + PyObject **dict = _PyObject_GetDictPtr(obj); + if (*dict == NULL) { + return -1; + } + Py_VISIT(*dict); + return 0; +} + +static inline void +PyObject_ClearManagedDict(PyObject *obj) +{ + PyObject **dict = _PyObject_GetDictPtr(obj); + if (*dict == NULL) { + return; + } + Py_CLEAR(*dict); +} +#endif + +// gh-108867 added PyThreadState_GetUnchecked() to Python 3.13.0a1 +// Python 3.5.2 added _PyThreadState_UncheckedGet(). +#if PY_VERSION_HEX >= 0x03050200 && PY_VERSION_HEX < 0x030D00A1 +static inline PyThreadState* +PyThreadState_GetUnchecked(void) +{ + return _PyThreadState_UncheckedGet(); +} +#endif + +// gh-110289 added PyUnicode_EqualToUTF8() and PyUnicode_EqualToUTF8AndSize() +// to Python 3.13.0a1 +#if PY_VERSION_HEX < 0x030D00A1 +static inline int +PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *str, Py_ssize_t str_len) +{ + Py_ssize_t len; + const void *utf8; + PyObject *exc_type, *exc_value, *exc_tb; + int res; + + // API cannot report errors so save/restore the exception + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); + + // Python 3.3.0a1 added PyUnicode_AsUTF8AndSize() +#if PY_VERSION_HEX >= 0x030300A1 + if (PyUnicode_IS_ASCII(unicode)) { + utf8 = PyUnicode_DATA(unicode); + len = PyUnicode_GET_LENGTH(unicode); + } + else { + utf8 = PyUnicode_AsUTF8AndSize(unicode, &len); + if (utf8 == NULL) { + // Memory allocation failure. The API cannot report error, + // so ignore the exception and return 0. + res = 0; + goto done; + } + } + + if (len != str_len) { + res = 0; + goto done; + } + res = (memcmp(utf8, str, (size_t)len) == 0); +#else + PyObject *bytes = PyUnicode_AsUTF8String(unicode); + if (bytes == NULL) { + // Memory allocation failure. The API cannot report error, + // so ignore the exception and return 0. + res = 0; + goto done; + } + +#if PY_VERSION_HEX >= 0x03000000 + len = PyBytes_GET_SIZE(bytes); + utf8 = PyBytes_AS_STRING(bytes); +#else + len = PyString_GET_SIZE(bytes); + utf8 = PyString_AS_STRING(bytes); +#endif + if (len != str_len) { + Py_DECREF(bytes); + res = 0; + goto done; + } + + res = (memcmp(utf8, str, (size_t)len) == 0); + Py_DECREF(bytes); +#endif + +done: + PyErr_Restore(exc_type, exc_value, exc_tb); + return res; +} + +static inline int +PyUnicode_EqualToUTF8(PyObject *unicode, const char *str) +{ + return PyUnicode_EqualToUTF8AndSize(unicode, str, (Py_ssize_t)strlen(str)); +} +#endif + + +// gh-111138 added PyList_Extend() and PyList_Clear() to Python 3.13.0a2 +#if PY_VERSION_HEX < 0x030D00A2 +static inline int +PyList_Extend(PyObject *list, PyObject *iterable) +{ + return PyList_SetSlice(list, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, iterable); +} + +static inline int +PyList_Clear(PyObject *list) +{ + return PyList_SetSlice(list, 0, PY_SSIZE_T_MAX, NULL); +} +#endif + +// gh-111262 added PyDict_Pop() and PyDict_PopString() to Python 3.13.0a2 +#if PY_VERSION_HEX < 0x030D00A2 +static inline int +PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result) +{ + PyObject *value; + + if (!PyDict_Check(dict)) { + PyErr_BadInternalCall(); + if (result) { + *result = NULL; + } + return -1; + } + + // bpo-16991 added _PyDict_Pop() to Python 3.5.0b2. + // Python 3.6.0b3 changed _PyDict_Pop() first argument type to PyObject*. + // Python 3.13.0a1 removed _PyDict_Pop(). +#if defined(PYPY_VERSION) || PY_VERSION_HEX < 0x030500b2 || PY_VERSION_HEX >= 0x030D0000 + value = PyObject_CallMethod(dict, "pop", "O", key); +#elif PY_VERSION_HEX < 0x030600b3 + value = _PyDict_Pop(_Py_CAST(PyDictObject*, dict), key, NULL); +#else + value = _PyDict_Pop(dict, key, NULL); +#endif + if (value == NULL) { + if (result) { + *result = NULL; + } + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_KeyError)) { + return -1; + } + PyErr_Clear(); + return 0; + } + if (result) { + *result = value; + } + else { + Py_DECREF(value); + } + return 1; +} + +static inline int +PyDict_PopString(PyObject *dict, const char *key, PyObject **result) +{ + PyObject *key_obj = PyUnicode_FromString(key); + if (key_obj == NULL) { + if (result != NULL) { + *result = NULL; + } + return -1; + } + + int res = PyDict_Pop(dict, key_obj, result); + Py_DECREF(key_obj); + return res; +} +#endif + + +#if PY_VERSION_HEX < 0x030200A4 +// Python 3.2.0a4 added Py_hash_t type +typedef Py_ssize_t Py_hash_t; +#endif + + +// gh-111545 added Py_HashPointer() to Python 3.13.0a3 +#if PY_VERSION_HEX < 0x030D00A3 +static inline Py_hash_t Py_HashPointer(const void *ptr) +{ +#if PY_VERSION_HEX >= 0x030900A4 && !defined(PYPY_VERSION) + return _Py_HashPointer(ptr); +#else + return _Py_HashPointer(_Py_CAST(void*, ptr)); +#endif +} +#endif + + +// Python 3.13a4 added a PyTime API. +// Use the private API added to Python 3.5. +#if PY_VERSION_HEX < 0x030D00A4 && PY_VERSION_HEX >= 0x03050000 +typedef _PyTime_t PyTime_t; +#define PyTime_MIN _PyTime_MIN +#define PyTime_MAX _PyTime_MAX + +static inline double PyTime_AsSecondsDouble(PyTime_t t) +{ return _PyTime_AsSecondsDouble(t); } + +static inline int PyTime_Monotonic(PyTime_t *result) +{ return _PyTime_GetMonotonicClockWithInfo(result, NULL); } + +static inline int PyTime_Time(PyTime_t *result) +{ return _PyTime_GetSystemClockWithInfo(result, NULL); } + +static inline int PyTime_PerfCounter(PyTime_t *result) +{ +#if PY_VERSION_HEX >= 0x03070000 && !defined(PYPY_VERSION) + return _PyTime_GetPerfCounterWithInfo(result, NULL); +#elif PY_VERSION_HEX >= 0x03070000 + // Call time.perf_counter_ns() and convert Python int object to PyTime_t. + // Cache time.perf_counter_ns() function for best performance. + static PyObject *func = NULL; + if (func == NULL) { + PyObject *mod = PyImport_ImportModule("time"); + if (mod == NULL) { + return -1; + } + + func = PyObject_GetAttrString(mod, "perf_counter_ns"); + Py_DECREF(mod); + if (func == NULL) { + return -1; + } + } + + PyObject *res = PyObject_CallNoArgs(func); + if (res == NULL) { + return -1; + } + long long value = PyLong_AsLongLong(res); + Py_DECREF(res); + + if (value == -1 && PyErr_Occurred()) { + return -1; + } + + Py_BUILD_ASSERT(sizeof(value) >= sizeof(PyTime_t)); + *result = (PyTime_t)value; + return 0; +#else + // Call time.perf_counter() and convert C double to PyTime_t. + // Cache time.perf_counter() function for best performance. + static PyObject *func = NULL; + if (func == NULL) { + PyObject *mod = PyImport_ImportModule("time"); + if (mod == NULL) { + return -1; + } + + func = PyObject_GetAttrString(mod, "perf_counter"); + Py_DECREF(mod); + if (func == NULL) { + return -1; + } + } + + PyObject *res = PyObject_CallNoArgs(func); + if (res == NULL) { + return -1; + } + double d = PyFloat_AsDouble(res); + Py_DECREF(res); + + if (d == -1.0 && PyErr_Occurred()) { + return -1; + } + + // Avoid floor() to avoid having to link to libm + *result = (PyTime_t)(d * 1e9); + return 0; +#endif +} + +#endif + +// gh-111389 added hash constants to Python 3.13.0a5. These constants were +// added first as private macros to Python 3.4.0b1 and PyPy 7.3.9. +#if (!defined(PyHASH_BITS) \ + && ((!defined(PYPY_VERSION) && PY_VERSION_HEX >= 0x030400B1) \ + || (defined(PYPY_VERSION) && PY_VERSION_HEX >= 0x03070000 \ + && PYPY_VERSION_NUM >= 0x07090000))) +# define PyHASH_BITS _PyHASH_BITS +# define PyHASH_MODULUS _PyHASH_MODULUS +# define PyHASH_INF _PyHASH_INF +# define PyHASH_IMAG _PyHASH_IMAG +#endif + + +// gh-111545 added Py_GetConstant() and Py_GetConstantBorrowed() +// to Python 3.13.0a6 +#if PY_VERSION_HEX < 0x030D00A6 && !defined(Py_CONSTANT_NONE) + +#define Py_CONSTANT_NONE 0 +#define Py_CONSTANT_FALSE 1 +#define Py_CONSTANT_TRUE 2 +#define Py_CONSTANT_ELLIPSIS 3 +#define Py_CONSTANT_NOT_IMPLEMENTED 4 +#define Py_CONSTANT_ZERO 5 +#define Py_CONSTANT_ONE 6 +#define Py_CONSTANT_EMPTY_STR 7 +#define Py_CONSTANT_EMPTY_BYTES 8 +#define Py_CONSTANT_EMPTY_TUPLE 9 + +static inline PyObject* Py_GetConstant(unsigned int constant_id) +{ + static PyObject* constants[Py_CONSTANT_EMPTY_TUPLE + 1] = {NULL}; + + if (constants[Py_CONSTANT_NONE] == NULL) { + constants[Py_CONSTANT_NONE] = Py_None; + constants[Py_CONSTANT_FALSE] = Py_False; + constants[Py_CONSTANT_TRUE] = Py_True; + constants[Py_CONSTANT_ELLIPSIS] = Py_Ellipsis; + constants[Py_CONSTANT_NOT_IMPLEMENTED] = Py_NotImplemented; + + constants[Py_CONSTANT_ZERO] = PyLong_FromLong(0); + if (constants[Py_CONSTANT_ZERO] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_ONE] = PyLong_FromLong(1); + if (constants[Py_CONSTANT_ONE] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_FromStringAndSize("", 0); + if (constants[Py_CONSTANT_EMPTY_STR] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize("", 0); + if (constants[Py_CONSTANT_EMPTY_BYTES] == NULL) { + goto fatal_error; + } + + constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); + if (constants[Py_CONSTANT_EMPTY_TUPLE] == NULL) { + goto fatal_error; + } + // goto dance to avoid compiler warnings about Py_FatalError() + goto init_done; + +fatal_error: + // This case should never happen + Py_FatalError("Py_GetConstant() failed to get constants"); + } + +init_done: + if (constant_id <= Py_CONSTANT_EMPTY_TUPLE) { + return Py_NewRef(constants[constant_id]); + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} + +static inline PyObject* Py_GetConstantBorrowed(unsigned int constant_id) +{ + PyObject *obj = Py_GetConstant(constant_id); + Py_XDECREF(obj); + return obj; +} +#endif + + +// gh-114329 added PyList_GetItemRef() to Python 3.13.0a4 +#if PY_VERSION_HEX < 0x030D00A4 +static inline PyObject * +PyList_GetItemRef(PyObject *op, Py_ssize_t index) +{ + PyObject *item = PyList_GetItem(op, index); + Py_XINCREF(item); + return item; +} +#endif + + +// gh-114329 added PyList_GetItemRef() to Python 3.13.0a4 +#if PY_VERSION_HEX < 0x030D00A4 +static inline int +PyDict_SetDefaultRef(PyObject *d, PyObject *key, PyObject *default_value, + PyObject **result) +{ + PyObject *value; + if (PyDict_GetItemRef(d, key, &value) < 0) { + // get error + if (result) { + *result = NULL; + } + return -1; + } + if (value != NULL) { + // present + if (result) { + *result = value; + } + else { + Py_DECREF(value); + } + return 1; + } + + // missing: set the item + if (PyDict_SetItem(d, key, default_value) < 0) { + // set error + if (result) { + *result = NULL; + } + return -1; + } + if (result) { + *result = Py_NewRef(default_value); + } + return 0; +} +#endif + + +// gh-116560 added PyLong_GetSign() to Python 3.14a4 +#if PY_VERSION_HEX < 0x030E00A1 +static inline int PyLong_GetSign(PyObject *obj, int *sign) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expect int, got %s", Py_TYPE(obj)->tp_name); + return -1; + } + + *sign = _PyLong_Sign(obj); + return 0; +} +#endif + + #ifdef __cplusplus } #endif From 31faa43406b9b2a4edaacc55ad637a695f0094f7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 17 Jun 2024 10:55:22 +0100 Subject: [PATCH 036/247] [mypyc] Inline fast paths of integer unboxing operations (#17266) This applies to `int` and native integer types. This speeds up this micro-benchmark by up to 80% (it spends most of the time unboxing integers): ``` # a is list[int]/list[i64]/... for i in a: if i == 789: n += 1 ``` The impact to compile time when self-compiling is below the noise floor. The generated binary is about 0.1% larger. Since integer unboxing can be performance-critical, this seems like a decent win. Closes mypyc/mypyc#987. Work on mypyc/mypyc#757. --- mypyc/common.py | 1 + mypyc/lib-rt/CPy.h | 147 +++++++++++++++++++++++++++++++++-- mypyc/lib-rt/int_ops.c | 133 ++----------------------------- mypyc/lib-rt/pythonsupport.c | 106 +++++++++++++++++++++++++ mypyc/lib-rt/pythonsupport.h | 73 ++++------------- mypyc/lib-rt/setup.py | 1 + 6 files changed, 269 insertions(+), 192 deletions(-) create mode 100644 mypyc/lib-rt/pythonsupport.c diff --git a/mypyc/common.py b/mypyc/common.py index d7610fe15c41..31567c689c34 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -79,6 +79,7 @@ "exc_ops.c", "misc_ops.c", "generic_ops.c", + "pythonsupport.c", ] diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 9e85647226fe..8aa5a77c180c 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -120,9 +120,6 @@ static inline size_t CPy_FindAttrOffset(PyTypeObject *trait, CPyVTableItem *vtab CPyTagged CPyTagged_FromSsize_t(Py_ssize_t value); CPyTagged CPyTagged_FromVoidPtr(void *ptr); CPyTagged CPyTagged_FromInt64(int64_t value); -CPyTagged CPyTagged_FromObject(PyObject *object); -CPyTagged CPyTagged_StealFromObject(PyObject *object); -CPyTagged CPyTagged_BorrowFromObject(PyObject *object); PyObject *CPyTagged_AsObject(CPyTagged x); PyObject *CPyTagged_StealAsObject(CPyTagged x); Py_ssize_t CPyTagged_AsSsize_t(CPyTagged x); @@ -148,18 +145,18 @@ CPyTagged CPyTagged_FromFloat(double f); PyObject *CPyLong_FromStrWithBase(PyObject *o, CPyTagged base); PyObject *CPyLong_FromStr(PyObject *o); PyObject *CPyBool_Str(bool b); -int64_t CPyLong_AsInt64(PyObject *o); +int64_t CPyLong_AsInt64_(PyObject *o); int64_t CPyInt64_Divide(int64_t x, int64_t y); int64_t CPyInt64_Remainder(int64_t x, int64_t y); -int32_t CPyLong_AsInt32(PyObject *o); +int32_t CPyLong_AsInt32_(PyObject *o); int32_t CPyInt32_Divide(int32_t x, int32_t y); int32_t CPyInt32_Remainder(int32_t x, int32_t y); void CPyInt32_Overflow(void); -int16_t CPyLong_AsInt16(PyObject *o); +int16_t CPyLong_AsInt16_(PyObject *o); int16_t CPyInt16_Divide(int16_t x, int16_t y); int16_t CPyInt16_Remainder(int16_t x, int16_t y); void CPyInt16_Overflow(void); -uint8_t CPyLong_AsUInt8(PyObject *o); +uint8_t CPyLong_AsUInt8_(PyObject *o); void CPyUInt8_Overflow(void); double CPyTagged_TrueDivide(CPyTagged x, CPyTagged y); @@ -199,6 +196,41 @@ static inline PyObject *CPyTagged_LongAsObject(CPyTagged x) { return (PyObject *)(x & ~CPY_INT_TAG); } +static inline CPyTagged CPyTagged_FromObject(PyObject *object) { + int overflow; + // The overflow check knows about CPyTagged's width + Py_ssize_t value = CPyLong_AsSsize_tAndOverflow(object, &overflow); + if (unlikely(overflow != 0)) { + Py_INCREF(object); + return ((CPyTagged)object) | CPY_INT_TAG; + } else { + return value << 1; + } +} + +static inline CPyTagged CPyTagged_StealFromObject(PyObject *object) { + int overflow; + // The overflow check knows about CPyTagged's width + Py_ssize_t value = CPyLong_AsSsize_tAndOverflow(object, &overflow); + if (unlikely(overflow != 0)) { + return ((CPyTagged)object) | CPY_INT_TAG; + } else { + Py_DECREF(object); + return value << 1; + } +} + +static inline CPyTagged CPyTagged_BorrowFromObject(PyObject *object) { + int overflow; + // The overflow check knows about CPyTagged's width + Py_ssize_t value = CPyLong_AsSsize_tAndOverflow(object, &overflow); + if (unlikely(overflow != 0)) { + return ((CPyTagged)object) | CPY_INT_TAG; + } else { + return value << 1; + } +} + static inline bool CPyTagged_TooBig(Py_ssize_t value) { // Micro-optimized for the common case where it fits. return (size_t)value > CPY_TAGGED_MAX @@ -286,6 +318,107 @@ static inline bool CPyTagged_IsLe(CPyTagged left, CPyTagged right) { } } +static inline int64_t CPyLong_AsInt64(PyObject *o) { + if (likely(PyLong_Check(o))) { + PyLongObject *lobj = (PyLongObject *)o; + Py_ssize_t size = Py_SIZE(lobj); + if (likely(size == 1)) { + // Fast path + return CPY_LONG_DIGIT(lobj, 0); + } else if (likely(size == 0)) { + return 0; + } + } + // Slow path + return CPyLong_AsInt64_(o); +} + +static inline int32_t CPyLong_AsInt32(PyObject *o) { + if (likely(PyLong_Check(o))) { + #if CPY_3_12_FEATURES + PyLongObject *lobj = (PyLongObject *)o; + size_t tag = CPY_LONG_TAG(lobj); + if (likely(tag == (1 << CPY_NON_SIZE_BITS))) { + // Fast path + return CPY_LONG_DIGIT(lobj, 0); + } else if (likely(tag == CPY_SIGN_ZERO)) { + return 0; + } + #else + PyLongObject *lobj = (PyLongObject *)o; + Py_ssize_t size = lobj->ob_base.ob_size; + if (likely(size == 1)) { + // Fast path + return CPY_LONG_DIGIT(lobj, 0); + } else if (likely(size == 0)) { + return 0; + } + #endif + } + // Slow path + return CPyLong_AsInt32_(o); +} + +static inline int16_t CPyLong_AsInt16(PyObject *o) { + if (likely(PyLong_Check(o))) { + #if CPY_3_12_FEATURES + PyLongObject *lobj = (PyLongObject *)o; + size_t tag = CPY_LONG_TAG(lobj); + if (likely(tag == (1 << CPY_NON_SIZE_BITS))) { + // Fast path + digit x = CPY_LONG_DIGIT(lobj, 0); + if (x < 0x8000) + return x; + } else if (likely(tag == CPY_SIGN_ZERO)) { + return 0; + } + #else + PyLongObject *lobj = (PyLongObject *)o; + Py_ssize_t size = lobj->ob_base.ob_size; + if (likely(size == 1)) { + // Fast path + digit x = lobj->ob_digit[0]; + if (x < 0x8000) + return x; + } else if (likely(size == 0)) { + return 0; + } + #endif + } + // Slow path + return CPyLong_AsInt16_(o); +} + +static inline uint8_t CPyLong_AsUInt8(PyObject *o) { + if (likely(PyLong_Check(o))) { + #if CPY_3_12_FEATURES + PyLongObject *lobj = (PyLongObject *)o; + size_t tag = CPY_LONG_TAG(lobj); + if (likely(tag == (1 << CPY_NON_SIZE_BITS))) { + // Fast path + digit x = CPY_LONG_DIGIT(lobj, 0); + if (x < 256) + return x; + } else if (likely(tag == CPY_SIGN_ZERO)) { + return 0; + } + #else + PyLongObject *lobj = (PyLongObject *)o; + Py_ssize_t size = lobj->ob_base.ob_size; + if (likely(size == 1)) { + // Fast path + digit x = lobj->ob_digit[0]; + if (x < 256) + return x; + } else if (likely(size == 0)) { + return 0; + } + #endif + } + // Slow path + return CPyLong_AsUInt8_(o); +} + static inline CPyTagged CPyTagged_Negate(CPyTagged num) { if (likely(CPyTagged_CheckShort(num) && num != (CPyTagged) ((Py_ssize_t)1 << (CPY_INT_BITS - 1)))) { diff --git a/mypyc/lib-rt/int_ops.c b/mypyc/lib-rt/int_ops.c index b1b3d6e125f3..9b5d4ef65fb1 100644 --- a/mypyc/lib-rt/int_ops.c +++ b/mypyc/lib-rt/int_ops.c @@ -44,41 +44,6 @@ CPyTagged CPyTagged_FromInt64(int64_t value) { } } -CPyTagged CPyTagged_FromObject(PyObject *object) { - int overflow; - // The overflow check knows about CPyTagged's width - Py_ssize_t value = CPyLong_AsSsize_tAndOverflow(object, &overflow); - if (unlikely(overflow != 0)) { - Py_INCREF(object); - return ((CPyTagged)object) | CPY_INT_TAG; - } else { - return value << 1; - } -} - -CPyTagged CPyTagged_StealFromObject(PyObject *object) { - int overflow; - // The overflow check knows about CPyTagged's width - Py_ssize_t value = CPyLong_AsSsize_tAndOverflow(object, &overflow); - if (unlikely(overflow != 0)) { - return ((CPyTagged)object) | CPY_INT_TAG; - } else { - Py_DECREF(object); - return value << 1; - } -} - -CPyTagged CPyTagged_BorrowFromObject(PyObject *object) { - int overflow; - // The overflow check knows about CPyTagged's width - Py_ssize_t value = CPyLong_AsSsize_tAndOverflow(object, &overflow); - if (unlikely(overflow != 0)) { - return ((CPyTagged)object) | CPY_INT_TAG; - } else { - return value << 1; - } -} - PyObject *CPyTagged_AsObject(CPyTagged x) { PyObject *value; if (unlikely(CPyTagged_CheckLong(x))) { @@ -420,18 +385,8 @@ CPyTagged CPyTagged_Lshift_(CPyTagged left, CPyTagged right) { return CPyTagged_StealFromObject(result); } -int64_t CPyLong_AsInt64(PyObject *o) { - if (likely(PyLong_Check(o))) { - PyLongObject *lobj = (PyLongObject *)o; - Py_ssize_t size = Py_SIZE(lobj); - if (likely(size == 1)) { - // Fast path - return CPY_LONG_DIGIT(lobj, 0); - } else if (likely(size == 0)) { - return 0; - } - } - // Slow path +// i64 unboxing slow path +int64_t CPyLong_AsInt64_(PyObject *o) { int overflow; int64_t result = PyLong_AsLongLongAndOverflow(o, &overflow); if (result == -1) { @@ -479,29 +434,8 @@ int64_t CPyInt64_Remainder(int64_t x, int64_t y) { return d; } -int32_t CPyLong_AsInt32(PyObject *o) { - if (likely(PyLong_Check(o))) { - #if CPY_3_12_FEATURES - PyLongObject *lobj = (PyLongObject *)o; - size_t tag = CPY_LONG_TAG(lobj); - if (likely(tag == (1 << CPY_NON_SIZE_BITS))) { - // Fast path - return CPY_LONG_DIGIT(lobj, 0); - } else if (likely(tag == CPY_SIGN_ZERO)) { - return 0; - } - #else - PyLongObject *lobj = (PyLongObject *)o; - Py_ssize_t size = lobj->ob_base.ob_size; - if (likely(size == 1)) { - // Fast path - return CPY_LONG_DIGIT(lobj, 0); - } else if (likely(size == 0)) { - return 0; - } - #endif - } - // Slow path +// i32 unboxing slow path +int32_t CPyLong_AsInt32_(PyObject *o) { int overflow; long result = PyLong_AsLongAndOverflow(o, &overflow); if (result > 0x7fffffffLL || result < -0x80000000LL) { @@ -557,33 +491,8 @@ void CPyInt32_Overflow() { PyErr_SetString(PyExc_OverflowError, "int too large to convert to i32"); } -int16_t CPyLong_AsInt16(PyObject *o) { - if (likely(PyLong_Check(o))) { - #if CPY_3_12_FEATURES - PyLongObject *lobj = (PyLongObject *)o; - size_t tag = CPY_LONG_TAG(lobj); - if (likely(tag == (1 << CPY_NON_SIZE_BITS))) { - // Fast path - digit x = CPY_LONG_DIGIT(lobj, 0); - if (x < 0x8000) - return x; - } else if (likely(tag == CPY_SIGN_ZERO)) { - return 0; - } - #else - PyLongObject *lobj = (PyLongObject *)o; - Py_ssize_t size = lobj->ob_base.ob_size; - if (likely(size == 1)) { - // Fast path - digit x = lobj->ob_digit[0]; - if (x < 0x8000) - return x; - } else if (likely(size == 0)) { - return 0; - } - #endif - } - // Slow path +// i16 unboxing slow path +int16_t CPyLong_AsInt16_(PyObject *o) { int overflow; long result = PyLong_AsLongAndOverflow(o, &overflow); if (result > 0x7fff || result < -0x8000) { @@ -639,34 +548,8 @@ void CPyInt16_Overflow() { PyErr_SetString(PyExc_OverflowError, "int too large to convert to i16"); } - -uint8_t CPyLong_AsUInt8(PyObject *o) { - if (likely(PyLong_Check(o))) { - #if CPY_3_12_FEATURES - PyLongObject *lobj = (PyLongObject *)o; - size_t tag = CPY_LONG_TAG(lobj); - if (likely(tag == (1 << CPY_NON_SIZE_BITS))) { - // Fast path - digit x = CPY_LONG_DIGIT(lobj, 0); - if (x < 256) - return x; - } else if (likely(tag == CPY_SIGN_ZERO)) { - return 0; - } - #else - PyLongObject *lobj = (PyLongObject *)o; - Py_ssize_t size = lobj->ob_base.ob_size; - if (likely(size == 1)) { - // Fast path - digit x = lobj->ob_digit[0]; - if (x < 256) - return x; - } else if (likely(size == 0)) { - return 0; - } - #endif - } - // Slow path +// u8 unboxing slow path +uint8_t CPyLong_AsUInt8_(PyObject *o) { int overflow; long result = PyLong_AsLongAndOverflow(o, &overflow); if (result < 0 || result >= 256) { diff --git a/mypyc/lib-rt/pythonsupport.c b/mypyc/lib-rt/pythonsupport.c new file mode 100644 index 000000000000..90fb69705a00 --- /dev/null +++ b/mypyc/lib-rt/pythonsupport.c @@ -0,0 +1,106 @@ +// Collects code that was copied in from cpython, for a couple of different reasons: +// * We wanted to modify it to produce a more efficient version for our uses +// * We needed to call it and it was static :( +// * We wanted to call it and needed to backport it + +#include "pythonsupport.h" + +#if CPY_3_12_FEATURES + +// Slow path of CPyLong_AsSsize_tAndOverflow (non-inlined) +Py_ssize_t +CPyLong_AsSsize_tAndOverflow_(PyObject *vv, int *overflow) +{ + PyLongObject *v = (PyLongObject *)vv; + size_t x, prev; + Py_ssize_t res; + Py_ssize_t i; + int sign; + + *overflow = 0; + + res = -1; + i = CPY_LONG_TAG(v); + + sign = 1; + x = 0; + if (i & CPY_SIGN_NEGATIVE) { + sign = -1; + } + i >>= CPY_NON_SIZE_BITS; + while (--i >= 0) { + prev = x; + x = (x << PyLong_SHIFT) + CPY_LONG_DIGIT(v, i); + if ((x >> PyLong_SHIFT) != prev) { + *overflow = sign; + goto exit; + } + } + /* Haven't lost any bits, but casting to long requires extra + * care. + */ + if (x <= (size_t)CPY_TAGGED_MAX) { + res = (Py_ssize_t)x * sign; + } + else if (sign < 0 && x == CPY_TAGGED_ABS_MIN) { + res = CPY_TAGGED_MIN; + } + else { + *overflow = sign; + /* res is already set to -1 */ + } + exit: + return res; +} + +#else + +// Slow path of CPyLong_AsSsize_tAndOverflow (non-inlined, Python 3.11 and earlier) +Py_ssize_t +CPyLong_AsSsize_tAndOverflow_(PyObject *vv, int *overflow) +{ + /* This version by Tim Peters */ + PyLongObject *v = (PyLongObject *)vv; + size_t x, prev; + Py_ssize_t res; + Py_ssize_t i; + int sign; + + *overflow = 0; + + res = -1; + i = Py_SIZE(v); + + sign = 1; + x = 0; + if (i < 0) { + sign = -1; + i = -(i); + } + while (--i >= 0) { + prev = x; + x = (x << PyLong_SHIFT) + CPY_LONG_DIGIT(v, i); + if ((x >> PyLong_SHIFT) != prev) { + *overflow = sign; + goto exit; + } + } + /* Haven't lost any bits, but casting to long requires extra + * care. + */ + if (x <= (size_t)CPY_TAGGED_MAX) { + res = (Py_ssize_t)x * sign; + } + else if (sign < 0 && x == CPY_TAGGED_ABS_MIN) { + res = CPY_TAGGED_MIN; + } + else { + *overflow = sign; + /* res is already set to -1 */ + } + exit: + return res; +} + + +#endif diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index f7d501f44a27..85f9ec64ac90 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -129,6 +129,9 @@ init_subclass(PyTypeObject *type, PyObject *kwds) return 0; } +Py_ssize_t +CPyLong_AsSsize_tAndOverflow_(PyObject *vv, int *overflow); + #if CPY_3_12_FEATURES static inline Py_ssize_t @@ -136,10 +139,8 @@ CPyLong_AsSsize_tAndOverflow(PyObject *vv, int *overflow) { /* This version by Tim Peters */ PyLongObject *v = (PyLongObject *)vv; - size_t x, prev; Py_ssize_t res; Py_ssize_t i; - int sign; *overflow = 0; @@ -154,35 +155,12 @@ CPyLong_AsSsize_tAndOverflow(PyObject *vv, int *overflow) } else if (i == ((1 << CPY_NON_SIZE_BITS) | CPY_SIGN_NEGATIVE)) { res = -(sdigit)CPY_LONG_DIGIT(v, 0); } else { - sign = 1; - x = 0; - if (i & CPY_SIGN_NEGATIVE) { - sign = -1; - } - i >>= CPY_NON_SIZE_BITS; - while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) + CPY_LONG_DIGIT(v, i); - if ((x >> PyLong_SHIFT) != prev) { - *overflow = sign; - goto exit; - } - } - /* Haven't lost any bits, but casting to long requires extra - * care (see comment above). - */ - if (x <= (size_t)CPY_TAGGED_MAX) { - res = (Py_ssize_t)x * sign; - } - else if (sign < 0 && x == CPY_TAGGED_ABS_MIN) { - res = CPY_TAGGED_MIN; - } - else { - *overflow = sign; - /* res is already set to -1 */ - } + // Slow path is moved to a non-inline helper function to + // limit size of generated code + int overflow_local; + res = CPyLong_AsSsize_tAndOverflow_(vv, &overflow_local); + *overflow = overflow_local; } - exit: return res; } @@ -204,10 +182,8 @@ CPyLong_AsSsize_tAndOverflow(PyObject *vv, int *overflow) { /* This version by Tim Peters */ PyLongObject *v = (PyLongObject *)vv; - size_t x, prev; Py_ssize_t res; Py_ssize_t i; - int sign; *overflow = 0; @@ -221,35 +197,12 @@ CPyLong_AsSsize_tAndOverflow(PyObject *vv, int *overflow) } else if (i == -1) { res = -(sdigit)CPY_LONG_DIGIT(v, 0); } else { - sign = 1; - x = 0; - if (i < 0) { - sign = -1; - i = -(i); - } - while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) + CPY_LONG_DIGIT(v, i); - if ((x >> PyLong_SHIFT) != prev) { - *overflow = sign; - goto exit; - } - } - /* Haven't lost any bits, but casting to long requires extra - * care (see comment above). - */ - if (x <= (size_t)CPY_TAGGED_MAX) { - res = (Py_ssize_t)x * sign; - } - else if (sign < 0 && x == CPY_TAGGED_ABS_MIN) { - res = CPY_TAGGED_MIN; - } - else { - *overflow = sign; - /* res is already set to -1 */ - } + // Slow path is moved to a non-inline helper function to + // limit size of generated code + int overflow_local; + res = CPyLong_AsSsize_tAndOverflow_(vv, &overflow_local); + *overflow = overflow_local; } - exit: return res; } diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index ef81b794c9bd..66b130581cb3 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -58,6 +58,7 @@ def run(self): "list_ops.c", "exc_ops.c", "generic_ops.c", + "pythonsupport.c", ], depends=["CPy.h", "mypyc_util.h", "pythonsupport.h"], extra_compile_args=["-Wno-unused-function", "-Wno-sign-compare"] + compile_args, From b20255276e72803da5c5f98cf6982b782cf5f4d7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 17 Jun 2024 12:03:01 +0100 Subject: [PATCH 037/247] [mypyc] Support Python 3.12 type alias syntax (PEP 695) (#17384) The main tricky bit is supporting uses of type alias objects at runtime. Python evaluates values of type aliases lazily, but there's no way to do this using public APIs, so we directly modify the `TypeAliasType` object that is used to represent a type alias at runtime in C. Unfortunately, this is fragile and will need to be updated each time CPython updates the internal representation of `TypeAliasType` objects. Wrap the target of the type alias within a lambda expression, so that we can easily create the lazy compute function in mypyc. This also reflects how this is implemented in CPython. Improve test stubs to avoid various false positives or confusing errors in tests when type checking runtime operations on types. This also makes some exisisting tests more realistic. Follow-up to #17357. --- mypy/checker.py | 4 ++ mypy/checkexpr.py | 4 +- mypy/fastparse.py | 8 +++- mypy/nodes.py | 4 +- mypy/semanal.py | 13 +++++- mypyc/irbuild/builder.py | 46 +++++++++++++++++++ mypyc/irbuild/classdef.py | 37 ++++----------- mypyc/irbuild/statement.py | 34 +++++++++++++- mypyc/irbuild/visitor.py | 3 +- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/misc_ops.c | 31 +++++++++++++ mypyc/primitives/misc_ops.py | 12 +++++ mypyc/test-data/fixtures/ir.py | 1 + mypyc/test-data/fixtures/typing-full.pyi | 5 +- mypyc/test-data/run-python312.test | 53 ++++++++++++++++++++++ test-data/unit/check-class-namedtuple.test | 3 +- test-data/unit/check-expressions.test | 6 ++- test-data/unit/check-generics.test | 1 + test-data/unit/check-newsemanal.test | 8 ++-- test-data/unit/check-python312.test | 28 ++++++++++-- test-data/unit/check-redefine.test | 9 ++-- test-data/unit/check-type-aliases.test | 9 ++-- test-data/unit/check-typevar-tuple.test | 6 ++- test-data/unit/check-union-or-syntax.test | 2 +- test-data/unit/deps.test | 6 +++ test-data/unit/fine-grained-python312.test | 1 + test-data/unit/fixtures/isinstance.pyi | 3 +- test-data/unit/fixtures/tuple.pyi | 28 ++++++------ test-data/unit/fixtures/type.pyi | 1 + test-data/unit/fixtures/typing-full.pyi | 13 ++++-- test-data/unit/lib-stub/types.pyi | 2 + test-data/unit/parse-python312.test | 9 ++-- 32 files changed, 310 insertions(+), 81 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 119aa9f3cea2..bf739e7d1242 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -125,6 +125,7 @@ TryStmt, TupleExpr, TypeAlias, + TypeAliasStmt, TypeInfo, TypeVarExpr, UnaryExpr, @@ -5289,6 +5290,9 @@ def remove_capture_conflicts(self, type_map: TypeMap, inferred_types: dict[Var, if node not in inferred_types or not is_subtype(typ, inferred_types[node]): del type_map[expr] + def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: + self.expr_checker.accept(o.value) + def make_fake_typeinfo( self, curr_module_fullname: str, diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 861c28e5b54c..4fd1a308e560 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -411,7 +411,9 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: result = self.alias_type_in_runtime_context( node, ctx=e, alias_definition=e.is_alias_rvalue or lvalue ) - elif isinstance(node, (TypeVarExpr, ParamSpecExpr, TypeVarTupleExpr)): + elif isinstance(node, TypeVarExpr): + return self.named_type("typing.TypeVar") + elif isinstance(node, (ParamSpecExpr, TypeVarTupleExpr)): result = self.object_type() else: if isinstance(node, PlaceholderNode): diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 70afe9010583..342cf36d69e8 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1791,7 +1791,13 @@ def visit_TypeAlias(self, n: ast_TypeAlias) -> TypeAliasStmt | AssignmentStmt: if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: type_params = self.translate_type_params(n.type_params) value = self.visit(n.value) - node = TypeAliasStmt(self.visit_Name(n.name), type_params, value) + # Since the value is evaluated lazily, wrap the value inside a lambda. + # This helps mypyc. + ret = ReturnStmt(value) + self.set_line(ret, n.value) + value_func = LambdaExpr(body=Block([ret])) + self.set_line(value_func, n.value) + node = TypeAliasStmt(self.visit_Name(n.name), type_params, value_func) return self.set_line(node, n) else: self.fail( diff --git a/mypy/nodes.py b/mypy/nodes.py index 850b1db87556..5d3a1d31aece 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1653,10 +1653,10 @@ class TypeAliasStmt(Statement): name: NameExpr type_args: list[TypeParam] - value: Expression # Will get translated into a type + value: LambdaExpr # Return value will get translated into a type invalid_recursive_alias: bool - def __init__(self, name: NameExpr, type_args: list[TypeParam], value: Expression) -> None: + def __init__(self, name: NameExpr, type_args: list[TypeParam], value: LambdaExpr) -> None: super().__init__() self.name = name self.type_args = type_args diff --git a/mypy/semanal.py b/mypy/semanal.py index d2f02d4835e2..03e6172bb325 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3766,6 +3766,10 @@ def analyze_alias( last_tvar_name_with_default = tvar_def.name tvar_defs.append(tvar_def) + if python_3_12_type_alias: + with self.allow_unbound_tvars_set(): + rvalue.accept(self) + analyzed, depends_on = analyze_type_alias( typ, self, @@ -5360,7 +5364,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: tag = self.track_incomplete_refs() res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias( s.name.name, - s.value, + s.value.expr(), allow_placeholder=True, declared_type_vars=type_params, all_declared_type_params_names=all_type_params_names, @@ -5443,6 +5447,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: current_node = existing.node if existing else alias_node assert isinstance(current_node, TypeAlias) self.disable_invalid_recursive_aliases(s, current_node, s.value) + s.name.accept(self) finally: self.pop_type_args(s.type_args) @@ -5457,7 +5462,11 @@ def visit_name_expr(self, expr: NameExpr) -> None: def bind_name_expr(self, expr: NameExpr, sym: SymbolTableNode) -> None: """Bind name expression to a symbol table node.""" - if isinstance(sym.node, TypeVarExpr) and self.tvar_scope.get_binding(sym): + if ( + isinstance(sym.node, TypeVarExpr) + and self.tvar_scope.get_binding(sym) + and not self.allow_unbound_tvars + ): self.fail(f'"{expr.name}" is a type variable and only valid in type context', expr) elif isinstance(sym.node, PlaceholderNode): self.process_placeholder(expr.name, "name", expr) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 1b4f551d4a2a..a9e1ce471953 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -24,6 +24,9 @@ ARG_POS, GDEF, LDEF, + PARAM_SPEC_KIND, + TYPE_VAR_KIND, + TYPE_VAR_TUPLE_KIND, ArgKind, CallExpr, Decorator, @@ -44,6 +47,7 @@ TupleExpr, TypeAlias, TypeInfo, + TypeParam, UnaryExpr, Var, ) @@ -1409,3 +1413,45 @@ def get_call_target_fullname(ref: RefExpr) -> str: if isinstance(target, Instance): return target.type.fullname return ref.fullname + + +def create_type_params( + builder: IRBuilder, typing_mod: Value, type_args: list[TypeParam], line: int +) -> list[Value]: + """Create objects representing various kinds of Python 3.12 type parameters. + + The "typing_mod" argument is the "_typing" module object. The type objects + are looked up from it. + + The returned list has one item for each "type_args" item, in the same order. + Each item is either a TypeVar, TypeVarTuple or ParamSpec instance. + """ + tvs = [] + type_var_imported: Value | None = None + for type_param in type_args: + if type_param.kind == TYPE_VAR_KIND: + if type_var_imported: + # Reuse previously imported value as a minor optimization + tvt = type_var_imported + else: + tvt = builder.py_get_attr(typing_mod, "TypeVar", line) + type_var_imported = tvt + elif type_param.kind == TYPE_VAR_TUPLE_KIND: + tvt = builder.py_get_attr(typing_mod, "TypeVarTuple", line) + else: + assert type_param.kind == PARAM_SPEC_KIND + tvt = builder.py_get_attr(typing_mod, "ParamSpec", line) + if type_param.kind != TYPE_VAR_TUPLE_KIND: + # To match runtime semantics, pass infer_variance=True + tv = builder.py_call( + tvt, + [builder.load_str(type_param.name), builder.true()], + line, + arg_kinds=[ARG_POS, ARG_NAMED], + arg_names=[None, "infer_variance"], + ) + else: + tv = builder.py_call(tvt, [builder.load_str(type_param.name)], line) + builder.init_type_var(tv, type_param.name, line) + tvs.append(tv) + return tvs diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 303ee8849244..2152da099e81 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -7,8 +7,6 @@ from typing import Callable, Final from mypy.nodes import ( - PARAM_SPEC_KIND, - TYPE_VAR_KIND, TYPE_VAR_TUPLE_KIND, AssignmentStmt, CallExpr, @@ -57,7 +55,7 @@ is_optional_type, object_rprimitive, ) -from mypyc.irbuild.builder import IRBuilder +from mypyc.irbuild.builder import IRBuilder, create_type_params from mypyc.irbuild.function import ( gen_property_getter_ir, gen_property_setter_ir, @@ -475,35 +473,20 @@ def make_generic_base_class( ) -> Value: """Construct Generic[...] base class object for a new-style generic class (Python 3.12).""" mod = builder.call_c(import_op, [builder.load_str("_typing")], line) - tvs = [] - type_var_imported: Value | None = None - for type_param in type_args: - unpack = False - if type_param.kind == TYPE_VAR_KIND: - if type_var_imported: - # Reuse previously imported value as a minor optimization - tvt = type_var_imported - else: - tvt = builder.py_get_attr(mod, "TypeVar", line) - type_var_imported = tvt - elif type_param.kind == TYPE_VAR_TUPLE_KIND: - tvt = builder.py_get_attr(mod, "TypeVarTuple", line) - unpack = True - else: - assert type_param.kind == PARAM_SPEC_KIND - tvt = builder.py_get_attr(mod, "ParamSpec", line) - tv = builder.py_call(tvt, [builder.load_str(type_param.name)], line) - builder.init_type_var(tv, type_param.name, line) - if unpack: + tvs = create_type_params(builder, mod, type_args, line) + args = [] + for tv, type_param in zip(tvs, type_args): + if type_param.kind == TYPE_VAR_TUPLE_KIND: # Evaluate *Ts for a TypeVarTuple it = builder.call_c(iter_op, [tv], line) tv = builder.call_c(next_op, [it], line) - tvs.append(tv) + args.append(tv) + gent = builder.py_get_attr(mod, "Generic", line) - if len(tvs) == 1: - arg = tvs[0] + if len(args) == 1: + arg = args[0] else: - arg = builder.new_tuple(tvs, line) + arg = builder.new_tuple(args, line) base = builder.call_c(py_get_item_op, [gent, arg], line) return base diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 2c17eb2bb14d..4d828b1b9d82 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -12,6 +12,8 @@ from typing import Callable, Sequence from mypy.nodes import ( + ARG_NAMED, + ARG_POS, AssertStmt, AssignmentStmt, AwaitExpr, @@ -37,6 +39,7 @@ TempNode, TryStmt, TupleExpr, + TypeAliasStmt, WhileStmt, WithStmt, YieldExpr, @@ -74,7 +77,7 @@ object_rprimitive, ) from mypyc.irbuild.ast_helpers import is_borrow_friendly_expr, process_conditional -from mypyc.irbuild.builder import IRBuilder, int_borrow_friendly_op +from mypyc.irbuild.builder import IRBuilder, create_type_params, int_borrow_friendly_op from mypyc.irbuild.for_helpers import for_loop_helper from mypyc.irbuild.generator import add_raise_exception_blocks_to_generator_class from mypyc.irbuild.nonlocalcontrol import ( @@ -105,7 +108,9 @@ coro_op, import_from_many_op, import_many_op, + import_op, send_op, + set_type_alias_compute_function_op, type_op, yield_from_except_op, ) @@ -1015,3 +1020,30 @@ def transform_await_expr(builder: IRBuilder, o: AwaitExpr) -> Value: def transform_match_stmt(builder: IRBuilder, m: MatchStmt) -> None: m.accept(MatchVisitor(builder, m)) + + +def transform_type_alias_stmt(builder: IRBuilder, s: TypeAliasStmt) -> None: + line = s.line + # Use "_typing" to avoid importing "typing", as the latter can be expensive. + # "_typing" includes everything we need here. + mod = builder.call_c(import_op, [builder.load_str("_typing")], line) + type_params = create_type_params(builder, mod, s.type_args, s.line) + + type_alias_type = builder.py_get_attr(mod, "TypeAliasType", line) + args = [builder.load_str(s.name.name), builder.none()] + arg_names: list[str | None] = [None, None] + arg_kinds = [ARG_POS, ARG_POS] + if s.type_args: + args.append(builder.new_tuple(type_params, line)) + arg_names.append("type_params") + arg_kinds.append(ARG_NAMED) + alias = builder.py_call(type_alias_type, args, line, arg_names=arg_names, arg_kinds=arg_kinds) + + # Use primitive to set function used to lazily compute type alias type value. + # The value needs to be lazily computed to match Python runtime behavior, but + # Python public APIs don't support this, so we use a C primitive. + compute_fn = s.value.accept(builder.visitor) + builder.builder.primitive_op(set_type_alias_compute_function_op, [alias, compute_fn], line) + + target = builder.get_assignment_target(s.name) + builder.assign(target, alias, line) diff --git a/mypyc/irbuild/visitor.py b/mypyc/irbuild/visitor.py index e7256f036e4c..05a033c3e6ad 100644 --- a/mypyc/irbuild/visitor.py +++ b/mypyc/irbuild/visitor.py @@ -137,6 +137,7 @@ transform_raise_stmt, transform_return_stmt, transform_try_stmt, + transform_type_alias_stmt, transform_while_stmt, transform_with_stmt, transform_yield_expr, @@ -251,7 +252,7 @@ def visit_match_stmt(self, stmt: MatchStmt) -> None: transform_match_stmt(self.builder, stmt) def visit_type_alias_stmt(self, stmt: TypeAliasStmt) -> None: - self.bail('The "type" statement is not yet supported by mypyc', stmt.line) + transform_type_alias_stmt(self.builder, stmt) # Expressions diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 8aa5a77c180c..2ec04e4c5b5c 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -901,6 +901,7 @@ PyObject *CPySingledispatch_RegisterFunction(PyObject *singledispatch_func, PyOb PyObject *CPy_GetAIter(PyObject *obj); PyObject *CPy_GetANext(PyObject *aiter); +void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_value); #ifdef __cplusplus } diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index f28eeb57e646..803123d436a2 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -940,3 +940,34 @@ PyObject *CPy_GetANext(PyObject *aiter) error: return NULL; } + +#ifdef CPY_3_12_FEATURES + +// Copied from Python 3.12.3, since this struct is internal to CPython. It defines +// the structure of typing.TypeAliasType objects. We need it since compute_value is +// not part of the public API, and we need to set it to match Python runtime semantics. +// +// IMPORTANT: This needs to be kept in sync with CPython! +typedef struct { + PyObject_HEAD + PyObject *name; + PyObject *type_params; + PyObject *compute_value; + PyObject *value; + PyObject *module; +} typealiasobject; + +void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_value) { + typealiasobject *obj = (typealiasobject *)alias; + if (obj->value != NULL) { + Py_DECREF(obj->value); + } + obj->value = NULL; + Py_INCREF(compute_value); + if (obj->compute_value != NULL) { + Py_DECREF(obj->compute_value); + } + obj->compute_value = compute_value; +} + +#endif diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index fea62bbb19c4..e9016e24c46d 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -265,3 +265,15 @@ return_type=c_pyssize_t_rprimitive, error_kind=ERR_NEVER, ) + +# Set the lazy value compute function of an TypeAliasType instance (Python 3.12+). +# This must only be used as part of initializing the object. Any existing value +# will be cleared. +set_type_alias_compute_function_op = custom_primitive_op( + name="set_type_alias_compute_function", + c_function_name="CPy_SetTypeAliasTypeComputeFunction", + # (alias object, value compute function) + arg_types=[object_rprimitive, object_rprimitive], + return_type=void_rtype, + error_kind=ERR_NEVER, +) diff --git a/mypyc/test-data/fixtures/ir.py b/mypyc/test-data/fixtures/ir.py index 6f0d8da90d57..ac95ffe2c047 100644 --- a/mypyc/test-data/fixtures/ir.py +++ b/mypyc/test-data/fixtures/ir.py @@ -45,6 +45,7 @@ def __ne__(self, x: object) -> bool: pass class type: def __init__(self, o: object) -> None: ... + def __or__(self, o: object) -> Any: ... __name__ : str __annotations__: Dict[str, Any] diff --git a/mypyc/test-data/fixtures/typing-full.pyi b/mypyc/test-data/fixtures/typing-full.pyi index 3ddc1f1bba08..8bb3b1398f87 100644 --- a/mypyc/test-data/fixtures/typing-full.pyi +++ b/mypyc/test-data/fixtures/typing-full.pyi @@ -10,6 +10,9 @@ from abc import abstractmethod, ABCMeta class GenericMeta(type): pass +class _SpecialForm: + def __getitem__(self, index): ... + cast = 0 overload = 0 Any = 0 @@ -19,7 +22,6 @@ TypeVar = 0 Generic = 0 Protocol = 0 Tuple = 0 -Callable = 0 _promote = 0 NamedTuple = 0 Type = 0 @@ -30,6 +32,7 @@ Literal = 0 TypedDict = 0 NoReturn = 0 NewType = 0 +Callable: _SpecialForm T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) diff --git a/mypyc/test-data/run-python312.test b/mypyc/test-data/run-python312.test index fbafeaf3e65f..5e8a388fd8d3 100644 --- a/mypyc/test-data/run-python312.test +++ b/mypyc/test-data/run-python312.test @@ -169,4 +169,57 @@ def test_class_with_value_restriction() -> None: assert r.x == 1 r2 = Restriction[str]('a') assert r2.x == 'a' + +type A = int + +def test_simple_type_alias() -> None: + assert isinstance(A, TypeAliasType) + assert getattr(A, "__value__") is int + assert str(A) == "A" + +type B = Fwd[int] +Fwd = list + +def test_forward_reference_in_alias() -> None: + assert isinstance(B, TypeAliasType) + assert getattr(B, "__value__") == list[int] + +type R = int | list[R] + +def test_recursive_type_alias() -> None: + assert isinstance(R, TypeAliasType) + assert getattr(R, "__value__") == (int | list[R]) +[typing fixtures/typing-full.pyi] + +[case testPEP695GenericTypeAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import Callable +from types import GenericAlias + +from testutil import assertRaises + +type A[T] = list[T] + +def test_generic_alias() -> None: + assert type(A[str]) is GenericAlias + assert str(A[str]) == "A[str]" + assert str(getattr(A, "__value__")) == "list[T]" + +type B[T, S] = dict[S, T] + +def test_generic_alias_with_two_args() -> None: + assert str(B[str, int]) == "B[str, int]" + assert str(getattr(B, "__value__")) == "dict[S, T]" + +type C[*Ts] = tuple[*Ts] + +def test_type_var_tuple_type_alias() -> None: + assert str(C[int, str]) == "C[int, str]" + assert str(getattr(C, "__value__")) == "tuple[typing.Unpack[Ts]]" + +type D[**P] = Callable[P, int] + +def test_param_spec_type_alias() -> None: + assert str(D[[int, str]]) == "D[[int, str]]" + assert str(getattr(D, "__value__")) == "typing.Callable[P, int]" [typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-class-namedtuple.test b/test-data/unit/check-class-namedtuple.test index f334b9011645..fd564c7e96cb 100644 --- a/test-data/unit/check-class-namedtuple.test +++ b/test-data/unit/check-class-namedtuple.test @@ -535,7 +535,7 @@ class Base(NamedTuple): self.x = 3 # E: Property "x" defined in "Base" is read-only self[1] # E: Tuple index out of range reveal_type(self[T]) # N: Revealed type is "builtins.int" \ - # E: No overload variant of "__getitem__" of "tuple" matches argument type "object" \ + # E: No overload variant of "__getitem__" of "tuple" matches argument type "TypeVar" \ # N: Possible overload variants: \ # N: def __getitem__(self, int, /) -> int \ # N: def __getitem__(self, slice, /) -> Tuple[int, ...] @@ -568,6 +568,7 @@ reveal_type(Base(1).bad_override()) # N: Revealed type is "builtins.int" reveal_type(takes_base(Base(1))) # N: Revealed type is "builtins.int" reveal_type(takes_base(Child(1))) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testNewNamedTupleIllegalNames] from typing import Callable, NamedTuple diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 04b3f7a131cc..4fc6e9a75c83 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -2449,5 +2449,7 @@ def f() -> int: # E: Missing return statement from typing import TypeVar T = TypeVar("T") x: int -x + T # E: Unsupported operand types for + ("int" and "object") -T() # E: "object" not callable +x + T # E: Unsupported left operand type for + ("int") +T() # E: "TypeVar" not callable +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index ea3f501fd949..abcb2a4bbc48 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -624,6 +624,7 @@ reveal_type(y) X = T # Error [builtins fixtures/list.pyi] +[typing fixtures/typing-full.pyi] [out] main:9:5: error: "Node" expects 2 type arguments, but 1 given main:11:5: error: "Node" expects 2 type arguments, but 3 given diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index 7cbed5637c3a..47e508ee1a6b 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -2184,8 +2184,7 @@ from typing import TypeVar, Generic, Any T = TypeVar('T', bound='B[Any]') # The "int" error is because of typing fixture. T = TypeVar('T', bound='C') # E: Cannot redefine "T" as a type variable \ - # E: Invalid assignment target \ - # E: "int" not callable + # E: Invalid assignment target class B(Generic[T]): x: T @@ -2194,6 +2193,8 @@ class C: ... x: B[int] # E: Type argument "int" of "B" must be a subtype of "B[Any]" y: B[B[Any]] reveal_type(y.x) # N: Revealed type is "__main__.B[Any]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testNewAnalyzerDuplicateTypeVarImportCycle] # flags: --disable-error-code used-before-def @@ -2216,12 +2217,13 @@ class C: ... x: B[int] y: B[B[Any]] reveal_type(y.x) +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [out] tmp/b.py:8: error: Type argument "int" of "B" must be a subtype of "B[Any]" tmp/b.py:10: note: Revealed type is "b.B[Any]" tmp/a.py:5: error: Cannot redefine "T" as a type variable tmp/a.py:5: error: Invalid assignment target -tmp/a.py:5: error: "int" not callable [case testNewAnalyzerDuplicateTypeVarImportCycleWithAliases] # flags: --disable-error-code used-before-def diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 06c5bada1e92..b3a3645dc9f8 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -41,7 +41,8 @@ reveal_type(g(1)) # E: Value of type "Coroutine[Any, Any, Any]" must be used \ [case test695TypeVar] from typing import Callable -type Alias1[T: int] = list[T] # E: PEP 695 type aliases are not yet supported +type Alias1[T: int] = list[T] # E: PEP 695 type aliases are not yet supported \ + # E: Name "T" is not defined type Alias2[**P] = Callable[P, int] # E: PEP 695 type aliases are not yet supported \ # E: Value of type "int" is not indexable \ # E: Name "P" is not defined @@ -52,7 +53,9 @@ class Cls1[T: int]: ... # E: PEP 695 generics are not yet supported class Cls2[**P]: ... # E: PEP 695 generics are not yet supported class Cls3[*Ts]: ... # E: PEP 695 generics are not yet supported -def func1[T: int](x: T) -> T: ... # E: PEP 695 generics are not yet supported +def func1[T: int](x: T) -> T: ... # E: PEP 695 generics are not yet supported \ + # E: Name "T" is not defined + def func2[**P](x: Callable[P, int]) -> Callable[P, str]: ... # E: PEP 695 generics are not yet supported \ # E: The first argument to Callable must be a list of types, parameter specification, or "..." \ # N: See https://mypy.readthedocs.io/en/stable/kinds_of_types.html#callable-types-and-lambdas \ @@ -504,6 +507,7 @@ reveal_type(a3) # N: Revealed type is "__main__.D[builtins.str, __main__.C[buil type A4 = int | str a4: A4 reveal_type(a4) # N: Revealed type is "Union[builtins.int, builtins.str]" +[builtins fixtures/type.pyi] [case testPEP695TypeAliasWithUnusedTypeParams] # flags: --enable-incomplete-feature=NewGenericSyntax @@ -531,6 +535,8 @@ a: A reveal_type(a) # N: Revealed type is "__main__.C" class C: pass +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testPEP695TypeAliasForwardReference3] # flags: --enable-incomplete-feature=NewGenericSyntax @@ -579,12 +585,15 @@ reveal_type(a) # N: Revealed type is "Any" [case testPEP695TypeAliasInvalidType] # flags: --enable-incomplete-feature=NewGenericSyntax -type A = int | 1 # E: Invalid type: try using Literal[1] instead? +type A = int | 1 # E: Invalid type: try using Literal[1] instead? \ + # E: Unsupported operand types for | ("Type[int]" and "int") + a: A reveal_type(a) # N: Revealed type is "Union[builtins.int, Any]" type B = int + str # E: Invalid type alias: expression is not a valid type b: B reveal_type(b) # N: Revealed type is "Any" +[builtins fixtures/type.pyi] [case testPEP695TypeAliasBoundForwardReference] # mypy: enable-incomplete-feature=NewGenericSyntax @@ -809,6 +818,7 @@ type C[**P] = Callable[P, int] f: C[[str, int | None]] reveal_type(f) # N: Revealed type is "def (builtins.str, Union[builtins.int, None]) -> builtins.int" [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testPEP695TypeVarTuple] # flags: --enable-incomplete-feature=NewGenericSyntax @@ -1062,7 +1072,7 @@ from typing import cast def f[T]( x: T = T # E: Name "T" is not defined \ - # E: Incompatible default for argument "x" (default has type "object", argument has type "T") + # E: Incompatible default for argument "x" (default has type "TypeVar", argument has type "T") ) -> T: return x @@ -1072,6 +1082,8 @@ def g[T](x: T = cast(T, None)) -> T: # E: Name "T" is not defined class C: def m[T](self, x: T = cast(T, None)) -> T: # E: Name "T" is not defined return x +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testPEP695ListComprehension] # mypy: enable-incomplete-feature=NewGenericSyntax @@ -1174,6 +1186,7 @@ class C[T]: pass type B[T] = C[T] | list[B[T]] b: B[int] reveal_type(b) # N: Revealed type is "Union[__main__.C[builtins.int], builtins.list[...]]" +[builtins fixtures/type.pyi] [case testPEP695BadRecursiveTypeAlias] # mypy: enable-incomplete-feature=NewGenericSyntax @@ -1184,6 +1197,8 @@ a: A reveal_type(a) # N: Revealed type is "Any" b: B reveal_type(b) # N: Revealed type is "Any" +[builtins fixtures/type.pyi] +[typing fixtures/typing-full.pyi] [case testPEP695RecursiveTypeAliasForwardReference] # mypy: enable-incomplete-feature=NewGenericSyntax @@ -1272,6 +1287,7 @@ reveal_type(a) # N: Revealed type is "builtins.list[Any]" type B = tuple[*Ts] # E: All type parameters should be declared ("Ts" not declared) type C = Callable[P, None] # E: All type parameters should be declared ("P" not declared) [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testPEP695NonGenericAliasToGenericClass] # mypy: enable-incomplete-feature=NewGenericSyntax @@ -1399,6 +1415,8 @@ c1: A3[C, int] c2: A3[D, str] c3: A3[C, N] # E: Value of type variable "S" of "A3" cannot be "N" c4: A3[int, str] # E: Type argument "int" of "A3" must be a subtype of "C" +[builtins fixtures/type.pyi] +[typing fixtures/typing-full.pyi] [case testPEP695TypeAliasInClassBodyOrFunction] # flags: --enable-incomplete-feature=NewGenericSyntax @@ -1451,7 +1469,7 @@ class E[T]: self.a: A reveal_type(E[str]().a) # N: Revealed type is "builtins.list[Any]" -[builtins fixtures/tuple.pyi] +[builtins fixtures/type.pyi] [typing fixtures/typing-full.pyi] [case testPEP695RedefineAsTypeAlias1] diff --git a/test-data/unit/check-redefine.test b/test-data/unit/check-redefine.test index e3f1b976d4e9..b7642d30efc8 100644 --- a/test-data/unit/check-redefine.test +++ b/test-data/unit/check-redefine.test @@ -270,14 +270,17 @@ def f() -> None: from typing import TypeVar def f() -> None: x = TypeVar('x') - x = 1 # E: Invalid assignment target - reveal_type(x) # N: Revealed type is "builtins.int" + x = 1 # E: Invalid assignment target \ + # E: Incompatible types in assignment (expression has type "int", variable has type "TypeVar") + reveal_type(x) # N: Revealed type is "typing.TypeVar" y = 1 # NOTE: '"int" not callable' is due to test stubs y = TypeVar('y') # E: Cannot redefine "y" as a type variable \ - # E: "int" not callable + # E: Incompatible types in assignment (expression has type "TypeVar", variable has type "int") def h(a: y) -> y: return a # E: Variable "y" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testCannotRedefineVarAsModule] # flags: --allow-redefinition diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 5eea1fb2b53e..6f9e9eda1d02 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -93,11 +93,9 @@ T = TypeVar('T') A = Tuple[T, T] if int(): - A = Union[T, int] # E: Cannot assign multiple types to name "A" without an explicit "Type[...]" annotation \ - # E: Value of type "int" is not indexable - # the second error is because of `Union = 0` in lib-stub/typing.pyi + A = Union[T, int] # E: Cannot assign multiple types to name "A" without an explicit "Type[...]" annotation [builtins fixtures/tuple.pyi] -[out] +[typing fixtures/typing-full.pyi] [case testProhibitUsingVariablesAsTypesAndAllowAliasesAsTypes] @@ -1202,8 +1200,7 @@ unbound_tvt_alias2: Ta10[int] # E: Bad number of arguments for type alias, expe reveal_type(unbound_tvt_alias2) # N: Revealed type is "def (*Any) -> builtins.str" class A(Generic[T]): - Ta11 = TypeAliasType("Ta11", Dict[str, T], type_params=(T,)) # E: Can't use bound type variable "T" to define generic alias \ - # E: "T" is a type variable and only valid in type context + Ta11 = TypeAliasType("Ta11", Dict[str, T], type_params=(T,)) # E: Can't use bound type variable "T" to define generic alias x: A.Ta11 = {"a": 1} reveal_type(x) # N: Revealed type is "builtins.dict[builtins.str, Any]" [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 0aff702e1b22..8f7dd12d9cd4 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -1100,9 +1100,10 @@ reveal_type(t.fn) # N: Revealed type is "def (builtins.int, builtins.int, built [builtins fixtures/tuple.pyi] [case testVariadicNamedTuple] -from typing import Tuple, Callable, NamedTuple, Generic +from typing import Tuple, Callable, NamedTuple, Generic, TypeVar from typing_extensions import TypeVarTuple, Unpack +T = TypeVar("T") Ts = TypeVarTuple("Ts") class A(NamedTuple, Generic[Unpack[Ts], T]): fn: Callable[[Unpack[Ts]], None] @@ -1129,9 +1130,10 @@ nt2 = A(fn=bad, val=42) # E: Argument "fn" to "A" has incompatible type "Callab [builtins fixtures/tuple.pyi] [case testVariadicTypedDict] -from typing import Tuple, Callable, Generic +from typing import Tuple, Callable, Generic, TypeVar from typing_extensions import TypeVarTuple, Unpack, TypedDict +T = TypeVar("T") Ts = TypeVarTuple("Ts") class A(TypedDict, Generic[Unpack[Ts], T]): fn: Callable[[Unpack[Ts]], None] diff --git a/test-data/unit/check-union-or-syntax.test b/test-data/unit/check-union-or-syntax.test index b5fd85cb7ed8..a1b63077eef9 100644 --- a/test-data/unit/check-union-or-syntax.test +++ b/test-data/unit/check-union-or-syntax.test @@ -189,7 +189,7 @@ def g(x: int | str | tuple[int, str] | C) -> None: # flags: --python-version 3.9 from typing import Union def f(x: Union[int, str, None]) -> None: - if isinstance(x, int | str): # E: Unsupported left operand type for | ("Type[int]") + if isinstance(x, int | str): reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" else: reveal_type(x) # N: Revealed type is "None" diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index f46cfebb113f..3364dee6c696 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -1443,7 +1443,13 @@ class C: pass class D: pass type E = D [out] + -> m + -> m -> m + -> m + -> m -> m -> m -> m +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/fine-grained-python312.test b/test-data/unit/fine-grained-python312.test index 70cf427d6798..3970c8cacfbf 100644 --- a/test-data/unit/fine-grained-python312.test +++ b/test-data/unit/fine-grained-python312.test @@ -75,6 +75,7 @@ from typing import Union as B from builtins import tuple as B [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [out] == main:4: error: Incompatible types in assignment (expression has type "int", variable has type "tuple[int, str]") diff --git a/test-data/unit/fixtures/isinstance.pyi b/test-data/unit/fixtures/isinstance.pyi index c1125c24b941..c1446492af9b 100644 --- a/test-data/unit/fixtures/isinstance.pyi +++ b/test-data/unit/fixtures/isinstance.pyi @@ -5,8 +5,9 @@ T = TypeVar('T') class object: def __init__(self) -> None: pass -class type: +class type(Generic[T]): def __init__(self, x) -> None: pass + def __or__(self, other: type) -> type: pass class tuple(Generic[T]): pass diff --git a/test-data/unit/fixtures/tuple.pyi b/test-data/unit/fixtures/tuple.pyi index eb89de8c86ef..3b62d7fc1513 100644 --- a/test-data/unit/fixtures/tuple.pyi +++ b/test-data/unit/fixtures/tuple.pyi @@ -3,8 +3,8 @@ import _typeshed from typing import Iterable, Iterator, TypeVar, Generic, Sequence, Optional, overload, Tuple, Type -T = TypeVar("T") -Tco = TypeVar('Tco', covariant=True) +_T = TypeVar("_T") +_Tco = TypeVar('_Tco', covariant=True) class object: def __init__(self) -> None: pass @@ -12,17 +12,17 @@ class object: class type: def __init__(self, *a: object) -> None: pass def __call__(self, *a: object) -> object: pass -class tuple(Sequence[Tco], Generic[Tco]): - def __new__(cls: Type[T], iterable: Iterable[Tco] = ...) -> T: ... - def __iter__(self) -> Iterator[Tco]: pass +class tuple(Sequence[_Tco], Generic[_Tco]): + def __new__(cls: Type[_T], iterable: Iterable[_Tco] = ...) -> _T: ... + def __iter__(self) -> Iterator[_Tco]: pass def __contains__(self, item: object) -> bool: pass @overload - def __getitem__(self, x: int) -> Tco: pass + def __getitem__(self, x: int) -> _Tco: pass @overload - def __getitem__(self, x: slice) -> Tuple[Tco, ...]: ... - def __mul__(self, n: int) -> Tuple[Tco, ...]: pass - def __rmul__(self, n: int) -> Tuple[Tco, ...]: pass - def __add__(self, x: Tuple[Tco, ...]) -> Tuple[Tco, ...]: pass + def __getitem__(self, x: slice) -> Tuple[_Tco, ...]: ... + def __mul__(self, n: int) -> Tuple[_Tco, ...]: pass + def __rmul__(self, n: int) -> Tuple[_Tco, ...]: pass + def __add__(self, x: Tuple[_Tco, ...]) -> Tuple[_Tco, ...]: pass def count(self, obj: object) -> int: pass class function: __name__: str @@ -40,13 +40,13 @@ class str: pass # For convenience class bytes: pass class bytearray: pass -class list(Sequence[T], Generic[T]): +class list(Sequence[_T], Generic[_T]): @overload - def __getitem__(self, i: int) -> T: ... + def __getitem__(self, i: int) -> _T: ... @overload - def __getitem__(self, s: slice) -> list[T]: ... + def __getitem__(self, s: slice) -> list[_T]: ... def __contains__(self, item: object) -> bool: ... - def __iter__(self) -> Iterator[T]: ... + def __iter__(self) -> Iterator[_T]: ... def isinstance(x: object, t: type) -> bool: pass diff --git a/test-data/unit/fixtures/type.pyi b/test-data/unit/fixtures/type.pyi index 084b7f8388d8..4ae8ed9ca6b1 100644 --- a/test-data/unit/fixtures/type.pyi +++ b/test-data/unit/fixtures/type.pyi @@ -27,6 +27,7 @@ class bool: pass class int: pass class str: pass class ellipsis: pass +class float: pass if sys.version_info >= (3, 10): # type: ignore def isinstance(obj: object, class_or_tuple: type | types.UnionType, /) -> bool: ... diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index 71d4dcb58853..9d61361fc16e 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -10,8 +10,11 @@ from abc import abstractmethod, ABCMeta class GenericMeta(type): pass -class _SpecialForm: ... -class TypeVar: ... +class _SpecialForm: + def __getitem__(self, index: Any) -> Any: ... +class TypeVar: + def __init__(self, name, *args, bound=None): ... + def __or__(self, other): ... class ParamSpec: ... class TypeVarTuple: ... @@ -19,12 +22,10 @@ def cast(t, o): ... def assert_type(o, t): ... overload = 0 Any = 0 -Union = 0 Optional = 0 Generic = 0 Protocol = 0 Tuple = 0 -Callable = 0 _promote = 0 Type = 0 no_type_check = 0 @@ -36,6 +37,8 @@ NoReturn = 0 NewType = 0 Self = 0 Unpack = 0 +Callable: _SpecialForm +Union: _SpecialForm T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) @@ -211,3 +214,5 @@ class TypeAliasType: def __init__( self, name: str, value: Any, *, type_params: Tuple[Union[TypeVar, ParamSpec, TypeVarTuple], ...] = () ) -> None: ... + + def __or__(self, other: Any) -> Any: ... diff --git a/test-data/unit/lib-stub/types.pyi b/test-data/unit/lib-stub/types.pyi index e4869dbc3093..dded0ba6cd9a 100644 --- a/test-data/unit/lib-stub/types.pyi +++ b/test-data/unit/lib-stub/types.pyi @@ -9,6 +9,8 @@ class ModuleType: __file__: str def __getattr__(self, name: str) -> Any: pass +class GenericAlias: ... + if sys.version_info >= (3, 10): class Union: def __or__(self, x) -> Union: ... diff --git a/test-data/unit/parse-python312.test b/test-data/unit/parse-python312.test index 28204ccd647b..90ee96f38deb 100644 --- a/test-data/unit/parse-python312.test +++ b/test-data/unit/parse-python312.test @@ -7,9 +7,12 @@ MypyFile:1( NameExpr(A) TypeParam( T) - IndexExpr:2( - NameExpr(C) - NameExpr(T)))) + LambdaExpr:2( + Block:-1( + ReturnStmt:2( + IndexExpr:2( + NameExpr(C) + NameExpr(T))))))) [case testPEP695GenericFunction] # mypy: enable-incomplete-feature=NewGenericSyntax From 06c7d2613a3eebac6a09befcad2b2fae6374d262 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Mon, 17 Jun 2024 15:22:14 +0200 Subject: [PATCH 038/247] stubgen: Gracefully handle invalid `Optional` and recognize aliases to PEP 604 unions (#17386) This Fixes 2 issues with invalid `Optional` (inspired by an error reported in #17197): - do not crash on empty `Optional` - treat `Optional` with more than one index as an unknown type instead of choosing the first type. It also fixes PEP 604 unions not being recognized as type aliases. --- mypy/stubgen.py | 6 ++++++ mypy/stubutil.py | 4 +++- test-data/unit/stubgen.test | 30 ++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 22028694ad6b..8478bd2135e4 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -314,6 +314,8 @@ def visit_index_expr(self, node: IndexExpr) -> str: return " | ".join([item.accept(self) for item in node.index.items]) return node.index.accept(self) if base_fullname == "typing.Optional": + if isinstance(node.index, TupleExpr): + return self.stubgen.add_name("_typeshed.Incomplete") return f"{node.index.accept(self)} | None" base = node.base.accept(self) index = node.index.accept(self) @@ -1060,6 +1062,10 @@ def is_alias_expression(self, expr: Expression, top_level: bool = True) -> bool: else: return False return all(self.is_alias_expression(i, top_level=False) for i in indices) + elif isinstance(expr, OpExpr) and expr.op == "|": + return self.is_alias_expression( + expr.left, top_level=False + ) and self.is_alias_expression(expr.right, top_level=False) else: return False diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 8e41d6862531..2f2db0dbbe53 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -257,7 +257,9 @@ def visit_unbound_type(self, t: UnboundType) -> str: if fullname == "typing.Union": return " | ".join([item.accept(self) for item in t.args]) if fullname == "typing.Optional": - return f"{t.args[0].accept(self)} | None" + if len(t.args) == 1: + return f"{t.args[0].accept(self)} | None" + return self.stubgen.add_name("_typeshed.Incomplete") if fullname in TYPING_BUILTIN_REPLACEMENTS: s = self.stubgen.add_name(TYPING_BUILTIN_REPLACEMENTS[fullname], require=True) if self.known_modules is not None and "." in s: diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index 916e2e3a8e17..5dcb0706a8cb 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -4366,3 +4366,33 @@ class Foo(Enum): class Bar(Enum): A = ... B = ... + +[case testGracefullyHandleInvalidOptionalUsage] +from typing import Optional + +x: Optional # invalid +y: Optional[int] # valid +z: Optional[int, str] # invalid +w: Optional[int | str] # valid +r: Optional[type[int | str]] + +X = Optional +Y = Optional[int] +Z = Optional[int, str] +W = Optional[int | str] +R = Optional[type[int | str]] + +[out] +from _typeshed import Incomplete +from typing import Optional + +x: Incomplete +y: int | None +z: Incomplete +w: int | str | None +r: type[int | str] | None +X = Optional +Y = int | None +Z = Incomplete +W = int | str | None +R = type[int | str] | None From ba5c2793b1f9bd253c0415492dffb703eb523306 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 18 Jun 2024 00:45:00 +0100 Subject: [PATCH 039/247] Allow new-style self-types in classmethods (#17381) Fixes https://github.com/python/mypy/issues/16547 Fixes https://github.com/python/mypy/issues/16410 Fixes https://github.com/python/mypy/issues/5570 From the upvotes on the issue it looks like an important use case. From what I see this is an omission in the original implementation, I don't see any additional unsafety (except for the same that exists for instance methods/variables). I also incorporate a small refactoring and remove couple unused `get_proper_type()` calls. The fix uncovered an unrelated issue with unions in descriptors, so I fix that one as well. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/checkexpr.py | 4 +- mypy/checkmember.py | 78 ++++++++++++++--------- test-data/unit/check-classes.test | 35 ++++++++++ test-data/unit/check-recursive-types.test | 2 +- test-data/unit/check-selftype.test | 61 ++++++++++++++++++ 5 files changed, 149 insertions(+), 31 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 4fd1a308e560..1cea4f6c19e6 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -3261,7 +3261,9 @@ def analyze_ordinary_member_access(self, e: MemberExpr, is_lvalue: bool) -> Type if isinstance(base, RefExpr) and isinstance(base.node, MypyFile): module_symbol_table = base.node.names if isinstance(base, RefExpr) and isinstance(base.node, Var): - is_self = base.node.is_self + # This is needed to special case self-types, so we don't need to track + # these flags separately in checkmember.py. + is_self = base.node.is_self or base.node.is_cls else: is_self = False diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 7525db25d9cd..0f117f5475ed 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -638,7 +638,7 @@ def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: Return: The return type of the appropriate ``__get__`` overload for the descriptor. """ - instance_type = get_proper_type(mx.original_type) + instance_type = get_proper_type(mx.self_type) orig_descriptor_type = descriptor_type descriptor_type = get_proper_type(descriptor_type) @@ -647,16 +647,6 @@ def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: return make_simplified_union( [analyze_descriptor_access(typ, mx) for typ in descriptor_type.items] ) - elif isinstance(instance_type, UnionType): - # map over the instance types - return make_simplified_union( - [ - analyze_descriptor_access( - descriptor_type, mx.copy_modified(original_type=original_type) - ) - for original_type in instance_type.relevant_items() - ] - ) elif not isinstance(descriptor_type, Instance): return orig_descriptor_type @@ -777,23 +767,10 @@ def analyze_var( if mx.is_lvalue and var.is_classvar: mx.msg.cant_assign_to_classvar(name, mx.context) t = freshen_all_functions_type_vars(typ) - if not (mx.is_self or mx.is_super) or supported_self_type( - get_proper_type(mx.original_type) - ): - t = expand_self_type(var, t, mx.original_type) - elif ( - mx.is_self - and original_itype.type != var.info - # If an attribute with Self-type was defined in a supertype, we need to - # rebind the Self type variable to Self type variable of current class... - and original_itype.type.self_type is not None - # ...unless `self` has an explicit non-trivial annotation. - and original_itype == mx.chk.scope.active_self_type() - ): - t = expand_self_type(var, t, original_itype.type.self_type) - t = get_proper_type(expand_type_by_instance(t, itype)) + t = expand_self_type_if_needed(t, mx, var, original_itype) + t = expand_type_by_instance(t, itype) freeze_all_type_vars(t) - result: Type = t + result = t typ = get_proper_type(typ) call_type: ProperType | None = None @@ -857,6 +834,50 @@ def analyze_var( return result +def expand_self_type_if_needed( + t: Type, mx: MemberContext, var: Var, itype: Instance, is_class: bool = False +) -> Type: + """Expand special Self type in a backwards compatible manner. + + This should ensure that mixing old-style and new-style self-types work + seamlessly. Also, re-bind new style self-types in subclasses if needed. + """ + original = get_proper_type(mx.self_type) + if not (mx.is_self or mx.is_super): + repl = mx.self_type + if is_class: + if isinstance(original, TypeType): + repl = original.item + elif isinstance(original, CallableType): + # Problematic access errors should have been already reported. + repl = erase_typevars(original.ret_type) + else: + repl = itype + return expand_self_type(var, t, repl) + elif supported_self_type( + # Support compatibility with plain old style T -> T and Type[T] -> T only. + get_proper_type(mx.self_type), + allow_instances=False, + allow_callable=False, + ): + repl = mx.self_type + if is_class and isinstance(original, TypeType): + repl = original.item + return expand_self_type(var, t, repl) + elif ( + mx.is_self + and itype.type != var.info + # If an attribute with Self-type was defined in a supertype, we need to + # rebind the Self type variable to Self type variable of current class... + and itype.type.self_type is not None + # ...unless `self` has an explicit non-trivial annotation. + and itype == mx.chk.scope.active_self_type() + ): + return expand_self_type(var, t, itype.type.self_type) + else: + return t + + def freeze_all_type_vars(member_type: Type) -> None: member_type.accept(FreezeTypeVarsVisitor()) @@ -1059,12 +1080,11 @@ def analyze_class_attribute_access( else: message = message_registry.GENERIC_INSTANCE_VAR_CLASS_ACCESS mx.msg.fail(message, mx.context) - + t = expand_self_type_if_needed(t, mx, node.node, itype, is_class=True) # Erase non-mapped variables, but keep mapped ones, even if there is an error. # In the above example this means that we infer following types: # C.x -> Any # C[int].x -> int - t = get_proper_type(expand_self_type(node.node, t, itype)) t = erase_typevars(expand_type_by_instance(t, isuper), {tv.id for tv in def_vars}) is_classmethod = (is_decorated and cast(Decorator, node.node).func.is_class) or ( diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 983cb8454a05..f37b0dd1dc41 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -1950,6 +1950,41 @@ class B: def foo(x: Union[A, B]) -> None: reveal_type(x.attr) # N: Revealed type is "builtins.str" +[case testDescriptorGetUnionRestricted] +from typing import Any, Union + +class getter: + def __get__(self, instance: X1, owner: Any) -> str: ... + +class X1: + prop = getter() + +class X2: + prop: str + +def foo(x: Union[X1, X2]) -> None: + reveal_type(x.prop) # N: Revealed type is "builtins.str" + +[case testDescriptorGetUnionType] +from typing import Any, Union, Type, overload + +class getter: + @overload + def __get__(self, instance: None, owner: Any) -> getter: ... + @overload + def __get__(self, instance: object, owner: Any) -> str: ... + def __get__(self, instance, owner): + ... + +class X1: + prop = getter() +class X2: + prop = getter() + +def foo(x: Type[Union[X1, X2]]) -> None: + reveal_type(x.prop) # N: Revealed type is "__main__.getter" + + -- _promote decorators -- ------------------- diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index 33cb9ccad9af..d5c8acd1bc15 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -440,7 +440,7 @@ from typing import NamedTuple, TypeVar, Tuple NT = NamedTuple("NT", [("x", NT), ("y", int)]) nt: NT reveal_type(nt) # N: Revealed type is "Tuple[..., builtins.int, fallback=__main__.NT]" -reveal_type(nt.x) # N: Revealed type is "Tuple[Tuple[..., builtins.int, fallback=__main__.NT], builtins.int, fallback=__main__.NT]" +reveal_type(nt.x) # N: Revealed type is "Tuple[..., builtins.int, fallback=__main__.NT]" reveal_type(nt[0]) # N: Revealed type is "Tuple[Tuple[..., builtins.int, fallback=__main__.NT], builtins.int, fallback=__main__.NT]" y: str if nt.x is not None: diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index e99b859bbcd0..fdd628b0271b 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2071,3 +2071,64 @@ p: Partial reveal_type(p()) # N: Revealed type is "Never" p2: Partial2 reveal_type(p2(42)) # N: Revealed type is "builtins.int" + +[case testAccessingSelfClassVarInClassMethod] +from typing import Self, ClassVar, Type, TypeVar + +T = TypeVar("T", bound="Foo") + +class Foo: + instance: ClassVar[Self] + @classmethod + def get_instance(cls) -> Self: + return reveal_type(cls.instance) # N: Revealed type is "Self`0" + @classmethod + def get_instance_old(cls: Type[T]) -> T: + return reveal_type(cls.instance) # N: Revealed type is "T`-1" + +class Bar(Foo): + extra: int + + @classmethod + def get_instance(cls) -> Self: + reveal_type(cls.instance.extra) # N: Revealed type is "builtins.int" + return cls.instance + + @classmethod + def other(cls) -> None: + reveal_type(cls.instance) # N: Revealed type is "Self`0" + reveal_type(cls.instance.extra) # N: Revealed type is "builtins.int" + +reveal_type(Bar.instance) # N: Revealed type is "__main__.Bar" +[builtins fixtures/classmethod.pyi] + +[case testAccessingSelfClassVarInClassMethodTuple] +from typing import Self, ClassVar, Tuple + +class C(Tuple[int, str]): + x: Self + y: ClassVar[Self] + + @classmethod + def bar(cls) -> None: + reveal_type(cls.y) # N: Revealed type is "Self`0" + @classmethod + def bar_self(self) -> Self: + return reveal_type(self.y) # N: Revealed type is "Self`0" + +c: C +reveal_type(c.x) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.C]" +reveal_type(c.y) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.C]" +reveal_type(C.y) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.C]" +C.x # E: Access to generic instance variables via class is ambiguous +[builtins fixtures/classmethod.pyi] + +[case testAccessingTypingSelfUnion] +from typing import Self, Union + +class C: + x: Self +class D: + x: int +x: Union[C, D] +reveal_type(x.x) # N: Revealed type is "Union[__main__.C, builtins.int]" From 59b2df4865d435f98b56d0692c6542e8cc8425a1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 18 Jun 2024 10:48:49 +0100 Subject: [PATCH 040/247] [PEP 695] Add more tests (#17397) --- test-data/unit/check-python312.test | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index b3a3645dc9f8..348f2d11f9a7 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1543,3 +1543,51 @@ a: A reveal_type(a) # N: Revealed type is "builtins.list[Any]" b: B reveal_type(b) # N: Revealed type is "Any" + +[case testPEP695GenericNamedTuple] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import NamedTuple + +# Invariant because of the signature of the generated _replace method +class N[T](NamedTuple): + x: T + y: int + +a: N[object] +reveal_type(a.x) # N: Revealed type is "builtins.object" +b: N[int] +reveal_type(b.x) # N: Revealed type is "builtins.int" +if int(): + a = b # E: Incompatible types in assignment (expression has type "N[int]", variable has type "N[object]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "N[object]", variable has type "N[int]") + +class M[T: (int, str)](NamedTuple): + x: T + +c: M[int] +d: M[str] +e: M[bool] # E: Value of type variable "T" of "M" cannot be "bool" + +[builtins fixtures/tuple.pyi] + +[case testPEP695GenericTypedDict] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import TypedDict + +class D[T](TypedDict): + x: T + y: int + +class E[T: str](TypedDict): + x: T + y: int + +a: D[object] +reveal_type(a["x"]) # N: Revealed type is "builtins.object" +b: D[int] +reveal_type(b["x"]) # N: Revealed type is "builtins.int" +c: E[str] +d: E[int] # E: Type argument "int" of "E" must be a subtype of "str" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] From 10f18a82b612b6127659cd64aa60c10b9cc7a904 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 18 Jun 2024 12:34:19 +0100 Subject: [PATCH 041/247] Improve fixtures for builtins.type and types.UnionType (#17400) In typeshed `builtins.type` is not generic, so it shouldn't be generic in fixtures either. Also replace `types.Union` with `types.UnionType` in test stubs, as the former doesn't exist. --- test-data/unit/check-classes.test | 2 +- test-data/unit/fixtures/isinstance.pyi | 2 +- test-data/unit/fixtures/isinstance_python3_10.pyi | 6 +++--- test-data/unit/fixtures/type.pyi | 2 +- test-data/unit/lib-stub/types.pyi | 5 +---- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index f37b0dd1dc41..427133eca10b 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -3466,7 +3466,7 @@ def foo(arg: Type[Any]): from typing import Type, Any def foo(arg: Type[Any]): reveal_type(arg.__str__) # N: Revealed type is "def () -> builtins.str" - reveal_type(arg.mro()) # N: Revealed type is "builtins.list[builtins.type[Any]]" + reveal_type(arg.mro()) # N: Revealed type is "builtins.list[builtins.type]" [builtins fixtures/type.pyi] [out] diff --git a/test-data/unit/fixtures/isinstance.pyi b/test-data/unit/fixtures/isinstance.pyi index c1446492af9b..12cef2035c2b 100644 --- a/test-data/unit/fixtures/isinstance.pyi +++ b/test-data/unit/fixtures/isinstance.pyi @@ -5,7 +5,7 @@ T = TypeVar('T') class object: def __init__(self) -> None: pass -class type(Generic[T]): +class type: def __init__(self, x) -> None: pass def __or__(self, other: type) -> type: pass diff --git a/test-data/unit/fixtures/isinstance_python3_10.pyi b/test-data/unit/fixtures/isinstance_python3_10.pyi index 7c919a216bfb..0918d10ab1ef 100644 --- a/test-data/unit/fixtures/isinstance_python3_10.pyi +++ b/test-data/unit/fixtures/isinstance_python3_10.pyi @@ -7,15 +7,15 @@ T = TypeVar('T') class object: def __init__(self) -> None: pass -class type(Generic[T]): +class type: def __init__(self, x) -> None: pass - def __or__(self, x) -> types.Union: pass + def __or__(self, x) -> types.UnionType: pass class tuple(Generic[T]): pass class function: pass -def isinstance(x: object, t: Union[Type[object], Tuple[Type[object], ...], types.Union]) -> bool: pass +def isinstance(x: object, t: Union[Type[object], Tuple[Type[object], ...], types.UnionType]) -> bool: pass def issubclass(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass class int: diff --git a/test-data/unit/fixtures/type.pyi b/test-data/unit/fixtures/type.pyi index 4ae8ed9ca6b1..0d93b2e1fcd6 100644 --- a/test-data/unit/fixtures/type.pyi +++ b/test-data/unit/fixtures/type.pyi @@ -13,7 +13,7 @@ class object: class list(Generic[T]): pass -class type(Generic[T]): +class type: __name__: str def __call__(self, *args: Any, **kwargs: Any) -> Any: pass def __or__(self, other: Union[type, None]) -> type: pass diff --git a/test-data/unit/lib-stub/types.pyi b/test-data/unit/lib-stub/types.pyi index dded0ba6cd9a..c3ac244c2a51 100644 --- a/test-data/unit/lib-stub/types.pyi +++ b/test-data/unit/lib-stub/types.pyi @@ -12,11 +12,8 @@ class ModuleType: class GenericAlias: ... if sys.version_info >= (3, 10): - class Union: - def __or__(self, x) -> Union: ... - class NoneType: ... class UnionType: - ... + def __or__(self, x) -> UnionType: ... From e1ff8aa30f291ec1613bc9893528067b269309bc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 19 Jun 2024 18:49:41 +0100 Subject: [PATCH 042/247] Consider overlap between instances and callables (#17389) Fixes https://github.com/python/mypy/issues/8869 The fix seems straightforward. --- mypy/meet.py | 18 ++++++++++++-- test-data/unit/check-statements.test | 35 ++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/mypy/meet.py b/mypy/meet.py index 48e5dfaa18ee..401200a11cc1 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -7,6 +7,7 @@ from mypy.maptype import map_instance_to_supertype from mypy.state import state from mypy.subtypes import ( + find_member, is_callable_compatible, is_equivalent, is_proper_subtype, @@ -477,9 +478,22 @@ def _type_object_overlap(left: Type, right: Type) -> bool: ignore_pos_arg_names=True, allow_partial_overlap=True, ) - elif isinstance(left, CallableType): + + call = None + other = None + if isinstance(left, CallableType) and isinstance(right, Instance): + call = find_member("__call__", right, right, is_operator=True) + other = left + if isinstance(right, CallableType) and isinstance(left, Instance): + call = find_member("__call__", left, left, is_operator=True) + other = right + if isinstance(get_proper_type(call), FunctionLike): + assert call is not None and other is not None + return _is_overlapping_types(call, other) + + if isinstance(left, CallableType): left = left.fallback - elif isinstance(right, CallableType): + if isinstance(right, CallableType): right = right.fallback if isinstance(left, LiteralType) and isinstance(right, LiteralType): diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 71cc80719779..34df5a8ab336 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -2307,3 +2307,38 @@ class Outer: class Inner: break # E: "break" outside loop [builtins fixtures/list.pyi] + +[case testCallableInstanceOverlapAllowed] +# flags: --warn-unreachable +from typing import Any, Callable, List + +class CAny: + def __call__(self) -> Any: ... +class CNone: + def __call__(self) -> None: ... +class CWrong: + def __call__(self, x: int) -> None: ... + +def describe(func: Callable[[], None]) -> str: + if isinstance(func, CAny): + return "CAny" + elif isinstance(func, CNone): + return "CNone" + elif isinstance(func, CWrong): + return "CWrong" # E: Statement is unreachable + else: + return "other" + +class C(CAny): + def __call__(self) -> None: ... + +def f(): + pass + +describe(CAny()) +describe(C()) +describe(CNone()) +describe(CWrong()) # E: Argument 1 to "describe" has incompatible type "CWrong"; expected "Callable[[], None]" \ + # N: "CWrong.__call__" has type "Callable[[Arg(int, 'x')], None]" +describe(f) +[builtins fixtures/isinstancelist.pyi] From 7cb733ad42eccaccd29380d46d5c222ccc2788cb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 19 Jun 2024 23:48:35 +0100 Subject: [PATCH 043/247] Re-work overload overlap logic (#17392) Fixes https://github.com/python/mypy/issues/5510 OK, so I noticed during last couple years, that every other time I change something about type variables, a few unsafe overload overlap errors either appears or disappears. At some point I almost stopped looking at them. The problem is that unsafe overload overlap detection for generic callables is currently ad-hoc. However, as I started working on it, I discovered a bunch of foundational problems (and few smaller issues), so I decided to re-work the unsafe overload overlap detection. Here is a detailed summary: * Currently return type compatibility is decided using regular subtype check. Although it is technically correct, in most cases there is nothing wrong if first overload returns `list[Subtype]` and second returns `list[Supertype]`. All the unsafe overload story is about runtime values, not static types, so we should use `is_subset()` instead of `is_subtype()`, which is IIUC easy to implement: we simply need to consider all invariant types covariant. * Current implementation only checks for overlap between parameters, i.e. it checks if there are some calls that are valid for both overloads. But we also need to check that those common calls will not be always caught by the first overload. I assume it was not checked because, naively, we already check elsewhere that first overload doesn't completely shadow the second one. But this is not the same: first overload may be not more general overall, but when narrowed to common calls, it may be more general. Example of such false-positive (this is an oversimplified version of what is often used in situations with many optional positional arguments): ```python @overload def foo(x: object) -> object: ... @overload def foo(x: int = ...) -> int: ... ``` * Currently overlap for generic callables is decided using some weird two-way unification procedure, where we actually keep going on (with non-unified variables, and/or ``) if the right to left unification fails. TBH I never understood this. What we need is to find some set of type variable values that makes two overloads unsafely overlapping. Constraint inference may be used as a (good) source of such guesses, but is not decisive in any way. So instead I simply try all combinations of upper bounds and values. The main benefit of such approach is that it is guaranteed false-positive free. If such algorithm finds an overlap it is definitely an overlap. There are however false negatives, but we can incrementally tighten them in the future. * I am making `Any` overlap nothing when considering overloads. Currently it overlaps everything (i.e. it is not different from `object`), but this violates the rule that replacing a precise type with `Any` should not generate an error. IOW I essentially treat `Any` as "too dynamic or not imported". * I extend `None` special-casing to be more uniform. Now essentially it only overlaps with explicitly optional types. This is important for descriptor-like signatures. * Finally, I did a cleanup in `is_overlapping_types()`, most notably flags were not passed down to various (recursive) helpers, and `ParamSpec`/`Parameters` were treated a bit arbitrary. Pros/cons of the outcome: * Pro: simple (even if not 100% accurate) mental model * Pro: all major classes of false positives eliminated * Pro: couple minor false negatives fixed * Con: two new false negatives added, more details below So here a two new false negatives and motivation on why I think they are OK. First example is ```python T = TypeVar("T") @overload def foo(x: str) -> int: ... @overload def foo(x: T) -> T: ... def foo(x): if isinstance(x, str): return 0 return x ``` This is obviously unsafe (consider `T = float`), but not flagged after this PR. I think this is ~fine for two reasons: * There is no good alternative for a user, the error is not very actionable. Using types like `(str | T) -> int | T` is a bad idea because unions with type variables are not only imprecise, but also highly problematic for inference. * The false negative is mostly affecting unbounded type variables, if a "suspicious" bound is used (like `bound=float` in this example), the error will be still reported. Second example is signatures like ```python @overload def foo(x: str, y: str) -> str: ... @overload def foo(*args: str) -> int: ... @overload def bar(*, x: str, y: str) -> str: ... @overload def bar(**kwds: str) -> int: ... ``` These are also unsafe because one can fool mypy with `x: tuple[str, ...] = ("x", "y"); foo(*x)` and `x: dict[str, str] = {"x": "x", "y": "y"}; bar(**x)`. I think this is OK because while such unsafe calls are quite rare, this kind of catch-all fallback as last overload is relatively common. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- mypy/checker.py | 195 +++++++++++------- mypy/constraints.py | 2 +- mypy/expandtype.py | 2 +- mypy/meet.py | 111 +++++----- mypy/messages.py | 10 +- mypy/semanal.py | 9 +- mypy/subtypes.py | 68 ++---- mypy/types.py | 9 +- mypy/typeshed/stdlib/builtins.pyi | 4 +- test-data/unit/check-async-await.test | 4 +- test-data/unit/check-classes.test | 18 +- test-data/unit/check-generics.test | 14 +- test-data/unit/check-overloading.test | 144 ++++++++----- .../unit/check-parameter-specification.test | 20 +- test-data/unit/check-selftype.test | 6 +- test-data/unit/pythoneval.test | 11 +- 16 files changed, 352 insertions(+), 275 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index bf739e7d1242..3a7f231ebf1d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -170,7 +170,6 @@ false_only, fixup_partial_type, function_type, - get_type_vars, is_literal_type_like, is_singleton_type, make_simplified_union, @@ -787,7 +786,16 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: type_vars = current_class.defn.type_vars if current_class else [] with state.strict_optional_set(True): if is_unsafe_overlapping_overload_signatures(sig1, sig2, type_vars): - self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, item.func) + flip_note = ( + j == 0 + and not is_unsafe_overlapping_overload_signatures( + sig2, sig1, type_vars + ) + and not overload_can_never_match(sig2, sig1) + ) + self.msg.overloaded_signatures_overlap( + i + 1, i + j + 2, flip_note, item.func + ) if impl_type is not None: assert defn.impl is not None @@ -1764,6 +1772,8 @@ def is_unsafe_overlapping_op( # second operand is the right argument -- we switch the order of # the arguments of the reverse method. + # TODO: this manipulation is dangerous if callables are generic. + # Shuffling arguments between callables can create meaningless types. forward_tweaked = forward_item.copy_modified( arg_types=[forward_base_erased, forward_item.arg_types[0]], arg_kinds=[nodes.ARG_POS] * 2, @@ -1790,7 +1800,9 @@ def is_unsafe_overlapping_op( current_class = self.scope.active_class() type_vars = current_class.defn.type_vars if current_class else [] - return is_unsafe_overlapping_overload_signatures(first, second, type_vars) + return is_unsafe_overlapping_overload_signatures( + first, second, type_vars, partial_only=False + ) def check_inplace_operator_method(self, defn: FuncBase) -> None: """Check an inplace operator method such as __iadd__. @@ -2185,7 +2197,7 @@ def get_op_other_domain(self, tp: FunctionLike) -> Type | None: if isinstance(tp, CallableType): if tp.arg_kinds and tp.arg_kinds[0] == ARG_POS: # For generic methods, domain comparison is tricky, as a first - # approximation erase all remaining type variables to bounds. + # approximation erase all remaining type variables. return erase_typevars(tp.arg_types[0], {v.id for v in tp.variables}) return None elif isinstance(tp, Overloaded): @@ -7827,68 +7839,112 @@ def are_argument_counts_overlapping(t: CallableType, s: CallableType) -> bool: return min_args <= max_args +def expand_callable_variants(c: CallableType) -> list[CallableType]: + """Expand a generic callable using all combinations of type variables' values/bounds.""" + for tv in c.variables: + # We need to expand self-type before other variables, because this is the only + # type variable that can have other type variables in the upper bound. + if tv.id.is_self(): + c = expand_type(c, {tv.id: tv.upper_bound}).copy_modified( + variables=[v for v in c.variables if not v.id.is_self()] + ) + break + + if not c.is_generic(): + # Fast path. + return [c] + + tvar_values = [] + for tvar in c.variables: + if isinstance(tvar, TypeVarType) and tvar.values: + tvar_values.append(tvar.values) + else: + tvar_values.append([tvar.upper_bound]) + + variants = [] + for combination in itertools.product(*tvar_values): + tvar_map = {tv.id: subst for (tv, subst) in zip(c.variables, combination)} + variants.append(expand_type(c, tvar_map).copy_modified(variables=[])) + return variants + + def is_unsafe_overlapping_overload_signatures( - signature: CallableType, other: CallableType, class_type_vars: list[TypeVarLikeType] + signature: CallableType, + other: CallableType, + class_type_vars: list[TypeVarLikeType], + partial_only: bool = True, ) -> bool: """Check if two overloaded signatures are unsafely overlapping or partially overlapping. - We consider two functions 's' and 't' to be unsafely overlapping if both - of the following are true: + We consider two functions 's' and 't' to be unsafely overlapping if three + conditions hold: + + 1. s's parameters are partially overlapping with t's. i.e. there are calls that are + valid for both signatures. + 2. for these common calls, some of t's parameters types are wider that s's. + 3. s's return type is NOT a subset of t's. - 1. s's parameters are all more precise or partially overlapping with t's - 2. s's return type is NOT a subtype of t's. + Note that we use subset rather than subtype relationship in these checks because: + * Overload selection happens at runtime, not statically. + * This results in more lenient behavior. + This can cause false negatives (e.g. if overloaded function returns an externally + visible attribute with invariant type), but such situations are rare. In general, + overloads in Python are generally unsafe, so we intentionally try to avoid giving + non-actionable errors (see more details in comments below). Assumes that 'signature' appears earlier in the list of overload alternatives then 'other' and that their argument counts are overlapping. """ # Try detaching callables from the containing class so that all TypeVars - # are treated as being free. - # - # This lets us identify cases where the two signatures use completely - # incompatible types -- e.g. see the testOverloadingInferUnionReturnWithMixedTypevars - # test case. + # are treated as being free, i.e. the signature is as seen from inside the class, + # where "self" is not yet bound to anything. signature = detach_callable(signature, class_type_vars) other = detach_callable(other, class_type_vars) - # Note: We repeat this check twice in both directions due to a slight - # asymmetry in 'is_callable_compatible'. When checking for partial overlaps, - # we attempt to unify 'signature' and 'other' both against each other. - # - # If 'signature' cannot be unified with 'other', we end early. However, - # if 'other' cannot be modified with 'signature', the function continues - # using the older version of 'other'. - # - # This discrepancy is unfortunately difficult to get rid of, so we repeat the - # checks twice in both directions for now. - # - # Note that we ignore possible overlap between type variables and None. This - # is technically unsafe, but unsafety is tiny and this prevents some common - # use cases like: - # @overload - # def foo(x: None) -> None: .. - # @overload - # def foo(x: T) -> Foo[T]: ... - return is_callable_compatible( - signature, - other, - is_compat=is_overlapping_types_no_promote_no_uninhabited_no_none, - is_proper_subtype=False, - is_compat_return=lambda l, r: not is_subtype_no_promote(l, r), - ignore_return=False, - check_args_covariantly=True, - allow_partial_overlap=True, - no_unify_none=True, - ) or is_callable_compatible( - other, - signature, - is_compat=is_overlapping_types_no_promote_no_uninhabited_no_none, - is_proper_subtype=False, - is_compat_return=lambda l, r: not is_subtype_no_promote(r, l), - ignore_return=False, - check_args_covariantly=False, - allow_partial_overlap=True, - no_unify_none=True, - ) + # Note: We repeat this check twice in both directions compensate for slight + # asymmetries in 'is_callable_compatible'. + + for sig_variant in expand_callable_variants(signature): + for other_variant in expand_callable_variants(other): + # Using only expanded callables may cause false negatives, we can add + # more variants (e.g. using inference between callables) in the future. + if is_subset_no_promote(sig_variant.ret_type, other_variant.ret_type): + continue + if not ( + is_callable_compatible( + sig_variant, + other_variant, + is_compat=is_overlapping_types_for_overload, + check_args_covariantly=False, + is_proper_subtype=False, + is_compat_return=lambda l, r: not is_subset_no_promote(l, r), + allow_partial_overlap=True, + ) + or is_callable_compatible( + other_variant, + sig_variant, + is_compat=is_overlapping_types_for_overload, + check_args_covariantly=True, + is_proper_subtype=False, + is_compat_return=lambda l, r: not is_subset_no_promote(r, l), + allow_partial_overlap=True, + ) + ): + continue + # Using the same `allow_partial_overlap` flag as before, can cause false + # negatives in case where star argument is used in a catch-all fallback overload. + # But again, practicality beats purity here. + if not partial_only or not is_callable_compatible( + other_variant, + sig_variant, + is_compat=is_subset_no_promote, + check_args_covariantly=True, + is_proper_subtype=False, + ignore_return=True, + allow_partial_overlap=True, + ): + return True + return False def detach_callable(typ: CallableType, class_type_vars: list[TypeVarLikeType]) -> CallableType: @@ -7897,21 +7953,11 @@ def detach_callable(typ: CallableType, class_type_vars: list[TypeVarLikeType]) - A callable normally keeps track of the type variables it uses within its 'variables' field. However, if the callable is from a method and that method is using a class type variable, the callable will not keep track of that type variable since it belongs to the class. - - This function will traverse the callable and find all used type vars and add them to the - variables field if it isn't already present. - - The caller can then unify on all type variables whether the callable is originally from - the class or not.""" + """ if not class_type_vars: # Fast path, nothing to update. return typ - seen_type_vars = set() - for t in typ.arg_types + [typ.ret_type]: - seen_type_vars |= set(get_type_vars(t)) - return typ.copy_modified( - variables=list(typ.variables) + [tv for tv in class_type_vars if tv in seen_type_vars] - ) + return typ.copy_modified(variables=list(typ.variables) + class_type_vars) def overload_can_never_match(signature: CallableType, other: CallableType) -> bool: @@ -8388,21 +8434,24 @@ def get_property_type(t: ProperType) -> ProperType: return t -def is_subtype_no_promote(left: Type, right: Type) -> bool: - return is_subtype(left, right, ignore_promotions=True) +def is_subset_no_promote(left: Type, right: Type) -> bool: + return is_subtype(left, right, ignore_promotions=True, always_covariant=True) -def is_overlapping_types_no_promote_no_uninhabited_no_none(left: Type, right: Type) -> bool: - # For the purpose of unsafe overload checks we consider list[Never] and list[int] - # non-overlapping. This is consistent with how we treat list[int] and list[str] as - # non-overlapping, despite [] belongs to both. Also this will prevent false positives - # for failed type inference during unification. +def is_overlapping_types_for_overload(left: Type, right: Type) -> bool: + # Note that among other effects 'overlap_for_overloads' flag will effectively + # ignore possible overlap between type variables and None. This is technically + # unsafe, but unsafety is tiny and this prevents some common use cases like: + # @overload + # def foo(x: None) -> None: .. + # @overload + # def foo(x: T) -> Foo[T]: ... return is_overlapping_types( left, right, ignore_promotions=True, - ignore_uninhabited=True, prohibit_none_typevar_overlap=True, + overlap_for_overloads=True, ) diff --git a/mypy/constraints.py b/mypy/constraints.py index 56ca51d19486..316f481ac870 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1055,7 +1055,7 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: # like U -> U, should be Callable[..., Any], but if U is a self-type, we can # allow it to leak, to be later bound to self. A bunch of existing code # depends on this old behaviour. - and not any(tv.id.raw_id == 0 for tv in cactual.variables) + and not any(tv.id.is_self() for tv in cactual.variables) ): # If the actual callable is generic, infer constraints in the opposite # direction, and indicate to the solver there are extra type variables diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 86875bc6079a..bff23c53defd 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -221,7 +221,7 @@ def visit_instance(self, t: Instance) -> Type: def visit_type_var(self, t: TypeVarType) -> Type: # Normally upper bounds can't contain other type variables, the only exception is # special type variable Self`0 <: C[T, S], where C is the class where Self is used. - if t.id.raw_id == 0: + if t.id.is_self(): t = t.copy_modified(upper_bound=t.upper_bound.accept(self)) repl = self.variables.get(t.id, t) if isinstance(repl, ProperType) and isinstance(repl, Instance): diff --git a/mypy/meet.py b/mypy/meet.py index 401200a11cc1..91abf43c0877 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -7,6 +7,7 @@ from mypy.maptype import map_instance_to_supertype from mypy.state import state from mypy.subtypes import ( + are_parameters_compatible, find_member, is_callable_compatible, is_equivalent, @@ -257,12 +258,16 @@ def is_literal_in_union(x: ProperType, y: ProperType) -> bool: ) +def is_object(t: ProperType) -> bool: + return isinstance(t, Instance) and t.type.fullname == "builtins.object" + + def is_overlapping_types( left: Type, right: Type, ignore_promotions: bool = False, prohibit_none_typevar_overlap: bool = False, - ignore_uninhabited: bool = False, + overlap_for_overloads: bool = False, seen_types: set[tuple[Type, Type]] | None = None, ) -> bool: """Can a value of type 'left' also be of type 'right' or vice-versa? @@ -270,6 +275,9 @@ def is_overlapping_types( If 'ignore_promotions' is True, we ignore promotions while checking for overlaps. If 'prohibit_none_typevar_overlap' is True, we disallow None from overlapping with TypeVars (in both strict-optional and non-strict-optional mode). + If 'overlap_for_overloads' is True, we check for overlaps more strictly (to avoid false + positives), for example: None only overlaps with explicitly optional types, Any + doesn't overlap with anything except object, we don't ignore positional argument names. """ if isinstance(left, TypeGuardedType) or isinstance( # type: ignore[misc] right, TypeGuardedType @@ -296,7 +304,7 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: right, ignore_promotions=ignore_promotions, prohibit_none_typevar_overlap=prohibit_none_typevar_overlap, - ignore_uninhabited=ignore_uninhabited, + overlap_for_overloads=overlap_for_overloads, seen_types=seen_types.copy(), ) @@ -325,7 +333,7 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: # 'Any' may or may not be overlapping with the other type if isinstance(left, AnyType) or isinstance(right, AnyType): - return True + return not overlap_for_overloads or is_object(left) or is_object(right) # We check for complete overlaps next as a general-purpose failsafe. # If this check fails, we start checking to see if there exists a @@ -345,11 +353,25 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: ): return True - if is_proper_subtype( - left, right, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited - ) or is_proper_subtype( - right, left, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited - ): + def is_none_object_overlap(t1: Type, t2: Type) -> bool: + t1, t2 = get_proper_types((t1, t2)) + return ( + isinstance(t1, NoneType) + and isinstance(t2, Instance) + and t2.type.fullname == "builtins.object" + ) + + if overlap_for_overloads: + if is_none_object_overlap(left, right) or is_none_object_overlap(right, left): + return False + + def _is_subtype(left: Type, right: Type) -> bool: + if overlap_for_overloads: + return is_proper_subtype(left, right, ignore_promotions=ignore_promotions) + else: + return is_subtype(left, right, ignore_promotions=ignore_promotions) + + if _is_subtype(left, right) or _is_subtype(right, left): return True # See the docstring for 'get_possible_variants' for more info on what the @@ -358,21 +380,6 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: left_possible = get_possible_variants(left) right_possible = get_possible_variants(right) - # First handle special cases relating to PEP 612: - # - comparing a `Parameters` to a `Parameters` - # - comparing a `Parameters` to a `ParamSpecType` - # - comparing a `ParamSpecType` to a `ParamSpecType` - # - # These should all always be considered overlapping equality checks. - # These need to be done before we move on to other TypeVarLike comparisons. - if isinstance(left, (Parameters, ParamSpecType)) and isinstance( - right, (Parameters, ParamSpecType) - ): - return True - # A `Parameters` does not overlap with anything else, however - if isinstance(left, Parameters) or isinstance(right, Parameters): - return False - # Now move on to checking multi-variant types like Unions. We also perform # the same logic if either type happens to be a TypeVar/ParamSpec/TypeVarTuple. # @@ -422,7 +429,7 @@ def is_none_typevarlike_overlap(t1: Type, t2: Type) -> bool: # into their 'Instance' fallbacks. if isinstance(left, TypedDictType) and isinstance(right, TypedDictType): - return are_typed_dicts_overlapping(left, right, ignore_promotions=ignore_promotions) + return are_typed_dicts_overlapping(left, right, _is_overlapping_types) elif typed_dict_mapping_pair(left, right): # Overlaps between TypedDicts and Mappings require dedicated logic. return typed_dict_mapping_overlap(left, right, overlapping=_is_overlapping_types) @@ -432,7 +439,7 @@ def is_none_typevarlike_overlap(t1: Type, t2: Type) -> bool: right = right.fallback if is_tuple(left) and is_tuple(right): - return are_tuples_overlapping(left, right, ignore_promotions=ignore_promotions) + return are_tuples_overlapping(left, right, _is_overlapping_types) elif isinstance(left, TupleType): left = tuple_fallback(left) elif isinstance(right, TupleType): @@ -469,13 +476,26 @@ def _type_object_overlap(left: Type, right: Type) -> bool: if isinstance(left, TypeType) or isinstance(right, TypeType): return _type_object_overlap(left, right) or _type_object_overlap(right, left) + if isinstance(left, Parameters) and isinstance(right, Parameters): + return are_parameters_compatible( + left, + right, + is_compat=_is_overlapping_types, + is_proper_subtype=False, + ignore_pos_arg_names=not overlap_for_overloads, + allow_partial_overlap=True, + ) + # A `Parameters` does not overlap with anything else, however + if isinstance(left, Parameters) or isinstance(right, Parameters): + return False + if isinstance(left, CallableType) and isinstance(right, CallableType): return is_callable_compatible( left, right, is_compat=_is_overlapping_types, is_proper_subtype=False, - ignore_pos_arg_names=True, + ignore_pos_arg_names=not overlap_for_overloads, allow_partial_overlap=True, ) @@ -514,11 +534,7 @@ def _type_object_overlap(left: Type, right: Type) -> bool: if isinstance(left, Instance) and isinstance(right, Instance): # First we need to handle promotions and structural compatibility for instances # that came as fallbacks, so simply call is_subtype() to avoid code duplication. - if is_subtype( - left, right, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited - ) or is_subtype( - right, left, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited - ): + if _is_subtype(left, right) or _is_subtype(right, left): return True if right.type.fullname == "builtins.int" and left.type.fullname in MYPYC_NATIVE_INT_NAMES: @@ -578,32 +594,21 @@ def is_overlapping_erased_types( def are_typed_dicts_overlapping( - left: TypedDictType, - right: TypedDictType, - *, - ignore_promotions: bool = False, - prohibit_none_typevar_overlap: bool = False, + left: TypedDictType, right: TypedDictType, is_overlapping: Callable[[Type, Type], bool] ) -> bool: """Returns 'true' if left and right are overlapping TypeDictTypes.""" # All required keys in left are present and overlapping with something in right for key in left.required_keys: if key not in right.items: return False - if not is_overlapping_types( - left.items[key], - right.items[key], - ignore_promotions=ignore_promotions, - prohibit_none_typevar_overlap=prohibit_none_typevar_overlap, - ): + if not is_overlapping(left.items[key], right.items[key]): return False # Repeat check in the other direction for key in right.required_keys: if key not in left.items: return False - if not is_overlapping_types( - left.items[key], right.items[key], ignore_promotions=ignore_promotions - ): + if not is_overlapping(left.items[key], right.items[key]): return False # The presence of any additional optional keys does not affect whether the two @@ -613,11 +618,7 @@ def are_typed_dicts_overlapping( def are_tuples_overlapping( - left: Type, - right: Type, - *, - ignore_promotions: bool = False, - prohibit_none_typevar_overlap: bool = False, + left: Type, right: Type, is_overlapping: Callable[[Type, Type], bool] ) -> bool: """Returns true if left and right are overlapping tuples.""" left, right = get_proper_types((left, right)) @@ -640,15 +641,7 @@ def are_tuples_overlapping( if len(left.items) != len(right.items): return False - return all( - is_overlapping_types( - l, - r, - ignore_promotions=ignore_promotions, - prohibit_none_typevar_overlap=prohibit_none_typevar_overlap, - ) - for l, r in zip(left.items, right.items) - ) + return all(is_overlapping(l, r) for l, r in zip(left.items, right.items)) def expand_tuple_if_possible(tup: TupleType, target: int) -> TupleType: diff --git a/mypy/messages.py b/mypy/messages.py index f01b0a726584..c3a34bd41aba 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1624,13 +1624,21 @@ def overload_inconsistently_applies_decorator(self, decorator: str, context: Con context, ) - def overloaded_signatures_overlap(self, index1: int, index2: int, context: Context) -> None: + def overloaded_signatures_overlap( + self, index1: int, index2: int, flip_note: bool, context: Context + ) -> None: self.fail( "Overloaded function signatures {} and {} overlap with " "incompatible return types".format(index1, index2), context, code=codes.OVERLOAD_OVERLAP, ) + if flip_note: + self.note( + "Flipping the order of overloads will fix this error", + context, + code=codes.OVERLOAD_OVERLAP, + ) def overloaded_signature_will_never_match( self, index1: int, index2: int, context: Context diff --git a/mypy/semanal.py b/mypy/semanal.py index 03e6172bb325..c7a22d20aac6 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1739,10 +1739,14 @@ def analyze_type_param( fullname = self.qualified_name(type_param.name) if type_param.upper_bound: upper_bound = self.anal_type(type_param.upper_bound) + # TODO: we should validate the upper bound is valid for a given kind. if upper_bound is None: return None else: - upper_bound = self.named_type("builtins.object") + if type_param.kind == TYPE_VAR_TUPLE_KIND: + upper_bound = self.named_type("builtins.tuple", [self.object_type()]) + else: + upper_bound = self.object_type() default = AnyType(TypeOfAny.from_omitted_generics) if type_param.kind == TYPE_VAR_KIND: values = [] @@ -1777,8 +1781,7 @@ def analyze_type_param( return TypeVarTupleExpr( name=type_param.name, fullname=fullname, - # Upper bound for *Ts is *tuple[object, ...], it can never be object. - upper_bound=tuple_fallback.copy_modified(), + upper_bound=upper_bound, tuple_fallback=tuple_fallback, default=default, is_new_style=True, diff --git a/mypy/subtypes.py b/mypy/subtypes.py index a5d1d5d8194a..649cbae4c831 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -92,8 +92,8 @@ def __init__( ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, # Supported for both proper and non-proper + always_covariant: bool = False, ignore_promotions: bool = False, - ignore_uninhabited: bool = False, # Proper subtype flags erase_instances: bool = False, keep_erased_types: bool = False, @@ -102,8 +102,8 @@ def __init__( self.ignore_type_params = ignore_type_params self.ignore_pos_arg_names = ignore_pos_arg_names self.ignore_declared_variance = ignore_declared_variance + self.always_covariant = always_covariant self.ignore_promotions = ignore_promotions - self.ignore_uninhabited = ignore_uninhabited self.erase_instances = erase_instances self.keep_erased_types = keep_erased_types self.options = options @@ -125,8 +125,8 @@ def is_subtype( ignore_type_params: bool = False, ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, + always_covariant: bool = False, ignore_promotions: bool = False, - ignore_uninhabited: bool = False, options: Options | None = None, ) -> bool: """Is 'left' subtype of 'right'? @@ -145,8 +145,8 @@ def is_subtype( ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, + always_covariant=always_covariant, ignore_promotions=ignore_promotions, - ignore_uninhabited=ignore_uninhabited, options=options, ) else: @@ -155,8 +155,8 @@ def is_subtype( ignore_type_params, ignore_pos_arg_names, ignore_declared_variance, + always_covariant, ignore_promotions, - ignore_uninhabited, options, } ), "Don't pass both context and individual flags" @@ -191,7 +191,6 @@ def is_proper_subtype( *, subtype_context: SubtypeContext | None = None, ignore_promotions: bool = False, - ignore_uninhabited: bool = False, erase_instances: bool = False, keep_erased_types: bool = False, ) -> bool: @@ -207,19 +206,12 @@ def is_proper_subtype( if subtype_context is None: subtype_context = SubtypeContext( ignore_promotions=ignore_promotions, - ignore_uninhabited=ignore_uninhabited, erase_instances=erase_instances, keep_erased_types=keep_erased_types, ) else: assert not any( - { - ignore_promotions, - ignore_uninhabited, - erase_instances, - keep_erased_types, - ignore_uninhabited, - } + {ignore_promotions, erase_instances, keep_erased_types} ), "Don't pass both context and individual flags" if type_state.is_assumed_proper_subtype(left, right): return True @@ -409,6 +401,7 @@ def build_subtype_kind(subtype_context: SubtypeContext, proper_subtype: bool) -> subtype_context.ignore_type_params, subtype_context.ignore_pos_arg_names, subtype_context.ignore_declared_variance, + subtype_context.always_covariant, subtype_context.ignore_promotions, subtype_context.erase_instances, subtype_context.keep_erased_types, @@ -447,11 +440,7 @@ def visit_none_type(self, left: NoneType) -> bool: return True def visit_uninhabited_type(self, left: UninhabitedType) -> bool: - # We ignore this for unsafe overload checks, so that and empty list and - # a list of int will be considered non-overlapping. - if isinstance(self.right, UninhabitedType): - return True - return not self.subtype_context.ignore_uninhabited + return True def visit_erased_type(self, left: ErasedType) -> bool: # This may be encountered during type inference. The result probably doesn't @@ -590,12 +579,15 @@ def visit_instance(self, left: Instance) -> bool: if tvar.variance == VARIANCE_NOT_READY and not tried_infer: infer_class_variances(right.type) tried_infer = True + if ( + self.subtype_context.always_covariant + and tvar.variance == INVARIANT + ): + variance = COVARIANT + else: + variance = tvar.variance if not check_type_parameter( - lefta, - righta, - tvar.variance, - self.proper_subtype, - self.subtype_context, + lefta, righta, variance, self.proper_subtype, self.subtype_context ): nominal = False else: @@ -687,6 +679,8 @@ def visit_parameters(self, left: Parameters) -> bool: is_proper_subtype=False, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, ) + elif isinstance(self.right, Instance): + return self.right.type.fullname == "builtins.object" else: return False @@ -1417,7 +1411,6 @@ def is_callable_compatible( check_args_covariantly: bool = False, allow_partial_overlap: bool = False, strict_concatenate: bool = False, - no_unify_none: bool = False, ) -> bool: """Is the left compatible with the right, using the provided compatibility check? @@ -1438,7 +1431,7 @@ def is_callable_compatible( configurable. For example, when checking the validity of overloads, it's useful to see if - the first overload alternative has more precise arguments then the second. + the first overload alternative has more precise arguments than the second. We would want to check the arguments covariantly in that case. Note! The following two function calls are NOT equivalent: @@ -1534,26 +1527,11 @@ def g(x: int) -> int: ... # (below) treats type variables on the two sides as independent. if left.variables: # Apply generic type variables away in left via type inference. - unified = unify_generic_callable( - left, right, ignore_return=ignore_return, no_unify_none=no_unify_none - ) + unified = unify_generic_callable(left, right, ignore_return=ignore_return) if unified is None: return False left = unified - # If we allow partial overlaps, we don't need to leave R generic: - # if we can find even just a single typevar assignment which - # would make these callables compatible, we should return True. - - # So, we repeat the above checks in the opposite direction. This also - # lets us preserve the 'symmetry' property of allow_partial_overlap. - if allow_partial_overlap and right.variables: - unified = unify_generic_callable( - right, left, ignore_return=ignore_return, no_unify_none=no_unify_none - ) - if unified is not None: - right = unified - # Check return types. if not ignore_return and not is_compat_return(left.ret_type, right.ret_type): return False @@ -1856,8 +1834,6 @@ def unify_generic_callable( target: NormalizedCallableType, ignore_return: bool, return_constraint_direction: int | None = None, - *, - no_unify_none: bool = False, ) -> NormalizedCallableType | None: """Try to unify a generic callable type with another callable type. @@ -1888,10 +1864,6 @@ def unify_generic_callable( type.ret_type, target.ret_type, return_constraint_direction ) constraints.extend(c) - if no_unify_none: - constraints = [ - c for c in constraints if not isinstance(get_proper_type(c.target), NoneType) - ] inferred_vars, _ = mypy.solve.solve_constraints( type.variables, constraints, allow_polymorphic=True ) diff --git a/mypy/types.py b/mypy/types.py index 0f8c48c8cb7d..3f764a5cc49e 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -501,7 +501,7 @@ class TypeVarId: # function type variables. # Metavariables are allocated unique ids starting from 1. - raw_id: int = 0 + raw_id: int # Level of the variable in type inference. Currently either 0 for # declared types, or 1 for type inference metavariables. @@ -545,6 +545,10 @@ def __hash__(self) -> int: def is_meta_var(self) -> bool: return self.meta_level > 0 + def is_self(self) -> bool: + # This is a special value indicating typing.Self variable. + return self.raw_id == 0 + class TypeVarLikeType(ProperType): __slots__ = ("name", "fullname", "id", "upper_bound", "default") @@ -3095,8 +3099,7 @@ def get_proper_type(typ: Type | None) -> ProperType | None: @overload -def get_proper_types(types: list[Type] | tuple[Type, ...]) -> list[ProperType]: # type: ignore[overload-overlap] - ... +def get_proper_types(types: list[Type] | tuple[Type, ...]) -> list[ProperType]: ... @overload diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index 4a6c4bbcae45..42c0b27baf68 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -1572,9 +1572,9 @@ def pow(base: float, exp: complex | _SupportsSomeKindOfPow, mod: None = None) -> @overload def pow(base: complex, exp: complex | _SupportsSomeKindOfPow, mod: None = None) -> complex: ... @overload -def pow(base: _SupportsPow2[_E, _T_co], exp: _E, mod: None = None) -> _T_co: ... +def pow(base: _SupportsPow2[_E, _T_co], exp: _E, mod: None = None) -> _T_co: ... # type: ignore[overload-overlap] @overload -def pow(base: _SupportsPow3NoneOnly[_E, _T_co], exp: _E, mod: None = None) -> _T_co: ... +def pow(base: _SupportsPow3NoneOnly[_E, _T_co], exp: _E, mod: None = None) -> _T_co: ... # type: ignore[overload-overlap] @overload def pow(base: _SupportsPow3[_E, _M, _T_co], exp: _E, mod: _M) -> _T_co: ... @overload diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index 713c82c752ce..876fe0c6be15 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -765,7 +765,8 @@ class Task(Future[T]): @overload def wait(fs: Iterable[FT]) -> Future[Tuple[List[FT], List[FT]]]: ... \ - # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def wait(fs: Iterable[Awaitable[T]]) -> Future[Tuple[List[Task[T]], List[Task[T]]]]: ... def wait(fs: Any) -> Any: @@ -789,6 +790,7 @@ async def precise2(futures: Iterable[Awaitable[int]]) -> None: done, pending = await wait(futures) reveal_type(done) # N: Revealed type is "builtins.list[__main__.Task[builtins.int]]" + [builtins fixtures/async_await.pyi] [typing fixtures/typing-async.pyi] diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 427133eca10b..e66eab5e2927 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -2427,10 +2427,10 @@ class B: [builtins fixtures/tuple.pyi] [case testReverseOperatorTypeVar1] -from typing import TypeVar, Any +from typing import TypeVar T = TypeVar("T", bound='Real') class Real: - def __add__(self, other: Any) -> str: ... + def __add__(self, other: object) -> str: ... class Fraction(Real): def __radd__(self, other: T) -> T: ... # E: Signatures of "__radd__" of "Fraction" and "__add__" of "T" are unsafely overlapping @@ -2465,7 +2465,7 @@ reveal_type(Real() + Fraction()) # N: Revealed type is "__main__.Real" reveal_type(Fraction() + Fraction()) # N: Revealed type is "builtins.str" [case testReverseOperatorTypeVar3] -from typing import TypeVar, Any +from typing import TypeVar T = TypeVar("T", bound='Real') class Real: def __add__(self, other: FractionChild) -> str: ... @@ -2701,14 +2701,12 @@ class X: [out] tmp/foo.pyi:6: error: Signatures of "__radd__" of "B" and "__add__" of "X" are unsafely overlapping -[case testUnsafeOverlappingWithLineNo] +[case testUnsafeOverlappingNotWithAny] from typing import TypeVar class Real: def __add__(self, other) -> str: ... class Fraction(Real): def __radd__(self, other: Real) -> Real: ... -[out] -main:5: error: Signatures of "__radd__" of "Fraction" and "__add__" of "Real" are unsafely overlapping [case testOverlappingNormalAndInplaceOperatorMethod] import typing @@ -4042,10 +4040,16 @@ def f(a: Type[User]) -> int: pass # E: Overloaded function signatures 1 and 2 o @overload def f(a: object) -> str: pass +# Note: plain type is equivalent to Type[Any] so no error here @overload -def g(a: Type[User]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def g(a: Type[User]) -> int: pass @overload def g(a: type) -> str: pass + +@overload +def h(a: Type[User]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def h(a: Type[object]) -> str: pass [builtins fixtures/classmethod.pyi] [out] diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index abcb2a4bbc48..d46d19946098 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -3028,7 +3028,7 @@ def dec(f: Callable[[T], S], g: Callable[[T], U]) -> Callable[[T], Tuple[S, U]]: def id(x: V) -> V: ... -reveal_type(dec(id, id)) # N: Revealed type is "def [T] (T`7) -> Tuple[T`7, T`7]" +reveal_type(dec(id, id)) # N: Revealed type is "def [T] (T`1) -> Tuple[T`1, T`1]" [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericEllipsisSelfSpecialCase] @@ -3264,8 +3264,8 @@ def transform( def dec(f: Callable[W, U]) -> Callable[W, U]: ... def dec2(f: Callable[Concatenate[str, W], U]) -> Callable[Concatenate[bytes, W], U]: ... -reveal_type(transform(dec)) # N: Revealed type is "def [P, T] (def (builtins.int, *P.args, **P.kwargs) -> T`9) -> def (builtins.int, *P.args, **P.kwargs) -> T`9" -reveal_type(transform(dec2)) # N: Revealed type is "def [W, T] (def (builtins.int, builtins.str, *W.args, **W.kwargs) -> T`13) -> def (builtins.int, builtins.bytes, *W.args, **W.kwargs) -> T`13" +reveal_type(transform(dec)) # N: Revealed type is "def [P, T] (def (builtins.int, *P.args, **P.kwargs) -> T`3) -> def (builtins.int, *P.args, **P.kwargs) -> T`3" +reveal_type(transform(dec2)) # N: Revealed type is "def [W, T] (def (builtins.int, builtins.str, *W.args, **W.kwargs) -> T`7) -> def (builtins.int, builtins.bytes, *W.args, **W.kwargs) -> T`7" [builtins fixtures/tuple.pyi] [case testNoAccidentalVariableClashInNestedGeneric] @@ -3319,8 +3319,8 @@ def id(x: U) -> U: ... def either(x: U, y: U) -> U: ... def pair(x: U, y: V) -> Tuple[U, V]: ... -reveal_type(dec(id)) # N: Revealed type is "def [T] (T`9) -> builtins.list[T`9]" -reveal_type(dec(either)) # N: Revealed type is "def [T] (T`11, T`11) -> builtins.list[T`11]" +reveal_type(dec(id)) # N: Revealed type is "def [T] (T`3) -> builtins.list[T`3]" +reveal_type(dec(either)) # N: Revealed type is "def [T] (T`5, T`5) -> builtins.list[T`5]" reveal_type(dec(pair)) # N: Revealed type is "def [U, V] (U`-1, V`-2) -> builtins.list[Tuple[U`-1, V`-2]]" [builtins fixtures/tuple.pyi] @@ -3338,8 +3338,8 @@ V = TypeVar("V") def id(x: U) -> U: ... def either(x: U, y: U) -> U: ... -reveal_type(dec(id)) # N: Revealed type is "def [T] (builtins.list[T`9]) -> T`9" -reveal_type(dec(either)) # N: Revealed type is "def [T] (builtins.list[T`11], builtins.list[T`11]) -> T`11" +reveal_type(dec(id)) # N: Revealed type is "def [T] (builtins.list[T`3]) -> T`3" +reveal_type(dec(either)) # N: Revealed type is "def [T] (builtins.list[T`5], builtins.list[T`5]) -> T`5" [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericVariadicPopOff] diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index bcb775ba5dac..03863b2978ba 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -373,7 +373,8 @@ def foo(t, s): pass class Wrapper(Generic[T]): @overload - def foo(self, t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def foo(self, t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def foo(self, t: T, s: T) -> str: ... def foo(self, t, s): pass @@ -384,7 +385,8 @@ class Dummy(Generic[T]): pass # cause the constraint solver to not infer T = object like it did in the # first example? @overload -def bar(d: Dummy[T], t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def bar(d: Dummy[T], t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def bar(d: Dummy[T], t: T, s: T) -> str: ... def bar(d: Dummy[T], t, s): pass @@ -1325,8 +1327,9 @@ def h(x: Sequence[str]) -> int: pass @overload def h(x: Sequence[T]) -> None: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader +# Safety of this highly depends on the implementation, so we lean towards being silent. @overload -def i(x: List[str]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def i(x: List[str]) -> int: pass @overload def i(x: List[T]) -> None: pass [builtins fixtures/list.pyi] @@ -1752,14 +1755,11 @@ reveal_type(f(d)) # N: Revealed type is "builtins.list[builtins.int]" from typing import overload, Any @overload -def f(*, x: int = 3, y: int = 3) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(*, x: int = 3, y: int = 3) -> int: ... @overload def f(**kwargs: str) -> str: ... def f(*args, **kwargs): pass -# Checking an overload flagged as unsafe is a bit weird, but this is the -# cleanest way to make sure 'Any' ambiguity checks work correctly with -# keyword arguments. a: Any i: int reveal_type(f(x=a, y=i)) # N: Revealed type is "builtins.int" @@ -2163,8 +2163,9 @@ from wrapper import * [file wrapper.pyi] from typing import overload +# Safety of this highly depends on the implementation, so we lean towards being silent. @overload -def foo1(*x: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(*x: int) -> int: ... @overload def foo1(x: int, y: int, z: int) -> str: ... @@ -2173,8 +2174,9 @@ def foo2(*x: int) -> int: ... @overload def foo2(x: int, y: str, z: int) -> str: ... +# Note: this is technically unsafe, but we don't report this for now. @overload -def bar1(x: int, y: int, z: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def bar1(x: int, y: int, z: int) -> str: ... @overload def bar1(*x: int) -> int: ... @@ -2248,7 +2250,7 @@ from wrapper import * from typing import overload @overload -def foo1(x: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(x: str) -> str: ... @overload def foo1(x: str, y: str = ...) -> int: ... @@ -2268,12 +2270,12 @@ from wrapper import * from typing import overload @overload -def foo1(*args: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(*args: int) -> int: ... @overload def foo1(**kwargs: int) -> str: ... @overload -def foo2(**kwargs: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo2(**kwargs: int) -> str: ... @overload def foo2(*args: int) -> int: ... [builtins fixtures/dict.pyi] @@ -2314,13 +2316,14 @@ def foo2(x: int, *args: int) -> str: ... @overload def foo2(*args2: str) -> int: ... +# The two examples are unsafe, but this is hard to detect. @overload -def foo3(*args: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo3(*args: int) -> int: ... @overload def foo3(x: int, *args2: int) -> str: ... @overload -def foo4(x: int, *args: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo4(x: int, *args: int) -> str: ... @overload def foo4(*args2: int) -> int: ... [builtins fixtures/tuple.pyi] @@ -2357,13 +2360,13 @@ def foo4(x: Other = ..., *args: str) -> int: ... from typing import overload @overload -def foo1(x: int = 0, y: int = 0) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(x: int = 0, y: int = 0) -> int: ... @overload def foo1(*xs: int) -> str: ... def foo1(*args): pass @overload -def foo2(*xs: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo2(*xs: int) -> str: ... @overload def foo2(x: int = 0, y: int = 0) -> int: ... def foo2(*args): pass @@ -2412,12 +2415,12 @@ from wrapper import * from typing import overload @overload -def foo1(x: str, y: str = ..., z: str = ...) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(x: str, y: str = ..., z: str = ...) -> str: ... @overload def foo1(*x: str) -> int: ... @overload -def foo2(*x: str) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo2(*x: str) -> int: ... @overload def foo2(x: str, y: str = ..., z: str = ...) -> str: ... @@ -2433,12 +2436,12 @@ from wrapper import * from typing import overload @overload -def foo1(x: str, y: str = ..., z: int = ...) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(x: str, y: str = ..., z: int = ...) -> str: ... @overload def foo1(*x: str) -> int: ... @overload -def foo2(x: str, y: str = ..., z: int = ...) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo2(x: str, y: str = ..., z: int = ...) -> str: ... @overload def foo2(*x: str) -> int: ... [builtins fixtures/tuple.pyi] @@ -2449,7 +2452,7 @@ from wrapper import * from typing import overload @overload -def foo1(*, x: str, y: str, z: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(*, x: str, y: str, z: str) -> str: ... @overload def foo1(**x: str) -> int: ... @@ -2481,12 +2484,12 @@ def foo2(**x: str) -> int: ... def foo2(*, x: str, y: str, z: int) -> str: ... @overload -def foo3(*, x: str, y: str, z: int = ...) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo3(*, x: str, y: str, z: int = ...) -> str: ... @overload def foo3(**x: str) -> int: ... @overload -def foo4(**x: str) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo4(**x: str) -> int: ... @overload def foo4(*, x: str, y: str, z: int = ...) -> str: ... [builtins fixtures/dict.pyi] @@ -2497,12 +2500,13 @@ from wrapper import * from typing import overload @overload -def foo1(x: str, *, y: str, z: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(x: str, *, y: str, z: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def foo1(**x: str) -> int: ... @overload -def foo2(**x: str) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo2(**x: str) -> int: ... @overload def foo2(x: str, *, y: str, z: str) -> str: ... @@ -2798,7 +2802,8 @@ def h(x: List[Union[C, D]]) -> str: ... def h(x): ... @overload -def i(x: List[Union[A, B]]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def i(x: List[Union[A, B]]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def i(x: List[Union[A, B, C]]) -> str: ... def i(x): ... @@ -2810,8 +2815,9 @@ from typing import TypeVar, overload T = TypeVar('T') +# Note: this is unsafe, but it is hard to detect. @overload -def f(x: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: int) -> str: ... @overload def f(x: T) -> T: ... def f(x): ... @@ -2827,14 +2833,15 @@ from typing import TypeVar, overload, List T = TypeVar('T') +# Note: first two examples are unsafe, but it is hard to detect. @overload -def f1(x: List[int]) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f1(x: List[int]) -> str: ... @overload def f1(x: List[T]) -> T: ... def f1(x): ... @overload -def f2(x: List[int]) -> List[str]: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f2(x: List[int]) -> List[str]: ... @overload def f2(x: List[T]) -> List[T]: ... def f2(x): ... @@ -2859,17 +2866,15 @@ from typing import TypeVar, overload, Generic T = TypeVar('T') class Wrapper(Generic[T]): + # Similar to above: this is unsafe, but it is hard to detect. @overload - def f(self, x: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def f(self, x: int) -> str: ... @overload def f(self, x: T) -> T: ... def f(self, x): ... - # TODO: This shouldn't trigger an error message? - # Related to testTypeCheckOverloadImplementationTypeVarDifferingUsage2? - # See https://github.com/python/mypy/issues/5510 @overload - def g(self, x: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def g(self, x: int) -> int: ... @overload def g(self, x: T) -> T: ... def g(self, x): ... @@ -2880,28 +2885,27 @@ from typing import TypeVar, overload, Generic, List T = TypeVar('T') class Wrapper(Generic[T]): + # Similar to above: first two examples are unsafe, but it is hard to detect. @overload - def f1(self, x: List[int]) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def f1(self, x: List[int]) -> str: ... @overload def f1(self, x: List[T]) -> T: ... def f1(self, x): ... @overload - def f2(self, x: List[int]) -> List[str]: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def f2(self, x: List[int]) -> List[str]: ... @overload def f2(self, x: List[T]) -> List[T]: ... def f2(self, x): ... - # TODO: This shouldn't trigger an error message? - # See https://github.com/python/mypy/issues/5510 @overload - def g1(self, x: List[int]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def g1(self, x: List[int]) -> int: ... @overload def g1(self, x: List[T]) -> T: ... def g1(self, x): ... @overload - def g2(self, x: List[int]) -> List[int]: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def g2(self, x: List[int]) -> List[int]: ... @overload def g2(self, x: List[T]) -> List[T]: ... def g2(self, x): ... @@ -3078,13 +3082,14 @@ class C: pass S = TypeVar('S', A, B) @overload -def f(x: S) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: S) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def f(x: Union[B, C]) -> str: ... def f(x): pass @overload -def g(x: Union[B, C]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def g(x: Union[B, C]) -> int: ... @overload def g(x: S) -> str: ... def g(x): pass @@ -3607,7 +3612,7 @@ def test(x: T) -> T: from typing import overload, Optional @overload -def f(x: None) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: None) -> int: ... @overload def f(x: object) -> str: ... def f(x): ... @@ -3632,7 +3637,7 @@ reveal_type(g(c)) # N: Revealed type is "builtins.str" from typing import overload, Optional @overload -def f(x: None) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: None) -> int: ... @overload def f(x: object) -> str: ... def f(x): ... @@ -3978,7 +3983,7 @@ from typing import overload, Any, Optional, Union class FakeAttribute: @overload - def dummy(self, instance: None, owner: Any) -> 'FakeAttribute': ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def dummy(self, instance: None, owner: Any) -> 'FakeAttribute': ... @overload def dummy(self, instance: object, owner: Any) -> int: ... def dummy(self, instance: Optional[object], owner: Any) -> Union['FakeAttribute', int]: ... @@ -4545,7 +4550,7 @@ reveal_type(Child().foo(3).child_only()) # N: Revealed type is "builtins.in [case testOverloadAndSelfTypesGenericNoOverlap] from typing import Generic, TypeVar, Any, overload, Self, Union -T = TypeVar("T", bound=Any) +T = TypeVar("T") class C(Generic[T]): @overload def get(self, obj: None) -> Self: ... @@ -4903,7 +4908,7 @@ T = TypeVar('T') def f() -> None: @overload - def g(x: str) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def g(x: str) -> int: ... @overload def g(x: T) -> T: ... def g(x): @@ -4944,7 +4949,7 @@ x: Any reveal_type(attr(x)) # N: Revealed type is "Any" attr("hi", 1) # E: No overload variant of "attr" matches argument types "str", "int" \ # N: Possible overload variants: \ - # N: def [T in (int, float)] attr(default: T = ..., blah: int = ...) -> T \ + # N: def [T in (int, float)] attr(default: T, blah: int = ...) -> T \ # N: def attr(default: Any = ...) -> int [file lib.pyi] from typing import overload, Any, TypeVar @@ -4952,7 +4957,7 @@ from typing import overload, Any, TypeVar T = TypeVar('T', int, float) @overload -def attr(default: T = ..., blah: int = ...) -> T: ... +def attr(default: T, blah: int = ...) -> T: ... @overload def attr(default: Any = ...) -> int: ... [out] @@ -5008,7 +5013,7 @@ children: List[Child] parents: List[Parent] @overload -def f(x: Child) -> List[Child]: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: Child) -> List[Child]: pass @overload def f(x: Parent) -> List[Parent]: pass def f(x: Union[Child, Parent]) -> Union[List[Child], List[Parent]]: @@ -5319,7 +5324,7 @@ def f1(g: G[A, B]) -> B: ... def f1(g: Any) -> Any: ... @overload -def f2(g: G[A, Any]) -> A: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f2(g: G[A, Any]) -> A: ... @overload def f2(g: G[A, B], x: int = ...) -> B: ... def f2(g: Any, x: int = ...) -> Any: ... @@ -6500,7 +6505,7 @@ P = ParamSpec("P") R = TypeVar("R") @overload -def func(x: Callable[Concatenate[Any, P], R]) -> Callable[P, R]: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def func(x: Callable[Concatenate[Any, P], R]) -> Callable[P, R]: ... @overload def func(x: Callable[P, R]) -> Callable[Concatenate[str, P], R]: ... def func(x: Callable[..., R]) -> Callable[..., R]: ... @@ -6710,3 +6715,38 @@ class B: def f(self, *args, **kwargs): pass [builtins fixtures/tuple.pyi] + +[case testOverloadsSafeOverlapAllowed] +from lib import * +[file lib.pyi] +from typing import overload + +@overload +def bar(x: object) -> object: ... +@overload +def bar(x: int = ...) -> int: ... + +[case testOverloadsInvariantOverlapAllowed] +from lib import * +[file lib.pyi] +from typing import overload, List + +@overload +def bar(x: List[int]) -> List[int]: ... +@overload +def bar(x: List[object]) -> List[object]: ... + +[case testOverloadsNoneAnyOverlapAllowed] +from lib import * +[file lib.pyi] +from typing import overload, Any + +@overload +def foo(x: None) -> int: ... +@overload +def foo(x: object) -> str: ... + +@overload +def bar(x: int) -> int: ... +@overload +def bar(x: Any) -> str: ... diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 37916c2155fe..e6d8cec3f0b0 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -901,8 +901,8 @@ class A: def func(self, action: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs) -> _R: ... -reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, action: def (*_P.args, **_P.kwargs) -> _R`15, *_P.args, **_P.kwargs) -> _R`15" -reveal_type(A().func) # N: Revealed type is "def [_P, _R] (action: def (*_P.args, **_P.kwargs) -> _R`19, *_P.args, **_P.kwargs) -> _R`19" +reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, action: def (*_P.args, **_P.kwargs) -> _R`6, *_P.args, **_P.kwargs) -> _R`6" +reveal_type(A().func) # N: Revealed type is "def [_P, _R] (action: def (*_P.args, **_P.kwargs) -> _R`10, *_P.args, **_P.kwargs) -> _R`10" def f(x: int) -> int: ... @@ -933,8 +933,8 @@ class A: def func(self, action: Job[_P, None]) -> Job[_P, None]: ... -reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`13, None]) -> __main__.Job[_P`13, None]" -reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`15, None]) -> __main__.Job[_P`15, None]" +reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`4, None]) -> __main__.Job[_P`4, None]" +reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`6, None]) -> __main__.Job[_P`6, None]" reveal_type(A().func(Job(lambda x: x))) # N: Revealed type is "__main__.Job[[x: Any], None]" def f(x: int, y: int) -> None: ... @@ -1096,7 +1096,7 @@ j = Job(generic_f) reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`-1]]" jf = j.into_callable() -reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`13)" +reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`4)" reveal_type(jf(1)) # N: Revealed type is "None" [builtins fixtures/paramspec.pyi] @@ -1115,10 +1115,10 @@ class Job(Generic[_P, _T]): def generic_f(x: _T) -> _T: ... j = Job(generic_f) -reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`12], _T`12]" +reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`3], _T`3]" jf = j.into_callable() -reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`13) -> _T`13" +reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`4) -> _T`4" reveal_type(jf(1)) # N: Revealed type is "builtins.int" [builtins fixtures/paramspec.pyi] @@ -1600,7 +1600,7 @@ from typing_extensions import Concatenate, ParamSpec P = ParamSpec("P") @overload -def command() -> Callable[[Callable[Concatenate[object, object, P], object]], None]: # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def command() -> Callable[[Callable[Concatenate[object, object, P], object]], None]: ... @overload @@ -1640,13 +1640,13 @@ U = TypeVar("U") def dec(f: Callable[P, T]) -> Callable[P, List[T]]: ... def test(x: U) -> U: ... reveal_type(dec) # N: Revealed type is "def [P, T] (f: def (*P.args, **P.kwargs) -> T`-2) -> def (*P.args, **P.kwargs) -> builtins.list[T`-2]" -reveal_type(dec(test)) # N: Revealed type is "def [T] (x: T`12) -> builtins.list[T`12]" +reveal_type(dec(test)) # N: Revealed type is "def [T] (x: T`3) -> builtins.list[T`3]" class A: ... TA = TypeVar("TA", bound=A) def test_with_bound(x: TA) -> TA: ... -reveal_type(dec(test_with_bound)) # N: Revealed type is "def [T <: __main__.A] (x: T`14) -> builtins.list[T`14]" +reveal_type(dec(test_with_bound)) # N: Revealed type is "def [T <: __main__.A] (x: T`5) -> builtins.list[T`5]" dec(test_with_bound)(0) # E: Value of type variable "T" of function cannot be "int" dec(test_with_bound)(A()) # OK [builtins fixtures/paramspec.pyi] diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index fdd628b0271b..1480c83b2272 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1793,7 +1793,7 @@ class C: def bar(self) -> Self: ... def foo(self, x: S) -> Tuple[Self, S]: ... -reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`7, x: S`8) -> Tuple[Self`7, S`8]" +reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`1, x: S`2) -> Tuple[Self`1, S`2]" reveal_type(C().foo(42)) # N: Revealed type is "Tuple[__main__.C, builtins.int]" [builtins fixtures/tuple.pyi] @@ -1807,7 +1807,7 @@ class C: def bar(self) -> Self: ... foo: Callable[[S, Self], Tuple[Self, S]] -reveal_type(C().foo) # N: Revealed type is "def [S] (S`7, __main__.C) -> Tuple[__main__.C, S`7]" +reveal_type(C().foo) # N: Revealed type is "def [S] (S`1, __main__.C) -> Tuple[__main__.C, S`1]" reveal_type(C().foo(42, C())) # N: Revealed type is "Tuple[__main__.C, builtins.int]" class This: ... [builtins fixtures/tuple.pyi] @@ -2032,7 +2032,7 @@ class Ben(Object): } @classmethod def doit(cls) -> Foo: - reveal_type(cls.MY_MAP) # N: Revealed type is "builtins.dict[builtins.str, def [Self <: __main__.Foo] (self: Self`10) -> Self`10]" + reveal_type(cls.MY_MAP) # N: Revealed type is "builtins.dict[builtins.str, def [Self <: __main__.Foo] (self: Self`4) -> Self`4]" foo_method = cls.MY_MAP["foo"] return foo_method(Foo()) [builtins fixtures/isinstancelist.pyi] diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index a76d3abd7114..3bf8613d2478 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -444,7 +444,7 @@ False [case testOverlappingOperatorMethods] class X: pass class A: - def __add__(self, x) -> int: + def __add__(self, x: object) -> int: if isinstance(x, X): return 1 return NotImplemented @@ -1942,13 +1942,13 @@ class Bar(Generic[P]): ... def bad(foo: Foo[[int]], bar: Bar[[int]]) -> bool: return foo == bar -def good1(foo1: Foo[[int]], foo2: Foo[[str]]) -> bool: +def bad1(foo1: Foo[[int]], foo2: Foo[[str]]) -> bool: return foo1 == foo2 -def good2(foo1: Foo[[int, str]], foo2: Foo[[int, bytes]]) -> bool: +def bad2(foo1: Foo[[int, str]], foo2: Foo[[int, bytes]]) -> bool: return foo1 == foo2 -def good3(foo1: Foo[[int]], foo2: Foo[[int, int]]) -> bool: +def bad3(foo1: Foo[[int]], foo2: Foo[[int, int]]) -> bool: return foo1 == foo2 def good4(foo1: Foo[[int]], foo2: Foo[[int]]) -> bool: @@ -1971,6 +1971,9 @@ def good9(foo1: Foo[Concatenate[int, P]], foo2: Foo[[int, str, bytes]], *args: P [out] _testStrictEqualitywithParamSpec.py:11: error: Non-overlapping equality check (left operand type: "Foo[[int]]", right operand type: "Bar[[int]]") +_testStrictEqualitywithParamSpec.py:14: error: Non-overlapping equality check (left operand type: "Foo[[int]]", right operand type: "Foo[[str]]") +_testStrictEqualitywithParamSpec.py:17: error: Non-overlapping equality check (left operand type: "Foo[[int, str]]", right operand type: "Foo[[int, bytes]]") +_testStrictEqualitywithParamSpec.py:20: error: Non-overlapping equality check (left operand type: "Foo[[int]]", right operand type: "Foo[[int, int]]") [case testInferenceOfDunderDictOnClassObjects] class Foo: ... From 6877d6fb668c24d984268b937c3eeed8c4cad296 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 20 Jun 2024 10:01:16 +0100 Subject: [PATCH 044/247] Make syntax of generic overload variants in messages close to PEP 695 (#17401) We used a custom syntax for type variable bounds and restrictions. Use PEP 695 syntax instead (or at least something closer to PEP 695 syntax). Co-authored-by: Ivan Levkivskyi --- mypy/messages.py | 4 ++-- test-data/unit/check-generic-subtyping.test | 2 +- test-data/unit/check-overloading.test | 12 ++++++------ test-data/unit/check-protocols.test | 4 ++-- test-data/unit/fine-grained.test | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index c3a34bd41aba..27f152413151 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2928,10 +2928,10 @@ def [T <: int] f(self, x: int, y: T) -> None isinstance(upper_bound, Instance) and upper_bound.type.fullname != "builtins.object" ): - tvars.append(f"{tvar.name} <: {format_type_bare(upper_bound, options)}") + tvars.append(f"{tvar.name}: {format_type_bare(upper_bound, options)}") elif tvar.values: tvars.append( - "{} in ({})".format( + "{}: ({})".format( tvar.name, ", ".join([format_type_bare(tp, options) for tp in tvar.values]), ) diff --git a/test-data/unit/check-generic-subtyping.test b/test-data/unit/check-generic-subtyping.test index fd40f128ff4a..90180e0f83f6 100644 --- a/test-data/unit/check-generic-subtyping.test +++ b/test-data/unit/check-generic-subtyping.test @@ -306,7 +306,7 @@ main:14: error: Signature of "f" incompatible with supertype "A" main:14: note: Superclass: main:14: note: def [S] f(self, x: int, y: S) -> None main:14: note: Subclass: -main:14: note: def [T1 <: str, S] f(self, x: T1, y: S) -> None +main:14: note: def [T1: str, S] f(self, x: T1, y: S) -> None -- Inheritance from generic types with implicit dynamic supertype -- -------------------------------------------------------------- diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 03863b2978ba..48d5996b226f 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -1276,7 +1276,7 @@ f('x')() # E: "str" not callable f(1)() # E: "bool" not callable f(1.1) # E: No overload variant of "f" matches argument type "float" \ # N: Possible overload variants: \ - # N: def [T <: str] f(x: T) -> T \ + # N: def [T: str] f(x: T) -> T \ # N: def f(x: int) -> bool f(mystr())() # E: "mystr" not callable [builtins fixtures/primitives.pyi] @@ -1298,8 +1298,8 @@ def g(x: U, y: V) -> None: f(x)() # E: "mystr" not callable f(y) # E: No overload variant of "f" matches argument type "V" \ # N: Possible overload variants: \ - # N: def [T <: str] f(x: T) -> T \ - # N: def [T <: str] f(x: List[T]) -> None + # N: def [T: str] f(x: T) -> T \ + # N: def [T: str] f(x: List[T]) -> None a = f([x]) reveal_type(a) # N: Revealed type is "None" f([y]) # E: Value of type variable "T" of "f" cannot be "V" @@ -1351,7 +1351,7 @@ f(b'1')() # E: "str" not callable f(1.0) # E: No overload variant of "f" matches argument type "float" \ # N: Possible overload variants: \ # N: def f(x: int) -> int \ - # N: def [AnyStr in (bytes, str)] f(x: AnyStr) -> str + # N: def [AnyStr: (bytes, str)] f(x: AnyStr) -> str @overload def g(x: AnyStr, *a: AnyStr) -> None: pass @@ -4949,7 +4949,7 @@ x: Any reveal_type(attr(x)) # N: Revealed type is "Any" attr("hi", 1) # E: No overload variant of "attr" matches argument types "str", "int" \ # N: Possible overload variants: \ - # N: def [T in (int, float)] attr(default: T, blah: int = ...) -> T \ + # N: def [T: (int, float)] attr(default: T, blah: int = ...) -> T \ # N: def attr(default: Any = ...) -> int [file lib.pyi] from typing import overload, Any, TypeVar @@ -4972,7 +4972,7 @@ x: Any reveal_type(attr(x)) # N: Revealed type is "Any" attr("hi", 1) # E: No overload variant of "attr" matches argument types "str", "int" \ # N: Possible overload variants: \ - # N: def [T <: int] attr(default: T = ..., blah: int = ...) -> T \ + # N: def [T: int] attr(default: T = ..., blah: int = ...) -> T \ # N: def attr(default: Any = ...) -> int [file lib.pyi] from typing import overload, TypeVar, Any diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index e73add454a67..ee7556461fd3 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -2182,7 +2182,7 @@ main:11: note: Following member(s) of "B" have conflicts: main:11: note: Expected: main:11: note: def [T] f(self, x: T) -> None main:11: note: Got: -main:11: note: def [S <: int, T] f(self, x: S, y: T) -> None +main:11: note: def [S: int, T] f(self, x: S, y: T) -> None [case testProtocolIncompatibilityWithGenericRestricted] from typing import Protocol, TypeVar @@ -2202,7 +2202,7 @@ main:11: note: Following member(s) of "B" have conflicts: main:11: note: Expected: main:11: note: def [T] f(self, x: T) -> None main:11: note: Got: -main:11: note: def [S in (int, str), T] f(self, x: S, y: T) -> None +main:11: note: def [S: (int, str), T] f(self, x: S, y: T) -> None [case testProtocolIncompatibilityWithManyOverloads] from typing import Protocol, overload diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index a87f8ceca15c..2a652e50b1e6 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -7138,7 +7138,7 @@ T = TypeVar('T', bound=str) a.py:2: error: No overload variant of "f" matches argument type "int" a.py:2: note: Possible overload variants: a.py:2: note: def f(x: C) -> None -a.py:2: note: def [c.T <: str] f(x: c.T) -> c.T +a.py:2: note: def [c.T: str] f(x: c.T) -> c.T [case testOverloadsGenericToNonGeneric] import a From 4ba2696466060435dbdac10c73ce94731370e252 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:45:37 +0200 Subject: [PATCH 045/247] Update typeshed (#17409) The automatic sync failed due to a merge conflict. Source commit: https://github.com/python/typeshed/commit/6dda799d8ad1d89e0f8aad7ac41d2d34bd838ace --- ...e-of-LiteralString-in-builtins-13743.patch | 38 ++-- mypy/typeshed/stdlib/VERSIONS | 3 +- mypy/typeshed/stdlib/_ast.pyi | 5 +- mypy/typeshed/stdlib/_curses.pyi | 3 +- mypy/typeshed/stdlib/_json.pyi | 2 +- mypy/typeshed/stdlib/_tkinter.pyi | 7 + mypy/typeshed/stdlib/_weakref.pyi | 5 +- mypy/typeshed/stdlib/ast.pyi | 3 + mypy/typeshed/stdlib/builtins.pyi | 14 +- mypy/typeshed/stdlib/configparser.pyi | 195 +++++++++++++----- mypy/typeshed/stdlib/dataclasses.pyi | 8 +- mypy/typeshed/stdlib/enum.pyi | 26 ++- mypy/typeshed/stdlib/glob.pyi | 10 +- mypy/typeshed/stdlib/io.pyi | 2 +- mypy/typeshed/stdlib/ipaddress.pyi | 6 +- mypy/typeshed/stdlib/itertools.pyi | 58 ++++++ mypy/typeshed/stdlib/json/encoder.pyi | 4 +- mypy/typeshed/stdlib/locale.pyi | 8 +- mypy/typeshed/stdlib/logging/__init__.pyi | 8 +- mypy/typeshed/stdlib/logging/handlers.pyi | 2 +- mypy/typeshed/stdlib/math.pyi | 2 +- mypy/typeshed/stdlib/mimetypes.pyi | 9 + mypy/typeshed/stdlib/mmap.pyi | 10 +- .../stdlib/multiprocessing/context.pyi | 16 +- .../stdlib/multiprocessing/managers.pyi | 4 + .../stdlib/multiprocessing/shared_memory.pyi | 6 +- .../stdlib/multiprocessing/sharedctypes.pyi | 45 ++-- mypy/typeshed/stdlib/os/__init__.pyi | 20 +- mypy/typeshed/stdlib/pathlib.pyi | 55 ++++- mypy/typeshed/stdlib/platform.pyi | 25 +++ mypy/typeshed/stdlib/shutil.pyi | 16 +- mypy/typeshed/stdlib/spwd.pyi | 5 + mypy/typeshed/stdlib/sqlite3/dbapi2.pyi | 6 +- mypy/typeshed/stdlib/tarfile.pyi | 53 +++-- mypy/typeshed/stdlib/telnetlib.pyi | 1 + mypy/typeshed/stdlib/time.pyi | 5 +- mypy/typeshed/stdlib/traceback.pyi | 72 +++++-- mypy/typeshed/stdlib/types.pyi | 9 +- mypy/typeshed/stdlib/typing.pyi | 37 +++- mypy/typeshed/stdlib/typing_extensions.pyi | 8 +- mypy/typeshed/stdlib/weakref.pyi | 5 +- mypy/typeshed/stdlib/xml/sax/handler.pyi | 2 +- 42 files changed, 630 insertions(+), 188 deletions(-) diff --git a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch index 6a0977dfc489..683b0c322b71 100644 --- a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch +++ b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch @@ -1,14 +1,14 @@ -From 5c00e362d40aa26e0a22a740f05a52d05edf0f91 Mon Sep 17 00:00:00 2001 +From 3ec9b878d6bbe3fae64a508a62372f10a886406f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 26 Sep 2022 12:55:07 -0700 Subject: [PATCH] Remove use of LiteralString in builtins (#13743) --- - mypy/typeshed/stdlib/builtins.pyi | 88 ------------------------------- - 1 file changed, 88 deletions(-) + mypy/typeshed/stdlib/builtins.pyi | 95 ------------------------------- + 1 file changed, 95 deletions(-) diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi -index b4765b26c..99919c64c 100644 +index 53e00ec6a..bad3250ef 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -61,7 +61,6 @@ from typing import ( # noqa: Y022 @@ -19,7 +19,7 @@ index b4765b26c..99919c64c 100644 ParamSpec, Self, TypeAlias, -@@ -434,31 +433,16 @@ class str(Sequence[str]): +@@ -435,31 +434,16 @@ class str(Sequence[str]): def __new__(cls, object: object = ...) -> Self: ... @overload def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... @@ -49,9 +49,9 @@ index b4765b26c..99919c64c 100644 - def format(self: LiteralString, *args: LiteralString, **kwargs: LiteralString) -> LiteralString: ... - @overload def format(self, *args: object, **kwargs: object) -> str: ... - def format_map(self, map: _FormatMapMapping) -> str: ... + def format_map(self, mapping: _FormatMapMapping, /) -> str: ... def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... -@@ -474,89 +458,32 @@ class str(Sequence[str]): +@@ -475,99 +459,35 @@ class str(Sequence[str]): def isspace(self) -> bool: ... def istitle(self) -> bool: ... def isupper(self) -> bool: ... @@ -75,10 +75,20 @@ index b4765b26c..99919c64c 100644 - def partition(self: LiteralString, sep: LiteralString, /) -> tuple[LiteralString, LiteralString, LiteralString]: ... - @overload def partition(self, sep: str, /) -> tuple[str, str, str]: ... # type: ignore[misc] -- @overload -- def replace(self: LiteralString, old: LiteralString, new: LiteralString, count: SupportsIndex = -1, /) -> LiteralString: ... -- @overload - def replace(self, old: str, new: str, count: SupportsIndex = -1, /) -> str: ... # type: ignore[misc] + if sys.version_info >= (3, 13): +- @overload +- def replace( +- self: LiteralString, old: LiteralString, new: LiteralString, /, count: SupportsIndex = -1 +- ) -> LiteralString: ... +- @overload + def replace(self, old: str, new: str, /, count: SupportsIndex = -1) -> str: ... # type: ignore[misc] + else: +- @overload +- def replace( +- self: LiteralString, old: LiteralString, new: LiteralString, count: SupportsIndex = -1, / +- ) -> LiteralString: ... +- @overload + def replace(self, old: str, new: str, count: SupportsIndex = -1, /) -> str: ... # type: ignore[misc] if sys.version_info >= (3, 9): - @overload - def removeprefix(self: LiteralString, prefix: LiteralString, /) -> LiteralString: ... @@ -141,7 +151,7 @@ index b4765b26c..99919c64c 100644 def zfill(self, width: SupportsIndex, /) -> str: ... # type: ignore[misc] @staticmethod @overload -@@ -567,9 +494,6 @@ class str(Sequence[str]): +@@ -578,9 +498,6 @@ class str(Sequence[str]): @staticmethod @overload def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ... @@ -151,7 +161,7 @@ index b4765b26c..99919c64c 100644 def __add__(self, value: str, /) -> str: ... # type: ignore[misc] # Incompatible with Sequence.__contains__ def __contains__(self, key: str, /) -> bool: ... # type: ignore[override] -@@ -578,25 +502,13 @@ class str(Sequence[str]): +@@ -589,25 +506,13 @@ class str(Sequence[str]): def __getitem__(self, key: SupportsIndex | slice, /) -> str: ... def __gt__(self, value: str, /) -> bool: ... def __hash__(self) -> int: ... @@ -178,5 +188,5 @@ index b4765b26c..99919c64c 100644 def __getnewargs__(self) -> tuple[str]: ... -- -2.39.3 (Apple Git-146) +2.45.2 diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index a8526aab9422..7b9ce2864484 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -65,9 +65,9 @@ array: 3.0- ast: 3.0- asynchat: 3.0-3.11 asyncio: 3.4- -asyncio.mixins: 3.10- asyncio.exceptions: 3.8- asyncio.format_helpers: 3.7- +asyncio.mixins: 3.10- asyncio.runners: 3.7- asyncio.staggered: 3.8- asyncio.taskgroups: 3.11- @@ -270,6 +270,7 @@ threading: 3.0- time: 3.0- timeit: 3.0- tkinter: 3.0- +tkinter.tix: 3.0-3.12 token: 3.0- tokenize: 3.0- tomllib: 3.11- diff --git a/mypy/typeshed/stdlib/_ast.pyi b/mypy/typeshed/stdlib/_ast.pyi index 51791b4099d5..d14c6d39a162 100644 --- a/mypy/typeshed/stdlib/_ast.pyi +++ b/mypy/typeshed/stdlib/_ast.pyi @@ -7,8 +7,11 @@ PyCF_ONLY_AST: Literal[1024] PyCF_TYPE_COMMENTS: Literal[4096] PyCF_ALLOW_TOP_LEVEL_AWAIT: Literal[8192] +if sys.version_info >= (3, 13): + PyCF_OPTIMIZED_AST: Literal[33792] + # Used for node end positions in constructor keyword arguments -_EndPositionT = typing_extensions.TypeVar("_EndPositionT", int, int | None, default=int | None) # noqa: Y023 +_EndPositionT = typing_extensions.TypeVar("_EndPositionT", int, int | None, default=int | None) # Alias used for fields that must always be valid identifiers # A string `x` counts as a valid identifier if both the following are True diff --git a/mypy/typeshed/stdlib/_curses.pyi b/mypy/typeshed/stdlib/_curses.pyi index 6f3fbd807fcc..eb1d7b9bde9f 100644 --- a/mypy/typeshed/stdlib/_curses.pyi +++ b/mypy/typeshed/stdlib/_curses.pyi @@ -63,8 +63,7 @@ A_COLOR: int A_DIM: int A_HORIZONTAL: int A_INVIS: int -if sys.platform != "darwin": - A_ITALIC: int +A_ITALIC: int A_LEFT: int A_LOW: int A_NORMAL: int diff --git a/mypy/typeshed/stdlib/_json.pyi b/mypy/typeshed/stdlib/_json.pyi index a6a62be184d8..069fb6eac4bf 100644 --- a/mypy/typeshed/stdlib/_json.pyi +++ b/mypy/typeshed/stdlib/_json.pyi @@ -45,5 +45,5 @@ class make_scanner: def __init__(self, context: make_scanner) -> None: ... def __call__(self, string: str, index: int) -> tuple[Any, int]: ... -def encode_basestring_ascii(s: str) -> str: ... +def encode_basestring_ascii(s: str, /) -> str: ... def scanstring(string: str, end: int, strict: bool = ...) -> tuple[str, int]: ... diff --git a/mypy/typeshed/stdlib/_tkinter.pyi b/mypy/typeshed/stdlib/_tkinter.pyi index 3340df424163..aea74c8be279 100644 --- a/mypy/typeshed/stdlib/_tkinter.pyi +++ b/mypy/typeshed/stdlib/_tkinter.pyi @@ -1,5 +1,7 @@ import sys +from collections.abc import Callable from typing import Any, ClassVar, Literal, final +from typing_extensions import TypeAlias # _tkinter is meant to be only used internally by tkinter, but some tkinter # functions e.g. return _tkinter.Tcl_Obj objects. Tcl_Obj represents a Tcl @@ -30,6 +32,8 @@ class Tcl_Obj: class TclError(Exception): ... +_TkinterTraceFunc: TypeAlias = Callable[[tuple[str, ...]], object] + # This class allows running Tcl code. Tkinter uses it internally a lot, and # it's often handy to drop a piece of Tcl code into a tkinter program. Example: # @@ -86,6 +90,9 @@ class TkappType: def unsetvar(self, *args, **kwargs): ... def wantobjects(self, *args, **kwargs): ... def willdispatch(self): ... + if sys.version_info >= (3, 12): + def gettrace(self, /) -> _TkinterTraceFunc | None: ... + def settrace(self, func: _TkinterTraceFunc | None, /) -> None: ... # These should be kept in sync with tkinter.tix constants, except ALL_EVENTS which doesn't match TCL_ALL_EVENTS ALL_EVENTS: Literal[-3] diff --git a/mypy/typeshed/stdlib/_weakref.pyi b/mypy/typeshed/stdlib/_weakref.pyi index 61365645d768..f142820c56c7 100644 --- a/mypy/typeshed/stdlib/_weakref.pyi +++ b/mypy/typeshed/stdlib/_weakref.pyi @@ -21,8 +21,9 @@ class ProxyType(Generic[_T]): # "weakproxy" def __getattr__(self, attr: str) -> Any: ... class ReferenceType(Generic[_T]): - __callback__: Callable[[ReferenceType[_T]], Any] - def __new__(cls, o: _T, callback: Callable[[ReferenceType[_T]], Any] | None = ..., /) -> Self: ... + __callback__: Callable[[Self], Any] + def __new__(cls, o: _T, callback: Callable[[Self], Any] | None = ..., /) -> Self: ... + def __init__(self, o: _T, callback: Callable[[Self], Any] | None = ..., /) -> None: ... def __call__(self) -> _T | None: ... def __eq__(self, value: object, /) -> bool: ... def __hash__(self) -> int: ... diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index 2525c3642a6f..90ede461fe3c 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -365,3 +365,6 @@ def walk(node: AST) -> Iterator[AST]: ... if sys.version_info >= (3, 9): def main() -> None: ... + +if sys.version_info >= (3, 14): + def compare(left: AST, right: AST, /, *, compare_attributes: bool = False) -> bool: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index 42c0b27baf68..28b0b11a8e5c 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -445,7 +445,7 @@ class str(Sequence[str]): def expandtabs(self, tabsize: SupportsIndex = 8) -> str: ... # type: ignore[misc] def find(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... def format(self, *args: object, **kwargs: object) -> str: ... - def format_map(self, map: _FormatMapMapping) -> str: ... + def format_map(self, mapping: _FormatMapMapping, /) -> str: ... def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... def isalnum(self) -> bool: ... def isalpha(self) -> bool: ... @@ -464,7 +464,10 @@ class str(Sequence[str]): def lower(self) -> str: ... # type: ignore[misc] def lstrip(self, chars: str | None = None, /) -> str: ... # type: ignore[misc] def partition(self, sep: str, /) -> tuple[str, str, str]: ... # type: ignore[misc] - def replace(self, old: str, new: str, count: SupportsIndex = -1, /) -> str: ... # type: ignore[misc] + if sys.version_info >= (3, 13): + def replace(self, old: str, new: str, /, count: SupportsIndex = -1) -> str: ... # type: ignore[misc] + else: + def replace(self, old: str, new: str, count: SupportsIndex = -1, /) -> str: ... # type: ignore[misc] if sys.version_info >= (3, 9): def removeprefix(self, prefix: str, /) -> str: ... # type: ignore[misc] def removesuffix(self, suffix: str, /) -> str: ... # type: ignore[misc] @@ -1126,6 +1129,9 @@ class property: fset: Callable[[Any, Any], None] | None fdel: Callable[[Any], None] | None __isabstractmethod__: bool + if sys.version_info >= (3, 13): + __name__: str + def __init__( self, fget: Callable[[Any], Any] | None = ..., @@ -1969,3 +1975,7 @@ if sys.version_info >= (3, 11): def split( self, condition: Callable[[_ExceptionT_co | Self], bool], / ) -> tuple[ExceptionGroup[_ExceptionT_co] | None, ExceptionGroup[_ExceptionT_co] | None]: ... + +if sys.version_info >= (3, 13): + class IncompleteInputError(SyntaxError): ... + class PythonFinalizationError(RuntimeError): ... diff --git a/mypy/typeshed/stdlib/configparser.pyi b/mypy/typeshed/stdlib/configparser.pyi index 07b57b17d56d..f38bb1de674d 100644 --- a/mypy/typeshed/stdlib/configparser.pyi +++ b/mypy/typeshed/stdlib/configparser.pyi @@ -5,7 +5,31 @@ from re import Pattern from typing import Any, ClassVar, Literal, TypeVar, overload from typing_extensions import TypeAlias -if sys.version_info >= (3, 12): +if sys.version_info >= (3, 13): + __all__ = ( + "NoSectionError", + "DuplicateOptionError", + "DuplicateSectionError", + "NoOptionError", + "InterpolationError", + "InterpolationDepthError", + "InterpolationMissingOptionError", + "InterpolationSyntaxError", + "ParsingError", + "MissingSectionHeaderError", + "ConfigParser", + "RawConfigParser", + "Interpolation", + "BasicInterpolation", + "ExtendedInterpolation", + "SectionProxy", + "ConverterMapping", + "DEFAULTSECT", + "MAX_INTERPOLATION_DEPTH", + "UNNAMED_SECTION", + "MultilineContinuationError", + ) +elif sys.version_info >= (3, 12): __all__ = ( "NoSectionError", "DuplicateOptionError", @@ -71,8 +95,9 @@ class Interpolation: class BasicInterpolation(Interpolation): ... class ExtendedInterpolation(Interpolation): ... -class LegacyInterpolation(Interpolation): - def before_get(self, parser: _Parser, section: str, option: str, value: str, vars: _Section) -> str: ... +if sys.version_info < (3, 13): + class LegacyInterpolation(Interpolation): + def before_get(self, parser: _Parser, section: str, option: str, value: str, vars: _Section) -> str: ... class RawConfigParser(_Parser): _SECT_TMPL: ClassVar[str] # undocumented @@ -86,54 +111,108 @@ class RawConfigParser(_Parser): BOOLEAN_STATES: ClassVar[Mapping[str, bool]] # undocumented default_section: str - @overload - def __init__( - self, - defaults: Mapping[str, str | None] | None = None, - dict_type: type[Mapping[str, str]] = ..., - *, - allow_no_value: Literal[True], - delimiters: Sequence[str] = ("=", ":"), - comment_prefixes: Sequence[str] = ("#", ";"), - inline_comment_prefixes: Sequence[str] | None = None, - strict: bool = True, - empty_lines_in_values: bool = True, - default_section: str = "DEFAULT", - interpolation: Interpolation | None = ..., - converters: _ConvertersMap = ..., - ) -> None: ... - @overload - def __init__( - self, - defaults: Mapping[str, str | None] | None, - dict_type: type[Mapping[str, str]], - allow_no_value: Literal[True], - *, - delimiters: Sequence[str] = ("=", ":"), - comment_prefixes: Sequence[str] = ("#", ";"), - inline_comment_prefixes: Sequence[str] | None = None, - strict: bool = True, - empty_lines_in_values: bool = True, - default_section: str = "DEFAULT", - interpolation: Interpolation | None = ..., - converters: _ConvertersMap = ..., - ) -> None: ... - @overload - def __init__( - self, - defaults: _Section | None = None, - dict_type: type[Mapping[str, str]] = ..., - allow_no_value: bool = False, - *, - delimiters: Sequence[str] = ("=", ":"), - comment_prefixes: Sequence[str] = ("#", ";"), - inline_comment_prefixes: Sequence[str] | None = None, - strict: bool = True, - empty_lines_in_values: bool = True, - default_section: str = "DEFAULT", - interpolation: Interpolation | None = ..., - converters: _ConvertersMap = ..., - ) -> None: ... + if sys.version_info >= (3, 13): + @overload + def __init__( + self, + defaults: Mapping[str, str | None] | None = None, + dict_type: type[Mapping[str, str]] = ..., + *, + allow_no_value: Literal[True], + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + allow_unnamed_section: bool = False, + ) -> None: ... + @overload + def __init__( + self, + defaults: Mapping[str, str | None] | None, + dict_type: type[Mapping[str, str]], + allow_no_value: Literal[True], + *, + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + allow_unnamed_section: bool = False, + ) -> None: ... + @overload + def __init__( + self, + defaults: _Section | None = None, + dict_type: type[Mapping[str, str]] = ..., + allow_no_value: bool = False, + *, + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + allow_unnamed_section: bool = False, + ) -> None: ... + else: + @overload + def __init__( + self, + defaults: Mapping[str, str | None] | None = None, + dict_type: type[Mapping[str, str]] = ..., + *, + allow_no_value: Literal[True], + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + ) -> None: ... + @overload + def __init__( + self, + defaults: Mapping[str, str | None] | None, + dict_type: type[Mapping[str, str]], + allow_no_value: Literal[True], + *, + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + ) -> None: ... + @overload + def __init__( + self, + defaults: _Section | None = None, + dict_type: type[Mapping[str, str]] = ..., + allow_no_value: bool = False, + *, + delimiters: Sequence[str] = ("=", ":"), + comment_prefixes: Sequence[str] = ("#", ";"), + inline_comment_prefixes: Sequence[str] | None = None, + strict: bool = True, + empty_lines_in_values: bool = True, + default_section: str = "DEFAULT", + interpolation: Interpolation | None = ..., + converters: _ConvertersMap = ..., + ) -> None: ... + def __len__(self) -> int: ... def __getitem__(self, key: str) -> SectionProxy: ... def __setitem__(self, key: str, value: _Section) -> None: ... @@ -300,7 +379,10 @@ class InterpolationSyntaxError(InterpolationError): ... class ParsingError(Error): source: str errors: list[tuple[int, str]] - if sys.version_info >= (3, 12): + if sys.version_info >= (3, 13): + def __init__(self, source: str, *args: object) -> None: ... + def combine(self, others: Iterable[ParsingError]) -> ParsingError: ... + elif sys.version_info >= (3, 12): def __init__(self, source: str) -> None: ... else: def __init__(self, source: str | None = None, filename: str | None = None) -> None: ... @@ -311,3 +393,12 @@ class MissingSectionHeaderError(ParsingError): lineno: int line: str def __init__(self, filename: str, lineno: int, line: str) -> None: ... + +if sys.version_info >= (3, 13): + class _UNNAMED_SECTION: ... + UNNAMED_SECTION: _UNNAMED_SECTION + + class MultilineContinuationError(ParsingError): + lineno: int + line: str + def __init__(self, filename: str, lineno: int, line: str) -> None: ... diff --git a/mypy/typeshed/stdlib/dataclasses.pyi b/mypy/typeshed/stdlib/dataclasses.pyi index 18c7e7b5a467..30489e6f8b3d 100644 --- a/mypy/typeshed/stdlib/dataclasses.pyi +++ b/mypy/typeshed/stdlib/dataclasses.pyi @@ -5,7 +5,7 @@ from _typeshed import DataclassInstance from builtins import type as Type # alias to avoid name clashes with fields named "type" from collections.abc import Callable, Iterable, Mapping from typing import Any, Generic, Literal, Protocol, TypeVar, overload -from typing_extensions import TypeAlias, TypeGuard +from typing_extensions import TypeAlias, TypeIs if sys.version_info >= (3, 9): from types import GenericAlias @@ -214,11 +214,9 @@ else: def fields(class_or_instance: DataclassInstance | type[DataclassInstance]) -> tuple[Field[Any], ...]: ... @overload -def is_dataclass(obj: DataclassInstance) -> Literal[True]: ... +def is_dataclass(obj: type) -> TypeIs[type[DataclassInstance]]: ... @overload -def is_dataclass(obj: type) -> TypeGuard[type[DataclassInstance]]: ... -@overload -def is_dataclass(obj: object) -> TypeGuard[DataclassInstance | type[DataclassInstance]]: ... +def is_dataclass(obj: object) -> TypeIs[DataclassInstance | type[DataclassInstance]]: ... class FrozenInstanceError(AttributeError): ... diff --git a/mypy/typeshed/stdlib/enum.pyi b/mypy/typeshed/stdlib/enum.pyi index 96cb2264ea20..5c82b07c4185 100644 --- a/mypy/typeshed/stdlib/enum.pyi +++ b/mypy/typeshed/stdlib/enum.pyi @@ -31,10 +31,12 @@ if sys.version_info >= (3, 11): "nonmember", "property", "verify", + "pickle_by_enum_name", + "pickle_by_global_name", ] -if sys.version_info >= (3, 11): - __all__ += ["pickle_by_enum_name", "pickle_by_global_name"] +if sys.version_info >= (3, 13): + __all__ += ["EnumDict"] _EnumMemberT = TypeVar("_EnumMemberT") _EnumerationT = TypeVar("_EnumerationT", bound=type[Enum]) @@ -74,6 +76,12 @@ class _EnumDict(dict[str, Any]): def update(self, members: SupportsKeysAndGetItem[str, Any], **more_members: Any) -> None: ... @overload def update(self, members: Iterable[tuple[str, Any]], **more_members: Any) -> None: ... + if sys.version_info >= (3, 13): + @property + def member_names(self) -> list[str]: ... + +if sys.version_info >= (3, 13): + EnumDict = _EnumDict # Structurally: Iterable[T], Reversible[T], Container[T] where T is the enum itself class EnumMeta(type): @@ -259,9 +267,9 @@ if sys.version_info >= (3, 11): def _generate_next_value_(name: str, start: int, count: int, last_values: list[str]) -> str: ... class EnumCheck(StrEnum): - CONTINUOUS: str - NAMED_FLAGS: str - UNIQUE: str + CONTINUOUS = "no skipped integer values" + NAMED_FLAGS = "multi-flag aliases may not contain unnamed flags" + UNIQUE = "one name per value" CONTINUOUS = EnumCheck.CONTINUOUS NAMED_FLAGS = EnumCheck.NAMED_FLAGS @@ -272,10 +280,10 @@ if sys.version_info >= (3, 11): def __call__(self, enumeration: _EnumerationT) -> _EnumerationT: ... class FlagBoundary(StrEnum): - STRICT: str - CONFORM: str - EJECT: str - KEEP: str + STRICT = "strict" + CONFORM = "conform" + EJECT = "eject" + KEEP = "keep" STRICT = FlagBoundary.STRICT CONFORM = FlagBoundary.CONFORM diff --git a/mypy/typeshed/stdlib/glob.pyi b/mypy/typeshed/stdlib/glob.pyi index 914ccc12ef1e..03cb5418e256 100644 --- a/mypy/typeshed/stdlib/glob.pyi +++ b/mypy/typeshed/stdlib/glob.pyi @@ -1,10 +1,13 @@ import sys from _typeshed import StrOrBytesPath -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from typing import AnyStr __all__ = ["escape", "glob", "iglob"] +if sys.version_info >= (3, 13): + __all__ += ["translate"] + def glob0(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... def glob1(dirname: AnyStr, pattern: AnyStr) -> list[AnyStr]: ... @@ -40,3 +43,8 @@ else: def escape(pathname: AnyStr) -> AnyStr: ... def has_magic(s: str | bytes) -> bool: ... # undocumented + +if sys.version_info >= (3, 13): + def translate( + pat: str, *, recursive: bool = False, include_hidden: bool = False, seps: Sequence[str] | None = None + ) -> str: ... diff --git a/mypy/typeshed/stdlib/io.pyi b/mypy/typeshed/stdlib/io.pyi index fdbbc8dddce9..01f3bfc06a27 100644 --- a/mypy/typeshed/stdlib/io.pyi +++ b/mypy/typeshed/stdlib/io.pyi @@ -75,7 +75,7 @@ class IOBase(metaclass=abc.ABCMeta): def __del__(self) -> None: ... @property def closed(self) -> bool: ... - def _checkClosed(self, msg: str | None = ...) -> None: ... # undocumented + def _checkClosed(self) -> None: ... # undocumented class RawIOBase(IOBase): def readall(self) -> bytes: ... diff --git a/mypy/typeshed/stdlib/ipaddress.pyi b/mypy/typeshed/stdlib/ipaddress.pyi index 98b1893d2a8a..03decc74e65e 100644 --- a/mypy/typeshed/stdlib/ipaddress.pyi +++ b/mypy/typeshed/stdlib/ipaddress.pyi @@ -147,7 +147,11 @@ class _BaseV4: @property def max_prefixlen(self) -> Literal[32]: ... -class IPv4Address(_BaseV4, _BaseAddress): ... +class IPv4Address(_BaseV4, _BaseAddress): + if sys.version_info >= (3, 13): + @property + def ipv6_mapped(self) -> IPv6Address: ... + class IPv4Network(_BaseV4, _BaseNetwork[IPv4Address]): ... class IPv4Interface(IPv4Address, _BaseInterface[IPv4Address, IPv4Network]): diff --git a/mypy/typeshed/stdlib/itertools.pyi b/mypy/typeshed/stdlib/itertools.pyi index 264064dcd682..16e04829c6cf 100644 --- a/mypy/typeshed/stdlib/itertools.pyi +++ b/mypy/typeshed/stdlib/itertools.pyi @@ -17,6 +17,10 @@ _T3 = TypeVar("_T3") _T4 = TypeVar("_T4") _T5 = TypeVar("_T5") _T6 = TypeVar("_T6") +_T7 = TypeVar("_T7") +_T8 = TypeVar("_T8") +_T9 = TypeVar("_T9") +_T10 = TypeVar("_T10") _Step: TypeAlias = SupportsFloat | SupportsInt | SupportsIndex | SupportsComplex @@ -214,6 +218,60 @@ class product(Iterator[_T_co]): /, ) -> product[tuple[_T1, _T2, _T3, _T4, _T5, _T6]]: ... @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + iter6: Iterable[_T6], + iter7: Iterable[_T7], + /, + ) -> product[tuple[_T1, _T2, _T3, _T4, _T5, _T6, _T7]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + iter6: Iterable[_T6], + iter7: Iterable[_T7], + iter8: Iterable[_T8], + /, + ) -> product[tuple[_T1, _T2, _T3, _T4, _T5, _T6, _T7, _T8]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + iter6: Iterable[_T6], + iter7: Iterable[_T7], + iter8: Iterable[_T8], + iter9: Iterable[_T9], + /, + ) -> product[tuple[_T1, _T2, _T3, _T4, _T5, _T6, _T7, _T8, _T9]]: ... + @overload + def __new__( + cls, + iter1: Iterable[_T1], + iter2: Iterable[_T2], + iter3: Iterable[_T3], + iter4: Iterable[_T4], + iter5: Iterable[_T5], + iter6: Iterable[_T6], + iter7: Iterable[_T7], + iter8: Iterable[_T8], + iter9: Iterable[_T9], + iter10: Iterable[_T10], + /, + ) -> product[tuple[_T1, _T2, _T3, _T4, _T5, _T6, _T7, _T8, _T9, _T10]]: ... + @overload def __new__(cls, *iterables: Iterable[_T1], repeat: int = 1) -> product[tuple[_T1, ...]]: ... def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... diff --git a/mypy/typeshed/stdlib/json/encoder.pyi b/mypy/typeshed/stdlib/json/encoder.pyi index c1062688bd93..473398a60b2a 100644 --- a/mypy/typeshed/stdlib/json/encoder.pyi +++ b/mypy/typeshed/stdlib/json/encoder.pyi @@ -10,8 +10,8 @@ INFINITY: float def py_encode_basestring(s: str) -> str: ... # undocumented def py_encode_basestring_ascii(s: str) -> str: ... # undocumented -def encode_basestring(s: str) -> str: ... # undocumented -def encode_basestring_ascii(s: str) -> str: ... # undocumented +def encode_basestring(s: str, /) -> str: ... # undocumented +def encode_basestring_ascii(s: str, /) -> str: ... # undocumented class JSONEncoder: item_separator: str diff --git a/mypy/typeshed/stdlib/locale.pyi b/mypy/typeshed/stdlib/locale.pyi index c18523e04361..58de65449572 100644 --- a/mypy/typeshed/stdlib/locale.pyi +++ b/mypy/typeshed/stdlib/locale.pyi @@ -96,7 +96,6 @@ __all__ = [ "getpreferredencoding", "Error", "setlocale", - "resetlocale", "localeconv", "strcoll", "strxfrm", @@ -121,6 +120,9 @@ if sys.version_info >= (3, 11): if sys.version_info < (3, 12): __all__ += ["format"] +if sys.version_info < (3, 13): + __all__ += ["resetlocale"] + if sys.platform != "win32": __all__ += ["LC_MESSAGES"] @@ -133,7 +135,9 @@ def getlocale(category: int = ...) -> tuple[_str | None, _str | None]: ... def setlocale(category: int, locale: _str | Iterable[_str | None] | None = None) -> _str: ... def getpreferredencoding(do_setlocale: bool = True) -> _str: ... def normalize(localename: _str) -> _str: ... -def resetlocale(category: int = ...) -> None: ... + +if sys.version_info < (3, 13): + def resetlocale(category: int = ...) -> None: ... if sys.version_info < (3, 12): def format( diff --git a/mypy/typeshed/stdlib/logging/__init__.pyi b/mypy/typeshed/stdlib/logging/__init__.pyi index 8b19444a5d01..4c6163257236 100644 --- a/mypy/typeshed/stdlib/logging/__init__.pyi +++ b/mypy/typeshed/stdlib/logging/__init__.pyi @@ -8,7 +8,7 @@ from string import Template from time import struct_time from types import FrameType, TracebackType from typing import Any, ClassVar, Generic, Literal, Protocol, TextIO, TypeVar, overload -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, deprecated if sys.version_info >= (3, 11): from types import GenericAlias @@ -572,7 +572,11 @@ fatal = critical def disable(level: int = 50) -> None: ... def addLevelName(level: int, levelName: str) -> None: ... -def getLevelName(level: _Level) -> Any: ... +@overload +def getLevelName(level: int) -> str: ... +@overload +@deprecated("The str -> int case is considered a mistake.") +def getLevelName(level: str) -> Any: ... if sys.version_info >= (3, 11): def getLevelNamesMapping() -> dict[str, int]: ... diff --git a/mypy/typeshed/stdlib/logging/handlers.pyi b/mypy/typeshed/stdlib/logging/handlers.pyi index 4c3dc913308c..4e97012abba1 100644 --- a/mypy/typeshed/stdlib/logging/handlers.pyi +++ b/mypy/typeshed/stdlib/logging/handlers.pyi @@ -46,7 +46,7 @@ class BaseRotatingHandler(FileHandler): def rotate(self, source: str, dest: str) -> None: ... class RotatingFileHandler(BaseRotatingHandler): - maxBytes: str # undocumented + maxBytes: int # undocumented backupCount: int # undocumented if sys.version_info >= (3, 9): def __init__( diff --git a/mypy/typeshed/stdlib/math.pyi b/mypy/typeshed/stdlib/math.pyi index 0e6565fcf588..2bb61e0669b4 100644 --- a/mypy/typeshed/stdlib/math.pyi +++ b/mypy/typeshed/stdlib/math.pyi @@ -125,4 +125,4 @@ if sys.version_info >= (3, 9): def ulp(x: _SupportsFloatOrIndex, /) -> float: ... if sys.version_info >= (3, 13): - def fma(x: _SupportsFloatOrIndex, y: _SupportsFloatOrIndex, z: _SupportsFloatOrIndex) -> float: ... + def fma(x: _SupportsFloatOrIndex, y: _SupportsFloatOrIndex, z: _SupportsFloatOrIndex, /) -> float: ... diff --git a/mypy/typeshed/stdlib/mimetypes.pyi b/mypy/typeshed/stdlib/mimetypes.pyi index e74b214d3ff1..517193e3516f 100644 --- a/mypy/typeshed/stdlib/mimetypes.pyi +++ b/mypy/typeshed/stdlib/mimetypes.pyi @@ -1,3 +1,4 @@ +import sys from _typeshed import StrPath from collections.abc import Sequence from typing import IO @@ -18,6 +19,9 @@ __all__ = [ "common_types", ] +if sys.version_info >= (3, 13): + __all__ += ["guess_file_type"] + def guess_type(url: StrPath, strict: bool = True) -> tuple[str | None, str | None]: ... def guess_all_extensions(type: str, strict: bool = True) -> list[str]: ... def guess_extension(type: str, strict: bool = True) -> str | None: ... @@ -25,6 +29,9 @@ def init(files: Sequence[str] | None = None) -> None: ... def read_mime_types(file: str) -> dict[str, str] | None: ... def add_type(type: str, ext: str, strict: bool = True) -> None: ... +if sys.version_info >= (3, 13): + def guess_file_type(path: StrPath, *, strict: bool = True) -> tuple[str | None, str | None]: ... + inited: bool knownfiles: list[str] suffix_map: dict[str, str] @@ -44,3 +51,5 @@ class MimeTypes: def read(self, filename: str, strict: bool = True) -> None: ... def readfp(self, fp: IO[str], strict: bool = True) -> None: ... def read_windows_registry(self, strict: bool = True) -> None: ... + if sys.version_info >= (3, 13): + def guess_file_type(self, path: StrPath, *, strict: bool = True) -> tuple[str | None, str | None]: ... diff --git a/mypy/typeshed/stdlib/mmap.pyi b/mypy/typeshed/stdlib/mmap.pyi index 93c4f408e5b6..7688970e5786 100644 --- a/mypy/typeshed/stdlib/mmap.pyi +++ b/mypy/typeshed/stdlib/mmap.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import ReadableBuffer, Unused from collections.abc import Iterable, Iterator, Sized -from typing import NoReturn, overload +from typing import Final, NoReturn, overload from typing_extensions import Self ACCESS_DEFAULT: int @@ -76,6 +76,8 @@ class mmap(Iterable[int], Sized): def __exit__(self, *args: Unused) -> None: ... def __buffer__(self, flags: int, /) -> memoryview: ... def __release_buffer__(self, buffer: memoryview, /) -> None: ... + if sys.version_info >= (3, 13): + def seekable(self) -> bool: ... if sys.platform != "win32": MADV_NORMAL: int @@ -111,3 +113,9 @@ if sys.platform != "linux" and sys.platform != "darwin" and sys.platform != "win if sys.version_info >= (3, 10) and sys.platform == "darwin": MADV_FREE_REUSABLE: int MADV_FREE_REUSE: int + +if sys.version_info >= (3, 13) and sys.platform != "win32": + MAP_32BIT: Final = 32768 + +if sys.version_info >= (3, 13) and sys.platform == "darwin": + MAP_TPRO: Final = 524288 diff --git a/mypy/typeshed/stdlib/multiprocessing/context.pyi b/mypy/typeshed/stdlib/multiprocessing/context.pyi index 9a45a81559c0..605be4686c1f 100644 --- a/mypy/typeshed/stdlib/multiprocessing/context.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/context.pyi @@ -93,16 +93,20 @@ class BaseContext: def Value(self, typecode_or_type: str | type[_CData], *args: Any, lock: bool | _LockLike = True) -> Any: ... @overload def Array( - self, typecode_or_type: type[c_char], size_or_initializer: int | Sequence[Any], *, lock: Literal[True] | _LockLike = True - ) -> SynchronizedString: ... + self, typecode_or_type: type[_SimpleCData[_T]], size_or_initializer: int | Sequence[Any], *, lock: Literal[False] + ) -> SynchronizedArray[_T]: ... @overload def Array( - self, typecode_or_type: type[_CT], size_or_initializer: int | Sequence[Any], *, lock: Literal[False] - ) -> SynchronizedArray[_CT]: ... + self, typecode_or_type: type[c_char], size_or_initializer: int | Sequence[Any], *, lock: Literal[True] | _LockLike = True + ) -> SynchronizedString: ... @overload def Array( - self, typecode_or_type: type[_CT], size_or_initializer: int | Sequence[Any], *, lock: Literal[True] | _LockLike = True - ) -> SynchronizedArray[_CT]: ... + self, + typecode_or_type: type[_SimpleCData[_T]], + size_or_initializer: int | Sequence[Any], + *, + lock: Literal[True] | _LockLike = True, + ) -> SynchronizedArray[_T]: ... @overload def Array( self, typecode_or_type: str, size_or_initializer: int | Sequence[Any], *, lock: Literal[True] | _LockLike = True diff --git a/mypy/typeshed/stdlib/multiprocessing/managers.pyi b/mypy/typeshed/stdlib/multiprocessing/managers.pyi index 9b2d2970112e..5d5b9cdcb913 100644 --- a/mypy/typeshed/stdlib/multiprocessing/managers.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/managers.pyi @@ -83,6 +83,8 @@ class DictProxy(BaseProxy, MutableMapping[_KT, _VT]): def keys(self) -> list[_KT]: ... # type: ignore[override] def items(self) -> list[tuple[_KT, _VT]]: ... # type: ignore[override] def values(self) -> list[_VT]: ... # type: ignore[override] + if sys.version_info >= (3, 13): + def __class_getitem__(cls, args: Any, /) -> Any: ... class BaseListProxy(BaseProxy, MutableSequence[_T]): __builtins__: ClassVar[dict[str, Any]] @@ -117,6 +119,8 @@ class BaseListProxy(BaseProxy, MutableSequence[_T]): class ListProxy(BaseListProxy[_T]): def __iadd__(self, value: Iterable[_T], /) -> Self: ... # type: ignore[override] def __imul__(self, value: SupportsIndex, /) -> Self: ... # type: ignore[override] + if sys.version_info >= (3, 13): + def __class_getitem__(cls, args: Any, /) -> Any: ... # Returned by BaseManager.get_server() class Server: diff --git a/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi b/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi index 0a6b113b194f..b63cedf85867 100644 --- a/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/shared_memory.pyi @@ -11,7 +11,11 @@ __all__ = ["SharedMemory", "ShareableList"] _SLT = TypeVar("_SLT", int, float, bool, str, bytes, None) class SharedMemory: - def __init__(self, name: str | None = None, create: bool = False, size: int = 0) -> None: ... + if sys.version_info >= (3, 13): + def __init__(self, name: str | None = None, create: bool = False, size: int = 0, *, track: bool = True) -> None: ... + else: + def __init__(self, name: str | None = None, create: bool = False, size: int = 0) -> None: ... + @property def buf(self) -> memoryview: ... @property diff --git a/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi b/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi index 4093a97e6ca3..2b96ff047470 100644 --- a/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi @@ -39,12 +39,20 @@ def Array( ) -> _CT: ... @overload def Array( - typecode_or_type: type[_CT], + typecode_or_type: type[c_char], size_or_initializer: int | Sequence[Any], *, lock: Literal[True] | _LockLike = True, ctx: BaseContext | None = None, -) -> SynchronizedArray[_CT]: ... +) -> SynchronizedString: ... +@overload +def Array( + typecode_or_type: type[_SimpleCData[_T]], + size_or_initializer: int | Sequence[Any], + *, + lock: Literal[True] | _LockLike = True, + ctx: BaseContext | None = None, +) -> SynchronizedArray[_T]: ... @overload def Array( typecode_or_type: str, @@ -65,9 +73,11 @@ def copy(obj: _CT) -> _CT: ... @overload def synchronized(obj: _SimpleCData[_T], lock: _LockLike | None = None, ctx: Any | None = None) -> Synchronized[_T]: ... @overload -def synchronized(obj: ctypes.Array[c_char], lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedString: ... +def synchronized(obj: ctypes.Array[c_char], lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedString: ... # type: ignore @overload -def synchronized(obj: ctypes.Array[_CT], lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedArray[_CT]: ... +def synchronized( + obj: ctypes.Array[_SimpleCData[_T]], lock: _LockLike | None = None, ctx: Any | None = None +) -> SynchronizedArray[_T]: ... @overload def synchronized(obj: _CT, lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedBase[_CT]: ... @@ -89,19 +99,30 @@ class SynchronizedBase(Generic[_CT]): class Synchronized(SynchronizedBase[_SimpleCData[_T]], Generic[_T]): value: _T -class SynchronizedArray(SynchronizedBase[ctypes.Array[_CT]], Generic[_CT]): +class SynchronizedArray(SynchronizedBase[ctypes.Array[_SimpleCData[_T]]], Generic[_T]): def __len__(self) -> int: ... @overload - def __getitem__(self, i: slice) -> list[_CT]: ... + def __getitem__(self, i: slice) -> list[_T]: ... @overload - def __getitem__(self, i: int) -> _CT: ... + def __getitem__(self, i: int) -> _T: ... @overload - def __setitem__(self, i: slice, value: Iterable[_CT]) -> None: ... + def __setitem__(self, i: slice, value: Iterable[_T]) -> None: ... @overload - def __setitem__(self, i: int, value: _CT) -> None: ... - def __getslice__(self, start: int, stop: int) -> list[_CT]: ... - def __setslice__(self, start: int, stop: int, values: Iterable[_CT]) -> None: ... + def __setitem__(self, i: int, value: _T) -> None: ... + def __getslice__(self, start: int, stop: int) -> list[_T]: ... + def __setslice__(self, start: int, stop: int, values: Iterable[_T]) -> None: ... + +class SynchronizedString(SynchronizedArray[bytes]): + @overload # type: ignore[override] + def __getitem__(self, i: slice) -> bytes: ... + @overload # type: ignore[override] + def __getitem__(self, i: int) -> bytes: ... + @overload # type: ignore[override] + def __setitem__(self, i: slice, value: bytes) -> None: ... + @overload # type: ignore[override] + def __setitem__(self, i: int, value: bytes) -> None: ... # type: ignore[override] + def __getslice__(self, start: int, stop: int) -> bytes: ... # type: ignore[override] + def __setslice__(self, start: int, stop: int, values: bytes) -> None: ... # type: ignore[override] -class SynchronizedString(SynchronizedArray[c_char]): value: bytes raw: bytes diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index 31c5d2aa3ee6..9b00117a5599 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -914,8 +914,8 @@ if sys.platform != "win32": def forkpty() -> tuple[int, int]: ... # some flavors of Unix def killpg(pgid: int, signal: int, /) -> None: ... def nice(increment: int, /) -> int: ... - if sys.platform != "darwin": - def plock(op: int, /) -> None: ... # ???op is int? + if sys.platform != "darwin" and sys.platform != "linux": + def plock(op: int, /) -> None: ... class _wrap_close(_TextIOWrapper): def __init__(self, stream: _TextIOWrapper, proc: Popen[str]) -> None: ... @@ -1141,16 +1141,16 @@ if sys.version_info >= (3, 10) and sys.platform == "linux": if sys.version_info >= (3, 12) and sys.platform == "linux": CLONE_FILES: int CLONE_FS: int - CLONE_NEWCGROUP: int - CLONE_NEWIPC: int - CLONE_NEWNET: int + CLONE_NEWCGROUP: int # Linux 4.6+ + CLONE_NEWIPC: int # Linux 2.6.19+ + CLONE_NEWNET: int # Linux 2.6.24+ CLONE_NEWNS: int - CLONE_NEWPID: int - CLONE_NEWTIME: int - CLONE_NEWUSER: int - CLONE_NEWUTS: int + CLONE_NEWPID: int # Linux 3.8+ + CLONE_NEWTIME: int # Linux 5.6+ + CLONE_NEWUSER: int # Linux 3.8+ + CLONE_NEWUTS: int # Linux 2.6.19+ CLONE_SIGHAND: int - CLONE_SYSVSEM: int + CLONE_SYSVSEM: int # Linux 2.6.26+ CLONE_THREAD: int CLONE_VM: int def unshare(flags: int) -> None: ... diff --git a/mypy/typeshed/stdlib/pathlib.pyi b/mypy/typeshed/stdlib/pathlib.pyi index 0013e221f2e1..c8c8dde0f33e 100644 --- a/mypy/typeshed/stdlib/pathlib.pyi +++ b/mypy/typeshed/stdlib/pathlib.pyi @@ -1,4 +1,5 @@ import sys +import types from _typeshed import ( OpenBinaryMode, OpenBinaryModeReading, @@ -14,7 +15,7 @@ from collections.abc import Callable, Generator, Iterator, Sequence from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper from os import PathLike, stat_result from types import TracebackType -from typing import IO, Any, BinaryIO, Literal, overload +from typing import IO, Any, BinaryIO, ClassVar, Literal, overload from typing_extensions import Self, deprecated if sys.version_info >= (3, 9): @@ -22,7 +23,14 @@ if sys.version_info >= (3, 9): __all__ = ["PurePath", "PurePosixPath", "PureWindowsPath", "Path", "PosixPath", "WindowsPath"] +if sys.version_info >= (3, 13): + __all__ += ["UnsupportedOperation"] + class PurePath(PathLike[str]): + if sys.version_info >= (3, 13): + parser: ClassVar[types.ModuleType] + def full_match(self, pattern: StrPath, *, case_sensitive: bool | None = None) -> bool: ... + @property def parts(self) -> tuple[str, ...]: ... @property @@ -94,8 +102,6 @@ class PureWindowsPath(PurePath): ... class Path(PurePath): def __new__(cls, *args: StrPath, **kwargs: Any) -> Self: ... - def __enter__(self) -> Self: ... - def __exit__(self, t: type[BaseException] | None, v: BaseException | None, tb: TracebackType | None) -> None: ... @classmethod def cwd(cls) -> Self: ... if sys.version_info >= (3, 10): @@ -105,17 +111,38 @@ class Path(PurePath): def stat(self) -> stat_result: ... def chmod(self, mode: int) -> None: ... - if sys.version_info >= (3, 12): - def exists(self, *, follow_symlinks: bool = True) -> bool: ... + if sys.version_info >= (3, 13): + @classmethod + def from_uri(cls, uri: str) -> Path: ... + def is_dir(self, *, follow_symlinks: bool = True) -> bool: ... + def is_file(self, *, follow_symlinks: bool = True) -> bool: ... + def read_text(self, encoding: str | None = None, errors: str | None = None, newline: str | None = None) -> str: ... + else: + def __enter__(self) -> Self: ... + def __exit__(self, t: type[BaseException] | None, v: BaseException | None, tb: TracebackType | None) -> None: ... + def is_dir(self) -> bool: ... + def is_file(self) -> bool: ... + def read_text(self, encoding: str | None = None, errors: str | None = None) -> str: ... + + if sys.version_info >= (3, 13): + def glob( + self, pattern: str, *, case_sensitive: bool | None = None, recurse_symlinks: bool = False + ) -> Generator[Self, None, None]: ... + def rglob( + self, pattern: str, *, case_sensitive: bool | None = None, recurse_symlinks: bool = False + ) -> Generator[Self, None, None]: ... + elif sys.version_info >= (3, 12): def glob(self, pattern: str, *, case_sensitive: bool | None = None) -> Generator[Self, None, None]: ... def rglob(self, pattern: str, *, case_sensitive: bool | None = None) -> Generator[Self, None, None]: ... else: - def exists(self) -> bool: ... def glob(self, pattern: str) -> Generator[Self, None, None]: ... def rglob(self, pattern: str) -> Generator[Self, None, None]: ... - def is_dir(self) -> bool: ... - def is_file(self) -> bool: ... + if sys.version_info >= (3, 12): + def exists(self, *, follow_symlinks: bool = True) -> bool: ... + else: + def exists(self) -> bool: ... + def is_symlink(self) -> bool: ... def is_socket(self) -> bool: ... def is_fifo(self) -> bool: ... @@ -186,8 +213,12 @@ class Path(PurePath): if sys.platform != "win32": # These methods do "exist" on Windows, but they always raise NotImplementedError, # so it's safer to pretend they don't exist - def owner(self) -> str: ... - def group(self) -> str: ... + if sys.version_info >= (3, 13): + def owner(self, *, follow_symlinks: bool = True) -> str: ... + def group(self, *, follow_symlinks: bool = True) -> str: ... + else: + def owner(self) -> str: ... + def group(self) -> str: ... # This method does "exist" on Windows on <3.12, but always raises NotImplementedError # On py312+, it works properly on Windows, as with all other platforms @@ -212,7 +243,6 @@ class Path(PurePath): def absolute(self) -> Self: ... def expanduser(self) -> Self: ... def read_bytes(self) -> bytes: ... - def read_text(self, encoding: str | None = None, errors: str | None = None) -> str: ... def samefile(self, other_path: StrPath) -> bool: ... def write_bytes(self, data: ReadableBuffer) -> int: ... if sys.version_info >= (3, 10): @@ -234,3 +264,6 @@ class Path(PurePath): class PosixPath(Path, PurePosixPath): ... class WindowsPath(Path, PureWindowsPath): ... + +if sys.version_info >= (3, 13): + class UnsupportedOperation(NotImplementedError): ... diff --git a/mypy/typeshed/stdlib/platform.pyi b/mypy/typeshed/stdlib/platform.pyi index f0e6d4123e1d..c47ecdc51df4 100644 --- a/mypy/typeshed/stdlib/platform.pyi +++ b/mypy/typeshed/stdlib/platform.pyi @@ -40,3 +40,28 @@ def platform(aliased: bool = ..., terse: bool = ...) -> str: ... if sys.version_info >= (3, 10): def freedesktop_os_release() -> dict[str, str]: ... + +if sys.version_info >= (3, 13): + class AndroidVer(NamedTuple): + release: str + api_level: int + manufacturer: str + model: str + device: str + is_emulator: bool + + class IOSVersionInfo(NamedTuple): + system: str + release: str + model: str + is_simulator: bool + + def android_ver( + release: str = "", + api_level: int = 0, + manufacturer: str = "", + model: str = "", + device: str = "", + is_emulator: bool = False, + ) -> AndroidVer: ... + def ios_ver(system: str = "", release: str = "", model: str = "", is_simulator: bool = False) -> IOSVersionInfo: ... diff --git a/mypy/typeshed/stdlib/shutil.pyi b/mypy/typeshed/stdlib/shutil.pyi index f6c8a390d85f..dcff18d110bd 100644 --- a/mypy/typeshed/stdlib/shutil.pyi +++ b/mypy/typeshed/stdlib/shutil.pyi @@ -78,13 +78,25 @@ class _RmtreeType(Protocol): avoids_symlink_attacks: bool if sys.version_info >= (3, 12): @overload - @deprecated("The `onerror` parameter is deprecated and will be removed in Python 3.14. Use `onexc` instead.") + @deprecated("The `onerror` parameter is deprecated. Use `onexc` instead.") + def __call__( + self, + path: StrOrBytesPath, + ignore_errors: bool, + onerror: _OnErrorCallback, + *, + onexc: None = None, + dir_fd: int | None = None, + ) -> None: ... + @overload + @deprecated("The `onerror` parameter is deprecated. Use `onexc` instead.") def __call__( self, path: StrOrBytesPath, ignore_errors: bool = False, - onerror: _OnErrorCallback | None = None, *, + onerror: _OnErrorCallback, + onexc: None = None, dir_fd: int | None = None, ) -> None: ... @overload diff --git a/mypy/typeshed/stdlib/spwd.pyi b/mypy/typeshed/stdlib/spwd.pyi index 67ad3bfc751b..3a5d39997dcc 100644 --- a/mypy/typeshed/stdlib/spwd.pyi +++ b/mypy/typeshed/stdlib/spwd.pyi @@ -36,6 +36,11 @@ if sys.platform != "win32": def sp_expire(self) -> int: ... @property def sp_flag(self) -> int: ... + # Deprecated aliases below. + @property + def sp_nam(self) -> str: ... + @property + def sp_pwd(self) -> str: ... def getspall() -> list[struct_spwd]: ... def getspnam(arg: str, /) -> struct_spwd: ... diff --git a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi index 068ce1514c3c..3cb4b93e88fe 100644 --- a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi +++ b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi @@ -428,7 +428,11 @@ class Connection: def executemany(self, sql: str, parameters: Iterable[_Parameters], /) -> Cursor: ... def executescript(self, sql_script: str, /) -> Cursor: ... def interrupt(self) -> None: ... - def iterdump(self) -> Generator[str, None, None]: ... + if sys.version_info >= (3, 13): + def iterdump(self, *, filter: str | None = None) -> Generator[str, None, None]: ... + else: + def iterdump(self) -> Generator[str, None, None]: ... + def rollback(self) -> None: ... def set_authorizer( self, authorizer_callback: Callable[[int, str | None, str | None, str | None, str | None], int] | None diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi index b6fe454eff78..e52099464174 100644 --- a/mypy/typeshed/stdlib/tarfile.pyi +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -142,22 +142,43 @@ class TarFile: errorlevel: int | None offset: int # undocumented extraction_filter: _FilterFunction | None - def __init__( - self, - name: StrOrBytesPath | None = None, - mode: Literal["r", "a", "w", "x"] = "r", - fileobj: _Fileobj | None = None, - format: int | None = None, - tarinfo: type[TarInfo] | None = None, - dereference: bool | None = None, - ignore_zeros: bool | None = None, - encoding: str | None = None, - errors: str = "surrogateescape", - pax_headers: Mapping[str, str] | None = None, - debug: int | None = None, - errorlevel: int | None = None, - copybufsize: int | None = None, # undocumented - ) -> None: ... + if sys.version_info >= (3, 13): + stream: bool + def __init__( + self, + name: StrOrBytesPath | None = None, + mode: Literal["r", "a", "w", "x"] = "r", + fileobj: _Fileobj | None = None, + format: int | None = None, + tarinfo: type[TarInfo] | None = None, + dereference: bool | None = None, + ignore_zeros: bool | None = None, + encoding: str | None = None, + errors: str = "surrogateescape", + pax_headers: Mapping[str, str] | None = None, + debug: int | None = None, + errorlevel: int | None = None, + copybufsize: int | None = None, # undocumented + stream: bool = False, + ) -> None: ... + else: + def __init__( + self, + name: StrOrBytesPath | None = None, + mode: Literal["r", "a", "w", "x"] = "r", + fileobj: _Fileobj | None = None, + format: int | None = None, + tarinfo: type[TarInfo] | None = None, + dereference: bool | None = None, + ignore_zeros: bool | None = None, + encoding: str | None = None, + errors: str = "surrogateescape", + pax_headers: Mapping[str, str] | None = None, + debug: int | None = None, + errorlevel: int | None = None, + copybufsize: int | None = None, # undocumented + ) -> None: ... + def __enter__(self) -> Self: ... def __exit__( self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None diff --git a/mypy/typeshed/stdlib/telnetlib.pyi b/mypy/typeshed/stdlib/telnetlib.pyi index d244d54f2fbf..294a1cb12b63 100644 --- a/mypy/typeshed/stdlib/telnetlib.pyi +++ b/mypy/typeshed/stdlib/telnetlib.pyi @@ -88,6 +88,7 @@ NOOPT: bytes class Telnet: host: str | None # undocumented + sock: socket.socket | None # undocumented def __init__(self, host: str | None = None, port: int = 0, timeout: float = ...) -> None: ... def open(self, host: str, port: int = 0, timeout: float = ...) -> None: ... def msg(self, msg: str, *args: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/time.pyi b/mypy/typeshed/stdlib/time.pyi index b7962f0751d6..71cdc4d78fdc 100644 --- a/mypy/typeshed/stdlib/time.pyi +++ b/mypy/typeshed/stdlib/time.pyi @@ -27,6 +27,9 @@ if sys.platform != "win32": if sys.platform == "darwin": CLOCK_UPTIME_RAW: int + if sys.version_info >= (3, 13): + CLOCK_UPTIME_RAW_APPROX: int + CLOCK_MONOTONIC_RAW_APPROX: int if sys.version_info >= (3, 9) and sys.platform == "linux": CLOCK_TAI: int @@ -94,7 +97,7 @@ if sys.platform != "win32": def clock_settime(clk_id: int, time: float, /) -> None: ... # Unix only if sys.platform != "win32": - def clock_gettime_ns(clock_id: int, /) -> int: ... + def clock_gettime_ns(clk_id: int, /) -> int: ... def clock_settime_ns(clock_id: int, time: int, /) -> int: ... if sys.platform == "linux": diff --git a/mypy/typeshed/stdlib/traceback.pyi b/mypy/typeshed/stdlib/traceback.pyi index 39803003cfe5..075c0f4b9de8 100644 --- a/mypy/typeshed/stdlib/traceback.pyi +++ b/mypy/typeshed/stdlib/traceback.pyi @@ -3,7 +3,7 @@ from _typeshed import SupportsWrite, Unused from collections.abc import Generator, Iterable, Iterator, Mapping from types import FrameType, TracebackType from typing import Any, Literal, overload -from typing_extensions import Self, TypeAlias +from typing_extensions import Self, TypeAlias, deprecated __all__ = [ "extract_stack", @@ -85,7 +85,13 @@ def format_list(extracted_list: Iterable[FrameSummary | _FrameSummaryTuple]) -> # undocumented def print_list(extracted_list: Iterable[FrameSummary | _FrameSummaryTuple], file: SupportsWrite[str] | None = None) -> None: ... -if sys.version_info >= (3, 10): +if sys.version_info >= (3, 13): + @overload + def format_exception_only(exc: BaseException | None, /, *, show_group: bool = False) -> list[str]: ... + @overload + def format_exception_only(exc: Unused, /, value: BaseException | None, *, show_group: bool = False) -> list[str]: ... + +elif sys.version_info >= (3, 10): @overload def format_exception_only(exc: BaseException | None, /) -> list[str]: ... @overload @@ -111,13 +117,20 @@ class TracebackException: __context__: TracebackException __suppress_context__: bool stack: StackSummary - exc_type: type[BaseException] filename: str lineno: int text: str offset: int msg: str - if sys.version_info >= (3, 11): + if sys.version_info >= (3, 13): + @property + def exc_type_str(self) -> str: ... + @property + @deprecated("Deprecated in 3.13. Use exc_type_str instead.") + def exc_type(self) -> type[BaseException] | None: ... + else: + exc_type: type[BaseException] + if sys.version_info >= (3, 13): def __init__( self, exc_type: type[BaseException], @@ -130,12 +143,15 @@ class TracebackException: compact: bool = False, max_group_width: int = 15, max_group_depth: int = 10, + save_exc_type: bool = True, _seen: set[int] | None = None, ) -> None: ... - @classmethod - def from_exception( - cls, - exc: BaseException, + elif sys.version_info >= (3, 11): + def __init__( + self, + exc_type: type[BaseException], + exc_value: BaseException, + exc_traceback: TracebackType | None, *, limit: int | None = None, lookup_lines: bool = True, @@ -143,7 +159,8 @@ class TracebackException: compact: bool = False, max_group_width: int = 15, max_group_depth: int = 10, - ) -> Self: ... + _seen: set[int] | None = None, + ) -> None: ... elif sys.version_info >= (3, 10): def __init__( self, @@ -157,6 +174,20 @@ class TracebackException: compact: bool = False, _seen: set[int] | None = None, ) -> None: ... + else: + def __init__( + self, + exc_type: type[BaseException], + exc_value: BaseException, + exc_traceback: TracebackType | None, + *, + limit: int | None = None, + lookup_lines: bool = True, + capture_locals: bool = False, + _seen: set[int] | None = None, + ) -> None: ... + + if sys.version_info >= (3, 11): @classmethod def from_exception( cls, @@ -166,19 +197,21 @@ class TracebackException: lookup_lines: bool = True, capture_locals: bool = False, compact: bool = False, + max_group_width: int = 15, + max_group_depth: int = 10, ) -> Self: ... - else: - def __init__( - self, - exc_type: type[BaseException], - exc_value: BaseException, - exc_traceback: TracebackType | None, + elif sys.version_info >= (3, 10): + @classmethod + def from_exception( + cls, + exc: BaseException, *, limit: int | None = None, lookup_lines: bool = True, capture_locals: bool = False, - _seen: set[int] | None = None, - ) -> None: ... + compact: bool = False, + ) -> Self: ... + else: @classmethod def from_exception( cls, exc: BaseException, *, limit: int | None = None, lookup_lines: bool = True, capture_locals: bool = False @@ -190,7 +223,10 @@ class TracebackException: else: def format(self, *, chain: bool = True) -> Generator[str, None, None]: ... - def format_exception_only(self) -> Generator[str, None, None]: ... + if sys.version_info >= (3, 13): + def format_exception_only(self, *, show_group: bool = False, _depth: int = 0) -> Generator[str, None, None]: ... + else: + def format_exception_only(self) -> Generator[str, None, None]: ... if sys.version_info >= (3, 11): def print(self, *, file: SupportsWrite[str] | None = None, chain: bool = True) -> None: ... diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 93cb89046366..9e9dc56b8529 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -81,7 +81,7 @@ class FunctionType: __name__: str __qualname__: str __annotations__: dict[str, Any] - __kwdefaults__: dict[str, Any] + __kwdefaults__: dict[str, Any] | None if sys.version_info >= (3, 10): @property def __builtins__(self) -> dict[str, Any]: ... @@ -358,6 +358,8 @@ class GeneratorType(Generator[_YieldT_co, _SendT_contra, _ReturnT_co]): ) -> _YieldT_co: ... @overload def throw(self, typ: BaseException, val: None = None, tb: TracebackType | None = ..., /) -> _YieldT_co: ... + if sys.version_info >= (3, 13): + def __class_getitem__(cls, item: Any, /) -> Any: ... @final class AsyncGeneratorType(AsyncGenerator[_YieldT_co, _SendT_contra]): @@ -401,6 +403,8 @@ class CoroutineType(Coroutine[_YieldT_co, _SendT_contra, _ReturnT_co]): ) -> _YieldT_co: ... @overload def throw(self, typ: BaseException, val: None = None, tb: TracebackType | None = ..., /) -> _YieldT_co: ... + if sys.version_info >= (3, 13): + def __class_getitem__(cls, item: Any, /) -> Any: ... @final class MethodType: @@ -587,6 +591,9 @@ if sys.version_info >= (3, 9): def __unpacked__(self) -> bool: ... @property def __typing_unpacked_tuple_args__(self) -> tuple[Any, ...] | None: ... + if sys.version_info >= (3, 10): + def __or__(self, value: Any, /) -> UnionType: ... + def __ror__(self, value: Any, /) -> UnionType: ... # GenericAlias delegates attr access to `__origin__` def __getattr__(self, name: str) -> Any: ... diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index 1b021d1eecbd..92427f91f022 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -21,7 +21,7 @@ from types import ( TracebackType, WrapperDescriptorType, ) -from typing_extensions import Never as _Never, ParamSpec as _ParamSpec +from typing_extensions import Never as _Never, ParamSpec as _ParamSpec, deprecated if sys.version_info >= (3, 9): from types import GenericAlias @@ -129,7 +129,7 @@ if sys.version_info >= (3, 12): __all__ += ["TypeAliasType", "override"] if sys.version_info >= (3, 13): - __all__ += ["get_protocol_members", "is_protocol", "NoDefault"] + __all__ += ["get_protocol_members", "is_protocol", "NoDefault", "TypeIs", "ReadOnly"] Any = object() @@ -183,6 +183,7 @@ class TypeVar: if sys.version_info >= (3, 11): def __typing_subst__(self, arg: Any) -> Any: ... if sys.version_info >= (3, 13): + def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... def has_default(self) -> bool: ... # Used for an undocumented mypy feature. Does not exist at runtime. @@ -989,7 +990,35 @@ class ForwardRef: else: def __init__(self, arg: str, is_argument: bool = True) -> None: ... - if sys.version_info >= (3, 9): + if sys.version_info >= (3, 13): + @overload + @deprecated( + "Failing to pass a value to the 'type_params' parameter of ForwardRef._evaluate() is deprecated, " + "as it leads to incorrect behaviour when evaluating a stringified annotation " + "that references a PEP 695 type parameter. It will be disallowed in Python 3.15." + ) + def _evaluate( + self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None, *, recursive_guard: frozenset[str] + ) -> Any | None: ... + @overload + def _evaluate( + self, + globalns: dict[str, Any] | None, + localns: dict[str, Any] | None, + type_params: tuple[TypeVar | ParamSpec | TypeVarTuple, ...], + *, + recursive_guard: frozenset[str], + ) -> Any | None: ... + elif sys.version_info >= (3, 12): + def _evaluate( + self, + globalns: dict[str, Any] | None, + localns: dict[str, Any] | None, + type_params: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] | None = None, + *, + recursive_guard: frozenset[str], + ) -> Any | None: ... + elif sys.version_info >= (3, 9): def _evaluate( self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None, recursive_guard: frozenset[str] ) -> Any | None: ... @@ -1036,3 +1065,5 @@ if sys.version_info >= (3, 13): class _NoDefaultType: ... NoDefault: _NoDefaultType + TypeIs: _SpecialForm + ReadOnly: _SpecialForm diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index 73fd2dc8cbb3..a7d2b2c2e083 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -419,6 +419,8 @@ if sys.version_info >= (3, 13): from typing import ( NoDefault as NoDefault, ParamSpec as ParamSpec, + ReadOnly as ReadOnly, + TypeIs as TypeIs, TypeVar as TypeVar, TypeVarTuple as TypeVarTuple, get_protocol_members as get_protocol_members, @@ -520,11 +522,11 @@ else: def has_default(self) -> bool: ... def __typing_prepare_subst__(self, alias: Any, args: Any) -> tuple[Any, ...]: ... + ReadOnly: _SpecialForm + TypeIs: _SpecialForm + class Doc: documentation: str def __init__(self, documentation: str, /) -> None: ... def __hash__(self) -> int: ... def __eq__(self, other: object) -> bool: ... - -ReadOnly: _SpecialForm -TypeIs: _SpecialForm diff --git a/mypy/typeshed/stdlib/weakref.pyi b/mypy/typeshed/stdlib/weakref.pyi index e345124237da..aaba7ffc98d9 100644 --- a/mypy/typeshed/stdlib/weakref.pyi +++ b/mypy/typeshed/stdlib/weakref.pyi @@ -41,7 +41,10 @@ _P = ParamSpec("_P") ProxyTypes: tuple[type[Any], ...] class WeakMethod(ref[_CallableT]): - def __new__(cls, meth: _CallableT, callback: Callable[[Self], object] | None = None) -> Self: ... + # `ref` is implemented in `C` so positional-only arguments are enforced, but not in `WeakMethod`. + def __new__( # pyright: ignore[reportInconsistentConstructor] + cls, meth: _CallableT, callback: Callable[[Self], Any] | None = None + ) -> Self: ... def __call__(self) -> _CallableT | None: ... def __eq__(self, other: object) -> bool: ... def __ne__(self, other: object) -> bool: ... diff --git a/mypy/typeshed/stdlib/xml/sax/handler.pyi b/mypy/typeshed/stdlib/xml/sax/handler.pyi index 30fe31d51374..7b7c69048efd 100644 --- a/mypy/typeshed/stdlib/xml/sax/handler.pyi +++ b/mypy/typeshed/stdlib/xml/sax/handler.pyi @@ -14,7 +14,7 @@ class ContentHandler: def startDocument(self) -> None: ... def endDocument(self) -> None: ... def startPrefixMapping(self, prefix: str | None, uri: str) -> None: ... - def endPrefixMapping(self, prefix) -> None: ... + def endPrefixMapping(self, prefix: str | None) -> None: ... def startElement(self, name: str, attrs: xmlreader.AttributesImpl) -> None: ... def endElement(self, name: str) -> None: ... def startElementNS(self, name: tuple[str, str], qname: str, attrs: xmlreader.AttributesNSImpl) -> None: ... From c4470f1a5b52c01b09c116f28b7ee12b658f746b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 20 Jun 2024 18:02:26 +0100 Subject: [PATCH 046/247] Make more type expressions valid in PEP 695 aliases and runtime contexts (#17404) Previously some type expressions, when used as the value of a PEP 695 type alias or in an expression context, generated errors, even if the code would work at runtime. Improve type inference of types in expression contexts (this includes PEP 695 type aliases) to better reflect runtime behavior. This is still not perfect, since we don't have precise types for everything in stubs. Use `typing._SpecialForm` as a fallback, as it supports indexing and `|` operations, which are supported for types. Also update stubs used in tests to better match typeshed stubs. In particular, provide `_SpecialForm` and define `Any = object()`, similar to typeshed. --- mypy/checkexpr.py | 26 ++++-- mypy/stubgenc.py | 2 +- mypyc/test-data/fixtures/typing-full.pyi | 6 +- test-data/unit/check-classes.test | 3 + test-data/unit/check-functions.test | 5 +- test-data/unit/check-generics.test | 7 +- test-data/unit/check-python312.test | 52 +++++++++++ .../check-type-object-type-inference.test | 3 +- test-data/unit/fixtures/typing-async.pyi | 4 +- test-data/unit/fixtures/typing-full.pyi | 7 +- test-data/unit/fixtures/typing-medium.pyi | 2 +- test-data/unit/fixtures/typing-namedtuple.pyi | 4 +- test-data/unit/fixtures/typing-override.pyi | 5 +- .../unit/fixtures/typing-typeddict-iror.pyi | 4 +- test-data/unit/fixtures/typing-typeddict.pyi | 4 +- test-data/unit/lib-stub/types.pyi | 4 +- test-data/unit/lib-stub/typing.pyi | 4 +- test-data/unit/pythoneval.test | 86 ++++++++++++++++++- 18 files changed, 196 insertions(+), 32 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 1cea4f6c19e6..734a9e1687bd 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -428,6 +428,9 @@ def analyze_var_ref(self, var: Var, context: Context) -> Type: if var.type: var_type = get_proper_type(var.type) if isinstance(var_type, Instance): + if var.fullname == "typing.Any": + # The typeshed type is 'object'; give a more useful type in runtime context + return self.named_type("typing._SpecialForm") if self.is_literal_context() and var_type.last_known_value is not None: return var_type.last_known_value if var.name in {"True", "False"}: @@ -4331,16 +4334,25 @@ def visit_index_with_type( return self.nonliteral_tuple_index_helper(left_type, index) elif isinstance(left_type, TypedDictType): return self.visit_typeddict_index_expr(left_type, e.index) - elif ( - isinstance(left_type, FunctionLike) - and left_type.is_type_obj() - and left_type.type_object().is_enum - ): - return self.visit_enum_index_expr(left_type.type_object(), e.index, e) - elif isinstance(left_type, TypeVarType) and not self.has_member( + elif isinstance(left_type, FunctionLike) and left_type.is_type_obj(): + if left_type.type_object().is_enum: + return self.visit_enum_index_expr(left_type.type_object(), e.index, e) + elif left_type.type_object().type_vars: + return self.named_type("types.GenericAlias") + elif ( + left_type.type_object().fullname == "builtins.type" + and self.chk.options.python_version >= (3, 9) + ): + # builtins.type is special: it's not generic in stubs, but it supports indexing + return self.named_type("typing._SpecialForm") + + if isinstance(left_type, TypeVarType) and not self.has_member( left_type.upper_bound, "__getitem__" ): return self.visit_index_with_type(left_type.upper_bound, e, original_type) + elif isinstance(left_type, Instance) and left_type.type.fullname == "typing._SpecialForm": + # Allow special forms to be indexed and used to create union types + return self.named_type("typing._SpecialForm") else: result, method_type = self.check_method_call_by_name( "__getitem__", left_type, [e.index], [ARG_POS], e, original_type=original_type diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 9acd3f171a41..bacb68f6d1c7 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -733,7 +733,7 @@ def generate_property_stub( def get_type_fullname(self, typ: type) -> str: """Given a type, return a string representation""" - if typ is Any: + if typ is Any: # type: ignore[comparison-overlap] return "Any" typename = getattr(typ, "__qualname__", typ.__name__) module_name = self.get_obj_module(typ) diff --git a/mypyc/test-data/fixtures/typing-full.pyi b/mypyc/test-data/fixtures/typing-full.pyi index 8bb3b1398f87..6b6aba6802b1 100644 --- a/mypyc/test-data/fixtures/typing-full.pyi +++ b/mypyc/test-data/fixtures/typing-full.pyi @@ -15,8 +15,7 @@ class _SpecialForm: cast = 0 overload = 0 -Any = 0 -Union = 0 +Any = object() Optional = 0 TypeVar = 0 Generic = 0 @@ -28,11 +27,12 @@ Type = 0 no_type_check = 0 ClassVar = 0 Final = 0 -Literal = 0 TypedDict = 0 NoReturn = 0 NewType = 0 Callable: _SpecialForm +Union: _SpecialForm +Literal: _SpecialForm T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index e66eab5e2927..82208d27df41 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -4790,12 +4790,15 @@ def g(x: Type[S]) -> str: return reveal_type(x * 0) # N: Revealed type is "builtins.str" [case testMetaclassGetitem] +import types + class M(type): def __getitem__(self, key) -> int: return 1 class A(metaclass=M): pass reveal_type(A[M]) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] [case testMetaclassSelfType] from typing import TypeVar, Type diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index ef6ca9f3b285..29cd977fe5d6 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1779,10 +1779,10 @@ def Arg(x, y): pass F = Callable[[Arg(int, 'x')], int] # E: Invalid argument constructor "__main__.Arg" [case testCallableParsingFromExpr] - from typing import Callable, List from mypy_extensions import Arg, VarArg, KwArg import mypy_extensions +import types # Needed for type checking def WrongArg(x, y): return y # Note that for this test, the 'Value of type "int" is not indexable' errors are silly, @@ -1799,11 +1799,10 @@ L = Callable[[Arg(name='x', type=int)], int] # ok # I have commented out the following test because I don't know how to expect the "defined here" note part of the error. # M = Callable[[Arg(gnome='x', type=int)], int] E: Invalid type alias: expression is not a valid type E: Unexpected keyword argument "gnome" for "Arg" N = Callable[[Arg(name=None, type=int)], int] # ok -O = Callable[[List[Arg(int)]], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: Type expected within [...] # E: The type "Type[List[Any]]" is not generic and not indexable +O = Callable[[List[Arg(int)]], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: Type expected within [...] P = Callable[[mypy_extensions.VarArg(int)], int] # ok Q = Callable[[Arg(int, type=int)], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "type" R = Callable[[Arg(int, 'x', name='y')], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "name" - [builtins fixtures/dict.pyi] [case testCallableParsing] diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index d46d19946098..b8cc0422b749 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -454,11 +454,13 @@ A[int, str, int]() # E: Type application has too many types (2 expected) [out] [case testInvalidTypeApplicationType] +import types a: A class A: pass a[A]() # E: Value of type "A" is not indexable A[A]() # E: The type "Type[A]" is not generic and not indexable -[out] +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testTypeApplicationArgTypes] from typing import TypeVar, Generic @@ -513,8 +515,9 @@ Alias[int]("a") # E: Argument 1 to "Node" has incompatible type "str"; expected [out] [case testTypeApplicationCrash] +import types type[int] # this was crashing, see #2302 (comment) # E: The type "Type[type]" is not generic and not indexable -[out] +[builtins fixtures/tuple.pyi] -- Generic type aliases diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 348f2d11f9a7..7c3d565b1b44 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1591,3 +1591,55 @@ c: E[str] d: E[int] # E: Type argument "int" of "E" must be a subtype of "str" [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] + +[case testPEP695TypeAliasWithDifferentTargetTypes] +# flags: --enable-incomplete-feature=NewGenericSyntax +import types # We need GenericAlias from here, and test stubs don't bring in 'types' +from typing import Any, Callable, List, Literal, TypedDict + +# Test that various type expressions don't generate false positives as type alias +# values, as they are type checked as expressions. There is a similar test case in +# pythoneval.test that uses typeshed stubs. + +class C[T]: pass + +class TD(TypedDict): + x: int + +type A1 = type[int] +type A2 = type[int] | None +type A3 = None | type[int] +type A4 = type[Any] + +type B1[**P, R] = Callable[P, R] | None +type B2[**P, R] = None | Callable[P, R] +type B3 = Callable[[str], int] +type B4 = Callable[..., int] + +type C1 = A1 | None +type C2 = None | A1 + +type D1 = Any | None +type D2 = None | Any + +type E1 = List[int] +type E2 = List[int] | None +type E3 = None | List[int] + +type F1 = Literal[1] +type F2 = Literal['x'] | None +type F3 = None | Literal[True] + +type G1 = tuple[int, Any] +type G2 = tuple[int, Any] | None +type G3 = None | tuple[int, Any] + +type H1 = TD +type H2 = TD | None +type H3 = None | TD + +type I1 = C[int] +type I2 = C[Any] | None +type I3 = None | C[TD] +[builtins fixtures/type.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-type-object-type-inference.test b/test-data/unit/check-type-object-type-inference.test index baeca1e22ac7..5a4afa0c9248 100644 --- a/test-data/unit/check-type-object-type-inference.test +++ b/test-data/unit/check-type-object-type-inference.test @@ -2,6 +2,7 @@ # flags: --python-version 3.9 from typing import TypeVar, Generic, Type from abc import abstractmethod +import types # Explicitly bring in stubs for 'types' T = TypeVar('T') class E(Generic[T]): @@ -37,5 +38,5 @@ def i(f: F): f.f(tuple[int,tuple[int,str]]).e( (27,(28,'z')) ) # OK reveal_type(f.f(tuple[int,tuple[int,str]]).e) # N: Revealed type is "def (t: Tuple[builtins.int, Tuple[builtins.int, builtins.str]]) -> builtins.str" -x = tuple[int,str][str] # E: The type "Type[Tuple[Any, ...]]" is not generic and not indexable +x = tuple[int,str][str] # False negative [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/fixtures/typing-async.pyi b/test-data/unit/fixtures/typing-async.pyi index 9897dfd0b270..03728f822316 100644 --- a/test-data/unit/fixtures/typing-async.pyi +++ b/test-data/unit/fixtures/typing-async.pyi @@ -10,7 +10,7 @@ from abc import abstractmethod, ABCMeta cast = 0 overload = 0 -Any = 0 +Any = object() Union = 0 Optional = 0 TypeVar = 0 @@ -125,3 +125,5 @@ class AsyncContextManager(Generic[T]): def __aenter__(self) -> Awaitable[T]: pass # Use Any because not all the precise types are in the fixtures. def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Awaitable[Any]: pass + +class _SpecialForm: pass diff --git a/test-data/unit/fixtures/typing-full.pyi b/test-data/unit/fixtures/typing-full.pyi index 9d61361fc16e..8e0116aab1c2 100644 --- a/test-data/unit/fixtures/typing-full.pyi +++ b/test-data/unit/fixtures/typing-full.pyi @@ -12,6 +12,8 @@ class GenericMeta(type): pass class _SpecialForm: def __getitem__(self, index: Any) -> Any: ... + def __or__(self, other): ... + def __ror__(self, other): ... class TypeVar: def __init__(self, name, *args, bound=None): ... def __or__(self, other): ... @@ -21,7 +23,7 @@ class TypeVarTuple: ... def cast(t, o): ... def assert_type(o, t): ... overload = 0 -Any = 0 +Any = object() Optional = 0 Generic = 0 Protocol = 0 @@ -31,7 +33,6 @@ Type = 0 no_type_check = 0 ClassVar = 0 Final = 0 -Literal = 0 TypedDict = 0 NoReturn = 0 NewType = 0 @@ -39,6 +40,7 @@ Self = 0 Unpack = 0 Callable: _SpecialForm Union: _SpecialForm +Literal: _SpecialForm T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) @@ -216,3 +218,4 @@ class TypeAliasType: ) -> None: ... def __or__(self, other: Any) -> Any: ... + def __ror__(self, other: Any) -> Any: ... diff --git a/test-data/unit/fixtures/typing-medium.pyi b/test-data/unit/fixtures/typing-medium.pyi index c19c5d5d96e2..c722a9ddb12c 100644 --- a/test-data/unit/fixtures/typing-medium.pyi +++ b/test-data/unit/fixtures/typing-medium.pyi @@ -8,7 +8,7 @@ cast = 0 overload = 0 -Any = 0 +Any = object() Union = 0 Optional = 0 TypeVar = 0 diff --git a/test-data/unit/fixtures/typing-namedtuple.pyi b/test-data/unit/fixtures/typing-namedtuple.pyi index f4744575fc09..bcdcfc44c3d2 100644 --- a/test-data/unit/fixtures/typing-namedtuple.pyi +++ b/test-data/unit/fixtures/typing-namedtuple.pyi @@ -1,6 +1,6 @@ TypeVar = 0 Generic = 0 -Any = 0 +Any = object() overload = 0 Type = 0 Literal = 0 @@ -26,3 +26,5 @@ class NamedTuple(tuple[Any, ...]): def __init__(self, typename: str, fields: Iterable[tuple[str, Any]] = ...) -> None: ... @overload def __init__(self, typename: str, fields: None = None, **kwargs: Any) -> None: ... + +class _SpecialForm: pass diff --git a/test-data/unit/fixtures/typing-override.pyi b/test-data/unit/fixtures/typing-override.pyi index 606ca63d4f0d..e9d2dfcf55c4 100644 --- a/test-data/unit/fixtures/typing-override.pyi +++ b/test-data/unit/fixtures/typing-override.pyi @@ -1,6 +1,6 @@ TypeVar = 0 Generic = 0 -Any = 0 +Any = object() overload = 0 Type = 0 Literal = 0 @@ -21,5 +21,6 @@ class Mapping(Iterable[KT], Generic[KT, T_co]): def keys(self) -> Iterable[T]: pass # Approximate return type def __getitem__(self, key: T) -> T_co: pass - def override(__arg: T) -> T: ... + +class _SpecialForm: pass diff --git a/test-data/unit/fixtures/typing-typeddict-iror.pyi b/test-data/unit/fixtures/typing-typeddict-iror.pyi index e452c8497109..845ac6cf208f 100644 --- a/test-data/unit/fixtures/typing-typeddict-iror.pyi +++ b/test-data/unit/fixtures/typing-typeddict-iror.pyi @@ -12,7 +12,7 @@ from abc import ABCMeta cast = 0 assert_type = 0 overload = 0 -Any = 0 +Any = object() Union = 0 Optional = 0 TypeVar = 0 @@ -64,3 +64,5 @@ class _TypedDict(Mapping[str, object]): def __ror__(self, __value: dict[str, Any]) -> dict[str, object]: ... # supposedly incompatible definitions of __or__ and __ior__ def __ior__(self, __value: Self) -> Self: ... # type: ignore[misc] + +class _SpecialForm: pass diff --git a/test-data/unit/fixtures/typing-typeddict.pyi b/test-data/unit/fixtures/typing-typeddict.pyi index 24a2f1328981..d136ac4ab8be 100644 --- a/test-data/unit/fixtures/typing-typeddict.pyi +++ b/test-data/unit/fixtures/typing-typeddict.pyi @@ -11,7 +11,7 @@ from abc import ABCMeta cast = 0 assert_type = 0 overload = 0 -Any = 0 +Any = object() Union = 0 Optional = 0 TypeVar = 0 @@ -71,3 +71,5 @@ class _TypedDict(Mapping[str, object]): def pop(self, k: NoReturn, default: T = ...) -> object: ... def update(self: T, __m: T) -> None: ... def __delitem__(self, k: NoReturn) -> None: ... + +class _SpecialForm: pass diff --git a/test-data/unit/lib-stub/types.pyi b/test-data/unit/lib-stub/types.pyi index c3ac244c2a51..3f713c31e417 100644 --- a/test-data/unit/lib-stub/types.pyi +++ b/test-data/unit/lib-stub/types.pyi @@ -9,7 +9,9 @@ class ModuleType: __file__: str def __getattr__(self, name: str) -> Any: pass -class GenericAlias: ... +class GenericAlias: + def __or__(self, o): ... + def __ror__(self, o): ... if sys.version_info >= (3, 10): class NoneType: diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 5f458ca687c0..3cb164140883 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -11,7 +11,7 @@ cast = 0 assert_type = 0 overload = 0 -Any = 0 +Any = object() Union = 0 Optional = 0 TypeVar = 0 @@ -63,3 +63,5 @@ class Coroutine(Awaitable[V], Generic[T, U, V]): pass def final(meth: T) -> T: pass def reveal_type(__obj: T) -> T: pass + +class _SpecialForm: pass diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 3bf8613d2478..222430c3ef55 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1606,8 +1606,8 @@ class Foo(Enum): Bar: Foo = Callable[[str], None] Baz: Foo = Callable[[Dict[str, "Missing"]], None] [out] -_testEnumValueWithPlaceholderNodeType.py:5: error: Incompatible types in assignment (expression has type "object", variable has type "Foo") -_testEnumValueWithPlaceholderNodeType.py:6: error: Incompatible types in assignment (expression has type "object", variable has type "Foo") +_testEnumValueWithPlaceholderNodeType.py:5: error: Incompatible types in assignment (expression has type "", variable has type "Foo") +_testEnumValueWithPlaceholderNodeType.py:6: error: Incompatible types in assignment (expression has type "", variable has type "Foo") _testEnumValueWithPlaceholderNodeType.py:6: error: Name "Missing" is not defined [case testTypeshedRecursiveTypesExample] @@ -1781,9 +1781,9 @@ C = str | int D: TypeAlias = str | int [out] _testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: Invalid type alias: expression is not a valid type -_testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: The type "Type[type]" is not generic and not indexable +_testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: Unsupported left operand type for | ("") _testTypeAliasNotSupportedWithNewStyleUnion.py:4: error: Invalid type alias: expression is not a valid type -_testTypeAliasNotSupportedWithNewStyleUnion.py:4: error: The type "Type[type]" is not generic and not indexable +_testTypeAliasNotSupportedWithNewStyleUnion.py:4: error: Unsupported left operand type for | ("Type[str]") _testTypeAliasNotSupportedWithNewStyleUnion.py:5: error: Invalid type alias: expression is not a valid type _testTypeAliasNotSupportedWithNewStyleUnion.py:5: error: Unsupported left operand type for | ("Type[str]") _testTypeAliasNotSupportedWithNewStyleUnion.py:6: error: Invalid type alias: expression is not a valid type @@ -2120,3 +2120,81 @@ def func( a2 = action # Error [out] _testPEP695VarianceInference.py:17: error: Incompatible types in assignment (expression has type "Job[None]", variable has type "Job[int]") + +[case testPEP695TypeAliasWithDifferentTargetTypes] +# flags: --python-version=3.12 --enable-incomplete-feature=NewGenericSyntax +from typing import Any, Callable, List, Literal, TypedDict, overload, TypeAlias, TypeVar, Never + +class C[T]: pass + +class O[T]: + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, x: int) -> None: ... + def __init__(self, x: int = 0) -> None: + pass + +class TD(TypedDict): + x: int + +S = TypeVar("S") +A = list[S] +B: TypeAlias = list[S] + +type A1 = type[int] +type A2 = type[int] | None +type A3 = None | type[int] +type A4 = type[Any] +type A5 = type[C] | None +type A6 = None | type[C] +type A7 = type[O] | None +type A8 = None | type[O] + +type B1[**P, R] = Callable[P, R] | None +type B2[**P, R] = None | Callable[P, R] +type B3 = Callable[[str], int] +type B4 = Callable[..., int] + +type C1 = A1 | None +type C2 = None | A1 + +type D1 = Any | None +type D2 = None | Any + +type E1 = List[int] +type E2 = List[int] | None +type E3 = None | List[int] + +type F1 = Literal[1] +type F2 = Literal['x'] | None +type F3 = None | Literal[True] + +type G1 = tuple[int, Any] +type G2 = tuple[int, Any] | None +type G3 = None | tuple[int, Any] + +type H1 = TD +type H2 = TD | None +type H3 = None | TD + +type I1 = C[int] +type I2 = C[Any] | None +type I3 = None | C[TD] +type I4 = O[int] | None +type I5 = None | O[int] + +type J1[T] = T | None +type J2[T] = None | T +type J3[*Ts] = tuple[*Ts] +type J4[T] = J1[T] | None +type J5[T] = None | J1[T] +type J6[*Ts] = J3[*Ts] | None + +type K1 = A[int] | None +type K2 = None | A[int] +type K3 = B[int] | None +type K4 = None | B[int] + +type L1 = Never +type L2 = list[Never] From f9d8f3ae9e4454777e0dd44380ba57bff7ef8ca2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 20 Jun 2024 18:57:31 +0100 Subject: [PATCH 047/247] Fix self-referential upper bound in new-style type variables (#17407) Fixes https://github.com/python/mypy/issues/17347 This copies old-style `TypeVar` logic 1:1 (I know it is ugly, but I don't think there is anything better now). Also while I am touching this code, I am removing `third_pass` argument (third pass is not a thing for ~5 years now). --- mypy/plugin.py | 1 - mypy/semanal.py | 13 ++++++------- test-data/unit/check-python312.test | 13 +++++++++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/mypy/plugin.py b/mypy/plugin.py index 38016191de8f..858795addb7f 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -328,7 +328,6 @@ def anal_type( allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, report_invalid_types: bool = True, - third_pass: bool = False, ) -> Type | None: """Analyze an unbound type. diff --git a/mypy/semanal.py b/mypy/semanal.py index c7a22d20aac6..f857c3e73381 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1738,10 +1738,12 @@ def analyze_type_param( ) -> TypeVarLikeExpr | None: fullname = self.qualified_name(type_param.name) if type_param.upper_bound: - upper_bound = self.anal_type(type_param.upper_bound) + upper_bound = self.anal_type(type_param.upper_bound, allow_placeholder=True) # TODO: we should validate the upper bound is valid for a given kind. if upper_bound is None: - return None + # This and below copies special-casing for old-style type variables, that + # is equally necessary for new-style classes to break a vicious circle. + upper_bound = PlaceholderType(None, [], context.line) else: if type_param.kind == TYPE_VAR_TUPLE_KIND: upper_bound = self.named_type("builtins.tuple", [self.object_type()]) @@ -1752,9 +1754,9 @@ def analyze_type_param( values = [] if type_param.values: for value in type_param.values: - analyzed = self.anal_type(value) + analyzed = self.anal_type(value, allow_placeholder=True) if analyzed is None: - return None + analyzed = PlaceholderType(None, [], context.line) values.append(analyzed) return TypeVarExpr( name=type_param.name, @@ -7192,7 +7194,6 @@ def anal_type( report_invalid_types: bool = True, prohibit_self_type: str | None = None, allow_type_any: bool = False, - third_pass: bool = False, ) -> Type | None: """Semantically analyze a type. @@ -7200,8 +7201,6 @@ def anal_type( typ: Type to analyze (if already analyzed, this is a no-op) allow_placeholder: If True, may return PlaceholderType if encountering an incomplete definition - third_pass: Unused; only for compatibility with old semantic - analyzer Return None only if some part of the type couldn't be bound *and* it referred to an incomplete namespace or definition. In this case also diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 7c3d565b1b44..27027d30a684 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1592,6 +1592,19 @@ d: E[int] # E: Type argument "int" of "E" must be a subtype of "str" [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] +[case testCurrentClassWorksAsBound] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import Protocol + +class Comparable[T: Comparable](Protocol): + def compare(self, other: T) -> bool: ... + +class Good: + def compare(self, other: Good) -> bool: ... + +x: Comparable[Good] +y: Comparable[int] # E: Type argument "int" of "Comparable" must be a subtype of "Comparable[Any]" + [case testPEP695TypeAliasWithDifferentTargetTypes] # flags: --enable-incomplete-feature=NewGenericSyntax import types # We need GenericAlias from here, and test stubs don't bring in 'types' From de4e9d612a7633b3a7d992ced5c15dbd47310296 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 21 Jun 2024 02:26:18 -0700 Subject: [PATCH 048/247] Fix isinstance checks with PEP 604 unions containing None (#17415) Fixes #17413 --- mypy/checker.py | 4 ++++ test-data/unit/check-union-or-syntax.test | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 3a7f231ebf1d..d2562d5dd722 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -7316,7 +7316,11 @@ def is_writable_attribute(self, node: Node) -> bool: def get_isinstance_type(self, expr: Expression) -> list[TypeRange] | None: if isinstance(expr, OpExpr) and expr.op == "|": left = self.get_isinstance_type(expr.left) + if left is None and is_literal_none(expr.left): + left = [TypeRange(NoneType(), is_upper_bound=False)] right = self.get_isinstance_type(expr.right) + if right is None and is_literal_none(expr.right): + right = [TypeRange(NoneType(), is_upper_bound=False)] if left is None or right is None: return None return left + right diff --git a/test-data/unit/check-union-or-syntax.test b/test-data/unit/check-union-or-syntax.test index a1b63077eef9..fcf679fff401 100644 --- a/test-data/unit/check-union-or-syntax.test +++ b/test-data/unit/check-union-or-syntax.test @@ -226,6 +226,17 @@ isinstance(5, str | list[str]) isinstance(5, ParameterizedAlias) [builtins fixtures/type.pyi] +[case testIsInstanceUnionNone] +# flags: --python-version 3.10 +def foo(value: str | bool | None): + assert not isinstance(value, str | None) + reveal_type(value) # N: Revealed type is "builtins.bool" + +def bar(value: object): + assert isinstance(value, str | None) + reveal_type(value) # N: Revealed type is "Union[builtins.str, None]" +[builtins fixtures/type.pyi] + # TODO: Get this test to pass [case testImplicit604TypeAliasWithCyclicImportNotInStub-xfail] From cc3492e45931d59666508ce81748dcdaa8ac436e Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 22 Jun 2024 12:38:30 -0700 Subject: [PATCH 049/247] Fix error reporting on cached run after uninstallation of third party library (#17420) Fixes https://github.com/python/mypy/issues/16766, fixes https://github.com/python/mypy/issues/17049 --- mypy/build.py | 7 +++++-- mypy/errors.py | 2 +- test-data/unit/check-incremental.test | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 3ceb473f0948..733f0685792e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -3467,8 +3467,11 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No for id in stale: graph[id].transitive_error = True for id in stale: - errors = manager.errors.file_messages(graph[id].xpath, formatter=manager.error_formatter) - manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False) + if graph[id].xpath not in manager.errors.ignored_files: + errors = manager.errors.file_messages( + graph[id].xpath, formatter=manager.error_formatter + ) + manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False) graph[id].write_cache() graph[id].mark_as_rechecked() diff --git a/mypy/errors.py b/mypy/errors.py index 7a937da39c20..d6dcd4e49e13 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -803,7 +803,7 @@ def blocker_module(self) -> str | None: def is_errors_for_file(self, file: str) -> bool: """Are there any errors for the given file?""" - return file in self.error_info_map + return file in self.error_info_map and file not in self.ignored_files def prefer_simple_messages(self) -> bool: """Should we generate simple/fast error messages? diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index ead896b8e458..24292bce3e21 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1833,6 +1833,21 @@ main:3: note: Revealed type is "builtins.int" main:3: note: Revealed type is "Any" +[case testIncrementalIgnoreErrors] +# flags: --config-file tmp/mypy.ini +import a +[file a.py] +import module_that_will_be_deleted +[file module_that_will_be_deleted.py] + +[file mypy.ini] +\[mypy] +\[mypy-a] +ignore_errors = True +[delete module_that_will_be_deleted.py.2] +[out1] +[out2] + [case testIncrementalNamedTupleInMethod] from ntcrash import nope [file ntcrash.py] From 9012fc9e954cb2ee0affd049f7c91b39c8fbc8e8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 22 Jun 2024 22:59:56 +0100 Subject: [PATCH 050/247] Some cleanup in partial plugin (#17423) Fixes https://github.com/python/mypy/issues/17405 Apart from fixing the crash I fix two obvious bugs I noticed while making this PR. --- mypy/checkexpr.py | 2 ++ mypy/plugins/functools.py | 31 +++++++++++++---- test-data/unit/check-functools.test | 52 +++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 734a9e1687bd..7ae23cfe516c 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1228,6 +1228,8 @@ def apply_function_plugin( formal_arg_exprs[formal].append(args[actual]) if arg_names: formal_arg_names[formal].append(arg_names[actual]) + else: + formal_arg_names[formal].append(None) formal_arg_kinds[formal].append(arg_kinds[actual]) if object_type is None: diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index 335123a4a108..4f2ed6f2361d 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -17,7 +17,6 @@ Type, TypeOfAny, UnboundType, - UninhabitedType, get_proper_type, ) @@ -132,6 +131,9 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: if fn_type is None: return ctx.default_return_type + # We must normalize from the start to have coherent view together with TypeChecker. + fn_type = fn_type.with_unpacked_kwargs().with_normalized_var_args() + defaulted = fn_type.copy_modified( arg_kinds=[ ( @@ -146,10 +148,25 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: # Make up a line number if we don't have one defaulted.set_line(ctx.default_return_type) - actual_args = [a for param in ctx.args[1:] for a in param] - actual_arg_kinds = [a for param in ctx.arg_kinds[1:] for a in param] - actual_arg_names = [a for param in ctx.arg_names[1:] for a in param] - actual_types = [a for param in ctx.arg_types[1:] for a in param] + # Flatten actual to formal mapping, since this is what check_call() expects. + actual_args = [] + actual_arg_kinds = [] + actual_arg_names = [] + actual_types = [] + seen_args = set() + for i, param in enumerate(ctx.args[1:], start=1): + for j, a in enumerate(param): + if a in seen_args: + # Same actual arg can map to multiple formals, but we need to include + # each one only once. + continue + # Here we rely on the fact that expressions are essentially immutable, so + # they can be compared by identity. + seen_args.add(a) + actual_args.append(a) + actual_arg_kinds.append(ctx.arg_kinds[i][j]) + actual_arg_names.append(ctx.arg_names[i][j]) + actual_types.append(ctx.arg_types[i][j]) # Create a valid context for various ad-hoc inspections in check_call(). call_expr = CallExpr( @@ -188,7 +205,7 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: for i, actuals in enumerate(formal_to_actual): if len(bound.arg_types) == len(fn_type.arg_types): arg_type = bound.arg_types[i] - if isinstance(get_proper_type(arg_type), UninhabitedType): + if not mypy.checker.is_valid_inferred_type(arg_type): arg_type = fn_type.arg_types[i] # bit of a hack else: # TODO: I assume that bound and fn_type have the same arguments. It appears this isn't @@ -210,7 +227,7 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: partial_names.append(fn_type.arg_names[i]) ret_type = bound.ret_type - if isinstance(get_proper_type(ret_type), UninhabitedType): + if not mypy.checker.is_valid_inferred_type(ret_type): ret_type = fn_type.ret_type # same kind of hack as above partially_applied = fn_type.copy_modified( diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index 283500f25a7d..79ae962a73e0 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -372,3 +372,55 @@ def foo(cls3: Type[B[T]]): reveal_type(functools.partial(cls3, 2)()) # N: Revealed type is "__main__.B[T`-1]" \ # E: Argument 1 to "B" has incompatible type "int"; expected "T" [builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialTypedDictUnpack] +from typing_extensions import TypedDict, Unpack +from functools import partial + +class Data(TypedDict, total=False): + x: int + +def f(**kwargs: Unpack[Data]) -> None: ... +def g(**kwargs: Unpack[Data]) -> None: + partial(f, **kwargs)() + +class MoreData(TypedDict, total=False): + x: int + y: int + +def f_more(**kwargs: Unpack[MoreData]) -> None: ... +def g_more(**kwargs: Unpack[MoreData]) -> None: + partial(f_more, **kwargs)() + +class Good(TypedDict, total=False): + y: int +class Bad(TypedDict, total=False): + y: str + +def h(**kwargs: Unpack[Data]) -> None: + bad: Bad + partial(f_more, **kwargs)(**bad) # E: Argument "y" to "f_more" has incompatible type "str"; expected "int" + good: Good + partial(f_more, **kwargs)(**good) +[builtins fixtures/dict.pyi] + +[case testFunctoolsPartialNestedGeneric] +from functools import partial +from typing import Generic, TypeVar, List + +T = TypeVar("T") +def get(n: int, args: List[T]) -> T: ... +first = partial(get, 0) + +x: List[str] +reveal_type(first(x)) # N: Revealed type is "builtins.str" +reveal_type(first([1])) # N: Revealed type is "builtins.int" + +first_kw = partial(get, n=0) +reveal_type(first_kw(args=[1])) # N: Revealed type is "builtins.int" + +# TODO: this is indeed invalid, but the error is incomprehensible. +first_kw([1]) # E: Too many positional arguments for "get" \ + # E: Too few arguments for "get" \ + # E: Argument 1 to "get" has incompatible type "List[int]"; expected "int" +[builtins fixtures/list.pyi] From abdaf6a571a4a539d755db1d6dcfdc45b69d97c5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 22 Jun 2024 23:19:22 +0100 Subject: [PATCH 051/247] Use (simplified) unions instead of joins for tuple fallbacks (#17408) Ref https://github.com/python/mypy/issues/12056 If `mypy_primer` will look good, I will add some logic to shorted unions in error messages. cc @JukkaL --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Alex Waygood --- mypy/checker.py | 3 ++ mypy/messages.py | 68 ++++++++++++++++++++++--- mypy/semanal_shared.py | 6 +-- mypy/typeops.py | 7 ++- test-data/unit/check-enum.test | 6 +-- test-data/unit/check-expressions.test | 2 +- test-data/unit/check-namedtuple.test | 6 +-- test-data/unit/check-newsemanal.test | 12 ++--- test-data/unit/check-statements.test | 2 +- test-data/unit/check-tuples.test | 4 +- test-data/unit/check-typevar-tuple.test | 6 +-- test-data/unit/check-unions.test | 57 +++++++++++++++++++++ test-data/unit/semanal-classes.test | 2 +- 13 files changed, 146 insertions(+), 35 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index d2562d5dd722..792e751691fd 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -49,6 +49,7 @@ SUGGESTED_TEST_FIXTURES, MessageBuilder, append_invariance_notes, + append_union_note, format_type, format_type_bare, format_type_distinctly, @@ -6814,6 +6815,8 @@ def check_subtype( ) if isinstance(subtype, Instance) and isinstance(supertype, Instance): notes = append_invariance_notes(notes, subtype, supertype) + if isinstance(subtype, UnionType) and isinstance(supertype, UnionType): + notes = append_union_note(notes, subtype, supertype, self.options) if extra_info: msg = msg.with_additional_msg(" (" + ", ".join(extra_info) + ")") diff --git a/mypy/messages.py b/mypy/messages.py index 27f152413151..62846c536f3d 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -90,6 +90,7 @@ UninhabitedType, UnionType, UnpackType, + flatten_nested_unions, get_proper_type, get_proper_types, ) @@ -145,6 +146,9 @@ "numbers.Integral", } +MAX_TUPLE_ITEMS = 10 +MAX_UNION_ITEMS = 10 + class MessageBuilder: """Helper class for reporting type checker error messages with parameters. @@ -2338,7 +2342,7 @@ def try_report_long_tuple_assignment_error( """ if isinstance(subtype, TupleType): if ( - len(subtype.items) > 10 + len(subtype.items) > MAX_TUPLE_ITEMS and isinstance(supertype, Instance) and supertype.type.fullname == "builtins.tuple" ): @@ -2347,7 +2351,7 @@ def try_report_long_tuple_assignment_error( self.generate_incompatible_tuple_error(lhs_types, subtype.items, context, msg) return True elif isinstance(supertype, TupleType) and ( - len(subtype.items) > 10 or len(supertype.items) > 10 + len(subtype.items) > MAX_TUPLE_ITEMS or len(supertype.items) > MAX_TUPLE_ITEMS ): if len(subtype.items) != len(supertype.items): if supertype_label is not None and subtype_label is not None: @@ -2370,7 +2374,7 @@ def try_report_long_tuple_assignment_error( def format_long_tuple_type(self, typ: TupleType) -> str: """Format very long tuple type using an ellipsis notation""" item_cnt = len(typ.items) - if item_cnt > 10: + if item_cnt > MAX_TUPLE_ITEMS: return "{}[{}, {}, ... <{} more items>]".format( "tuple" if self.options.use_lowercase_names() else "Tuple", format_type_bare(typ.items[0], self.options), @@ -2497,11 +2501,21 @@ def format(typ: Type) -> str: def format_list(types: Sequence[Type]) -> str: return ", ".join(format(typ) for typ in types) - def format_union(types: Sequence[Type]) -> str: + def format_union_items(types: Sequence[Type]) -> list[str]: formatted = [format(typ) for typ in types if format(typ) != "None"] + if len(formatted) > MAX_UNION_ITEMS and verbosity == 0: + more = len(formatted) - MAX_UNION_ITEMS // 2 + formatted = formatted[: MAX_UNION_ITEMS // 2] + else: + more = 0 + if more: + formatted.append(f"<{more} more items>") if any(format(typ) == "None" for typ in types): formatted.append("None") - return " | ".join(formatted) + return formatted + + def format_union(types: Sequence[Type]) -> str: + return " | ".join(format_union_items(types)) def format_literal_value(typ: LiteralType) -> str: if typ.is_enum_literal(): @@ -2605,6 +2619,9 @@ def format_literal_value(typ: LiteralType) -> str: elif isinstance(typ, LiteralType): return f"Literal[{format_literal_value(typ)}]" elif isinstance(typ, UnionType): + typ = get_proper_type(ignore_last_known_values(typ)) + if not isinstance(typ, UnionType): + return format(typ) literal_items, union_items = separate_union_literals(typ) # Coalesce multiple Literal[] members. This also changes output order. @@ -2624,7 +2641,7 @@ def format_literal_value(typ: LiteralType) -> str: return ( f"{literal_str} | {format_union(union_items)}" if options.use_or_syntax() - else f"Union[{format_list(union_items)}, {literal_str}]" + else f"Union[{', '.join(format_union_items(union_items))}, {literal_str}]" ) else: return literal_str @@ -2645,7 +2662,7 @@ def format_literal_value(typ: LiteralType) -> str: s = ( format_union(typ.items) if options.use_or_syntax() - else f"Union[{format_list(typ.items)}]" + else f"Union[{', '.join(format_union_items(typ.items))}]" ) return s elif isinstance(typ, NoneType): @@ -3182,6 +3199,23 @@ def append_invariance_notes( return notes +def append_union_note( + notes: list[str], arg_type: UnionType, expected_type: UnionType, options: Options +) -> list[str]: + """Point to specific union item(s) that may cause failure in subtype check.""" + non_matching = [] + items = flatten_nested_unions(arg_type.items) + if len(items) < MAX_UNION_ITEMS: + return notes + for item in items: + if not is_subtype(item, expected_type): + non_matching.append(item) + if non_matching: + types = ", ".join([format_type(typ, options) for typ in non_matching]) + notes.append(f"Item{plural_s(non_matching)} in the first union not in the second: {types}") + return notes + + def append_numbers_notes( notes: list[str], arg_type: Instance, expected_type: Instance ) -> list[str]: @@ -3235,3 +3269,23 @@ def format_key_list(keys: list[str], *, short: bool = False) -> str: return f"{td}key {formatted_keys[0]}" else: return f"{td}keys ({', '.join(formatted_keys)})" + + +def ignore_last_known_values(t: UnionType) -> Type: + """This will avoid types like str | str in error messages. + + last_known_values are kept during union simplification, but may cause + weird formatting for e.g. tuples of literals. + """ + union_items: list[Type] = [] + seen_instances = set() + for item in t.items: + if isinstance(item, ProperType) and isinstance(item, Instance): + erased = item.copy_modified(last_known_value=None) + if erased in seen_instances: + continue + seen_instances.add(erased) + union_items.append(erased) + else: + union_items.append(item) + return UnionType.make_union(union_items, t.line, t.column) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index 01d8e9aafffb..db19f074911f 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -8,7 +8,6 @@ from mypy_extensions import trait -from mypy import join from mypy.errorcodes import LITERAL_REQ, ErrorCode from mypy.nodes import ( CallExpr, @@ -30,6 +29,7 @@ from mypy.plugin import SemanticAnalyzerPluginInterface from mypy.tvar_scope import TypeVarLikeScope from mypy.type_visitor import ANY_STRATEGY, BoolTypeQuery +from mypy.typeops import make_simplified_union from mypy.types import ( TPDICT_FB_NAMES, AnyType, @@ -58,7 +58,7 @@ # Priorities for ordering of patches within the "patch" phase of semantic analysis # (after the main pass): -# Fix fallbacks (does joins) +# Fix fallbacks (does subtype checks). PRIORITY_FALLBACKS: Final = 1 @@ -304,7 +304,7 @@ def calculate_tuple_fallback(typ: TupleType) -> None: raise NotImplementedError else: items.append(item) - fallback.args = (join.join_type_list(items),) + fallback.args = (make_simplified_union(items),) class _NamedTypeCallback(Protocol): diff --git a/mypy/typeops.py b/mypy/typeops.py index 62c850452516..4fe187f811ca 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -95,8 +95,6 @@ def is_recursive_pair(s: Type, t: Type) -> bool: def tuple_fallback(typ: TupleType) -> Instance: """Return fallback type for a tuple.""" - from mypy.join import join_type_list - info = typ.partial_fallback.type if info.fullname != "builtins.tuple": return typ.partial_fallback @@ -115,8 +113,9 @@ def tuple_fallback(typ: TupleType) -> Instance: raise NotImplementedError else: items.append(item) - # TODO: we should really use a union here, tuple types are special. - return Instance(info, [join_type_list(items)], extra_attrs=typ.partial_fallback.extra_attrs) + return Instance( + info, [make_simplified_union(items)], extra_attrs=typ.partial_fallback.extra_attrs + ) def get_self_type(func: CallableType, default_self: Instance | TupleType) -> Type | None: diff --git a/test-data/unit/check-enum.test b/test-data/unit/check-enum.test index d53935085325..78a114eda764 100644 --- a/test-data/unit/check-enum.test +++ b/test-data/unit/check-enum.test @@ -1010,7 +1010,7 @@ _empty: Final = Empty.token def func(x: Union[int, None, Empty] = _empty) -> int: boom = x + 42 # E: Unsupported left operand type for + ("None") \ # E: Unsupported left operand type for + ("Empty") \ - # N: Left operand is of type "Union[int, None, Empty]" + # N: Left operand is of type "Union[int, Empty, None]" if x is _empty: reveal_type(x) # N: Revealed type is "Literal[__main__.Empty.token]" return 0 @@ -1056,7 +1056,7 @@ _empty = Empty.token def func(x: Union[int, None, Empty] = _empty) -> int: boom = x + 42 # E: Unsupported left operand type for + ("None") \ # E: Unsupported left operand type for + ("Empty") \ - # N: Left operand is of type "Union[int, None, Empty]" + # N: Left operand is of type "Union[int, Empty, None]" if x is _empty: reveal_type(x) # N: Revealed type is "Literal[__main__.Empty.token]" return 0 @@ -1084,7 +1084,7 @@ _empty = Empty.token def func(x: Union[int, None, Empty] = _empty) -> int: boom = x + 42 # E: Unsupported left operand type for + ("None") \ # E: Unsupported left operand type for + ("Empty") \ - # N: Left operand is of type "Union[int, None, Empty]" + # N: Left operand is of type "Union[int, Empty, None]" if x is _empty: reveal_type(x) # N: Revealed type is "Literal[__main__.Empty.token]" return 0 diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 4fc6e9a75c83..f9bd60f4dcc8 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1640,7 +1640,7 @@ from typing import Generator def g() -> Generator[int, None, None]: x = yield from () # E: Function does not return a value (it only ever returns None) x = yield from (0, 1, 2) # E: Function does not return a value (it only ever returns None) - x = yield from (0, "ERROR") # E: Incompatible types in "yield from" (actual type "object", expected type "int") \ + x = yield from (0, "ERROR") # E: Incompatible types in "yield from" (actual type "Union[int, str]", expected type "int") \ # E: Function does not return a value (it only ever returns None) x = yield from ("ERROR",) # E: Incompatible types in "yield from" (actual type "str", expected type "int") \ # E: Function does not return a value (it only ever returns None) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 2007d574f922..e9d156754d9c 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1249,7 +1249,7 @@ nti: NT[int] reveal_type(nti * x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" nts: NT[str] -reveal_type(nts * x) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +reveal_type(nts * x) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] @@ -1310,9 +1310,9 @@ reveal_type(foo(nti, nts)) # N: Revealed type is "Tuple[builtins.int, builtins. reveal_type(foo(nts, nti)) # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=__main__.NT[builtins.object]]" reveal_type(foo(nti, x)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(foo(nts, x)) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +reveal_type(foo(nts, x)) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" reveal_type(foo(x, nti)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(foo(x, nts)) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +reveal_type(foo(x, nts)) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index 47e508ee1a6b..511c7b003015 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -1947,7 +1947,7 @@ class NTStr(NamedTuple): y: str t1: T -reveal_type(t1.__iter__) # N: Revealed type is "def () -> typing.Iterator[__main__.A]" +reveal_type(t1.__iter__) # N: Revealed type is "def () -> typing.Iterator[Union[__main__.B, __main__.C]]" t2: NTInt reveal_type(t2.__iter__) # N: Revealed type is "def () -> typing.Iterator[builtins.int]" @@ -1960,7 +1960,6 @@ t: Union[Tuple[int, int], Tuple[str, str]] for x in t: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [builtins fixtures/for.pyi] -[out] [case testNewAnalyzerFallbackUpperBoundCheckAndFallbacks] from typing import TypeVar, Generic, Tuple @@ -1973,10 +1972,9 @@ S = TypeVar('S', bound='Tuple[G[A], ...]') class GG(Generic[S]): pass -g: GG[Tuple[G[B], G[C]]] \ - # E: Type argument "Tuple[G[B], G[C]]" of "GG" must be a subtype of "Tuple[G[A], ...]" \ - # E: Type argument "B" of "G" must be a subtype of "A" \ - # E: Type argument "C" of "G" must be a subtype of "A" +g: GG[Tuple[G[B], G[C]]] # E: Type argument "Tuple[G[B], G[C]]" of "GG" must be a subtype of "Tuple[G[A], ...]" \ + # E: Type argument "B" of "G" must be a subtype of "A" \ + # E: Type argument "C" of "G" must be a subtype of "A" T = TypeVar('T', bound=A, covariant=True) @@ -1984,7 +1982,7 @@ class G(Generic[T]): pass t: Tuple[G[B], G[C]] # E: Type argument "B" of "G" must be a subtype of "A" \ # E: Type argument "C" of "G" must be a subtype of "A" -reveal_type(t.__iter__) # N: Revealed type is "def () -> typing.Iterator[builtins.object]" +reveal_type(t.__iter__) # N: Revealed type is "def () -> typing.Iterator[__main__.G[__main__.B]]" [builtins fixtures/tuple.pyi] [case testNewAnalyzerClassKeywordsForward] diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 34df5a8ab336..d1464423e90f 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -1339,7 +1339,7 @@ from typing import Generator def g() -> Generator[int, None, None]: yield from () yield from (0, 1, 2) - yield from (0, "ERROR") # E: Incompatible types in "yield from" (actual type "object", expected type "int") + yield from (0, "ERROR") # E: Incompatible types in "yield from" (actual type "Union[int, str]", expected type "int") yield from ("ERROR",) # E: Incompatible types in "yield from" (actual type "str", expected type "int") [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index ad4893c2890a..bf36977b56e3 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1408,8 +1408,8 @@ y = "" reveal_type(t[x]) # N: Revealed type is "Union[builtins.int, builtins.str]" t[y] # E: No overload variant of "__getitem__" of "tuple" matches argument type "str" \ # N: Possible overload variants: \ - # N: def __getitem__(self, int, /) -> object \ - # N: def __getitem__(self, slice, /) -> Tuple[object, ...] + # N: def __getitem__(self, int, /) -> Union[int, str] \ + # N: def __getitem__(self, slice, /) -> Tuple[Union[int, str], ...] [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 8f7dd12d9cd4..49298114e069 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -24,7 +24,7 @@ def g(a: Tuple[Unpack[Ts]], b: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: reveal_type(g(args, args)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" reveal_type(g(args, args2)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" -reveal_type(g(args, args3)) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +reveal_type(g(args, args3)) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" reveal_type(g(any, any)) # N: Revealed type is "builtins.tuple[Any, ...]" [builtins fixtures/tuple.pyi] @@ -989,7 +989,7 @@ from typing_extensions import Unpack def pipeline(*xs: Unpack[Tuple[int, Unpack[Tuple[float, ...]], bool]]) -> None: for x in xs: - reveal_type(x) # N: Revealed type is "builtins.float" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] [case testFixedUnpackItemInInstanceArguments] @@ -1715,7 +1715,7 @@ vt: Tuple[int, Unpack[Tuple[float, ...]], int] reveal_type(vt + (1, 2)) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.int, Literal[1]?, Literal[2]?]" reveal_type((1, 2) + vt) # N: Revealed type is "Tuple[Literal[1]?, Literal[2]?, builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.int]" -reveal_type(vt + vt) # N: Revealed type is "builtins.tuple[builtins.float, ...]" +reveal_type(vt + vt) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.float], ...]" reveal_type(vtf + (1, 2)) # N: Revealed type is "Tuple[Unpack[builtins.tuple[builtins.float, ...]], Literal[1]?, Literal[2]?]" reveal_type((1, 2) + vtf) # N: Revealed type is "Tuple[Literal[1]?, Literal[2]?, Unpack[builtins.tuple[builtins.float, ...]]]" diff --git a/test-data/unit/check-unions.test b/test-data/unit/check-unions.test index 2ca2f1ba9eb3..329896f7a1a7 100644 --- a/test-data/unit/check-unions.test +++ b/test-data/unit/check-unions.test @@ -1289,3 +1289,60 @@ x: str = a_class_or_none.field a_or_none: Optional[A] y: int = a_or_none.field [builtins fixtures/list.pyi] + +[case testLargeUnionsShort] +from typing import Union + +class C1: ... +class C2: ... +class C3: ... +class C4: ... +class C5: ... +class C6: ... +class C7: ... +class C8: ... +class C9: ... +class C10: ... +class C11: ... + +u: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11] +x: int = u # E: Incompatible types in assignment (expression has type "Union[C1, C2, C3, C4, C5, <6 more items>]", variable has type "int") + +[case testLargeUnionsLongIfNeeded] +from typing import Union + +class C1: ... +class C2: ... +class C3: ... +class C4: ... +class C5: ... +class C6: ... +class C7: ... +class C8: ... +class C9: ... +class C10: ... + +x: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, int] +y: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, str] +x = y # E: Incompatible types in assignment (expression has type "Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, str]", variable has type "Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, int]") \ + # N: Item in the first union not in the second: "str" + +[case testLargeUnionsNoneShown] +from typing import Union + +class C1: ... +class C2: ... +class C3: ... +class C4: ... +class C5: ... +class C6: ... +class C7: ... +class C8: ... +class C9: ... +class C10: ... +class C11: ... + +x: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11] +y: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, None] +x = y # E: Incompatible types in assignment (expression has type "Union[C1, C2, C3, C4, C5, <6 more items>, None]", variable has type "Union[C1, C2, C3, C4, C5, <6 more items>]") \ + # N: Item in the first union not in the second: "None" diff --git a/test-data/unit/semanal-classes.test b/test-data/unit/semanal-classes.test index 951791e23490..b14358509f85 100644 --- a/test-data/unit/semanal-classes.test +++ b/test-data/unit/semanal-classes.test @@ -585,7 +585,7 @@ MypyFile:1( TupleType( Tuple[builtins.int, builtins.str]) BaseType( - builtins.tuple[builtins.object, ...]) + builtins.tuple[Union[builtins.int, builtins.str], ...]) PassStmt:2())) [case testBaseClassFromIgnoredModule] From 1b116dfbe37a4503e0541d6bd6f5dd5c815ab36d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 23 Jun 2024 08:47:52 +0100 Subject: [PATCH 052/247] Fix explicit type for partial (#17424) Fixes https://github.com/python/mypy/issues/17301 --- mypy/plugins/functools.py | 17 +++++++++++++--- test-data/unit/check-functools.test | 31 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index 4f2ed6f2361d..e41afe2fde02 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -6,6 +6,7 @@ import mypy.checker import mypy.plugin +import mypy.semanal from mypy.argmap import map_actuals_to_formals from mypy.nodes import ARG_POS, ARG_STAR2, ArgKind, Argument, CallExpr, FuncItem, Var from mypy.plugins.common import add_method_to_class @@ -24,6 +25,8 @@ _ORDERING_METHODS: Final = {"__lt__", "__le__", "__gt__", "__ge__"} +PARTIAL = "functools.partial" + class _MethodInfo(NamedTuple): is_static: bool @@ -142,7 +145,8 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: else (ArgKind.ARG_NAMED_OPT if k == ArgKind.ARG_NAMED else k) ) for k in fn_type.arg_kinds - ] + ], + ret_type=ctx.api.named_generic_type(PARTIAL, [fn_type.ret_type]), ) if defaulted.line < 0: # Make up a line number if we don't have one @@ -188,6 +192,13 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: bound = get_proper_type(bound) if not isinstance(bound, CallableType): return ctx.default_return_type + wrapped_ret_type = get_proper_type(bound.ret_type) + if not isinstance(wrapped_ret_type, Instance) or wrapped_ret_type.type.fullname != PARTIAL: + return ctx.default_return_type + if not mypy.semanal.refers_to_fullname(ctx.args[0][0], PARTIAL): + # If the first argument is partial, above call will trigger the plugin + # again, in between the wrapping above an unwrapping here. + bound = bound.copy_modified(ret_type=wrapped_ret_type.args[0]) formal_to_actual = map_actuals_to_formals( actual_kinds=actual_arg_kinds, @@ -237,7 +248,7 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: ret_type=ret_type, ) - ret = ctx.api.named_generic_type("functools.partial", [ret_type]) + ret = ctx.api.named_generic_type(PARTIAL, [ret_type]) ret = ret.copy_with_extra_attr("__mypy_partial", partially_applied) return ret @@ -247,7 +258,7 @@ def partial_call_callback(ctx: mypy.plugin.MethodContext) -> Type: if ( not isinstance(ctx.api, mypy.checker.TypeChecker) # use internals or not isinstance(ctx.type, Instance) - or ctx.type.type.fullname != "functools.partial" + or ctx.type.type.fullname != PARTIAL or not ctx.type.extra_attrs or "__mypy_partial" not in ctx.type.extra_attrs.attrs ): diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index 79ae962a73e0..997f5bc70c7d 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -347,6 +347,37 @@ reveal_type(functools.partial(fn3, 2)()) # E: "str" not callable \ # E: Argument 1 to "partial" has incompatible type "Union[Callable[[int], int], str]"; expected "Callable[..., int]" [builtins fixtures/tuple.pyi] +[case testFunctoolsPartialExplicitType] +from functools import partial +from typing import Type, TypeVar, Callable + +T = TypeVar("T") +def generic(string: str, integer: int, resulting_type: Type[T]) -> T: ... + +p: partial[str] = partial(generic, resulting_type=str) +q: partial[bool] = partial(generic, resulting_type=str) # E: Argument "resulting_type" to "generic" has incompatible type "Type[str]"; expected "Type[bool]" + +pc: Callable[..., str] = partial(generic, resulting_type=str) +qc: Callable[..., bool] = partial(generic, resulting_type=str) # E: Incompatible types in assignment (expression has type "partial[str]", variable has type "Callable[..., bool]") \ + # N: "partial[str].__call__" has type "Callable[[VarArg(Any), KwArg(Any)], str]" +[builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialNestedPartial] +from functools import partial +from typing import Any + +def foo(x: int) -> int: ... +p = partial(partial, foo) +reveal_type(p()(1)) # N: Revealed type is "builtins.int" +p()("no") # E: Argument 1 to "foo" has incompatible type "str"; expected "int" + +q = partial(partial, partial, foo) +q()()("no") # E: Argument 1 to "foo" has incompatible type "str"; expected "int" + +r = partial(partial, foo, 1) +reveal_type(r()()) # N: Revealed type is "builtins.int" +[builtins fixtures/tuple.pyi] + [case testFunctoolsPartialTypeObject] import functools from typing import Type, Generic, TypeVar From 79b1c8d6a467cd829bf6b9e3919fbcef7b50eb19 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 23 Jun 2024 15:12:41 +0100 Subject: [PATCH 053/247] Fix previous partial fix (#17429) This is a bit unfortunate, but the best we can probably do. cc @hauntsaninja --- mypy/plugins/functools.py | 34 ++++++++++++++++++++++------- test-data/unit/check-functools.test | 13 +++++++++++ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index e41afe2fde02..19be71ca36df 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -137,6 +137,20 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: # We must normalize from the start to have coherent view together with TypeChecker. fn_type = fn_type.with_unpacked_kwargs().with_normalized_var_args() + last_context = ctx.api.type_context[-1] + if not fn_type.is_type_obj(): + # We wrap the return type to get use of a possible type context provided by caller. + # We cannot do this in case of class objects, since otherwise the plugin may get + # falsely triggered when evaluating the constructed call itself. + ret_type: Type = ctx.api.named_generic_type(PARTIAL, [fn_type.ret_type]) + wrapped_return = True + else: + ret_type = fn_type.ret_type + # Instead, for class objects we ignore any type context to avoid spurious errors, + # since the type context will be partial[X] etc., not X. + ctx.api.type_context[-1] = None + wrapped_return = False + defaulted = fn_type.copy_modified( arg_kinds=[ ( @@ -146,7 +160,7 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: ) for k in fn_type.arg_kinds ], - ret_type=ctx.api.named_generic_type(PARTIAL, [fn_type.ret_type]), + ret_type=ret_type, ) if defaulted.line < 0: # Make up a line number if we don't have one @@ -189,16 +203,20 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: arg_names=actual_arg_names, context=call_expr, ) + if not wrapped_return: + # Restore previously ignored context. + ctx.api.type_context[-1] = last_context + bound = get_proper_type(bound) if not isinstance(bound, CallableType): return ctx.default_return_type - wrapped_ret_type = get_proper_type(bound.ret_type) - if not isinstance(wrapped_ret_type, Instance) or wrapped_ret_type.type.fullname != PARTIAL: - return ctx.default_return_type - if not mypy.semanal.refers_to_fullname(ctx.args[0][0], PARTIAL): - # If the first argument is partial, above call will trigger the plugin - # again, in between the wrapping above an unwrapping here. - bound = bound.copy_modified(ret_type=wrapped_ret_type.args[0]) + + if wrapped_return: + # Reverse the wrapping we did above. + ret_type = get_proper_type(bound.ret_type) + if not isinstance(ret_type, Instance) or ret_type.type.fullname != PARTIAL: + return ctx.default_return_type + bound = bound.copy_modified(ret_type=ret_type.args[0]) formal_to_actual = map_actuals_to_formals( actual_kinds=actual_arg_kinds, diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index 997f5bc70c7d..e4b3e4cffdc1 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -455,3 +455,16 @@ first_kw([1]) # E: Too many positional arguments for "get" \ # E: Too few arguments for "get" \ # E: Argument 1 to "get" has incompatible type "List[int]"; expected "int" [builtins fixtures/list.pyi] + +[case testFunctoolsPartialClassObjectMatchingPartial] +from functools import partial + +class A: + def __init__(self, var: int, b: int, c: int) -> None: ... + +p = partial(A, 1) +reveal_type(p) # N: Revealed type is "functools.partial[__main__.A]" +p(1, "no") # E: Argument 2 to "A" has incompatible type "str"; expected "int" + +q: partial[A] = partial(A, 1) # OK +[builtins fixtures/tuple.pyi] From 39b9b899178e6a30e7e8664c12f0eb610b8a44a5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 23 Jun 2024 22:09:22 +0100 Subject: [PATCH 054/247] Always allow lambda calls (#17430) See https://github.com/python/mypy/pull/17408 for context. --- mypy/checkexpr.py | 2 ++ mypy/nodes.py | 4 +++- mypy/plugins/functools.py | 2 +- test-data/unit/check-functions.test | 16 ++++++++++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 7ae23cfe516c..fdc0f94b3997 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -36,6 +36,7 @@ ARG_STAR, ARG_STAR2, IMPLICITLY_ABSTRACT, + LAMBDA_NAME, LITERAL_TYPE, REVEAL_LOCALS, REVEAL_TYPE, @@ -599,6 +600,7 @@ def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) -> and self.chk.in_checked_function() and isinstance(callee_type, CallableType) and callee_type.implicit + and callee_type.name != LAMBDA_NAME ): if fullname is None and member is not None: assert object_type is not None diff --git a/mypy/nodes.py b/mypy/nodes.py index 5d3a1d31aece..d215bcfce098 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -175,6 +175,8 @@ def get_nongen_builtins(python_version: tuple[int, int]) -> dict[str, str]: "typing_extensions.runtime_checkable", ) +LAMBDA_NAME: Final = "" + class Node(Context): """Common base class for all non-type parse tree nodes.""" @@ -2262,7 +2264,7 @@ class LambdaExpr(FuncItem, Expression): @property def name(self) -> str: - return "" + return LAMBDA_NAME def expr(self) -> Expression: """Return the expression (the body) of the lambda.""" diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index 19be71ca36df..9589c6aeca8b 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -25,7 +25,7 @@ _ORDERING_METHODS: Final = {"__lt__", "__le__", "__gt__", "__ge__"} -PARTIAL = "functools.partial" +PARTIAL: Final = "functools.partial" class _MethodInfo(NamedTuple): diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 29cd977fe5d6..93540e203c36 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -3366,3 +3366,19 @@ class C(B): ) -> None: ... [builtins fixtures/tuple.pyi] + +[case testLambdaAlwaysAllowed] +# flags: --disallow-untyped-calls +from typing import Callable, Optional + +def func() -> Optional[str]: ... +var: Optional[str] + +factory: Callable[[], Optional[str]] +for factory in ( + lambda: var, + func, +): + reveal_type(factory) # N: Revealed type is "def () -> Union[builtins.str, None]" + var = factory() +[builtins fixtures/tuple.pyi] From 18945af2a86af79ae9317fc716a034549682728d Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 24 Jun 2024 12:34:58 +0300 Subject: [PATCH 055/247] Suppress second error message with `:=` and `[truthy-bool]` (#15941) Closes https://github.com/python/mypy/issues/15685 CC @ikonst Co-authored-by: Ilya Priven --- mypy/checker.py | 26 +++++++++++++++++++++----- test-data/unit/check-errorcodes.test | 6 ++++++ test-data/unit/check-python38.test | 3 +-- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 792e751691fd..4f20c6ee8493 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5762,7 +5762,9 @@ def combine_maps(list_maps: list[TypeMap]) -> TypeMap: else_map = {} return if_map, else_map - def find_isinstance_check(self, node: Expression) -> tuple[TypeMap, TypeMap]: + def find_isinstance_check( + self, node: Expression, *, in_boolean_context: bool = True + ) -> tuple[TypeMap, TypeMap]: """Find any isinstance checks (within a chain of ands). Includes implicit and explicit checks for None and calls to callable. Also includes TypeGuard and TypeIs functions. @@ -5773,15 +5775,24 @@ def find_isinstance_check(self, node: Expression) -> tuple[TypeMap, TypeMap]: If either of the values in the tuple is None, then that particular branch can never occur. + If `in_boolean_context=True` is passed, it means that we handle + a walrus expression. We treat rhs values + in expressions like `(a := A())` specially: + for example, some errors are suppressed. + May return {}, {}. Can return None, None in situations involving NoReturn. """ - if_map, else_map = self.find_isinstance_check_helper(node) + if_map, else_map = self.find_isinstance_check_helper( + node, in_boolean_context=in_boolean_context + ) new_if_map = self.propagate_up_typemap_info(if_map) new_else_map = self.propagate_up_typemap_info(else_map) return new_if_map, new_else_map - def find_isinstance_check_helper(self, node: Expression) -> tuple[TypeMap, TypeMap]: + def find_isinstance_check_helper( + self, node: Expression, *, in_boolean_context: bool = True + ) -> tuple[TypeMap, TypeMap]: if is_true_literal(node): return {}, None if is_false_literal(node): @@ -6050,7 +6061,9 @@ def has_no_custom_eq_checks(t: Type) -> bool: if else_assignment_map is not None: else_map.update(else_assignment_map) - if_condition_map, else_condition_map = self.find_isinstance_check(node.value) + if_condition_map, else_condition_map = self.find_isinstance_check( + node.value, in_boolean_context=False + ) if if_condition_map is not None: if_map.update(if_condition_map) @@ -6112,7 +6125,10 @@ def has_no_custom_eq_checks(t: Type) -> bool: # Restrict the type of the variable to True-ish/False-ish in the if and else branches # respectively original_vartype = self.lookup_type(node) - self._check_for_truthy_type(original_vartype, node) + if in_boolean_context: + # We don't check `:=` values in expressions like `(a := A())`, + # because they produce two error messages. + self._check_for_truthy_type(original_vartype, node) vartype = try_expanding_sum_type_to_union(original_vartype, "builtins.bool") if_type = true_only(vartype) diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index 9d49480539e0..961815b11817 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -892,6 +892,12 @@ if a: any_or_object: Union[object, Any] if any_or_object: pass + +if (my_foo := Foo()): # E: "__main__.my_foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + pass + +if my_a := (a or Foo()): # E: "__main__.Foo" returns "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + pass [builtins fixtures/list.pyi] [case testTruthyFunctions] diff --git a/test-data/unit/check-python38.test b/test-data/unit/check-python38.test index 0f1cbb6e81c4..dfb918defb0a 100644 --- a/test-data/unit/check-python38.test +++ b/test-data/unit/check-python38.test @@ -297,8 +297,7 @@ def f(x: int = (c := 4)) -> int: z2: NT # E: Variable "NT" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases - if Alias := int: # E: Function "Alias" could always be true in boolean context \ - # E: Function "int" could always be true in boolean context + if Alias := int: # E: Function "Alias" could always be true in boolean context z3: Alias # E: Variable "Alias" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases From 620e28148afb4c8c04fbc0255e0c04769431c6b2 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 24 Jun 2024 18:38:37 +0300 Subject: [PATCH 056/247] Do not report plugin-generated methods with `explicit-override` (#17433) Closes https://github.com/typeddjango/django-stubs/issues/2226 Closes https://github.com/python/mypy/issues/17417 Closes https://github.com/python/mypy/pull/17370 Closes https://github.com/python/mypy/issues/17224 This is an alternative to https://github.com/python/mypy/pull/17418 Thanks a lot to @sterliakov, I took a dataclasses test case from #17370 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/checker.py | 9 ++++++- test-data/unit/check-custom-plugin.test | 33 +++++++++++++++++++++++++ test-data/unit/check-dataclasses.test | 14 +++++++++++ test-data/unit/plugins/add_method.py | 23 +++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 test-data/unit/plugins/add_method.py diff --git a/mypy/checker.py b/mypy/checker.py index 4f20c6ee8493..2df74cf7be8d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1938,8 +1938,15 @@ def check_explicit_override_decorator( found_method_base_classes: list[TypeInfo] | None, context: Context | None = None, ) -> None: + plugin_generated = False + if defn.info and (node := defn.info.get(defn.name)) and node.plugin_generated: + # Do not report issues for plugin generated nodes, + # they can't realistically use `@override` for their methods. + plugin_generated = True + if ( - found_method_base_classes + not plugin_generated + and found_method_base_classes and not defn.is_explicit_override and defn.name not in ("__init__", "__new__") and not is_private(defn.name) diff --git a/test-data/unit/check-custom-plugin.test b/test-data/unit/check-custom-plugin.test index 63529cf165ce..2b3b3f4a8695 100644 --- a/test-data/unit/check-custom-plugin.test +++ b/test-data/unit/check-custom-plugin.test @@ -1050,6 +1050,39 @@ reveal_type(my_class.stmethod) # N: Revealed type is "Overload(def (arg: builti \[mypy] plugins=/test-data/unit/plugins/add_overloaded_method.py +[case testAddMethodPluginExplicitOverride] +# flags: --python-version 3.12 --config-file tmp/mypy.ini +from typing import override, TypeVar + +T = TypeVar('T', bound=type) + +def inject_foo(t: T) -> T: + # Imitates: + # t.foo_implicit = some_method + return t + +class BaseWithoutFoo: pass + +@inject_foo +class ChildWithFoo(BaseWithoutFoo): pass +reveal_type(ChildWithFoo.foo_implicit) # N: Revealed type is "def (self: __main__.ChildWithFoo)" + +@inject_foo +class SomeWithFoo(ChildWithFoo): pass +reveal_type(SomeWithFoo.foo_implicit) # N: Revealed type is "def (self: __main__.SomeWithFoo)" + +class ExplicitOverride(SomeWithFoo): + @override + def foo_implicit(self) -> None: pass + +class ImplicitOverride(SomeWithFoo): + def foo_implicit(self) -> None: pass # E: Method "foo_implicit" is not using @override but is overriding a method in class "__main__.SomeWithFoo" +[file mypy.ini] +\[mypy] +plugins=/test-data/unit/plugins/add_method.py +enable_error_code = explicit-override +[typing fixtures/typing-override.pyi] + [case testCustomErrorCodePlugin] # flags: --config-file tmp/mypy.ini --show-error-codes def main() -> int: diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 924f9c7bb5be..f26ccd9a4854 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -2475,3 +2475,17 @@ class Base: class Child(Base): y: int [builtins fixtures/dataclasses.pyi] + + +[case testDataclassInheritanceWorksWithExplicitOverridesAndOrdering] +# flags: --enable-error-code explicit-override +from dataclasses import dataclass + +@dataclass(order=True) +class Base: + x: int + +@dataclass(order=True) +class Child(Base): + y: int +[builtins fixtures/dataclasses.pyi] diff --git a/test-data/unit/plugins/add_method.py b/test-data/unit/plugins/add_method.py new file mode 100644 index 000000000000..f3a7ebdb95ed --- /dev/null +++ b/test-data/unit/plugins/add_method.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import Callable + +from mypy.plugin import ClassDefContext, Plugin +from mypy.plugins.common import add_method +from mypy.types import NoneType + + +class AddOverrideMethodPlugin(Plugin): + def get_class_decorator_hook_2(self, fullname: str) -> Callable[[ClassDefContext], bool] | None: + if fullname == "__main__.inject_foo": + return add_extra_methods_hook + return None + + +def add_extra_methods_hook(ctx: ClassDefContext) -> bool: + add_method(ctx, "foo_implicit", [], NoneType()) + return True + + +def plugin(version: str) -> type[AddOverrideMethodPlugin]: + return AddOverrideMethodPlugin From 6c1d8671ce6eaf2c955fa986cbad51d6e6726d5d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 24 Jun 2024 20:57:29 +0100 Subject: [PATCH 057/247] Fix ParamSpec inference against TypeVarTuple (#17431) Fixes https://github.com/python/mypy/issues/17278 Fixes https://github.com/python/mypy/issues/17127 --- mypy/constraints.py | 6 ++- mypy/expandtype.py | 14 ++++++- mypy/semanal_typeargs.py | 12 +----- mypy/types.py | 13 +++++- test-data/unit/check-typevar-tuple.test | 53 +++++++++++++++++++++++++ 5 files changed, 85 insertions(+), 13 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 316f481ac870..49a2aea8fa05 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1071,7 +1071,11 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: # (with literal '...'). if not template.is_ellipsis_args: unpack_present = find_unpack_in_list(template.arg_types) - if unpack_present is not None: + # When both ParamSpec and TypeVarTuple are present, things become messy + # quickly. For now, we only allow ParamSpec to "capture" TypeVarTuple, + # but not vice versa. + # TODO: infer more from prefixes when possible. + if unpack_present is not None and not cactual.param_spec(): # We need to re-normalize args to the form they appear in tuples, # for callables we always pack the suffix inside another tuple. unpack = template.arg_types[unpack_present] diff --git a/mypy/expandtype.py b/mypy/expandtype.py index bff23c53defd..5c4d6af9458e 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -270,6 +270,13 @@ def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: repl = self.variables.get(t.id, t) if isinstance(repl, TypeVarTupleType): return repl + elif isinstance(repl, ProperType) and isinstance(repl, (AnyType, UninhabitedType)): + # Some failed inference scenarios will try to set all type variables to Never. + # Instead of being picky and require all the callers to wrap them, + # do this here instead. + # Note: most cases when this happens are handled in expand unpack below, but + # in rare cases (e.g. ParamSpec containing Unpack star args) it may be skipped. + return t.tuple_fallback.copy_modified(args=[repl]) raise NotImplementedError def visit_unpack_type(self, t: UnpackType) -> Type: @@ -348,7 +355,7 @@ def visit_callable_type(self, t: CallableType) -> CallableType: # the replacement is ignored. if isinstance(repl, Parameters): # We need to expand both the types in the prefix and the ParamSpec itself - return t.copy_modified( + expanded = t.copy_modified( arg_types=self.expand_types(t.arg_types[:-2]) + repl.arg_types, arg_kinds=t.arg_kinds[:-2] + repl.arg_kinds, arg_names=t.arg_names[:-2] + repl.arg_names, @@ -358,6 +365,11 @@ def visit_callable_type(self, t: CallableType) -> CallableType: imprecise_arg_kinds=(t.imprecise_arg_kinds or repl.imprecise_arg_kinds), variables=[*repl.variables, *t.variables], ) + var_arg = expanded.var_arg() + if var_arg is not None and isinstance(var_arg.typ, UnpackType): + # Sometimes we get new unpacks after expanding ParamSpec. + expanded.normalize_trivial_unpack() + return expanded elif isinstance(repl, ParamSpecType): # We're substituting one ParamSpec for another; this can mean that the prefix # changes, e.g. substitute Concatenate[int, P] in place of Q. diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index 02cb1b1f6128..dbf5136afa1b 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -15,7 +15,7 @@ from mypy.message_registry import INVALID_PARAM_SPEC_LOCATION, INVALID_PARAM_SPEC_LOCATION_NOTE from mypy.messages import format_type from mypy.mixedtraverser import MixedTraverserVisitor -from mypy.nodes import ARG_STAR, Block, ClassDef, Context, FakeInfo, FuncItem, MypyFile +from mypy.nodes import Block, ClassDef, Context, FakeInfo, FuncItem, MypyFile from mypy.options import Options from mypy.scope import Scope from mypy.subtypes import is_same_type, is_subtype @@ -104,15 +104,7 @@ def visit_tuple_type(self, t: TupleType) -> None: def visit_callable_type(self, t: CallableType) -> None: super().visit_callable_type(t) - # Normalize trivial unpack in var args as *args: *tuple[X, ...] -> *args: X - if t.is_var_arg: - star_index = t.arg_kinds.index(ARG_STAR) - star_type = t.arg_types[star_index] - if isinstance(star_type, UnpackType): - p_type = get_proper_type(star_type.type) - if isinstance(p_type, Instance): - assert p_type.type.fullname == "builtins.tuple" - t.arg_types[star_index] = p_type.args[0] + t.normalize_trivial_unpack() def visit_instance(self, t: Instance) -> None: super().visit_instance(t) diff --git a/mypy/types.py b/mypy/types.py index 3f764a5cc49e..52f8a8d63f09 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2084,6 +2084,17 @@ def param_spec(self) -> ParamSpecType | None: prefix = Parameters(self.arg_types[:-2], self.arg_kinds[:-2], self.arg_names[:-2]) return arg_type.copy_modified(flavor=ParamSpecFlavor.BARE, prefix=prefix) + def normalize_trivial_unpack(self) -> None: + # Normalize trivial unpack in var args as *args: *tuple[X, ...] -> *args: X in place. + if self.is_var_arg: + star_index = self.arg_kinds.index(ARG_STAR) + star_type = self.arg_types[star_index] + if isinstance(star_type, UnpackType): + p_type = get_proper_type(star_type.type) + if isinstance(p_type, Instance): + assert p_type.type.fullname == "builtins.tuple" + self.arg_types[star_index] = p_type.args[0] + def with_unpacked_kwargs(self) -> NormalizedCallableType: if not self.unpack_kwargs: return cast(NormalizedCallableType, self) @@ -2113,7 +2124,7 @@ def with_normalized_var_args(self) -> Self: if not isinstance(unpacked, TupleType): # Note that we don't normalize *args: *tuple[X, ...] -> *args: X, # this should be done once in semanal_typeargs.py for user-defined types, - # and we ourselves should never construct such type. + # and we ourselves rarely construct such type. return self unpack_index = find_unpack_in_list(unpacked.items) if unpack_index == 0 and len(unpacked.items) > 1: diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 49298114e069..ea692244597c 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2407,3 +2407,56 @@ reveal_type(x) # N: Revealed type is "__main__.C[builtins.str, builtins.int]" reveal_type(C(f)) # N: Revealed type is "__main__.C[builtins.str, builtins.int, builtins.int, builtins.int, builtins.int]" C[()] # E: At least 1 type argument(s) expected, none given [builtins fixtures/tuple.pyi] + +[case testTypeVarTupleAgainstParamSpecActualSuccess] +from typing import Generic, TypeVar, TypeVarTuple, Unpack, Callable, Tuple, List +from typing_extensions import ParamSpec + +R = TypeVar("R") +P = ParamSpec("P") + +class CM(Generic[R]): ... +def cm(fn: Callable[P, R]) -> Callable[P, CM[R]]: ... + +Ts = TypeVarTuple("Ts") +@cm +def test(*args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: ... + +reveal_type(test) # N: Revealed type is "def [Ts] (*args: Unpack[Ts`-1]) -> __main__.CM[Tuple[Unpack[Ts`-1]]]" +reveal_type(test(1, 2, 3)) # N: Revealed type is "__main__.CM[Tuple[Literal[1]?, Literal[2]?, Literal[3]?]]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleAgainstParamSpecActualFailedNoCrash] +from typing import Generic, TypeVar, TypeVarTuple, Unpack, Callable, Tuple, List +from typing_extensions import ParamSpec + +R = TypeVar("R") +P = ParamSpec("P") + +class CM(Generic[R]): ... +def cm(fn: Callable[P, List[R]]) -> Callable[P, CM[R]]: ... + +Ts = TypeVarTuple("Ts") +@cm # E: Argument 1 to "cm" has incompatible type "Callable[[VarArg(Unpack[Ts])], Tuple[Unpack[Ts]]]"; expected "Callable[[VarArg(Never)], List[Never]]" +def test(*args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: ... + +reveal_type(test) # N: Revealed type is "def (*args: Never) -> __main__.CM[Never]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleAgainstParamSpecActualPrefix] +from typing import Generic, TypeVar, TypeVarTuple, Unpack, Callable, Tuple, List +from typing_extensions import ParamSpec, Concatenate + +R = TypeVar("R") +P = ParamSpec("P") +T = TypeVar("T") + +class CM(Generic[R]): ... +def cm(fn: Callable[Concatenate[T, P], R]) -> Callable[Concatenate[List[T], P], CM[R]]: ... + +Ts = TypeVarTuple("Ts") +@cm +def test(x: T, *args: Unpack[Ts]) -> Tuple[T, Unpack[Ts]]: ... + +reveal_type(test) # N: Revealed type is "def [T, Ts] (builtins.list[T`2], *args: Unpack[Ts`-2]) -> __main__.CM[Tuple[T`2, Unpack[Ts`-2]]]" +[builtins fixtures/tuple.pyi] From d39f0234a18762a9b261a28763c7bea706633ce7 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 24 Jun 2024 17:22:46 -0700 Subject: [PATCH 058/247] Add changelog entry for 1.10.1 (#17436) --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0ea19866892..9d5919cafe33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -165,6 +165,9 @@ This feature was contributed by Shantanu (PR [16756](https://github.com/python/m Please see [git log](https://github.com/python/typeshed/commits/main?after=7c8e82fe483a40ec4cb0a2505cfdb0f3e7cc81d9+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. +#### Mypy 1.10.1 + +- Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) #### Acknowledgements Thanks to all mypy contributors who contributed to this release: From b88fdbd32fe0a45d40531a6504317aa3fd48489e Mon Sep 17 00:00:00 2001 From: InSync Date: Wed, 26 Jun 2024 14:04:04 +0700 Subject: [PATCH 059/247] Fix typo in `error_code_list2.rst` (#17443) --- docs/source/error_code_list2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst index 2b765e412913..0655ef2d35d8 100644 --- a/docs/source/error_code_list2.rst +++ b/docs/source/error_code_list2.rst @@ -389,7 +389,7 @@ Example: # Are you missing an await? asyncio.create_task(f()) -You can assign the value to a temporary, otherwise unused to variable to +You can assign the value to a temporary, otherwise unused variable to silence the error: .. code-block:: python From 69042d3fca754380965fc63def0a21a346895667 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 29 Jun 2024 20:46:17 +0100 Subject: [PATCH 060/247] Allow mixing ParamSpec and TypeVarTuple in Generic (#17450) Fixes https://github.com/python/mypy/issues/16696 Fixes https://github.com/python/mypy/issues/16695 I think there are no good reasons to not allow this anymore. Also I am using this opportunity to tighten a bit invalid instances/aliases where a regular type variable is replaced with parameters and vice versa. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/erasetype.py | 13 ++---------- mypy/nodes.py | 3 --- mypy/semanal_typeargs.py | 26 ++++++++++++++++++------ mypy/typevars.py | 13 ++---------- mypy/typevartuples.py | 14 +++++++++++++ test-data/unit/check-typevar-tuple.test | 27 +++++++++++++++++++++++++ 6 files changed, 65 insertions(+), 31 deletions(-) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index b41eefcd4821..5d95b221af15 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -34,6 +34,7 @@ get_proper_type, get_proper_types, ) +from mypy.typevartuples import erased_vars def erase_type(typ: Type) -> ProperType: @@ -77,17 +78,7 @@ def visit_deleted_type(self, t: DeletedType) -> ProperType: return t def visit_instance(self, t: Instance) -> ProperType: - args: list[Type] = [] - for tv in t.type.defn.type_vars: - # Valid erasure for *Ts is *tuple[Any, ...], not just Any. - if isinstance(tv, TypeVarTupleType): - args.append( - UnpackType( - tv.tuple_fallback.copy_modified(args=[AnyType(TypeOfAny.special_form)]) - ) - ) - else: - args.append(AnyType(TypeOfAny.special_form)) + args = erased_vars(t.type.defn.type_vars, TypeOfAny.special_form) return Instance(t.type, args, t.line) def visit_type_var(self, t: TypeVarType) -> ProperType: diff --git a/mypy/nodes.py b/mypy/nodes.py index d215bcfce098..2eb39d4baaf6 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3165,9 +3165,6 @@ def add_type_vars(self) -> None: self.type_var_tuple_prefix = i self.type_var_tuple_suffix = len(self.defn.type_vars) - i - 1 self.type_vars.append(vd.name) - assert not ( - self.has_param_spec_type and self.has_type_var_tuple_type - ), "Mixing type var tuples and param specs not supported yet" @property def name(self) -> str: diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index dbf5136afa1b..646bb28a3b6e 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -39,6 +39,7 @@ get_proper_types, split_with_prefix_and_suffix, ) +from mypy.typevartuples import erased_vars class TypeArgumentAnalyzer(MixedTraverserVisitor): @@ -89,7 +90,14 @@ def visit_type_alias_type(self, t: TypeAliasType) -> None: return self.seen_aliases.add(t) assert t.alias is not None, f"Unfixed type alias {t.type_ref}" - is_error = self.validate_args(t.alias.name, tuple(t.args), t.alias.alias_tvars, t) + is_error, is_invalid = self.validate_args( + t.alias.name, tuple(t.args), t.alias.alias_tvars, t + ) + if is_invalid: + # If there is an arity error (e.g. non-Parameters used for ParamSpec etc.), + # then it is safer to erase the arguments completely, to avoid crashes later. + # TODO: can we move this logic to typeanal.py? + t.args = erased_vars(t.alias.alias_tvars, TypeOfAny.from_error) if not is_error: # If there was already an error for the alias itself, there is no point in checking # the expansion, most likely it will result in the same kind of error. @@ -113,7 +121,9 @@ def visit_instance(self, t: Instance) -> None: info = t.type if isinstance(info, FakeInfo): return # https://github.com/python/mypy/issues/11079 - self.validate_args(info.name, t.args, info.defn.type_vars, t) + _, is_invalid = self.validate_args(info.name, t.args, info.defn.type_vars, t) + if is_invalid: + t.args = tuple(erased_vars(info.defn.type_vars, TypeOfAny.from_error)) if t.type.fullname == "builtins.tuple" and len(t.args) == 1: # Normalize Tuple[*Tuple[X, ...], ...] -> Tuple[X, ...] arg = t.args[0] @@ -125,7 +135,7 @@ def visit_instance(self, t: Instance) -> None: def validate_args( self, name: str, args: tuple[Type, ...], type_vars: list[TypeVarLikeType], ctx: Context - ) -> bool: + ) -> tuple[bool, bool]: if any(isinstance(v, TypeVarTupleType) for v in type_vars): prefix = next(i for (i, v) in enumerate(type_vars) if isinstance(v, TypeVarTupleType)) tvt = type_vars[prefix] @@ -136,10 +146,11 @@ def validate_args( args = start + (TupleType(list(middle), tvt.tuple_fallback),) + end is_error = False + is_invalid = False for (i, arg), tvar in zip(enumerate(args), type_vars): if isinstance(tvar, TypeVarType): if isinstance(arg, ParamSpecType): - is_error = True + is_invalid = True self.fail( INVALID_PARAM_SPEC_LOCATION.format(format_type(arg, self.options)), ctx, @@ -152,7 +163,7 @@ def validate_args( ) continue if isinstance(arg, Parameters): - is_error = True + is_invalid = True self.fail( f"Cannot use {format_type(arg, self.options)} for regular type variable," " only for ParamSpec", @@ -205,13 +216,16 @@ def validate_args( if not isinstance( get_proper_type(arg), (ParamSpecType, Parameters, AnyType, UnboundType) ): + is_invalid = True self.fail( "Can only replace ParamSpec with a parameter types list or" f" another ParamSpec, got {format_type(arg, self.options)}", ctx, code=codes.VALID_TYPE, ) - return is_error + if is_invalid: + is_error = True + return is_error, is_invalid def visit_unpack_type(self, typ: UnpackType) -> None: super().visit_unpack_type(typ) diff --git a/mypy/typevars.py b/mypy/typevars.py index 3d74a40c303f..e871973104a2 100644 --- a/mypy/typevars.py +++ b/mypy/typevars.py @@ -3,7 +3,6 @@ from mypy.erasetype import erase_typevars from mypy.nodes import TypeInfo from mypy.types import ( - AnyType, Instance, ParamSpecType, ProperType, @@ -15,6 +14,7 @@ TypeVarType, UnpackType, ) +from mypy.typevartuples import erased_vars def fill_typevars(typ: TypeInfo) -> Instance | TupleType: @@ -64,16 +64,7 @@ def fill_typevars(typ: TypeInfo) -> Instance | TupleType: def fill_typevars_with_any(typ: TypeInfo) -> Instance | TupleType: """Apply a correct number of Any's as type arguments to a type.""" - args: list[Type] = [] - for tv in typ.defn.type_vars: - # Valid erasure for *Ts is *tuple[Any, ...], not just Any. - if isinstance(tv, TypeVarTupleType): - args.append( - UnpackType(tv.tuple_fallback.copy_modified(args=[AnyType(TypeOfAny.special_form)])) - ) - else: - args.append(AnyType(TypeOfAny.special_form)) - inst = Instance(typ, args) + inst = Instance(typ, erased_vars(typ.defn.type_vars, TypeOfAny.special_form)) if typ.tuple_type is None: return inst erased_tuple_type = erase_typevars(typ.tuple_type, {tv.id for tv in typ.defn.type_vars}) diff --git a/mypy/typevartuples.py b/mypy/typevartuples.py index af2effbd4035..2a9998c10746 100644 --- a/mypy/typevartuples.py +++ b/mypy/typevartuples.py @@ -5,9 +5,12 @@ from typing import Sequence from mypy.types import ( + AnyType, Instance, ProperType, Type, + TypeVarLikeType, + TypeVarTupleType, UnpackType, get_proper_type, split_with_prefix_and_suffix, @@ -30,3 +33,14 @@ def extract_unpack(types: Sequence[Type]) -> ProperType | None: if isinstance(types[0], UnpackType): return get_proper_type(types[0].type) return None + + +def erased_vars(type_vars: Sequence[TypeVarLikeType], type_of_any: int) -> list[Type]: + args: list[Type] = [] + for tv in type_vars: + # Valid erasure for *Ts is *tuple[Any, ...], not just Any. + if isinstance(tv, TypeVarTupleType): + args.append(UnpackType(tv.tuple_fallback.copy_modified(args=[AnyType(type_of_any)]))) + else: + args.append(AnyType(type_of_any)) + return args diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index ea692244597c..f49e1b3c6613 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2460,3 +2460,30 @@ def test(x: T, *args: Unpack[Ts]) -> Tuple[T, Unpack[Ts]]: ... reveal_type(test) # N: Revealed type is "def [T, Ts] (builtins.list[T`2], *args: Unpack[Ts`-2]) -> __main__.CM[Tuple[T`2, Unpack[Ts`-2]]]" [builtins fixtures/tuple.pyi] + +[case testMixingTypeVarTupleAndParamSpec] +from typing import Generic, ParamSpec, TypeVarTuple, Unpack, Callable, TypeVar + +P = ParamSpec("P") +Ts = TypeVarTuple("Ts") + +class A(Generic[P, Unpack[Ts]]): ... +class B(Generic[Unpack[Ts], P]): ... + +a: A[[int, str], int, str] +reveal_type(a) # N: Revealed type is "__main__.A[[builtins.int, builtins.str], builtins.int, builtins.str]" +b: B[int, str, [int, str]] +reveal_type(b) # N: Revealed type is "__main__.B[builtins.int, builtins.str, [builtins.int, builtins.str]]" + +x: A[int, str, [int, str]] # E: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "int" +reveal_type(x) # N: Revealed type is "__main__.A[Any, Unpack[builtins.tuple[Any, ...]]]" +y: B[[int, str], int, str] # E: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "str" +reveal_type(y) # N: Revealed type is "__main__.B[Unpack[builtins.tuple[Any, ...]], Any]" + +R = TypeVar("R") +class C(Generic[P, R]): + fn: Callable[P, None] + +c: C[int, str] # E: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "int" +reveal_type(c.fn) # N: Revealed type is "def (*Any, **Any)" +[builtins fixtures/tuple.pyi] From a27447d7cfb4ea2f31fd0dcce13dc4be23abb6eb Mon Sep 17 00:00:00 2001 From: Michael Carlstrom Date: Sat, 29 Jun 2024 19:27:59 -0400 Subject: [PATCH 061/247] Add Literal support for docstrings (#17441) (Explain how this PR changes mypy.) Updates the is_valid_type regex to include quotes and . --- mypy/stubdoc.py | 2 +- mypy/stubgenc.py | 1 + mypy/test/teststubgen.py | 11 +++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py index 8c0a4dab696f..928d024514f3 100644 --- a/mypy/stubdoc.py +++ b/mypy/stubdoc.py @@ -20,7 +20,7 @@ Sig: _TypeAlias = Tuple[str, str] -_TYPE_RE: Final = re.compile(r"^[a-zA-Z_][\w\[\], ]*(\.[a-zA-Z_][\w\[\], ]*)*$") +_TYPE_RE: Final = re.compile(r"^[a-zA-Z_][\w\[\], .\"\']*(\.[a-zA-Z_][\w\[\], ]*)*$") _ARG_NAME_RE: Final = re.compile(r"\**[A-Za-z_][A-Za-z0-9_]*$") diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index bacb68f6d1c7..0aa6088a4e02 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -252,6 +252,7 @@ def __init__( "Iterable", "Iterator", "List", + "Literal", "NamedTuple", "Optional", "Tuple", diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 3669772854cb..05a3809179bd 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -1357,6 +1357,17 @@ def test_is_valid_type(self) -> None: assert is_valid_type("List[int]") assert is_valid_type("Dict[str, int]") assert is_valid_type("None") + assert is_valid_type("Literal[26]") + assert is_valid_type("Literal[0x1A]") + assert is_valid_type('Literal["hello world"]') + assert is_valid_type('Literal[b"hello world"]') + assert is_valid_type('Literal[u"hello world"]') + assert is_valid_type("Literal[True]") + assert is_valid_type("Literal[Color.RED]") + assert is_valid_type("Literal[None]") + assert is_valid_type( + 'Literal[26, 0x1A, "hello world", b"hello world", u"hello world", True, Color.RED, None]' + ) assert not is_valid_type("foo-bar") assert not is_valid_type("x->y") assert not is_valid_type("True") From e4de4e32eeb87c33a35b5abe88676cf3f34f388f Mon Sep 17 00:00:00 2001 From: Eric Mark Martin Date: Sat, 29 Jun 2024 19:32:51 -0400 Subject: [PATCH 062/247] Include keyword only args when generating signatures in stubgenc (#17448) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, signatures generated for callable by the `InspectionStubGenerator` won’t include keywords only arguments or their defaults. This change includes them in the generated signatures. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/stubgenc.py | 56 +++++++++++++++++++++++++++---------- mypy/test/teststubgen.py | 29 +++++++++++++++++++ test-data/unit/stubgen.test | 9 ++++++ 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 0aa6088a4e02..7ab500b4fe12 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -12,7 +12,7 @@ import keyword import os.path from types import FunctionType, ModuleType -from typing import Any, Mapping +from typing import Any, Callable, Mapping from mypy.fastparse import parse_type_comment from mypy.moduleinspect import is_c_module @@ -292,6 +292,8 @@ def get_default_function_sig(self, func: object, ctx: FunctionContext) -> Functi varargs = argspec.varargs kwargs = argspec.varkw annotations = argspec.annotations + kwonlyargs = argspec.kwonlyargs + kwonlydefaults = argspec.kwonlydefaults def get_annotation(key: str) -> str | None: if key not in annotations: @@ -304,27 +306,51 @@ def get_annotation(key: str) -> str | None: return argtype arglist: list[ArgSig] = [] + # Add the arguments to the signature - for i, arg in enumerate(args): - # Check if the argument has a default value - if defaults and i >= len(args) - len(defaults): - default_value = defaults[i - (len(args) - len(defaults))] - if arg in annotations: - argtype = annotations[arg] + def add_args( + args: list[str], get_default_value: Callable[[int, str], object | None] + ) -> None: + for i, arg in enumerate(args): + # Check if the argument has a default value + default_value = get_default_value(i, arg) + if default_value is not None: + if arg in annotations: + argtype = annotations[arg] + else: + argtype = self.get_type_annotation(default_value) + if argtype == "None": + # None is not a useful annotation, but we can infer that the arg + # is optional + incomplete = self.add_name("_typeshed.Incomplete") + argtype = f"{incomplete} | None" + + arglist.append(ArgSig(arg, argtype, default=True)) else: - argtype = self.get_type_annotation(default_value) - if argtype == "None": - # None is not a useful annotation, but we can infer that the arg - # is optional - incomplete = self.add_name("_typeshed.Incomplete") - argtype = f"{incomplete} | None" - arglist.append(ArgSig(arg, argtype, default=True)) + arglist.append(ArgSig(arg, get_annotation(arg), default=False)) + + def get_pos_default(i: int, _arg: str) -> Any | None: + if defaults and i >= len(args) - len(defaults): + return defaults[i - (len(args) - len(defaults))] else: - arglist.append(ArgSig(arg, get_annotation(arg), default=False)) + return None + + add_args(args, get_pos_default) # Add *args if present if varargs: arglist.append(ArgSig(f"*{varargs}", get_annotation(varargs))) + # if we have keyword only args, then wee need to add "*" + elif kwonlyargs: + arglist.append(ArgSig("*")) + + def get_kw_default(_i: int, arg: str) -> Any | None: + if kwonlydefaults: + return kwonlydefaults.get(arg) + else: + return None + + add_args(kwonlyargs, get_kw_default) # Add **kwargs if present if kwargs: diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 05a3809179bd..e65a16c8f395 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -845,6 +845,35 @@ class TestClassVariableCls: assert_equal(gen.get_imports().splitlines(), ["from typing import ClassVar"]) assert_equal(output, ["class C:", " x: ClassVar[int] = ..."]) + def test_non_c_generate_signature_with_kw_only_args(self) -> None: + class TestClass: + def test( + self, arg0: str, *, keyword_only: str, keyword_only_with_default: int = 7 + ) -> None: + pass + + output: list[str] = [] + mod = ModuleType(TestClass.__module__, "") + gen = InspectionStubGenerator(mod.__name__, known_modules=[mod.__name__], module=mod) + gen.is_c_module = False + gen.generate_function_stub( + "test", + TestClass.test, + output=output, + class_info=ClassInfo( + self_var="self", + cls=TestClass, + name="TestClass", + docstring=getattr(TestClass, "__doc__", None), + ), + ) + assert_equal( + output, + [ + "def test(self, arg0: str, *, keyword_only: str, keyword_only_with_default: int = ...) -> None: ..." + ], + ) + def test_generate_c_type_inheritance(self) -> None: class TestClass(KeyError): pass diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index 5dcb0706a8cb..94d0edb2ae37 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -361,6 +361,15 @@ def g(x, *, y=1, z=2): ... def f(x, *, y: int = 1) -> None: ... def g(x, *, y: int = 1, z: int = 2) -> None: ... +[case testKeywordOnlyArg_inspect] +def f(x, *, y=1): ... +def g(x, *, y=1, z=2): ... +def h(x, *, y, z=2): ... +[out] +def f(x, *, y: int = ...): ... +def g(x, *, y: int = ..., z: int = ...): ... +def h(x, *, y, z: int = ...): ... + [case testProperty] class A: @property From 177c8ee7b8166b3dcf89c034a676ef5818edbc38 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 30 Jun 2024 00:34:26 +0100 Subject: [PATCH 063/247] Fix strict optional handling in attrs plugin (#17451) Fixes https://github.com/python/mypy/issues/13794 Fix is trivial (but unlike for dataclasses, the calls to type ops are scattered, so I simply put the whole hook inside a single `with`). --- mypy/plugins/attrs.py | 17 ++++++++++++++++- test-data/unit/check-plugin-attrs.test | 20 ++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/mypy/plugins/attrs.py b/mypy/plugins/attrs.py index db976385ee56..b67a285af11d 100644 --- a/mypy/plugins/attrs.py +++ b/mypy/plugins/attrs.py @@ -55,6 +55,7 @@ deserialize_and_fixup_type, ) from mypy.server.trigger import make_wildcard_trigger +from mypy.state import state from mypy.typeops import get_type_vars, make_simplified_union, map_type_from_supertype from mypy.types import ( AnyType, @@ -317,9 +318,23 @@ def attr_class_maker_callback( See https://www.attrs.org/en/stable/how-does-it-work.html for information on how attrs works. - If this returns False, some required metadata was not ready yet and we need another + If this returns False, some required metadata was not ready yet, and we need another pass. """ + with state.strict_optional_set(ctx.api.options.strict_optional): + # This hook is called during semantic analysis, but it uses a bunch of + # type-checking ops, so it needs the strict optional set properly. + return attr_class_maker_callback_impl( + ctx, auto_attribs_default, frozen_default, slots_default + ) + + +def attr_class_maker_callback_impl( + ctx: mypy.plugin.ClassDefContext, + auto_attribs_default: bool | None, + frozen_default: bool, + slots_default: bool, +) -> bool: info = ctx.cls.info init = _get_decorator_bool_argument(ctx, "init", True) diff --git a/test-data/unit/check-plugin-attrs.test b/test-data/unit/check-plugin-attrs.test index b96c00730a74..0c653d608187 100644 --- a/test-data/unit/check-plugin-attrs.test +++ b/test-data/unit/check-plugin-attrs.test @@ -2475,3 +2475,23 @@ class B: reveal_type(B.__hash__) # N: Revealed type is "None" [builtins fixtures/plugin_attrs.pyi] + +[case testAttrsStrictOptionalSetProperly] +from typing import Generic, Optional, TypeVar + +import attr + +T = TypeVar("T") + +@attr.mutable() +class Parent(Generic[T]): + run_type: Optional[int] = None + +@attr.mutable() +class Child(Parent[float]): + pass + +Parent(run_type = None) +c = Child(run_type = None) +reveal_type(c.run_type) # N: Revealed type is "Union[builtins.int, None]" +[builtins fixtures/plugin_attrs.pyi] From 02d3667442e0c8e63989e0276d1e78754cc84b2a Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sun, 30 Jun 2024 16:15:34 -0400 Subject: [PATCH 064/247] Fix typechecking for async generators (#17452) Fixes #10534 This PR fixes a bug in typechecking asynchronous generators. Mypy currently typechecks a generator/comprehension as `AsyncGenerator` if the leftmost expression contains `await`, or if it contains an `async for`. However, there are other situations where we should get async generator: If there is an `await` expression in any of the conditions or in any sequence except for the leftmost one, the generator/comprehension should also be typechecked as `AsyncGenerator`. I've implemented this change in Mypy and added a test case to assert this behavior. If I enter the test cases into a regular repl, I can confirm that the runtime representation is generator/async_generator as the test case expects. According to the [language reference](https://docs.python.org/3/reference/expressions.html#grammar-token-python-grammar-comp_for): > If a comprehension contains either async for clauses or await expressions or other asynchronous comprehensions it is called an asynchronous comprehension. Confusingly, the documentation itself is actually not quite correct either, as pointed out in https://github.com/python/cpython/issues/114104 Alongside this change, I've made a PR to update the docs to be more precise: https://github.com/python/cpython/pull/121175 has more details. --- mypy/checkexpr.py | 9 +++++++-- test-data/unit/check-async-await.test | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index fdc0f94b3997..c4ab8a081acc 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -5585,8 +5585,13 @@ def visit_set_comprehension(self, e: SetComprehension) -> Type: def visit_generator_expr(self, e: GeneratorExpr) -> Type: # If any of the comprehensions use async for, the expression will return an async generator - # object, or if the left-side expression uses await. - if any(e.is_async) or has_await_expression(e.left_expr): + # object, or await is used anywhere but in the leftmost sequence. + if ( + any(e.is_async) + or has_await_expression(e.left_expr) + or any(has_await_expression(sequence) for sequence in e.sequences[1:]) + or any(has_await_expression(cond) for condlist in e.condlists for cond in condlist) + ): typ = "typing.AsyncGenerator" # received type is always None in async generator expressions additional_args: list[Type] = [NoneType()] diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index 876fe0c6be15..0ef08e5a0775 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -573,6 +573,25 @@ async def return_f() -> AsyncGenerator[int, None]: [builtins fixtures/dict.pyi] [typing fixtures/typing-async.pyi] +[case testImplicitAsyncGenerator] +from typing import List + +async def get_list() -> List[int]: + return [1] + +async def predicate() -> bool: + return True + +async def test_implicit_generators() -> None: + reveal_type(await predicate() for _ in [1]) # N: Revealed type is "typing.AsyncGenerator[builtins.bool, None]" + reveal_type(x for x in [1] if await predicate()) # N: Revealed type is "typing.AsyncGenerator[builtins.int, None]" + reveal_type(x for x in await get_list()) # N: Revealed type is "typing.Generator[builtins.int, None, None]" + reveal_type(x for _ in [1] for x in await get_list()) # N: Revealed type is "typing.AsyncGenerator[builtins.int, None]" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + + -- The full matrix of coroutine compatibility -- ------------------------------------------ From c346c5425891c1eac5e4eba4c1eb4e84ceeb2a06 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 30 Jun 2024 17:14:04 -0700 Subject: [PATCH 065/247] Use Python 3.12 for mypy_primer (#17456) I'd like for us to cover projects that use Python 3.12 only syntax, like homeassistant --- .github/workflows/mypy_primer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index 07a1d0863eb2..0c77d3a255d8 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -39,7 +39,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies run: | python -m pip install -U pip From 98717718be97bf54a102f9d844ae7185aedaa7ac Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Jun 2024 17:24:36 -0700 Subject: [PATCH 066/247] Sync typeshed (#17458) Source commit: https://github.com/python/typeshed/commit/dcab6e88883c629ede9637fb011958f8b4918f52 --- mypy/typeshed/stdlib/VERSIONS | 1 + mypy/typeshed/stdlib/_interpchannels.pyi | 84 +++++ mypy/typeshed/stdlib/argparse.pyi | 8 +- mypy/typeshed/stdlib/asyncio/events.pyi | 93 +++--- mypy/typeshed/stdlib/asyncio/tasks.pyi | 5 +- mypy/typeshed/stdlib/asyncio/unix_events.pyi | 290 ++++++++++-------- .../stdlib/asyncio/windows_events.pyi | 5 +- mypy/typeshed/stdlib/dataclasses.pyi | 2 +- mypy/typeshed/stdlib/posixpath.pyi | 16 +- mypy/typeshed/stdlib/subprocess.pyi | 8 + mypy/typeshed/stdlib/tarfile.pyi | 24 +- mypy/typeshed/stdlib/zipimport.pyi | 2 + 12 files changed, 347 insertions(+), 191 deletions(-) create mode 100644 mypy/typeshed/stdlib/_interpchannels.pyi diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index 7b9ce2864484..89754f65f3fa 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -34,6 +34,7 @@ _dummy_thread: 3.0-3.8 _dummy_threading: 3.0-3.8 _heapq: 3.0- _imp: 3.0- +_interpchannels: 3.13- _json: 3.0- _locale: 3.0- _lsprof: 3.0- diff --git a/mypy/typeshed/stdlib/_interpchannels.pyi b/mypy/typeshed/stdlib/_interpchannels.pyi new file mode 100644 index 000000000000..b77fe321a071 --- /dev/null +++ b/mypy/typeshed/stdlib/_interpchannels.pyi @@ -0,0 +1,84 @@ +from _typeshed import structseq +from typing import Final, Literal, SupportsIndex, final +from typing_extensions import Buffer, Self + +class ChannelError(RuntimeError): ... +class ChannelClosedError(ChannelError): ... +class ChannelEmptyError(ChannelError): ... +class ChannelNotEmptyError(ChannelError): ... +class ChannelNotFoundError(ChannelError): ... + +# Mark as final, since instantiating ChannelID is not supported. +@final +class ChannelID: + @property + def end(self) -> Literal["send", "recv", "both"]: ... + @property + def send(self) -> Self: ... + @property + def recv(self) -> Self: ... + def __eq__(self, other: object) -> bool: ... + def __ge__(self, other: ChannelID) -> bool: ... + def __gt__(self, other: ChannelID) -> bool: ... + def __hash__(self) -> int: ... + def __index__(self) -> int: ... + def __int__(self) -> int: ... + def __le__(self, other: ChannelID) -> bool: ... + def __lt__(self, other: ChannelID) -> bool: ... + def __ne__(self, other: object) -> bool: ... + +@final +class ChannelInfo(structseq[int], tuple[bool, bool, bool, int, int, int, int, int]): + __match_args__: Final = ( + "open", + "closing", + "closed", + "count", + "num_interp_send", + "num_interp_send_released", + "num_interp_recv", + "num_interp_recv_released", + ) + @property + def open(self) -> bool: ... + @property + def closing(self) -> bool: ... + @property + def closed(self) -> bool: ... + @property + def count(self) -> int: ... # type: ignore[override] + @property + def num_interp_send(self) -> int: ... + @property + def num_interp_send_released(self) -> int: ... + @property + def num_interp_recv(self) -> int: ... + @property + def num_interp_recv_released(self) -> int: ... + @property + def num_interp_both(self) -> int: ... + @property + def num_interp_both_recv_released(self) -> int: ... + @property + def num_interp_both_send_released(self) -> int: ... + @property + def num_interp_both_released(self) -> int: ... + @property + def recv_associated(self) -> bool: ... + @property + def recv_released(self) -> bool: ... + @property + def send_associated(self) -> bool: ... + @property + def send_released(self) -> bool: ... + +def create() -> ChannelID: ... +def destroy(cid: SupportsIndex) -> None: ... +def list_all() -> list[ChannelID]: ... +def list_interpreters(cid: SupportsIndex, *, send: bool) -> list[int]: ... +def send(cid: SupportsIndex, obj: object, *, blocking: bool = True, timeout: float | None = None) -> None: ... +def send_buffer(cid: SupportsIndex, obj: Buffer, *, blocking: bool = True, timeout: float | None = None) -> None: ... +def recv(cid: SupportsIndex, default: object = ...) -> object: ... +def close(cid: SupportsIndex, *, send: bool = False, recv: bool = False) -> None: ... +def get_info(cid: SupportsIndex) -> ChannelInfo: ... +def release(cid: SupportsIndex, *, send: bool = False, recv: bool = False, force: bool = False) -> None: ... diff --git a/mypy/typeshed/stdlib/argparse.pyi b/mypy/typeshed/stdlib/argparse.pyi index 1956d08c9933..bc781ec8e61d 100644 --- a/mypy/typeshed/stdlib/argparse.pyi +++ b/mypy/typeshed/stdlib/argparse.pyi @@ -32,6 +32,7 @@ _T = TypeVar("_T") _ActionT = TypeVar("_ActionT", bound=Action) _ArgumentParserT = TypeVar("_ArgumentParserT", bound=ArgumentParser) _N = TypeVar("_N") +_ActionType: TypeAlias = Callable[[str], Any] | FileType | str # more precisely, Literal["store", "store_const", "store_true", # "store_false", "append", "append_const", "count", "help", "version", # "extend"], but using this would make it hard to annotate callers @@ -89,7 +90,7 @@ class _ActionsContainer: nargs: int | _NArgsStr | _SUPPRESS_T | None = None, const: Any = ..., default: Any = ..., - type: Callable[[str], _T] | FileType = ..., + type: _ActionType = ..., choices: Iterable[_T] | None = ..., required: bool = ..., help: str | None = ..., @@ -313,7 +314,7 @@ class Action(_AttributeHolder): nargs: int | str | None const: Any default: Any - type: Callable[[str], Any] | FileType | None + type: _ActionType | None choices: Iterable[Any] | None required: bool help: str | None @@ -699,6 +700,7 @@ class _SubParsersAction(Action, Generic[_ArgumentParserT]): add_help: bool = ..., allow_abbrev: bool = ..., exit_on_error: bool = ..., + **kwargs: Any, # Accepting any additional kwargs for custom parser classes ) -> _ArgumentParserT: ... elif sys.version_info >= (3, 9): def add_parser( @@ -721,6 +723,7 @@ class _SubParsersAction(Action, Generic[_ArgumentParserT]): add_help: bool = ..., allow_abbrev: bool = ..., exit_on_error: bool = ..., + **kwargs: Any, # Accepting any additional kwargs for custom parser classes ) -> _ArgumentParserT: ... else: def add_parser( @@ -742,6 +745,7 @@ class _SubParsersAction(Action, Generic[_ArgumentParserT]): conflict_handler: str = ..., add_help: bool = ..., allow_abbrev: bool = ..., + **kwargs: Any, # Accepting any additional kwargs for custom parser classes ) -> _ArgumentParserT: ... def _get_subactions(self) -> list[Action]: ... diff --git a/mypy/typeshed/stdlib/asyncio/events.pyi b/mypy/typeshed/stdlib/asyncio/events.pyi index c0345eb1b5b5..8c2664666835 100644 --- a/mypy/typeshed/stdlib/asyncio/events.pyi +++ b/mypy/typeshed/stdlib/asyncio/events.pyi @@ -16,23 +16,40 @@ from .tasks import Task from .transports import BaseTransport, DatagramTransport, ReadTransport, SubprocessTransport, Transport, WriteTransport from .unix_events import AbstractChildWatcher -__all__ = ( - "AbstractEventLoopPolicy", - "AbstractEventLoop", - "AbstractServer", - "Handle", - "TimerHandle", - "get_event_loop_policy", - "set_event_loop_policy", - "get_event_loop", - "set_event_loop", - "new_event_loop", - "get_child_watcher", - "set_child_watcher", - "_set_running_loop", - "get_running_loop", - "_get_running_loop", -) +if sys.version_info >= (3, 14): + __all__ = ( + "AbstractEventLoopPolicy", + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "get_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "set_event_loop", + "new_event_loop", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", + ) +else: + __all__ = ( + "AbstractEventLoopPolicy", + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "get_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "set_event_loop", + "new_event_loop", + "get_child_watcher", + "set_child_watcher", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", + ) _T = TypeVar("_T") _Ts = TypeVarTuple("_Ts") @@ -541,18 +558,19 @@ class AbstractEventLoopPolicy: @abstractmethod def new_event_loop(self) -> AbstractEventLoop: ... # Child processes handling (Unix only). - if sys.version_info >= (3, 12): - @abstractmethod - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def get_child_watcher(self) -> AbstractChildWatcher: ... - @abstractmethod - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... - else: - @abstractmethod - def get_child_watcher(self) -> AbstractChildWatcher: ... - @abstractmethod - def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @abstractmethod + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def get_child_watcher(self) -> AbstractChildWatcher: ... + @abstractmethod + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... + else: + @abstractmethod + def get_child_watcher(self) -> AbstractChildWatcher: ... + @abstractmethod + def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy, metaclass=ABCMeta): def get_event_loop(self) -> AbstractEventLoop: ... @@ -565,15 +583,16 @@ def get_event_loop() -> AbstractEventLoop: ... def set_event_loop(loop: AbstractEventLoop | None) -> None: ... def new_event_loop() -> AbstractEventLoop: ... -if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def get_child_watcher() -> AbstractChildWatcher: ... - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... +if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def get_child_watcher() -> AbstractChildWatcher: ... + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... -else: - def get_child_watcher() -> AbstractChildWatcher: ... - def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... + else: + def get_child_watcher() -> AbstractChildWatcher: ... + def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... def _set_running_loop(loop: AbstractEventLoop | None, /) -> None: ... def _get_running_loop() -> AbstractEventLoop: ... diff --git a/mypy/typeshed/stdlib/asyncio/tasks.pyi b/mypy/typeshed/stdlib/asyncio/tasks.pyi index c16a1919b7c8..4613bca70c1a 100644 --- a/mypy/typeshed/stdlib/asyncio/tasks.pyi +++ b/mypy/typeshed/stdlib/asyncio/tasks.pyi @@ -70,7 +70,10 @@ _T4 = TypeVar("_T4") _T5 = TypeVar("_T5") _T6 = TypeVar("_T6") _FT = TypeVar("_FT", bound=Future[Any]) -_FutureLike: TypeAlias = Future[_T] | Generator[Any, None, _T] | Awaitable[_T] +if sys.version_info >= (3, 12): + _FutureLike: TypeAlias = Future[_T] | Awaitable[_T] +else: + _FutureLike: TypeAlias = Future[_T] | Generator[Any, None, _T] | Awaitable[_T] _TaskYieldType: TypeAlias = Future[object] | None FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED diff --git a/mypy/typeshed/stdlib/asyncio/unix_events.pyi b/mypy/typeshed/stdlib/asyncio/unix_events.pyi index e9274b853290..3a2c62646121 100644 --- a/mypy/typeshed/stdlib/asyncio/unix_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/unix_events.pyi @@ -13,51 +13,54 @@ _Ts = TypeVarTuple("_Ts") # This is also technically not available on Win, # but other parts of typeshed need this definition. # So, it is special cased. -if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - class AbstractChildWatcher: - @abstractmethod - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - @abstractmethod - def remove_child_handler(self, pid: int) -> bool: ... - @abstractmethod - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - @abstractmethod - def close(self) -> None: ... - @abstractmethod - def __enter__(self) -> Self: ... - @abstractmethod - def __exit__( - self, typ: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None - ) -> None: ... - @abstractmethod - def is_active(self) -> bool: ... - -else: - class AbstractChildWatcher: - @abstractmethod - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - @abstractmethod - def remove_child_handler(self, pid: int) -> bool: ... - @abstractmethod - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - @abstractmethod - def close(self) -> None: ... - @abstractmethod - def __enter__(self) -> Self: ... - @abstractmethod - def __exit__( - self, typ: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None - ) -> None: ... - @abstractmethod - def is_active(self) -> bool: ... +if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class AbstractChildWatcher: + @abstractmethod + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + @abstractmethod + def remove_child_handler(self, pid: int) -> bool: ... + @abstractmethod + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def close(self) -> None: ... + @abstractmethod + def __enter__(self) -> Self: ... + @abstractmethod + def __exit__( + self, typ: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None + ) -> None: ... + @abstractmethod + def is_active(self) -> bool: ... + + else: + class AbstractChildWatcher: + @abstractmethod + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + @abstractmethod + def remove_child_handler(self, pid: int) -> bool: ... + @abstractmethod + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def close(self) -> None: ... + @abstractmethod + def __enter__(self) -> Self: ... + @abstractmethod + def __exit__( + self, typ: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None + ) -> None: ... + @abstractmethod + def is_active(self) -> bool: ... if sys.platform != "win32": - if sys.version_info >= (3, 9): + if sys.version_info >= (3, 14): + __all__ = ("SelectorEventLoop", "DefaultEventLoopPolicy") + elif sys.version_info >= (3, 9): __all__ = ( "SelectorEventLoop", "AbstractChildWatcher", @@ -79,118 +82,137 @@ if sys.platform != "win32": "DefaultEventLoopPolicy", ) - # Doesn't actually have ABCMeta metaclass at runtime, but mypy complains if we don't have it in the stub. - # See discussion in #7412 - class BaseChildWatcher(AbstractChildWatcher, metaclass=ABCMeta): - def close(self) -> None: ... - def is_active(self) -> bool: ... - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - - if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - class SafeChildWatcher(BaseChildWatcher): - def __enter__(self) -> Self: ... - def __exit__(self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + # Doesn't actually have ABCMeta metaclass at runtime, but mypy complains if we don't have it in the stub. + # See discussion in #7412 + class BaseChildWatcher(AbstractChildWatcher, metaclass=ABCMeta): + def close(self) -> None: ... + def is_active(self) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - class FastChildWatcher(BaseChildWatcher): - def __enter__(self) -> Self: ... - def __exit__(self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class SafeChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... - else: - class SafeChildWatcher(BaseChildWatcher): - def __enter__(self) -> Self: ... - def __exit__(self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class FastChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... - class FastChildWatcher(BaseChildWatcher): - def __enter__(self) -> Self: ... - def __exit__(self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... + else: + # Doesn't actually have ABCMeta metaclass at runtime, but mypy complains if we don't have it in the stub. + # See discussion in #7412 + class BaseChildWatcher(AbstractChildWatcher, metaclass=ABCMeta): + def close(self) -> None: ... + def is_active(self) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + + class SafeChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + + class FastChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... class _UnixSelectorEventLoop(BaseSelectorEventLoop): ... class _UnixDefaultEventLoopPolicy(BaseDefaultEventLoopPolicy): - if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def get_child_watcher(self) -> AbstractChildWatcher: ... - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... - else: - def get_child_watcher(self) -> AbstractChildWatcher: ... - def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def get_child_watcher(self) -> AbstractChildWatcher: ... + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... + else: + def get_child_watcher(self) -> AbstractChildWatcher: ... + def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... SelectorEventLoop = _UnixSelectorEventLoop DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy - if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - class MultiLoopChildWatcher(AbstractChildWatcher): - def is_active(self) -> bool: ... - def close(self) -> None: ... - def __enter__(self) -> Self: ... - def __exit__( - self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None - ) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class MultiLoopChildWatcher(AbstractChildWatcher): + def is_active(self) -> bool: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - else: - class MultiLoopChildWatcher(AbstractChildWatcher): - def is_active(self) -> bool: ... + else: + class MultiLoopChildWatcher(AbstractChildWatcher): + def is_active(self) -> bool: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + + if sys.version_info < (3, 14): + class ThreadedChildWatcher(AbstractChildWatcher): + def is_active(self) -> Literal[True]: ... def close(self) -> None: ... def __enter__(self) -> Self: ... def __exit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None ) -> None: ... + def __del__(self) -> None: ... def add_child_handler( self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] ) -> None: ... def remove_child_handler(self, pid: int) -> bool: ... def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - class ThreadedChildWatcher(AbstractChildWatcher): - def is_active(self) -> Literal[True]: ... - def close(self) -> None: ... - def __enter__(self) -> Self: ... - def __exit__( - self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None - ) -> None: ... - def __del__(self) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - - if sys.version_info >= (3, 9): - class PidfdChildWatcher(AbstractChildWatcher): - def __enter__(self) -> Self: ... - def __exit__( - self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None - ) -> None: ... - def is_active(self) -> bool: ... - def close(self) -> None: ... - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... + if sys.version_info >= (3, 9): + class PidfdChildWatcher(AbstractChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def is_active(self) -> bool: ... + def close(self) -> None: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... diff --git a/mypy/typeshed/stdlib/asyncio/windows_events.pyi b/mypy/typeshed/stdlib/asyncio/windows_events.pyi index 9c150ee16beb..97aa52ff8b9a 100644 --- a/mypy/typeshed/stdlib/asyncio/windows_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/windows_events.pyi @@ -74,8 +74,9 @@ if sys.platform == "win32": class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): _loop_factory: ClassVar[type[SelectorEventLoop]] - def get_child_watcher(self) -> NoReturn: ... - def set_child_watcher(self, watcher: Any) -> NoReturn: ... + if sys.version_info < (3, 14): + def get_child_watcher(self) -> NoReturn: ... + def set_child_watcher(self, watcher: Any) -> NoReturn: ... class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): _loop_factory: ClassVar[type[ProactorEventLoop]] diff --git a/mypy/typeshed/stdlib/dataclasses.pyi b/mypy/typeshed/stdlib/dataclasses.pyi index 30489e6f8b3d..626608e8a59d 100644 --- a/mypy/typeshed/stdlib/dataclasses.pyi +++ b/mypy/typeshed/stdlib/dataclasses.pyi @@ -108,7 +108,7 @@ class _DefaultFactory(Protocol[_T_co]): class Field(Generic[_T]): name: str - type: Type[_T] + type: Type[_T] | str | Any default: _T | Literal[_MISSING_TYPE.MISSING] default_factory: _DefaultFactory[_T] | Literal[_MISSING_TYPE.MISSING] repr: bool diff --git a/mypy/typeshed/stdlib/posixpath.pyi b/mypy/typeshed/stdlib/posixpath.pyi index e5f5fa0d813c..31406f8df950 100644 --- a/mypy/typeshed/stdlib/posixpath.pyi +++ b/mypy/typeshed/stdlib/posixpath.pyi @@ -77,11 +77,7 @@ pathsep: LiteralString defpath: LiteralString devnull: LiteralString -# Overloads are necessary to work around python/mypy#3644. -@overload -def abspath(path: PathLike[AnyStr]) -> AnyStr: ... -@overload -def abspath(path: AnyStr) -> AnyStr: ... +def abspath(path: PathLike[AnyStr] | AnyStr) -> AnyStr: ... @overload def basename(p: PathLike[AnyStr]) -> AnyStr: ... @overload @@ -90,14 +86,8 @@ def basename(p: AnyOrLiteralStr) -> AnyOrLiteralStr: ... def dirname(p: PathLike[AnyStr]) -> AnyStr: ... @overload def dirname(p: AnyOrLiteralStr) -> AnyOrLiteralStr: ... -@overload -def expanduser(path: PathLike[AnyStr]) -> AnyStr: ... -@overload -def expanduser(path: AnyStr) -> AnyStr: ... -@overload -def expandvars(path: PathLike[AnyStr]) -> AnyStr: ... -@overload -def expandvars(path: AnyStr) -> AnyStr: ... +def expanduser(path: PathLike[AnyStr] | AnyStr) -> AnyStr: ... +def expandvars(path: PathLike[AnyStr] | AnyStr) -> AnyStr: ... @overload def normcase(s: PathLike[AnyStr]) -> AnyStr: ... @overload diff --git a/mypy/typeshed/stdlib/subprocess.pyi b/mypy/typeshed/stdlib/subprocess.pyi index 6234ecc02b48..b01bac2455ce 100644 --- a/mypy/typeshed/stdlib/subprocess.pyi +++ b/mypy/typeshed/stdlib/subprocess.pyi @@ -889,6 +889,7 @@ if sys.version_info >= (3, 11): start_new_session: bool = False, pass_fds: Collection[int] = ..., *, + encoding: str | None = None, timeout: float | None = None, text: bool | None = None, user: str | int | None = None, @@ -920,6 +921,7 @@ elif sys.version_info >= (3, 10): start_new_session: bool = False, pass_fds: Collection[int] = ..., *, + encoding: str | None = None, timeout: float | None = None, text: bool | None = None, user: str | int | None = None, @@ -950,6 +952,7 @@ elif sys.version_info >= (3, 9): start_new_session: bool = False, pass_fds: Collection[int] = ..., *, + encoding: str | None = None, timeout: float | None = None, text: bool | None = None, user: str | int | None = None, @@ -978,6 +981,7 @@ else: start_new_session: bool = False, pass_fds: Collection[int] = ..., *, + encoding: str | None = None, timeout: float | None = None, text: bool | None = None, ) -> int: ... @@ -1005,6 +1009,7 @@ if sys.version_info >= (3, 11): pass_fds: Collection[int] = ..., timeout: float | None = ..., *, + encoding: str | None = None, text: bool | None = None, user: str | int | None = None, group: str | int | None = None, @@ -1036,6 +1041,7 @@ elif sys.version_info >= (3, 10): pass_fds: Collection[int] = ..., timeout: float | None = ..., *, + encoding: str | None = None, text: bool | None = None, user: str | int | None = None, group: str | int | None = None, @@ -1066,6 +1072,7 @@ elif sys.version_info >= (3, 9): pass_fds: Collection[int] = ..., timeout: float | None = ..., *, + encoding: str | None = None, text: bool | None = None, user: str | int | None = None, group: str | int | None = None, @@ -1094,6 +1101,7 @@ else: pass_fds: Collection[int] = ..., timeout: float | None = ..., *, + encoding: str | None = None, text: bool | None = None, ) -> int: ... diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi index e52099464174..d6adf21c1900 100644 --- a/mypy/typeshed/stdlib/tarfile.pyi +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -103,10 +103,13 @@ PAX_NAME_FIELDS: set[str] ENCODING: str +_FileCreationModes: TypeAlias = Literal["a", "w", "x"] + +@overload def open( name: StrOrBytesPath | None = None, mode: str = "r", - fileobj: IO[bytes] | None = None, # depends on mode + fileobj: IO[bytes] | None = None, bufsize: int = 10240, *, format: int | None = ..., @@ -121,6 +124,25 @@ def open( compresslevel: int | None = ..., preset: Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] | None = ..., ) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None = None, + mode: _FileCreationModes = ..., + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + compresslevel: int | None = ..., + preset: int | None = ..., +) -> TarFile: ... class ExFileObject(io.BufferedReader): def __init__(self, tarfile: TarFile, tarinfo: TarInfo) -> None: ... diff --git a/mypy/typeshed/stdlib/zipimport.pyi b/mypy/typeshed/stdlib/zipimport.pyi index 158d573cac74..f53b09e188eb 100644 --- a/mypy/typeshed/stdlib/zipimport.pyi +++ b/mypy/typeshed/stdlib/zipimport.pyi @@ -28,5 +28,7 @@ class zipimporter: def is_package(self, fullname: str) -> bool: ... def load_module(self, fullname: str) -> ModuleType: ... if sys.version_info >= (3, 10): + def exec_module(self, module: ModuleType) -> None: ... + def create_module(self, spec: ModuleSpec) -> None: ... def find_spec(self, fullname: str, target: ModuleType | None = None) -> ModuleSpec | None: ... def invalidate_caches(self) -> None: ... From 4ae632b6f07b33dc5937a3b1dd28af98a080bc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 2 Jul 2024 01:15:20 +0200 Subject: [PATCH 067/247] Fix type comments crash inside generic definitions (#16849) Closes https://github.com/python/mypy/issues/16649 It's the first time I am contributing to mypy so I am not very familiar with how it works entirely behind the scene. The issue that I had is that a crash happens when using tuple type comments inside functions/classes that depend on a *constrained* type variable. After investigation, the reason is that the type checker generates all possible definitions (since constraints are known) and expands the functions definitions and bodies accordingly. However, by doing so, a tuple type comment ('# type: (int, float)') would have a FakeInfo, so `ExpandTypeVisitor` would fail since it queries `t.type.fullname`. By the way, feel free to change where my test should lie. --- mypy/expandtype.py | 12 ++++++++++- test-data/unit/check-typevar-values.test | 26 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 5c4d6af9458e..9336be54437b 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -2,7 +2,7 @@ from typing import Final, Iterable, Mapping, Sequence, TypeVar, cast, overload -from mypy.nodes import ARG_STAR, Var +from mypy.nodes import ARG_STAR, FakeInfo, Var from mypy.state import state from mypy.types import ( ANY_STRATEGY, @@ -208,6 +208,16 @@ def visit_erased_type(self, t: ErasedType) -> Type: def visit_instance(self, t: Instance) -> Type: args = self.expand_types_with_unpack(list(t.args)) + + if isinstance(t.type, FakeInfo): + # The type checker expands function definitions and bodies + # if they depend on constrained type variables but the body + # might contain a tuple type comment (e.g., # type: (int, float)), + # in which case 't.type' is not yet available. + # + # See: https://github.com/python/mypy/issues/16649 + return t.copy_modified(args=args) + if t.type.fullname == "builtins.tuple": # Normalize Tuple[*Tuple[X, ...], ...] -> Tuple[X, ...] arg = args[0] diff --git a/test-data/unit/check-typevar-values.test b/test-data/unit/check-typevar-values.test index effaf620f1f0..8b961d88d23d 100644 --- a/test-data/unit/check-typevar-values.test +++ b/test-data/unit/check-typevar-values.test @@ -706,3 +706,29 @@ Func = Callable[[], T] class A: ... class B: ... + +[case testTypeCommentInGenericTypeWithConstrainedTypeVar] +from typing import Generic, TypeVar + +NT = TypeVar("NT", int, float) + +class Foo1(Generic[NT]): + p = 1 # type: int + +class Foo2(Generic[NT]): + p, q = 1, 2.0 # type: (int, float) + +class Foo3(Generic[NT]): + def bar(self) -> None: + p = 1 # type: int + +class Foo4(Generic[NT]): + def bar(self) -> None: + p, q = 1, 2.0 # type: (int, float) + +def foo3(x: NT) -> None: + p = 1 # type: int + +def foo4(x: NT) -> None: + p, q = 1, 2.0 # type: (int, float) +[builtins fixtures/tuple.pyi] From d1d3c780c7a2d30b2a038903289ea7487303a218 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 Jul 2024 16:28:56 -0700 Subject: [PATCH 068/247] Further improvements to functools.partial handling (#17425) - Fixes another crash case / type inference in that case - Fix a false positive when calling the partially applied function with kwargs - TypeTraverse / comment / daemon test follow up ilevkivskyi mentioned on the original PR See also https://github.com/python/mypy/pull/17423 --- mypy/plugins/functools.py | 31 ++++--- mypy/type_visitor.py | 1 + mypy/types.py | 3 +- test-data/unit/check-functools.test | 121 ++++++++++++++++++++++------ test-data/unit/fine-grained.test | 48 +++++++++++ 5 files changed, 169 insertions(+), 35 deletions(-) diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index 9589c6aeca8b..6650af637519 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -245,11 +245,14 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: partial_kinds.append(fn_type.arg_kinds[i]) partial_types.append(arg_type) partial_names.append(fn_type.arg_names[i]) - elif actuals: - if any(actual_arg_kinds[j] == ArgKind.ARG_POS for j in actuals): + else: + assert actuals + if any(actual_arg_kinds[j] in (ArgKind.ARG_POS, ArgKind.ARG_STAR) for j in actuals): + # Don't add params for arguments passed positionally continue + # Add defaulted params for arguments passed via keyword kind = actual_arg_kinds[actuals[0]] - if kind == ArgKind.ARG_NAMED: + if kind == ArgKind.ARG_NAMED or kind == ArgKind.ARG_STAR2: kind = ArgKind.ARG_NAMED_OPT partial_kinds.append(kind) partial_types.append(arg_type) @@ -286,15 +289,25 @@ def partial_call_callback(ctx: mypy.plugin.MethodContext) -> Type: if len(ctx.arg_types) != 2: # *args, **kwargs return ctx.default_return_type - args = [a for param in ctx.args for a in param] - arg_kinds = [a for param in ctx.arg_kinds for a in param] - arg_names = [a for param in ctx.arg_names for a in param] + # See comments for similar actual to formal code above + actual_args = [] + actual_arg_kinds = [] + actual_arg_names = [] + seen_args = set() + for i, param in enumerate(ctx.args): + for j, a in enumerate(param): + if a in seen_args: + continue + seen_args.add(a) + actual_args.append(a) + actual_arg_kinds.append(ctx.arg_kinds[i][j]) + actual_arg_names.append(ctx.arg_names[i][j]) result = ctx.api.expr_checker.check_call( callee=partial_type, - args=args, - arg_kinds=arg_kinds, - arg_names=arg_names, + args=actual_args, + arg_kinds=actual_arg_kinds, + arg_names=actual_arg_names, context=ctx.context, ) return result[0] diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index d0876629fc08..e685c49904bc 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -213,6 +213,7 @@ def visit_instance(self, t: Instance) -> Type: line=t.line, column=t.column, last_known_value=last_known_value, + extra_attrs=t.extra_attrs, ) def visit_type_var(self, t: TypeVarType) -> Type: diff --git a/mypy/types.py b/mypy/types.py index 52f8a8d63f09..2e7cbfd4e733 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1417,8 +1417,7 @@ def __init__( self._hash = -1 # Additional attributes defined per instance of this type. For example modules - # have different attributes per instance of types.ModuleType. This is intended - # to be "short-lived", we don't serialize it, and even don't store as variable type. + # have different attributes per instance of types.ModuleType. self.extra_attrs = extra_attrs def accept(self, visitor: TypeVisitor[T]) -> T: diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index e4b3e4cffdc1..710d3e66dfad 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -191,6 +191,7 @@ functools.partial(1) # E: "int" not callable \ [case testFunctoolsPartialStar] import functools +from typing import List def foo(a: int, b: str, *args: int, d: str, **kwargs: int) -> int: ... @@ -215,6 +216,13 @@ def bar(*a: bytes, **k: int): p1("a", **k) # E: Argument 2 to "foo" has incompatible type "**Dict[str, int]"; expected "str" p1(**k) # E: Argument 1 to "foo" has incompatible type "**Dict[str, int]"; expected "str" p1(*a) # E: List or tuple expected as variadic arguments + + +def baz(a: int, b: int) -> int: ... +def test_baz(xs: List[int]): + p3 = functools.partial(baz, *xs) + p3() + p3(1) # E: Too many arguments for "baz" [builtins fixtures/dict.pyi] [case testFunctoolsPartialGeneric] @@ -408,33 +416,83 @@ def foo(cls3: Type[B[T]]): from typing_extensions import TypedDict, Unpack from functools import partial -class Data(TypedDict, total=False): - x: int - -def f(**kwargs: Unpack[Data]) -> None: ... -def g(**kwargs: Unpack[Data]) -> None: - partial(f, **kwargs)() - -class MoreData(TypedDict, total=False): - x: int - y: int +class D1(TypedDict, total=False): + a1: int + +def fn1(a1: int) -> None: ... # N: "fn1" defined here +def main1(**d1: Unpack[D1]) -> None: + partial(fn1, **d1)() + partial(fn1, **d1)(**d1) + partial(fn1, **d1)(a1=1) + partial(fn1, **d1)(a1="asdf") # E: Argument "a1" to "fn1" has incompatible type "str"; expected "int" + partial(fn1, **d1)(oops=1) # E: Unexpected keyword argument "oops" for "fn1" + +def fn2(**kwargs: Unpack[D1]) -> None: ... # N: "fn2" defined here +def main2(**d1: Unpack[D1]) -> None: + partial(fn2, **d1)() + partial(fn2, **d1)(**d1) + partial(fn2, **d1)(a1=1) + partial(fn2, **d1)(a1="asdf") # E: Argument "a1" to "fn2" has incompatible type "str"; expected "int" + partial(fn2, **d1)(oops=1) # E: Unexpected keyword argument "oops" for "fn2" + +class D2(TypedDict, total=False): + a1: int + a2: str + +class A2Good(TypedDict, total=False): + a2: str +class A2Bad(TypedDict, total=False): + a2: int + +def fn3(a1: int, a2: str) -> None: ... # N: "fn3" defined here +def main3(a2good: A2Good, a2bad: A2Bad, **d2: Unpack[D2]) -> None: + partial(fn3, **d2)() + partial(fn3, **d2)(a1=1, a2="asdf") + + partial(fn3, **d2)(**d2) + + partial(fn3, **d2)(a1="asdf") # E: Argument "a1" to "fn3" has incompatible type "str"; expected "int" + partial(fn3, **d2)(a1=1, a2="asdf", oops=1) # E: Unexpected keyword argument "oops" for "fn3" + + partial(fn3, **d2)(**a2good) + partial(fn3, **d2)(**a2bad) # E: Argument "a2" to "fn3" has incompatible type "int"; expected "str" + +def fn4(**kwargs: Unpack[D2]) -> None: ... # N: "fn4" defined here +def main4(a2good: A2Good, a2bad: A2Bad, **d2: Unpack[D2]) -> None: + partial(fn4, **d2)() + partial(fn4, **d2)(a1=1, a2="asdf") + + partial(fn4, **d2)(**d2) + + partial(fn4, **d2)(a1="asdf") # E: Argument "a1" to "fn4" has incompatible type "str"; expected "int" + partial(fn4, **d2)(a1=1, a2="asdf", oops=1) # E: Unexpected keyword argument "oops" for "fn4" + + partial(fn3, **d2)(**a2good) + partial(fn3, **d2)(**a2bad) # E: Argument "a2" to "fn3" has incompatible type "int"; expected "str" + +def main5(**d2: Unpack[D2]) -> None: + partial(fn1, **d2)() # E: Extra argument "a2" from **args for "fn1" + partial(fn2, **d2)() # E: Extra argument "a2" from **args for "fn2" + +def main6(a2good: A2Good, a2bad: A2Bad, **d1: Unpack[D1]) -> None: + partial(fn3, **d1)() # E: Missing positional argument "a1" in call to "fn3" + partial(fn3, **d1)("asdf") # E: Too many positional arguments for "fn3" \ + # E: Too few arguments for "fn3" \ + # E: Argument 1 to "fn3" has incompatible type "str"; expected "int" + partial(fn3, **d1)(a2="asdf") + partial(fn3, **d1)(**a2good) + partial(fn3, **d1)(**a2bad) # E: Argument "a2" to "fn3" has incompatible type "int"; expected "str" + + partial(fn4, **d1)() + partial(fn4, **d1)("asdf") # E: Too many positional arguments for "fn4" \ + # E: Argument 1 to "fn4" has incompatible type "str"; expected "int" + partial(fn4, **d1)(a2="asdf") + partial(fn4, **d1)(**a2good) + partial(fn4, **d1)(**a2bad) # E: Argument "a2" to "fn4" has incompatible type "int"; expected "str" -def f_more(**kwargs: Unpack[MoreData]) -> None: ... -def g_more(**kwargs: Unpack[MoreData]) -> None: - partial(f_more, **kwargs)() - -class Good(TypedDict, total=False): - y: int -class Bad(TypedDict, total=False): - y: str - -def h(**kwargs: Unpack[Data]) -> None: - bad: Bad - partial(f_more, **kwargs)(**bad) # E: Argument "y" to "f_more" has incompatible type "str"; expected "int" - good: Good - partial(f_more, **kwargs)(**good) [builtins fixtures/dict.pyi] + [case testFunctoolsPartialNestedGeneric] from functools import partial from typing import Generic, TypeVar, List @@ -456,6 +514,21 @@ first_kw([1]) # E: Too many positional arguments for "get" \ # E: Argument 1 to "get" has incompatible type "List[int]"; expected "int" [builtins fixtures/list.pyi] +[case testFunctoolsPartialHigherOrder] +from functools import partial +from typing import Callable + +def fn(a: int, b: str, c: bytes) -> int: ... + +def callback1(fn: Callable[[str, bytes], int]) -> None: ... +def callback2(fn: Callable[[str, int], int]) -> None: ... + +callback1(partial(fn, 1)) +# TODO: false negative +# https://github.com/python/mypy/issues/17461 +callback2(partial(fn, 1)) +[builtins fixtures/tuple.pyi] + [case testFunctoolsPartialClassObjectMatchingPartial] from functools import partial diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 2a652e50b1e6..2ad31311a402 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10497,3 +10497,51 @@ from pkg.sub import modb [out] == + +[case testFineGrainedFunctoolsPartial] +import m + +[file m.py] +from typing import Callable +from partial import p1 + +reveal_type(p1) +p1("a") +p1("a", 3) +p1("a", c=3) +p1(1, 3) +p1(1, "a", 3) +p1(a=1, b="a", c=3) +[builtins fixtures/dict.pyi] + +[file partial.py] +from typing import Callable +import functools + +def foo(a: int, b: str, c: int = 5) -> int: ... +p1 = foo + +[file partial.py.2] +from typing import Callable +import functools + +def foo(a: int, b: str, c: int = 5) -> int: ... +p1 = functools.partial(foo, 1) + +[out] +m.py:4: note: Revealed type is "def (a: builtins.int, b: builtins.str, c: builtins.int =) -> builtins.int" +m.py:5: error: Too few arguments +m.py:5: error: Argument 1 has incompatible type "str"; expected "int" +m.py:6: error: Argument 1 has incompatible type "str"; expected "int" +m.py:6: error: Argument 2 has incompatible type "int"; expected "str" +m.py:7: error: Too few arguments +m.py:7: error: Argument 1 has incompatible type "str"; expected "int" +m.py:8: error: Argument 2 has incompatible type "int"; expected "str" +== +m.py:4: note: Revealed type is "functools.partial[builtins.int]" +m.py:8: error: Argument 1 to "foo" has incompatible type "int"; expected "str" +m.py:9: error: Too many arguments for "foo" +m.py:9: error: Argument 1 to "foo" has incompatible type "int"; expected "str" +m.py:9: error: Argument 2 to "foo" has incompatible type "str"; expected "int" +m.py:10: error: Unexpected keyword argument "a" for "foo" +partial.py:4: note: "foo" defined here From f297917fc9eba13cc2a5b7e2e276394d94c03c8d Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 2 Jul 2024 08:51:36 -0700 Subject: [PATCH 069/247] Mention --enable-incomplete-feature=NewGenericSyntax (#17462) --- mypy/fastparse.py | 16 +++++++++++++--- test-data/unit/check-python312.test | 28 ++++++++++++++-------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 342cf36d69e8..01f6ed4733ae 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -954,7 +954,9 @@ def do_func_def( else: self.fail( ErrorMessage( - "PEP 695 generics are not yet supported", code=codes.VALID_TYPE + "PEP 695 generics are not yet supported. " + "Use --enable-incomplete-feature=NewGenericSyntax for experimental support", + code=codes.VALID_TYPE, ), n.type_params[0].lineno, n.type_params[0].col_offset, @@ -1145,7 +1147,11 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: explicit_type_params = self.translate_type_params(n.type_params) else: self.fail( - ErrorMessage("PEP 695 generics are not yet supported", code=codes.VALID_TYPE), + ErrorMessage( + "PEP 695 generics are not yet supported. " + "Use --enable-incomplete-feature=NewGenericSyntax for experimental support", + code=codes.VALID_TYPE, + ), n.type_params[0].lineno, n.type_params[0].col_offset, blocker=False, @@ -1801,7 +1807,11 @@ def visit_TypeAlias(self, n: ast_TypeAlias) -> TypeAliasStmt | AssignmentStmt: return self.set_line(node, n) else: self.fail( - ErrorMessage("PEP 695 type aliases are not yet supported", code=codes.VALID_TYPE), + ErrorMessage( + "PEP 695 type aliases are not yet supported. " + "Use --enable-incomplete-feature=NewGenericSyntax for experimental support", + code=codes.VALID_TYPE, + ), n.lineno, n.col_offset, blocker=False, diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 27027d30a684..5307f47d539a 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1,10 +1,10 @@ [case test695TypeAlias] -type MyInt = int # E: PEP 695 type aliases are not yet supported +type MyInt = int # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support def f(x: MyInt) -> MyInt: return reveal_type(x) # N: Revealed type is "builtins.int" -type MyList[T] = list[T] # E: PEP 695 type aliases are not yet supported \ +type MyList[T] = list[T] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "T" is not defined def g(x: MyList[int]) -> MyList[int]: # E: Variable "__main__.MyList" is not valid as a type \ @@ -17,7 +17,7 @@ def h(x: MyInt2) -> MyInt2: return reveal_type(x) # N: Revealed type is "builtins.int" [case test695Class] -class MyGen[T]: # E: PEP 695 generics are not yet supported +class MyGen[T]: # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support def __init__(self, x: T) -> None: # E: Name "T" is not defined self.x = x @@ -25,13 +25,13 @@ def f(x: MyGen[int]): # E: "MyGen" expects no type arguments, but 1 given reveal_type(x.x) # N: Revealed type is "Any" [case test695Function] -def f[T](x: T) -> T: # E: PEP 695 generics are not yet supported \ +def f[T](x: T) -> T: # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "T" is not defined return reveal_type(x) # N: Revealed type is "Any" reveal_type(f(1)) # N: Revealed type is "Any" -async def g[T](x: T) -> T: # E: PEP 695 generics are not yet supported \ +async def g[T](x: T) -> T: # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "T" is not defined return reveal_type(x) # N: Revealed type is "Any" @@ -41,26 +41,26 @@ reveal_type(g(1)) # E: Value of type "Coroutine[Any, Any, Any]" must be used \ [case test695TypeVar] from typing import Callable -type Alias1[T: int] = list[T] # E: PEP 695 type aliases are not yet supported \ +type Alias1[T: int] = list[T] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "T" is not defined -type Alias2[**P] = Callable[P, int] # E: PEP 695 type aliases are not yet supported \ +type Alias2[**P] = Callable[P, int] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Value of type "int" is not indexable \ # E: Name "P" is not defined -type Alias3[*Ts] = tuple[*Ts] # E: PEP 695 type aliases are not yet supported \ +type Alias3[*Ts] = tuple[*Ts] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "Ts" is not defined -class Cls1[T: int]: ... # E: PEP 695 generics are not yet supported -class Cls2[**P]: ... # E: PEP 695 generics are not yet supported -class Cls3[*Ts]: ... # E: PEP 695 generics are not yet supported +class Cls1[T: int]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support +class Cls2[**P]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support +class Cls3[*Ts]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support -def func1[T: int](x: T) -> T: ... # E: PEP 695 generics are not yet supported \ +def func1[T: int](x: T) -> T: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "T" is not defined -def func2[**P](x: Callable[P, int]) -> Callable[P, str]: ... # E: PEP 695 generics are not yet supported \ +def func2[**P](x: Callable[P, int]) -> Callable[P, str]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: The first argument to Callable must be a list of types, parameter specification, or "..." \ # N: See https://mypy.readthedocs.io/en/stable/kinds_of_types.html#callable-types-and-lambdas \ # E: Name "P" is not defined -def func3[*Ts](x: tuple[*Ts]) -> tuple[int, *Ts]: ... # E: PEP 695 generics are not yet supported \ +def func3[*Ts](x: tuple[*Ts]) -> tuple[int, *Ts]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "Ts" is not defined [builtins fixtures/tuple.pyi] From 55a08120b7be56e7d33cf2e9d240d30a8f608f83 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 2 Jul 2024 19:36:16 +0100 Subject: [PATCH 070/247] Infer unions for ternary expressions (#17427) Ref https://github.com/python/mypy/issues/12056 cc @JukkaL Again, let's check the primer... --- mypy/checkexpr.py | 19 +++-- mypyc/test-data/irbuild-any.test | 4 +- test-data/unit/check-errorcodes.test | 2 +- test-data/unit/check-expressions.test | 17 ++-- test-data/unit/check-functions.test | 17 +++- test-data/unit/check-inference-context.test | 6 +- test-data/unit/check-inference.test | 10 ++- test-data/unit/check-optional.test | 2 +- test-data/unit/check-tuples.test | 87 ++++++++++++--------- 9 files changed, 99 insertions(+), 65 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index c4ab8a081acc..3532e18b93b2 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -5766,16 +5766,15 @@ def visit_conditional_expr(self, e: ConditionalExpr, allow_none_return: bool = F context=if_type_fallback, allow_none_return=allow_none_return, ) - - # Only create a union type if the type context is a union, to be mostly - # compatible with older mypy versions where we always did a join. - # - # TODO: Always create a union or at least in more cases? - if isinstance(get_proper_type(self.type_context[-1]), UnionType): - res: Type = make_simplified_union([if_type, full_context_else_type]) - else: - res = join.join_types(if_type, else_type) - + res: Type = make_simplified_union([if_type, else_type]) + if has_uninhabited_component(res) and not isinstance( + get_proper_type(self.type_context[-1]), UnionType + ): + # In rare cases with empty collections join may give a better result. + alternative = join.join_types(if_type, else_type) + p_alt = get_proper_type(alternative) + if not isinstance(p_alt, Instance) or p_alt.type.fullname != "builtins.object": + res = alternative return res def analyze_cond_branch( diff --git a/mypyc/test-data/irbuild-any.test b/mypyc/test-data/irbuild-any.test index 0d14e1a5dfc8..3bfb1587fb3b 100644 --- a/mypyc/test-data/irbuild-any.test +++ b/mypyc/test-data/irbuild-any.test @@ -151,7 +151,9 @@ def f4(a, n, b): a :: object n :: int b :: bool - r0, r1, r2, r3 :: object + r0 :: union[object, int] + r1, r2 :: object + r3 :: union[int, object] r4 :: int L0: if b goto L1 else goto L2 :: bool diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index 961815b11817..c4d72388fba9 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -460,7 +460,7 @@ a: D = {'x': ''} # E: Incompatible types (expression has type "str", TypedDict b: D = {'y': ''} # E: Missing key "x" for TypedDict "D" [typeddict-item] \ # E: Extra key "y" for TypedDict "D" [typeddict-unknown-key] c = D(x=0) if int() else E(x=0, y=0) -c = {} # E: Expected TypedDict key "x" but found no keys [typeddict-item] +c = {} # E: Missing key "x" for TypedDict "D" [typeddict-item] d: D = {'x': '', 'y': 1} # E: Extra key "y" for TypedDict "D" [typeddict-unknown-key] \ # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") [typeddict-item] diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index f9bd60f4dcc8..61cee1d00c58 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1470,10 +1470,9 @@ if int(): [case testConditionalExpressionUnion] from typing import Union -reveal_type(1 if bool() else 2) # N: Revealed type is "builtins.int" -reveal_type(1 if bool() else '') # N: Revealed type is "builtins.object" -x: Union[int, str] = reveal_type(1 if bool() else '') \ - # N: Revealed type is "Union[Literal[1]?, Literal['']?]" +reveal_type(1 if bool() else 2) # N: Revealed type is "Union[Literal[1]?, Literal[2]?]" +reveal_type(1 if bool() else '') # N: Revealed type is "Union[Literal[1]?, Literal['']?]" +x: Union[int, str] = reveal_type(1 if bool() else '') # N: Revealed type is "Union[Literal[1]?, Literal['']?]" class A: pass class B(A): @@ -1487,17 +1486,17 @@ b = B() c = C() d = D() reveal_type(a if bool() else b) # N: Revealed type is "__main__.A" -reveal_type(b if bool() else c) # N: Revealed type is "builtins.object" -reveal_type(c if bool() else b) # N: Revealed type is "builtins.object" -reveal_type(c if bool() else a) # N: Revealed type is "builtins.object" -reveal_type(d if bool() else b) # N: Revealed type is "__main__.A" +reveal_type(b if bool() else c) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(c if bool() else b) # N: Revealed type is "Union[__main__.C, __main__.B]" +reveal_type(c if bool() else a) # N: Revealed type is "Union[__main__.C, __main__.A]" +reveal_type(d if bool() else b) # N: Revealed type is "Union[__main__.D, __main__.B]" [builtins fixtures/bool.pyi] [case testConditionalExpressionUnionWithAny] from typing import Union, Any a: Any x: Union[int, str] = reveal_type(a if int() else 1) # N: Revealed type is "Union[Any, Literal[1]?]" -reveal_type(a if int() else 1) # N: Revealed type is "Any" +reveal_type(a if int() else 1) # N: Revealed type is "Union[Any, Literal[1]?]" [case testConditionalExpressionStatementNoReturn] from typing import List, Union diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 93540e203c36..6c895c86e899 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -2250,13 +2250,26 @@ def dec(f: Callable[[A, str], None]) -> Callable[[A, int], None]: pass [out] [case testUnknownFunctionNotCallable] +from typing import TypeVar + def f() -> None: pass def g(x: int) -> None: pass h = f if bool() else g -reveal_type(h) # N: Revealed type is "builtins.function" -h(7) # E: Cannot call function of unknown type +reveal_type(h) # N: Revealed type is "Union[def (), def (x: builtins.int)]" +h(7) # E: Too many arguments for "f" + +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +h2 = join(f, g) +reveal_type(h2) # N: Revealed type is "builtins.function" +h2(7) # E: Cannot call function of unknown type + +h3 = join(g, f) +reveal_type(h3) # N: Revealed type is "builtins.function" +h3(7) # E: Cannot call function of unknown type [builtins fixtures/bool.pyi] [case testFunctionWithNameUnderscore] diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index afe6548df2d4..17ae6d9934b7 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -701,7 +701,7 @@ class A: pass class B(A): pass class C(A): pass def f(func: Callable[[T], S], *z: T, r: Optional[S] = None) -> S: pass -reveal_type(f(lambda x: 0 if isinstance(x, B) else 1)) # N: Revealed type is "builtins.int" +reveal_type(f(lambda x: 0 if isinstance(x, B) else 1)) # N: Revealed type is "Union[Literal[0]?, Literal[1]?]" f(lambda x: 0 if isinstance(x, B) else 1, A())() # E: "int" not callable f(lambda x: x if isinstance(x, B) else B(), A(), r=B())() # E: "B" not callable f( @@ -1391,7 +1391,7 @@ from typing import Union, List, Any def f(x: Union[List[str], Any]) -> None: a = x if x else [] - reveal_type(a) # N: Revealed type is "Union[builtins.list[Union[builtins.str, Any]], builtins.list[builtins.str], Any]" + reveal_type(a) # N: Revealed type is "Union[builtins.list[builtins.str], Any, builtins.list[Union[builtins.str, Any]]]" [builtins fixtures/list.pyi] [case testConditionalExpressionWithEmptyIteableAndUnionWithAny] @@ -1399,7 +1399,7 @@ from typing import Union, Iterable, Any def f(x: Union[Iterable[str], Any]) -> None: a = x if x else [] - reveal_type(a) # N: Revealed type is "Union[builtins.list[Union[builtins.str, Any]], typing.Iterable[builtins.str], Any]" + reveal_type(a) # N: Revealed type is "Union[typing.Iterable[builtins.str], Any, builtins.list[Union[builtins.str, Any]]]" [builtins fixtures/list.pyi] [case testInferMultipleAnyUnionCovariant] diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index fcd03f8efe01..0dbefbc774a3 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1438,18 +1438,22 @@ class Wrapper: def f(cond: bool) -> Any: f = Wrapper if cond else lambda x: x - reveal_type(f) # N: Revealed type is "def (x: Any) -> Any" + reveal_type(f) # N: Revealed type is "Union[def (x: Any) -> __main__.Wrapper, def (x: Any) -> Any]" return f(3) def g(cond: bool) -> Any: f = lambda x: x if cond else Wrapper - reveal_type(f) # N: Revealed type is "def (x: Any) -> Any" + reveal_type(f) # N: Revealed type is "def (x: Any) -> Union[Any, def (x: Any) -> __main__.Wrapper]" + return f(3) + +def h(cond: bool) -> Any: + f = (lambda x: x) if cond else Wrapper + reveal_type(f) # N: Revealed type is "Union[def (x: Any) -> Any, def (x: Any) -> __main__.Wrapper]" return f(3) -- Boolean operators -- ----------------- - [case testOrOperationWithGenericOperands] from typing import List a: List[A] diff --git a/test-data/unit/check-optional.test b/test-data/unit/check-optional.test index 70f3c4486e14..f80aa5115bc3 100644 --- a/test-data/unit/check-optional.test +++ b/test-data/unit/check-optional.test @@ -395,7 +395,7 @@ def lookup_field(name, obj): attr = None [case testTernaryWithNone] -reveal_type(None if bool() else 0) # N: Revealed type is "Union[Literal[0]?, None]" +reveal_type(None if bool() else 0) # N: Revealed type is "Union[None, Literal[0]?]" [builtins fixtures/bool.pyi] [case testListWithNone] diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index bf36977b56e3..972bccf8c24b 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1228,68 +1228,76 @@ x, y = g(z) # E: Argument 1 to "g" has incompatible type "int"; expected "Tuple[ [out] [case testFixedTupleJoinVarTuple] -from typing import Tuple +from typing import Tuple, TypeVar class A: pass class B(A): pass fixtup: Tuple[B, B] +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + vartup_b: Tuple[B, ...] -reveal_type(fixtup if int() else vartup_b) # N: Revealed type is "builtins.tuple[__main__.B, ...]" -reveal_type(vartup_b if int() else fixtup) # N: Revealed type is "builtins.tuple[__main__.B, ...]" +reveal_type(join(fixtup, vartup_b)) # N: Revealed type is "builtins.tuple[__main__.B, ...]" +reveal_type(join(vartup_b, fixtup)) # N: Revealed type is "builtins.tuple[__main__.B, ...]" vartup_a: Tuple[A, ...] -reveal_type(fixtup if int() else vartup_a) # N: Revealed type is "builtins.tuple[__main__.A, ...]" -reveal_type(vartup_a if int() else fixtup) # N: Revealed type is "builtins.tuple[__main__.A, ...]" - +reveal_type(join(fixtup, vartup_a)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(vartup_a, fixtup)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" [builtins fixtures/tuple.pyi] [out] [case testFixedTupleJoinList] -from typing import Tuple, List +from typing import Tuple, List, TypeVar class A: pass class B(A): pass fixtup: Tuple[B, B] +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + lst_b: List[B] -reveal_type(fixtup if int() else lst_b) # N: Revealed type is "typing.Sequence[__main__.B]" -reveal_type(lst_b if int() else fixtup) # N: Revealed type is "typing.Sequence[__main__.B]" +reveal_type(join(fixtup, lst_b)) # N: Revealed type is "typing.Sequence[__main__.B]" +reveal_type(join(lst_b, fixtup)) # N: Revealed type is "typing.Sequence[__main__.B]" lst_a: List[A] -reveal_type(fixtup if int() else lst_a) # N: Revealed type is "typing.Sequence[__main__.A]" -reveal_type(lst_a if int() else fixtup) # N: Revealed type is "typing.Sequence[__main__.A]" +reveal_type(join(fixtup, lst_a)) # N: Revealed type is "typing.Sequence[__main__.A]" +reveal_type(join(lst_a, fixtup)) # N: Revealed type is "typing.Sequence[__main__.A]" [builtins fixtures/tuple.pyi] [out] [case testEmptyTupleJoin] -from typing import Tuple, List +from typing import Tuple, List, TypeVar class A: pass empty = () +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + fixtup: Tuple[A] -reveal_type(fixtup if int() else empty) # N: Revealed type is "builtins.tuple[__main__.A, ...]" -reveal_type(empty if int() else fixtup) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(fixtup, empty)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(empty, fixtup)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" vartup: Tuple[A, ...] -reveal_type(empty if int() else vartup) # N: Revealed type is "builtins.tuple[__main__.A, ...]" -reveal_type(vartup if int() else empty) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(vartup, empty)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(empty, vartup)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" lst: List[A] -reveal_type(empty if int() else lst) # N: Revealed type is "typing.Sequence[__main__.A]" -reveal_type(lst if int() else empty) # N: Revealed type is "typing.Sequence[__main__.A]" +reveal_type(join(empty, lst)) # N: Revealed type is "typing.Sequence[__main__.A]" +reveal_type(join(lst, empty)) # N: Revealed type is "typing.Sequence[__main__.A]" [builtins fixtures/tuple.pyi] [out] [case testTupleSubclassJoin] -from typing import Tuple, NamedTuple +from typing import Tuple, NamedTuple, TypeVar class NTup(NamedTuple): a: bool @@ -1302,32 +1310,38 @@ ntup: NTup subtup: SubTuple vartup: SubVarTuple -reveal_type(ntup if int() else vartup) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(subtup if int() else vartup) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +reveal_type(join(ntup, vartup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(subtup, vartup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" [builtins fixtures/tuple.pyi] [out] [case testTupleJoinIrregular] -from typing import Tuple +from typing import Tuple, TypeVar tup1: Tuple[bool, int] tup2: Tuple[bool] -reveal_type(tup1 if int() else tup2) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(tup2 if int() else tup1) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +reveal_type(join(tup1, tup2)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(tup2, tup1)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(tup1 if int() else ()) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(() if int() else tup1) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(tup1, ())) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join((), tup1)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(tup2 if int() else ()) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" -reveal_type(() if int() else tup2) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +reveal_type(join(tup2, ())) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +reveal_type(join((), tup2)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" [builtins fixtures/tuple.pyi] [out] [case testTupleSubclassJoinIrregular] -from typing import Tuple, NamedTuple +from typing import Tuple, NamedTuple, TypeVar class NTup1(NamedTuple): a: bool @@ -1342,14 +1356,17 @@ tup1: NTup1 tup2: NTup2 subtup: SubTuple -reveal_type(tup1 if int() else tup2) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" -reveal_type(tup2 if int() else tup1) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +reveal_type(join(tup1, tup2)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +reveal_type(join(tup2, tup1)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" -reveal_type(tup1 if int() else subtup) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(subtup if int() else tup1) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(tup1, subtup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(subtup, tup1)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(tup2 if int() else subtup) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(subtup if int() else tup2) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(tup2, subtup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(subtup, tup2)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" [builtins fixtures/tuple.pyi] [out] From 45afd73974ef9c558f753d3a55e2997610b9827a Mon Sep 17 00:00:00 2001 From: Raphael Krupinski Date: Wed, 3 Jul 2024 14:56:52 +0200 Subject: [PATCH 071/247] Fix help message for --no-namespace-packages. (#17472) Fixes #17466 Change help message of '--no-namespace-packages' option Co-authored-by: Raphael Krupinski <10319569-mattesilver@users.noreply.gitlab.com> --- mypy/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/main.py b/mypy/main.py index 489ef8fd9a7b..05044335ecee 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -565,7 +565,7 @@ def add_invertible_flag( "--no-namespace-packages", dest="namespace_packages", default=True, - help="Support namespace packages (PEP 420, __init__.py-less)", + help="Disable support for namespace packages (PEP 420, __init__.py-less)", group=imports_group, ) imports_group.add_argument( From cb7b96d32d7ca2ba791106f61e1adf5c0d7b0ea4 Mon Sep 17 00:00:00 2001 From: Max Muoto Date: Wed, 3 Jul 2024 12:05:40 -0500 Subject: [PATCH 072/247] Add `__replace__` for dataclasses in 3.13 (#17469) Fixes https://github.com/python/mypy/issues/17471 --- mypy/plugins/dataclasses.py | 15 ++++++++++++ test-data/unit/check-dataclasses.test | 34 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index dd2eceab217f..edfc6840fc37 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -385,6 +385,9 @@ def transform(self) -> bool: self._add_dataclass_fields_magic_attribute() self._add_internal_replace_method(attributes) + if self._api.options.python_version >= (3, 13): + self._add_dunder_replace(attributes) + if "__post_init__" in info.names: self._add_internal_post_init_method(attributes) @@ -395,6 +398,18 @@ def transform(self) -> bool: return True + def _add_dunder_replace(self, attributes: list[DataclassAttribute]) -> None: + """Add a `__replace__` method to the class, which is used to replace attributes in the `copy` module.""" + args = [attr.to_argument(self._cls.info, of="replace") for attr in attributes] + type_vars = [tv for tv in self._cls.type_vars] + add_method_to_class( + self._api, + self._cls, + "__replace__", + args=args, + return_type=Instance(self._cls.info, type_vars), + ) + def _add_internal_replace_method(self, attributes: list[DataclassAttribute]) -> None: """ Stashes the signature of 'dataclasses.replace(...)' for this specific dataclass diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index f26ccd9a4854..0f726242b25b 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -2489,3 +2489,37 @@ class Base: class Child(Base): y: int [builtins fixtures/dataclasses.pyi] + +[case testDunderReplacePresent] +# flags: --python-version 3.13 +from dataclasses import dataclass + +@dataclass +class Coords: + x: int + y: int + + +replaced = Coords(2, 4).__replace__(x=2, y=5) +reveal_type(replaced) # N: Revealed type is "__main__.Coords" + +replaced = Coords(2, 4).__replace__(x=2) +reveal_type(replaced) # N: Revealed type is "__main__.Coords" + +Coords(2, 4).__replace__(x="asdf") # E: Argument "x" to "__replace__" of "Coords" has incompatible type "str"; expected "int" +Coords(2, 4).__replace__(23) # E: Too many positional arguments for "__replace__" of "Coords" +Coords(2, 4).__replace__(23, 25) # E: Too many positional arguments for "__replace__" of "Coords" +Coords(2, 4).__replace__(x=23, y=25, z=42) # E: Unexpected keyword argument "z" for "__replace__" of "Coords" + +from typing import Generic, TypeVar +T = TypeVar('T') + +@dataclass +class Gen(Generic[T]): + x: T + +replaced_2 = Gen(2).__replace__(x=2) +reveal_type(replaced_2) # N: Revealed type is "__main__.Gen[builtins.int]" +Gen(2).__replace__(x="not an int") # E: Argument "x" to "__replace__" of "Gen" has incompatible type "str"; expected "int" + +[builtins fixtures/tuple.pyi] From 1882ed78aed75da6882145bb6c4747688b72eec0 Mon Sep 17 00:00:00 2001 From: Max Muoto Date: Wed, 3 Jul 2024 12:37:14 -0500 Subject: [PATCH 073/247] Have namedtuple `__replace__` return `Self` (#17475) --- mypy/semanal_namedtuple.py | 4 +++- test-data/unit/check-namedtuple.test | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 768dd265b338..bf526a1ee990 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -57,6 +57,7 @@ TYPED_NAMEDTUPLE_NAMES, AnyType, CallableType, + Instance, LiteralType, TupleType, Type, @@ -631,9 +632,10 @@ def add_method( args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], ) if self.options.python_version >= (3, 13): + type_vars = [tv for tv in info.defn.type_vars] add_method( "__replace__", - ret=None, + ret=Instance(info, type_vars), args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], ) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index e9d156754d9c..f10217b9aa5f 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1407,9 +1407,23 @@ from typing import NamedTuple class A(NamedTuple): x: int -A(x=0).__replace__(x=1) +replaced = A(x=0).__replace__(x=1) +reveal_type(replaced) # N: Revealed type is "__main__.A" + A(x=0).__replace__(x="asdf") # E: Argument "x" to "__replace__" of "A" has incompatible type "str"; expected "int" A(x=0).__replace__(y=1) # E: Unexpected keyword argument "y" for "__replace__" of "A" + +from typing import TypeVar, Generic + +T = TypeVar("T") + +class GenericA(NamedTuple, Generic[T]): + x: T + +replaced_2 = GenericA(x=0).__replace__(x=1) +reveal_type(replaced_2) # N: Revealed type is "__main__.GenericA" +GenericA(x=0).__replace__(x="abc") # E: Argument "x" to "__replace__" of "GenericA" has incompatible type "str"; expected "int" + [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] From 606971807fad1de26ebc575d327d4c1c33f71c0e Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:17:14 +0200 Subject: [PATCH 074/247] Bump version to 1.12.0+dev (#17467) The release branch has been cut: https://github.com/python/mypy/tree/release-1.11 --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index f2615b77109d..8e00b4cce702 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.11.0+dev" +__version__ = "1.12.0+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From ec00fb8db7f5c72faccde55a7f48dc6023c65411 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 5 Jul 2024 19:03:16 +0300 Subject: [PATCH 075/247] Refactor: remove temporary `def` from `semanal_classprop.check_protocol_status` (#17486) There's no need to create and call a temporary function, when we can just call an existing method: faster and simplier. --- mypy/semanal_classprop.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mypy/semanal_classprop.py b/mypy/semanal_classprop.py index b5f1b2181761..c5ad34122f6c 100644 --- a/mypy/semanal_classprop.py +++ b/mypy/semanal_classprop.py @@ -122,11 +122,12 @@ def check_protocol_status(info: TypeInfo, errors: Errors) -> None: if info.is_protocol: for type in info.bases: if not type.type.is_protocol and type.type.fullname != "builtins.object": - - def report(message: str, severity: str) -> None: - errors.report(info.line, info.column, message, severity=severity) - - report("All bases of a protocol must be protocols", "error") + errors.report( + info.line, + info.column, + "All bases of a protocol must be protocols", + severity="error", + ) def calculate_class_vars(info: TypeInfo) -> None: From 2e0b8687599fa033e78978dc78c42afa86934913 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Jul 2024 21:05:46 +0100 Subject: [PATCH 076/247] Revert "Have namedtuple `__replace__` return `Self`" (#17496) Reverts python/mypy#17475 --- mypy/semanal_namedtuple.py | 4 +--- test-data/unit/check-namedtuple.test | 16 +--------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index bf526a1ee990..768dd265b338 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -57,7 +57,6 @@ TYPED_NAMEDTUPLE_NAMES, AnyType, CallableType, - Instance, LiteralType, TupleType, Type, @@ -632,10 +631,9 @@ def add_method( args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], ) if self.options.python_version >= (3, 13): - type_vars = [tv for tv in info.defn.type_vars] add_method( "__replace__", - ret=Instance(info, type_vars), + ret=None, args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], ) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index f10217b9aa5f..e9d156754d9c 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1407,23 +1407,9 @@ from typing import NamedTuple class A(NamedTuple): x: int -replaced = A(x=0).__replace__(x=1) -reveal_type(replaced) # N: Revealed type is "__main__.A" - +A(x=0).__replace__(x=1) A(x=0).__replace__(x="asdf") # E: Argument "x" to "__replace__" of "A" has incompatible type "str"; expected "int" A(x=0).__replace__(y=1) # E: Unexpected keyword argument "y" for "__replace__" of "A" - -from typing import TypeVar, Generic - -T = TypeVar("T") - -class GenericA(NamedTuple, Generic[T]): - x: T - -replaced_2 = GenericA(x=0).__replace__(x=1) -reveal_type(replaced_2) # N: Revealed type is "__main__.GenericA" -GenericA(x=0).__replace__(x="abc") # E: Argument "x" to "__replace__" of "GenericA" has incompatible type "str"; expected "int" - [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] From 1ea867624d392f90755c7ccd3b410690ad87e891 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 6 Jul 2024 13:44:04 -0700 Subject: [PATCH 077/247] Run Python 3.13 tests in CI (with failures allowed) (#17484) https://github.com/python/mypy/issues/17264 --- .github/workflows/docs.yml | 2 +- .github/workflows/test.yml | 18 ++++++++++++++++++ test-requirements.in | 2 -- test-requirements.txt | 34 ++++------------------------------ 4 files changed, 23 insertions(+), 33 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f13a3de1f2e3..8bded1d380aa 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -38,7 +38,7 @@ jobs: with: python-version: '3.8' - name: Install tox - run: pip install --upgrade 'setuptools!=50' tox==4.11.0 + run: pip install tox==4.11.0 - name: Setup tox environment run: tox run -e ${{ env.TOXENV }} --notest - name: Test diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 98a737a78b3b..01d5876635b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,6 +147,24 @@ jobs: - name: Test run: tox run -e ${{ matrix.toxenv }} --skip-pkg-install -- ${{ matrix.tox_extra_args }} + python-nightly: + runs-on: ubuntu-latest + name: Test suite with Python nightly + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.13-dev' + - name: Install tox + run: pip install setuptools==68.2.2 tox==4.11.0 + - name: Setup tox environment + run: tox run -e py --notest + - name: Test + run: tox run -e py --skip-pkg-install -- "-n 4" + continue-on-error: true + - name: Mark as a success + run: exit 0 + python_32bits: runs-on: ubuntu-latest name: Test mypyc suite with 32-bit Python diff --git a/test-requirements.in b/test-requirements.in index 637f5b948055..5a888811bfcd 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -8,8 +8,6 @@ black==24.3.0 # must match version in .pre-commit-config.yaml filelock>=3.3.0 # lxml 4.9.3 switched to manylinux_2_28, the wheel builder still uses manylinux2014 lxml>=4.9.1,<4.9.3; (python_version<'3.11' or sys_platform!='win32') and python_version<'3.12' -pre-commit -pre-commit-hooks==4.5.0 psutil>=4.0 pytest>=8.1.0 pytest-xdist>=1.34.0 diff --git a/test-requirements.txt b/test-requirements.txt index 9005daab2876..75a970c5bf0e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,22 +8,14 @@ attrs==23.1.0 # via -r test-requirements.in black==24.3.0 # via -r test-requirements.in -cfgv==3.4.0 - # via pre-commit click==8.1.7 # via black coverage==7.3.2 # via pytest-cov -distlib==0.3.7 - # via virtualenv execnet==2.0.2 # via pytest-xdist filelock==3.12.4 - # via - # -r test-requirements.in - # virtualenv -identify==2.5.30 - # via pre-commit + # via -r test-requirements.in iniconfig==2.0.0 # via pytest lxml==4.9.2 ; (python_version < "3.11" or sys_platform != "win32") and python_version < "3.12" @@ -32,8 +24,6 @@ mypy-extensions==1.0.0 # via # -r mypy-requirements.txt # black -nodeenv==1.8.0 - # via pre-commit packaging==23.2 # via # black @@ -41,15 +31,9 @@ packaging==23.2 pathspec==0.11.2 # via black platformdirs==3.11.0 - # via - # black - # virtualenv + # via black pluggy==1.4.0 # via pytest -pre-commit==3.5.0 - # via -r test-requirements.in -pre-commit-hooks==4.5.0 - # via -r test-requirements.in psutil==5.9.6 # via -r test-requirements.in pytest==8.1.1 @@ -61,12 +45,6 @@ pytest-cov==4.1.0 # via -r test-requirements.in pytest-xdist==3.3.1 # via -r test-requirements.in -pyyaml==6.0.1 - # via pre-commit -ruamel-yaml==0.17.40 - # via pre-commit-hooks -ruamel-yaml-clib==0.2.8 - # via ruamel-yaml ruff==0.2.0 # via -r test-requirements.in tomli==2.0.1 @@ -75,13 +53,9 @@ types-psutil==5.9.5.17 # via -r build-requirements.txt types-setuptools==68.2.0.0 # via -r build-requirements.txt -typing-extensions==4.8.0 +typing-extensions==4.12.2 # via -r mypy-requirements.txt -virtualenv==20.24.5 - # via pre-commit # The following packages are considered to be unsafe in a requirements file: setuptools==68.2.2 - # via - # -r test-requirements.in - # nodeenv + # via -r test-requirements.in From 4ccf216492392ef673f4847ed8779fb54e826bd7 Mon Sep 17 00:00:00 2001 From: InSync Date: Sun, 7 Jul 2024 03:49:05 +0700 Subject: [PATCH 078/247] Fix cross-variable type-narrowing example (#17488) From [Type narrowing § Limitations](https://github.com/python/mypy/blob/606971807fad1de26ebc575d327d4c1c33f71c0e/docs/source/type_narrowing.rst#limitations): ```python def f(a: str | None, b: str | None) -> str: if a is not None or b is not None: return a or b # Incompatible return value type (got "str | None", expected "str") return 'spam' ``` A trivial counter-example is `f('', None)`, which returns `None`. Ironically, this somewhat makes Mypy's diagnostic "correct". I propose that `str` be replaced with a custom class `C` whose `__bool__()` is not defined (does it have to be `@final` too?): ```python class C: pass def f(a: C | None, b: C | None) -> C: if a is not None or b is not None: return a or b # Incompatible return value type (got "C | None", expected "C") return C() ``` --- docs/source/type_narrowing.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/source/type_narrowing.rst b/docs/source/type_narrowing.rst index 4c5c2851edd0..d698f35c44bc 100644 --- a/docs/source/type_narrowing.rst +++ b/docs/source/type_narrowing.rst @@ -368,14 +368,18 @@ Limitations Mypy's analysis is limited to individual symbols and it will not track relationships between symbols. For example, in the following code it's easy to deduce that if :code:`a` is None then :code:`b` must not be, -therefore :code:`a or b` will always be a string, but Mypy will not be able to tell that: +therefore :code:`a or b` will always be an instance of :code:`C`, +but Mypy will not be able to tell that: .. code-block:: python - def f(a: str | None, b: str | None) -> str: + class C: + pass + + def f(a: C | None, b: C | None) -> C: if a is not None or b is not None: - return a or b # Incompatible return value type (got "str | None", expected "str") - return 'spam' + return a or b # Incompatible return value type (got "C | None", expected "C") + return C() Tracking these sort of cross-variable conditions in a type checker would add significant complexity and performance overhead. @@ -385,9 +389,9 @@ or rewrite the function to be slightly more verbose: .. code-block:: python - def f(a: str | None, b: str | None) -> str: + def f(a: C | None, b: C | None) -> C: if a is not None: return a elif b is not None: return b - return 'spam' + return C() From e5b3b563ea3f6d1f83d0fe3552965c8b05ea4c3d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Jul 2024 22:04:07 +0100 Subject: [PATCH 079/247] Fix daemon crash on invalid type in TypedDict (#17495) Fixes https://github.com/python/mypy/issues/10007 Fixes https://github.com/python/mypy/issues/17477 This fixes the crash as proposed in https://github.com/python/mypy/pull/13732, but also fixes some inconsistencies in `Any` types exposed by the fix. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/semanal.py | 21 +++++++++++++++++ mypy/semanal_typeddict.py | 6 +++-- mypy/stats.py | 4 ++++ mypy/types.py | 5 +++- test-data/unit/check-flags.test | 4 ++-- test-data/unit/check-semanal-error.test | 31 ++++++++++++++++++++++++- test-data/unit/check-typeddict.test | 20 ++++++++++++++++ test-data/unit/reports.test | 16 ++++++------- test-data/unit/semanal-typeddict.test | 2 +- 9 files changed, 94 insertions(+), 15 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index f857c3e73381..f36149076fe6 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3935,6 +3935,9 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: # When this type alias gets "inlined", the Any is not explicit anymore, # so we need to replace it with non-explicit Anys. res = make_any_non_explicit(res) + if self.options.disallow_any_unimported and has_any_from_unimported_type(res): + self.msg.unimported_type_becomes_any("Type alias target", res, s) + res = make_any_non_unimported(res) # Note: with the new (lazy) type alias representation we only need to set no_args to True # if the expected number of arguments is non-zero, so that aliases like `A = List` work # but not aliases like `A = TypeAliasType("A", List)` as these need explicit type params. @@ -5407,6 +5410,9 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: # When this type alias gets "inlined", the Any is not explicit anymore, # so we need to replace it with non-explicit Anys. res = make_any_non_explicit(res) + if self.options.disallow_any_unimported and has_any_from_unimported_type(res): + self.msg.unimported_type_becomes_any("Type alias target", res, s) + res = make_any_non_unimported(res) eager = self.is_func_scope() if isinstance(res, ProperType) and isinstance(res, Instance) and not res.args: fix_instance(res, self.fail, self.note, disallow_any=False, options=self.options) @@ -7433,6 +7439,21 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: return t.copy_modified(args=[a.accept(self) for a in t.args]) +def make_any_non_unimported(t: Type) -> Type: + """Replace all Any types that come from unimported types with special form Any.""" + return t.accept(MakeAnyNonUnimported()) + + +class MakeAnyNonUnimported(TrivialSyntheticTypeTranslator): + def visit_any(self, t: AnyType) -> Type: + if t.type_of_any == TypeOfAny.from_unimported_type: + return t.copy_modified(TypeOfAny.special_form, missing_import_name=None) + return t + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + return t.copy_modified(args=[a.accept(self) for a in t.args]) + + def apply_semantic_analyzer_patches(patches: list[tuple[int, Callable[[], None]]]) -> None: """Call patch callbacks in the right order. diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index eee98d4d20fa..7b8d874337a2 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -310,11 +310,11 @@ def analyze_typeddict_classdef_fields( # Append stmt, name, and type in this case... fields.append(name) statements.append(stmt) - if stmt.type is None: + if stmt.unanalyzed_type is None: types.append(AnyType(TypeOfAny.unannotated)) else: analyzed = self.api.anal_type( - stmt.type, + stmt.unanalyzed_type, allow_required=True, allow_placeholder=not self.api.is_func_scope(), prohibit_self_type="TypedDict item type", @@ -322,6 +322,8 @@ def analyze_typeddict_classdef_fields( if analyzed is None: return None, [], [], set() # Need to defer types.append(analyzed) + if not has_placeholder(analyzed): + stmt.type = analyzed # ...despite possible minor failures that allow further analysis. if stmt.type is None or hasattr(stmt, "new_syntax") and not stmt.new_syntax: self.fail(TPDICT_CLASS_ERROR, stmt) diff --git a/mypy/stats.py b/mypy/stats.py index b167a41b0e34..9c69a245741b 100644 --- a/mypy/stats.py +++ b/mypy/stats.py @@ -203,7 +203,11 @@ def visit_assignment_stmt(self, o: AssignmentStmt) -> None: # Type variable definition -- not a real assignment. return if o.type: + # If there is an explicit type, don't visit the l.h.s. as an expression + # to avoid double-counting and mishandling special forms. self.type(o.type) + o.rvalue.accept(self) + return elif self.inferred and not self.all_nodes: # if self.all_nodes is set, lvalues will be visited later for lvalue in o.lvalues: diff --git a/mypy/types.py b/mypy/types.py index 2e7cbfd4e733..89609e8d0546 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1120,15 +1120,18 @@ def copy_modified( # Mark with Bogus because _dummy is just an object (with type Any) type_of_any: int = _dummy_int, original_any: Bogus[AnyType | None] = _dummy, + missing_import_name: Bogus[str | None] = _dummy, ) -> AnyType: if type_of_any == _dummy_int: type_of_any = self.type_of_any if original_any is _dummy: original_any = self.source_any + if missing_import_name is _dummy: + missing_import_name = self.missing_import_name return AnyType( type_of_any=type_of_any, source_any=original_any, - missing_import_name=self.missing_import_name, + missing_import_name=missing_import_name, line=self.line, column=self.column, ) diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index 62711d5f0071..4f327a2f0edc 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -924,9 +924,9 @@ class A(List[Unchecked]): # E: Base type becomes "List[Any]" due to an unfollowe from missing import Unchecked from typing import List -X = List[Unchecked] +X = List[Unchecked] # E: Type alias target becomes "List[Any]" due to an unfollowed import -def f(x: X) -> None: # E: Argument 1 to "f" becomes "List[Any]" due to an unfollowed import +def f(x: X) -> None: pass [builtins fixtures/list.pyi] diff --git a/test-data/unit/check-semanal-error.test b/test-data/unit/check-semanal-error.test index c6cf45d96691..d7ab272aed6c 100644 --- a/test-data/unit/check-semanal-error.test +++ b/test-data/unit/check-semanal-error.test @@ -151,4 +151,33 @@ class C: x: P[int] = C() [builtins fixtures/tuple.pyi] -[out] + +[case testSemanalDoesNotLeakSyntheticTypes] +# flags: --cache-fine-grained +from typing import Generic, NamedTuple, TypedDict, TypeVar +from dataclasses import dataclass + +T = TypeVar('T') +class Wrap(Generic[T]): pass + +invalid_1: 1 + 2 # E: Invalid type comment or annotation +invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +class A: + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +class B(NamedTuple): + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +class C(TypedDict): + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +@dataclass +class D: + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index fa77d98e4a34..d35ec8ddd80e 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -2362,6 +2362,26 @@ Foo = TypedDict('Foo', {'camelCaseKey': str}) value: Foo = {} # E: Missing key "camelCaseKey" for TypedDict "Foo" [builtins fixtures/dict.pyi] +[case testTypedDictWithDeferredFieldTypeEval] +from typing import Generic, TypeVar, TypedDict, NotRequired + +class Foo(TypedDict): + y: NotRequired[int] + x: Outer[Inner[ForceDeferredEval]] + +var: Foo +reveal_type(var) # N: Revealed type is "TypedDict('__main__.Foo', {'y'?: builtins.int, 'x': __main__.Outer[__main__.Inner[__main__.ForceDeferredEval]]})" + +T1 = TypeVar("T1") +class Outer(Generic[T1]): pass + +T2 = TypeVar("T2", bound="ForceDeferredEval") +class Inner(Generic[T2]): pass + +class ForceDeferredEval: pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + -- Required[] [case testDoesRecognizeRequiredInTypedDictWithClass] diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 16061d9c32bf..81e24240af2d 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -81,19 +81,19 @@ def foo(a: int) -> MyDict: return {"a": a} md: MyDict = MyDict(**foo(42)) [outfile build/cobertura.xml] - + $PWD - + - + - + @@ -155,9 +155,9 @@ z: NestedGen[Any] [outfile report/types-of-anys.txt] Name Unannotated Explicit Unimported Omitted Generics Error Special Form Implementation Artifact ----------------------------------------------------------------------------------------------------------------- - n 0 4 0 8 0 0 0 + n 0 2 0 8 0 0 0 ----------------------------------------------------------------------------------------------------------------- -Total 0 4 0 8 0 0 0 +Total 0 2 0 8 0 0 0 [case testTypeVarTreatedAsEmptyLine] # cmd: mypy --html-report report n.py @@ -371,9 +371,9 @@ z = g.does_not_exist() # type: ignore # Error [outfile report/types-of-anys.txt] Name Unannotated Explicit Unimported Omitted Generics Error Special Form Implementation Artifact ----------------------------------------------------------------------------------------------------------------- - n 2 4 2 1 3 0 0 + n 2 3 1 1 3 0 0 ----------------------------------------------------------------------------------------------------------------- -Total 2 4 2 1 3 0 0 +Total 2 3 1 1 3 0 0 [case testAnyExpressionsReportUnqualifiedError] # cmd: mypy --any-exprs-report report n.py diff --git a/test-data/unit/semanal-typeddict.test b/test-data/unit/semanal-typeddict.test index b9eb6e0c2b13..9ce89155c308 100644 --- a/test-data/unit/semanal-typeddict.test +++ b/test-data/unit/semanal-typeddict.test @@ -42,4 +42,4 @@ MypyFile:1( NameExpr(x) TempNode:4( Any) - str?))) + builtins.str))) From 1acdfd073010a13a7e3b982097a9a1a7a5d9a544 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Jul 2024 22:38:54 +0100 Subject: [PATCH 080/247] Fix crash on NamedTuple with method and error in function (#17498) Fixes https://github.com/python/mypy/issues/16814 This one is tricky and may expose some other bugs. But IMO this is strictly correct thing to do. --- mypy/semanal_main.py | 2 ++ test-data/unit/check-namedtuple.test | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/mypy/semanal_main.py b/mypy/semanal_main.py index 1185a3821553..09a1223be6aa 100644 --- a/mypy/semanal_main.py +++ b/mypy/semanal_main.py @@ -291,6 +291,8 @@ def process_top_level_function( deferred, incomplete, progress = semantic_analyze_target( target, module, state, node, active_type, final_iteration, patches ) + if not incomplete: + state.manager.incomplete_namespaces.discard(module) if final_iteration: assert not deferred, "Must not defer during final iteration" if not progress: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index e9d156754d9c..147270dff72e 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1423,3 +1423,27 @@ class Foo(typing.NamedTuple): reveal_type(x) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] + +[case testNameErrorInNamedTupleNestedInFunction1] +from typing import NamedTuple + +def bar() -> None: + class MyNamedTuple(NamedTuple): + a: int + def foo(self) -> None: + ... + int_set: Set[int] # E: Name "Set" is not defined \ + # N: Did you forget to import it from "typing"? (Suggestion: "from typing import Set") +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNameErrorInNamedTupleNestedInFunction2] +from typing import NamedTuple + +def bar() -> None: + class MyNamedTuple(NamedTuple): + a: int + def foo(self) -> None: + misspelled_var_name # E: Name "misspelled_var_name" is not defined +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] From 9c0a6f9ba355b307cd87c861619040e4b4691af4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Jul 2024 23:57:05 +0100 Subject: [PATCH 081/247] Fix crash on self-type in callable protocol (#17499) Fixes https://github.com/python/mypy/issues/16629 This is really ad-hoc, but a proper fix would be much more hard, and this currently we have a crash in a relatively common scenario. --- mypy/solve.py | 12 ++++++++++++ test-data/unit/check-selftype.test | 28 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/mypy/solve.py b/mypy/solve.py index bb87b6576ada..8a1495a9a246 100644 --- a/mypy/solve.py +++ b/mypy/solve.py @@ -553,6 +553,11 @@ def pre_validate_solutions( """ new_solutions: list[Type | None] = [] for t, s in zip(original_vars, solutions): + if is_callable_protocol(t.upper_bound): + # This is really ad-hoc, but a proper fix would be much more complex, + # and otherwise this may cause crash in a relatively common scenario. + new_solutions.append(s) + continue if s is not None and not is_subtype(s, t.upper_bound): bound_satisfies_all = True for c in constraints: @@ -567,3 +572,10 @@ def pre_validate_solutions( continue new_solutions.append(s) return new_solutions + + +def is_callable_protocol(t: Type) -> bool: + proper_t = get_proper_type(t) + if isinstance(proper_t, Instance) and proper_t.type.is_protocol: + return "__call__" in proper_t.type.protocol_members + return False diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 1480c83b2272..9601852ef823 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2132,3 +2132,31 @@ class D: x: int x: Union[C, D] reveal_type(x.x) # N: Revealed type is "Union[__main__.C, builtins.int]" + +[case testCallableProtocolTypingSelf] +from typing import Protocol, Self + +class MyProtocol(Protocol): + __name__: str + + def __call__( + self: Self, + ) -> None: ... + +def test() -> None: ... +value: MyProtocol = test + +[case testCallableProtocolOldSelf] +from typing import Protocol, TypeVar + +Self = TypeVar("Self", bound="MyProtocol") + +class MyProtocol(Protocol): + __name__: str + + def __call__( + self: Self, + ) -> None: ... + +def test() -> None: ... +value: MyProtocol = test From 6d45f3cb58ce4781b7e7f47469358af9441ea48b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 7 Jul 2024 10:59:20 +0100 Subject: [PATCH 082/247] Experimental: allow inline/anonymous TypedDicts (#17457) Fixes https://github.com/python/mypy/issues/9884 I was always a bit skeptical about this thing, since it feels more like TypeScript than Python, but it is second most upvoted issue. Also (this specific) implementation is like 60 lines of code plus tests, so why not. I know there is no PEP etc., but IMO this syntax is obvious and it just works. cc @JukkaL --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/source/command_line.rst | 21 +++++++++++ docs/source/typed_dict.rst | 38 +++++++++++++++++++ mypy/checker.py | 6 ++- mypy/exprtotype.py | 31 ++++++++++++++- mypy/fastparse.py | 20 +++++++++- mypy/message_registry.py | 1 + mypy/nodes.py | 1 + mypy/options.py | 3 +- mypy/semanal_typeddict.py | 15 ++------ mypy/typeanal.py | 55 +++++++++++++++++++++++---- mypy/types.py | 4 +- test-data/unit/check-literal.test | 19 +++++----- test-data/unit/check-python312.test | 15 +++++++- test-data/unit/check-typeddict.test | 58 ++++++++++++++++++++++++++++- 14 files changed, 247 insertions(+), 40 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 50a6ef65f4d0..906231dc7e42 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1055,6 +1055,27 @@ List of currently incomplete/experimental features: # Without PreciseTupleTypes: tuple[int, ...] # With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int] +* ``NewGenericSyntax``: this feature enables support for syntax defined + by :pep:`695`. For example: + + .. code-block:: python + + class Container[T]: # defines a generic class + content: T + + def first[T](items: list[T]) -> T: # defines a generic function + return items[0] + + type Items[T] = list[tuple[T, T]] # defines a generic type alias + +* ``InlineTypedDict``: this feature enables non-standard syntax for inline + :ref:`TypedDicts `, for example: + + .. code-block:: python + + def test_values() -> {"int": int, "str": str}: + return {"int": 42, "str": "test"} + Miscellaneous ************* diff --git a/docs/source/typed_dict.rst b/docs/source/typed_dict.rst index e5ce2927db4d..c379b5449eae 100644 --- a/docs/source/typed_dict.rst +++ b/docs/source/typed_dict.rst @@ -248,3 +248,41 @@ section of the docs has a full description with an example, but in short, you wi need to give each TypedDict the same key where each value has a unique :ref:`Literal type `. Then, check that key to distinguish between your TypedDicts. + +Inline TypedDict types +---------------------- + +.. note:: + + This is an experimental (non-standard) feature. Use + ``--enable-incomplete-feature=InlineTypedDict`` to enable. + +Sometimes you may want to define a complex nested JSON schema, or annotate +a one-off function that returns a TypedDict. In such cases it may be convenient +to use inline TypedDict syntax. For example: + +.. code-block:: python + + def test_values() -> {"int": int, "str": str}: + return {"int": 42, "str": "test"} + + class Response(TypedDict): + status: int + msg: str + # Using inline syntax here avoids defining two additional TypedDicts. + content: {"items": list[{"key": str, "value": str}]} + +Inline TypedDicts can also by used as targets of type aliases, but due to +ambiguity with a regular variables it is only allowed for (newer) explicit +type alias forms: + +.. code-block:: python + + from typing import TypeAlias + + X = {"a": int, "b": int} # creates a variable with type dict[str, type[int]] + Y: TypeAlias = {"a": int, "b": int} # creates a type alias + type Z = {"a": int, "b": int} # same as above (Python 3.12+ only) + +Also, due to incompatibility with runtime type-checking it is strongly recommended +to *not* use inline syntax in union types. diff --git a/mypy/checker.py b/mypy/checker.py index 2df74cf7be8d..0ae499916ec6 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2971,7 +2971,8 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.msg.annotation_in_unchecked_function(context=s) def check_type_alias_rvalue(self, s: AssignmentStmt) -> None: - alias_type = self.expr_checker.accept(s.rvalue) + with self.msg.filter_errors(): + alias_type = self.expr_checker.accept(s.rvalue) self.store_type(s.lvalues[-1], alias_type) def check_assignment( @@ -5311,7 +5312,8 @@ def remove_capture_conflicts(self, type_map: TypeMap, inferred_types: dict[Var, del type_map[expr] def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: - self.expr_checker.accept(o.value) + with self.msg.filter_errors(): + self.expr_checker.accept(o.value) def make_fake_typeinfo( self, diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index d9bdf2e2b20b..92316d11926d 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -4,9 +4,11 @@ from mypy.fastparse import parse_type_string from mypy.nodes import ( + MISSING_FALLBACK, BytesExpr, CallExpr, ComplexExpr, + DictExpr, EllipsisExpr, Expression, FloatExpr, @@ -29,9 +31,11 @@ AnyType, CallableArgument, EllipsisType, + Instance, ProperType, RawExpressionType, Type, + TypedDictType, TypeList, TypeOfAny, UnboundType, @@ -55,7 +59,7 @@ def _extract_argument_name(expr: Expression) -> str | None: def expr_to_unanalyzed_type( expr: Expression, - options: Options | None = None, + options: Options, allow_new_syntax: bool = False, _parent: Expression | None = None, allow_unpack: bool = False, @@ -67,6 +71,8 @@ def expr_to_unanalyzed_type( If allow_new_syntax is True, allow all type syntax independent of the target Python version (used in stubs). + + # TODO: a lot of code here is duplicated in fastparse.py, refactor this. """ # The `parent` parameter is used in recursive calls to provide context for # understanding whether an CallableArgument is ok. @@ -116,7 +122,7 @@ def expr_to_unanalyzed_type( elif ( isinstance(expr, OpExpr) and expr.op == "|" - and ((options and options.python_version >= (3, 10)) or allow_new_syntax) + and ((options.python_version >= (3, 10)) or allow_new_syntax) ): return UnionType( [ @@ -206,5 +212,26 @@ def expr_to_unanalyzed_type( return UnpackType( expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax), from_star_syntax=True ) + elif isinstance(expr, DictExpr): + if not expr.items: + raise TypeTranslationError() + items: dict[str, Type] = {} + extra_items_from = [] + for item_name, value in expr.items: + if not isinstance(item_name, StrExpr): + if item_name is None: + extra_items_from.append( + expr_to_unanalyzed_type(value, options, allow_new_syntax, expr) + ) + continue + raise TypeTranslationError() + items[item_name.value] = expr_to_unanalyzed_type( + value, options, allow_new_syntax, expr + ) + result = TypedDictType( + items, set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column + ) + result.extra_items_from = extra_items_from + return result else: raise TypeTranslationError() diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 01f6ed4733ae..75c4bd46550c 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -17,6 +17,7 @@ ARG_POS, ARG_STAR, ARG_STAR2, + MISSING_FALLBACK, PARAM_SPEC_KIND, TYPE_VAR_KIND, TYPE_VAR_TUPLE_KIND, @@ -42,7 +43,6 @@ EllipsisExpr, Expression, ExpressionStmt, - FakeInfo, FloatExpr, ForStmt, FuncDef, @@ -116,6 +116,7 @@ RawExpressionType, TupleType, Type, + TypedDictType, TypeList, TypeOfAny, UnboundType, @@ -190,7 +191,6 @@ def ast3_parse( # There is no way to create reasonable fallbacks at this stage, # they must be patched later. -MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal") _dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1) TYPE_IGNORE_PATTERN: Final = re.compile(r"[^#]*#\s*type:\s*ignore\s*(.*)") @@ -2106,6 +2106,22 @@ def visit_Tuple(self, n: ast3.Tuple) -> Type: column=self.convert_column(n.col_offset), ) + def visit_Dict(self, n: ast3.Dict) -> Type: + if not n.keys: + return self.invalid_type(n) + items: dict[str, Type] = {} + extra_items_from = [] + for item_name, value in zip(n.keys, n.values): + if not isinstance(item_name, ast3.Constant) or not isinstance(item_name.value, str): + if item_name is None: + extra_items_from.append(self.visit(value)) + continue + return self.invalid_type(n) + items[item_name.value] = self.visit(value) + result = TypedDictType(items, set(), _dummy_fallback, n.lineno, n.col_offset) + result.extra_items_from = extra_items_from + return result + # Attribute(expr value, identifier attr, expr_context ctx) def visit_Attribute(self, n: Attribute) -> Type: before_dot = self.visit(n.value) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index befacc9e6182..06199e70d6b4 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -138,6 +138,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage: TYPEDDICT_KEY_MUST_BE_STRING_LITERAL: Final = ErrorMessage( "Expected TypedDict key to be string literal" ) +TYPEDDICT_OVERRIDE_MERGE: Final = 'Overwriting TypedDict field "{}" while merging' MALFORMED_ASSERT: Final = ErrorMessage("Assertion is always true, perhaps remove parentheses?") DUPLICATE_TYPE_SIGNATURES: Final = ErrorMessage("Function has duplicate type signatures") DESCRIPTOR_SET_NOT_CALLABLE: Final = ErrorMessage("{}.__set__ is not callable") diff --git a/mypy/nodes.py b/mypy/nodes.py index 2eb39d4baaf6..4a5c7240fa83 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3480,6 +3480,7 @@ def __getattribute__(self, attr: str) -> type: VAR_NO_INFO: Final[TypeInfo] = FakeInfo("Var is lacking info") CLASSDEF_NO_INFO: Final[TypeInfo] = FakeInfo("ClassDef is lacking info") FUNC_NO_INFO: Final[TypeInfo] = FakeInfo("FuncBase for non-methods lack info") +MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal") class TypeAlias(SymbolNode): diff --git a/mypy/options.py b/mypy/options.py index 5ef6bc2a35e7..bff096d82c15 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -74,7 +74,8 @@ class BuildType: UNPACK: Final = "Unpack" PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax" -INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX)) +INLINE_TYPEDDICT: Final = "InlineTypedDict" +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX, INLINE_TYPEDDICT)) COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK)) diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 7b8d874337a2..e639871364ce 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -8,6 +8,7 @@ from mypy.errorcodes import ErrorCode from mypy.expandtype import expand_type from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type +from mypy.message_registry import TYPEDDICT_OVERRIDE_MERGE from mypy.messages import MessageBuilder from mypy.nodes import ( ARG_NAMED, @@ -216,7 +217,7 @@ def add_keys_and_types_from_base( valid_items = self.map_items_to_base(valid_items, tvars, base_args) for key in base_items: if key in keys: - self.fail(f'Overwriting TypedDict field "{key}" while merging', ctx) + self.fail(TYPEDDICT_OVERRIDE_MERGE.format(key), ctx) keys.extend(valid_items.keys()) types.extend(valid_items.values()) required_keys.update(base_typed_dict.required_keys) @@ -507,17 +508,7 @@ def parse_typeddict_fields_with_types( field_type_expr, self.options, self.api.is_stub_file ) except TypeTranslationError: - if ( - isinstance(field_type_expr, CallExpr) - and isinstance(field_type_expr.callee, RefExpr) - and field_type_expr.callee.fullname in TPDICT_NAMES - ): - self.fail_typeddict_arg( - "Inline TypedDict types not supported; use assignment to define TypedDict", - field_type_expr, - ) - else: - self.fail_typeddict_arg("Invalid field type", field_type_expr) + self.fail_typeddict_arg("Use dict literal for nested TypedDict", field_type_expr) return [], [], False analyzed = self.api.anal_type( type, diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6651af7dad4f..f63aef30a09a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -10,7 +10,11 @@ from mypy import errorcodes as codes, message_registry, nodes from mypy.errorcodes import ErrorCode from mypy.expandtype import expand_type -from mypy.message_registry import INVALID_PARAM_SPEC_LOCATION, INVALID_PARAM_SPEC_LOCATION_NOTE +from mypy.message_registry import ( + INVALID_PARAM_SPEC_LOCATION, + INVALID_PARAM_SPEC_LOCATION_NOTE, + TYPEDDICT_OVERRIDE_MERGE, +) from mypy.messages import ( MessageBuilder, format_type, @@ -25,6 +29,7 @@ ARG_POS, ARG_STAR, ARG_STAR2, + MISSING_FALLBACK, SYMBOL_FUNCBASE_TYPES, ArgKind, Context, @@ -43,7 +48,7 @@ check_arg_names, get_nongen_builtins, ) -from mypy.options import Options +from mypy.options import INLINE_TYPEDDICT, Options from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface from mypy.semanal_shared import ( SemanticAnalyzerCoreInterface, @@ -1220,10 +1225,45 @@ def visit_tuple_type(self, t: TupleType) -> Type: return TupleType(self.anal_array(t.items, allow_unpack=True), fallback, t.line) def visit_typeddict_type(self, t: TypedDictType) -> Type: - items = { - item_name: self.anal_type(item_type) for (item_name, item_type) in t.items.items() - } - return TypedDictType(items, set(t.required_keys), t.fallback) + req_keys = set() + items = {} + for item_name, item_type in t.items.items(): + analyzed = self.anal_type(item_type, allow_required=True) + if isinstance(analyzed, RequiredType): + if analyzed.required: + req_keys.add(item_name) + analyzed = analyzed.item + else: + # Keys are required by default. + req_keys.add(item_name) + items[item_name] = analyzed + if t.fallback.type is MISSING_FALLBACK: # anonymous/inline TypedDict + if INLINE_TYPEDDICT not in self.options.enable_incomplete_feature: + self.fail( + "Inline TypedDict is experimental," + " must be enabled with --enable-incomplete-feature=InlineTypedDict", + t, + ) + required_keys = req_keys + fallback = self.named_type("typing._TypedDict") + for typ in t.extra_items_from: + analyzed = self.analyze_type(typ) + p_analyzed = get_proper_type(analyzed) + if not isinstance(p_analyzed, TypedDictType): + if not isinstance(p_analyzed, (AnyType, PlaceholderType)): + self.fail("Can only merge-in other TypedDict", t, code=codes.VALID_TYPE) + continue + for sub_item_name, sub_item_type in p_analyzed.items.items(): + if sub_item_name in items: + self.fail(TYPEDDICT_OVERRIDE_MERGE.format(sub_item_name), t) + continue + items[sub_item_name] = sub_item_type + if sub_item_name in p_analyzed.required_keys: + req_keys.add(sub_item_name) + else: + required_keys = t.required_keys + fallback = t.fallback + return TypedDictType(items, required_keys, fallback, t.line, t.column) def visit_raw_expression_type(self, t: RawExpressionType) -> Type: # We should never see a bare Literal. We synthesize these raw literals @@ -1761,11 +1801,12 @@ def anal_type( allow_param_spec: bool = False, allow_unpack: bool = False, allow_ellipsis: bool = False, + allow_required: bool = False, ) -> Type: if nested: self.nesting_level += 1 old_allow_required = self.allow_required - self.allow_required = False + self.allow_required = allow_required old_allow_ellipsis = self.allow_ellipsis self.allow_ellipsis = allow_ellipsis old_allow_unpack = self.allow_unpack diff --git a/mypy/types.py b/mypy/types.py index 89609e8d0546..91b40536f1cf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2521,11 +2521,12 @@ class TypedDictType(ProperType): TODO: The fallback structure is perhaps overly complicated. """ - __slots__ = ("items", "required_keys", "fallback") + __slots__ = ("items", "required_keys", "fallback", "extra_items_from") items: dict[str, Type] # item_name -> item_type required_keys: set[str] fallback: Instance + extra_items_from: list[ProperType] # only used during semantic analysis def __init__( self, @@ -2541,6 +2542,7 @@ def __init__( self.fallback = fallback self.can_be_true = len(self.items) > 0 self.can_be_false = len(self.required_keys) == 0 + self.extra_items_from = [] def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_typeddict_type(self) diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index 8f8aaf6a3982..6d76ce176aaf 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -608,36 +608,35 @@ e: Literal[dummy()] # E: Invalid type: Literal[...] cannot contain a [case testLiteralDisallowCollections] from typing_extensions import Literal -a: Literal[{"a": 1, "b": 2}] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +a: Literal[{"a": 1, "b": 2}] # E: Parameter 1 of Literal[...] is invalid b: Literal[{1, 2, 3}] # E: Invalid type: Literal[...] cannot contain arbitrary expressions -c: {"a": 1, "b": 2} # E: Invalid type comment or annotation +c: {"a": 1, "b": 2} # E: Inline TypedDict is experimental, must be enabled with --enable-incomplete-feature=InlineTypedDict \ + # E: Invalid type: try using Literal[1] instead? \ + # E: Invalid type: try using Literal[2] instead? d: {1, 2, 3} # E: Invalid type comment or annotation [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testLiteralDisallowCollections2] - from typing_extensions import Literal a: (1, 2, 3) # E: Syntax error in type annotation \ # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) b: Literal[[1, 2, 3]] # E: Parameter 1 of Literal[...] is invalid c: [1, 2, 3] # E: Bracketed expression "[...]" is not valid as a type [builtins fixtures/tuple.pyi] -[out] [case testLiteralDisallowCollectionsTypeAlias] - from typing_extensions import Literal -at = Literal[{"a": 1, "b": 2}] # E: Invalid type alias: expression is not a valid type +at = Literal[{"a": 1, "b": 2}] # E: Parameter 1 of Literal[...] is invalid bt = {"a": 1, "b": 2} -a: at # E: Variable "__main__.at" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +a: at +reveal_type(a) # N: Revealed type is "Any" b: bt # E: Variable "__main__.bt" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases [builtins fixtures/dict.pyi] -[out] +[typing fixtures/typing-typeddict.pyi] [case testLiteralDisallowCollectionsTypeAlias2] - from typing_extensions import Literal at = Literal[{1, 2, 3}] # E: Invalid type alias: expression is not a valid type bt = {1, 2, 3} diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 5307f47d539a..073ef7f4bdec 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -585,8 +585,7 @@ reveal_type(a) # N: Revealed type is "Any" [case testPEP695TypeAliasInvalidType] # flags: --enable-incomplete-feature=NewGenericSyntax -type A = int | 1 # E: Invalid type: try using Literal[1] instead? \ - # E: Unsupported operand types for | ("Type[int]" and "int") +type A = int | 1 # E: Invalid type: try using Literal[1] instead? a: A reveal_type(a) # N: Revealed type is "Union[builtins.int, Any]" @@ -1656,3 +1655,15 @@ type I2 = C[Any] | None type I3 = None | C[TD] [builtins fixtures/type.pyi] [typing fixtures/typing-full.pyi] + +[case testTypedDictInlineYesNewStyleAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax --enable-incomplete-feature=InlineTypedDict +type X[T] = {"item": T, "other": X[T] | None} +x: X[str] +reveal_type(x) # N: Revealed type is "TypedDict({'item': builtins.str, 'other': Union[..., None]})" +if x["other"] is not None: + reveal_type(x["other"]["item"]) # N: Revealed type is "builtins.str" + +type Y[T] = {"item": T, **Y[T]} # E: Overwriting TypedDict field "item" while merging +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index d35ec8ddd80e..6a5120159c2d 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -78,7 +78,7 @@ p = Point(x='meaning_of_life', y=1337) # E: Incompatible types (expression has [case testCannotCreateTypedDictInstanceWithInlineTypedDict] from mypy_extensions import TypedDict D = TypedDict('D', { - 'x': TypedDict('E', { # E: Inline TypedDict types not supported; use assignment to define TypedDict + 'x': TypedDict('E', { # E: Use dict literal for nested TypedDict 'y': int }) }) @@ -3570,3 +3570,59 @@ class Test: run(test2, other="yes", **params) run(test2, other=0, **params) # E: Argument "other" to "run" has incompatible type "int"; expected "str" [builtins fixtures/tuple.pyi] + +[case testTypedDictInlineNoOldStyleAlias] +# flags: --enable-incomplete-feature=InlineTypedDict +X = {"int": int, "str": str} +reveal_type(X) # N: Revealed type is "builtins.dict[builtins.str, def () -> builtins.object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineYesMidStyleAlias] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing_extensions import TypeAlias +X: TypeAlias = {"int": int, "str": str} +x: X +reveal_type(x) # N: # N: Revealed type is "TypedDict({'int': builtins.int, 'str': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNoEmpty] +# flags: --enable-incomplete-feature=InlineTypedDict +x: {} # E: Invalid type comment or annotation +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNotRequired] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing import NotRequired + +x: {"one": int, "other": NotRequired[int]} +x = {"one": 1} # OK +y: {"one": int, "other": int} +y = {"one": 1} # E: Expected TypedDict keys ("one", "other") but found only key "one" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNestedSchema] +# flags: --enable-incomplete-feature=InlineTypedDict +def nested() -> {"one": str, "other": {"a": int, "b": int}}: + if bool(): + return {"one": "yes", "other": {"a": 1, "b": 2}} # OK + else: + return {"one": "no", "other": {"a": 1, "b": "2"}} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineMergeAnother] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing import TypeVar +from typing_extensions import TypeAlias + +T = TypeVar("T") +X: TypeAlias = {"item": T} +x: {"a": int, **X[str], "b": int} +reveal_type(x) # N: Revealed type is "TypedDict({'a': builtins.int, 'b': builtins.int, 'item': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] From 4c54801afefa545bee6ee205578f4f53c6afeedb Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 7 Jul 2024 22:00:14 +0200 Subject: [PATCH 083/247] Fix `_PyObject_FastCall` for Python 3.13 (#17502) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_PyObject_FastCall` will be removed in 3.13. It can safely replaced by `PyObject_Vectorcall` (available since `3.9`) / `_PyObject_Vectorcall` (available since `3.8`). https://github.com/python/cpython/issues/106023#issuecomment-1613963489 https://peps.python.org/pep-0590/ Fixes ```cpp /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h: In function ‘update_bases’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:62:20: error: implicit declaration of function ‘_PyObject_FastCall’; did you mean ‘PyObject_Call’? [-Werror=implicit-function-declaration] (diff) 62 | new_base = _PyObject_FastCall(meth, stack, 1); (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:62:18: error: assignment to ‘PyObject *’ {aka ‘struct _object *’} from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion] (diff) 62 | new_base = _PyObject_FastCall(meth, stack, 1); (diff) | ^ (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h: In function ‘init_subclass’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:111:11: error: assignment to ‘PyObject *’ {aka ‘struct _object *’} from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion] (diff) 111 | super = _PyObject_FastCall((PyObject *)&PySuper_Type, args, 2); (diff) | ^ (diff) ``` --- mypyc/lib-rt/pythonsupport.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 85f9ec64ac90..87e034f1fc51 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -59,7 +59,7 @@ update_bases(PyObject *bases) } continue; } - new_base = _PyObject_FastCall(meth, stack, 1); + new_base = _PyObject_Vectorcall(meth, stack, 1, NULL); Py_DECREF(meth); if (!new_base) { goto error; @@ -108,7 +108,7 @@ init_subclass(PyTypeObject *type, PyObject *kwds) PyObject *super, *func, *result; PyObject *args[2] = {(PyObject *)type, (PyObject *)type}; - super = _PyObject_FastCall((PyObject *)&PySuper_Type, args, 2); + super = _PyObject_Vectorcall((PyObject *)&PySuper_Type, args, 2, NULL); if (super == NULL) { return -1; } From d4f7e5cd67c83f08f1e2dab418725351ff15da1b Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 7 Jul 2024 22:06:02 +0200 Subject: [PATCH 084/247] Fix `gen_is_coroutine` for Python 3.13 (#17501) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `_PyInterpreterFrame` struct was changed in https://github.com/python/cpython/pull/105727 to store the code object in `f_executable` instead of `f_code`. Fixes ```cpp /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h: In function ‘_PyGen_GetCode’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:403:17: error: ‘_PyInterpreterFrame’ has no member named ‘f_code’ (diff) 403 | return frame->f_code; (diff) | ^~ (diff) ``` --- mypyc/lib-rt/mypyc_util.h | 3 +++ mypyc/lib-rt/pythonsupport.h | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h index 3c888a581a33..9967f0a13b4f 100644 --- a/mypyc/lib-rt/mypyc_util.h +++ b/mypyc/lib-rt/mypyc_util.h @@ -115,4 +115,7 @@ static inline void CPyLong_SetUnsignedSize(PyLongObject *o, Py_ssize_t n) { #endif +// Are we targeting Python 3.13 or newer? +#define CPY_3_13_FEATURES (PY_VERSION_HEX >= 0x030d0000) + #endif diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 87e034f1fc51..c5423ab0fab5 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -392,7 +392,30 @@ _CPyObject_HasAttrId(PyObject *v, _Py_Identifier *name) { _PyObject_CallMethodIdObjArgs((self), (name), (arg), NULL) #endif -#if CPY_3_12_FEATURES +#if CPY_3_13_FEATURES + +// These are copied from genobject.c in Python 3.13 + +/* Returns a borrowed reference */ +static inline PyCodeObject * +_PyGen_GetCode(PyGenObject *gen) { + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); + return _PyFrame_GetCode(frame); +} + +static int +gen_is_coroutine(PyObject *o) +{ + if (PyGen_CheckExact(o)) { + PyCodeObject *code = _PyGen_GetCode((PyGenObject*)o); + if (code->co_flags & CO_ITERABLE_COROUTINE) { + return 1; + } + } + return 0; +} + +#elif CPY_3_12_FEATURES // These are copied from genobject.c in Python 3.12 From 9175ce59322759d3e02727fd804008ee2234bf65 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 7 Jul 2024 22:10:09 +0200 Subject: [PATCH 085/247] Fix `_PyList_Extend` for Python 3.13 (#17503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace `_PyList_Extend` with `PyList_Extend` from `pythoncapi_compat.h`. https://github.com/python/cpython/issues/111138 https://docs.python.org/dev/c-api/list.html#c.PyList_Extend Fixes ```cpp /home/runner/work/mypy/mypy/mypyc/lib-rt/list_ops.c: In function ‘CPyList_Extend’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/list_ops.c:259:12: error: implicit declaration of function ‘_PyList_Extend’; did you mean ‘CPyList_Extend’? [-Werror=implicit-function-declaration] (diff) 259 | return _PyList_Extend((PyListObject *)o1, o2); (diff) | ^~~~~~~~~~~~~~ (diff) | CPyList_Extend (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/list_ops.c:259:12: error: returning ‘int’ from a function with return type ‘PyObject *’ {aka ‘struct _object *’} makes pointer from integer without a cast [-Werror=int-conversion] (diff) 259 | return _PyList_Extend((PyListObject *)o1, o2); (diff) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c: In function ‘CPyDict_Keys’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c:233:21: error: initialization of ‘PyObject *’ {aka ‘struct _object *’} from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion] (diff) 233 | PyObject *res = _PyList_Extend((PyListObject *)list, view); (diff) | ^~~~~~~~~~~~~~ (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c: In function ‘CPyDict_Values’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c:253:21: error: initialization of ‘PyObject *’ {aka ‘struct _object *’} from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion] (diff) 253 | PyObject *res = _PyList_Extend((PyListObject *)list, view); (diff) | ^~~~~~~~~~~~~~ (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c: In function ‘CPyDict_Items’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c:273:21: error: initialization of ‘PyObject *’ {aka ‘struct _object *’} from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion] (diff) 273 | PyObject *res = _PyList_Extend((PyListObject *)list, view); (diff) | ^~~~~~~~~~~~~~ (diff) ``` --- mypyc/lib-rt/dict_ops.c | 15 ++++++--------- mypyc/lib-rt/list_ops.c | 5 ++++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mypyc/lib-rt/dict_ops.c b/mypyc/lib-rt/dict_ops.c index c0cc8d5a7f87..031df8f63c49 100644 --- a/mypyc/lib-rt/dict_ops.c +++ b/mypyc/lib-rt/dict_ops.c @@ -230,12 +230,11 @@ PyObject *CPyDict_Keys(PyObject *dict) { if (view == NULL) { return NULL; } - PyObject *res = _PyList_Extend((PyListObject *)list, view); + int res = PyList_Extend(list, view); Py_DECREF(view); - if (res == NULL) { + if (res < 0) { return NULL; } - Py_DECREF(res); return list; } @@ -250,12 +249,11 @@ PyObject *CPyDict_Values(PyObject *dict) { if (view == NULL) { return NULL; } - PyObject *res = _PyList_Extend((PyListObject *)list, view); + int res = PyList_Extend(list, view); Py_DECREF(view); - if (res == NULL) { + if (res < 0) { return NULL; } - Py_DECREF(res); return list; } @@ -270,12 +268,11 @@ PyObject *CPyDict_Items(PyObject *dict) { if (view == NULL) { return NULL; } - PyObject *res = _PyList_Extend((PyListObject *)list, view); + int res = PyList_Extend(list, view); Py_DECREF(view); - if (res == NULL) { + if (res < 0) { return NULL; } - Py_DECREF(res); return list; } diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index df87228a0d10..d297ece8f417 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -256,7 +256,10 @@ int CPyList_Insert(PyObject *list, CPyTagged index, PyObject *value) } PyObject *CPyList_Extend(PyObject *o1, PyObject *o2) { - return _PyList_Extend((PyListObject *)o1, o2); + if (PyList_Extend(o1, o2) < 0) { + return NULL; + } + Py_RETURN_NONE; } // Return -2 or error, -1 if not found, or index of first match otherwise. From 7f67090b10694657bc72f39bdeaa3b189c7be7fc Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 7 Jul 2024 23:56:33 +0200 Subject: [PATCH 086/247] Fix `_PyObject_LookupAttrId` for Python 3.13 (#17505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_PyObject_LookupAttrId` was removed / replaced with `PyObject_GetOptionalAttrString` in https://github.com/python/cpython/pull/106522. https://docs.python.org/dev/c-api/object.html#c.PyObject_GetOptionalAttrString Fixes ```cpp /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h: In function ‘update_bases’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:51:13: error: implicit declaration of function ‘_PyObject_LookupAttrId’; did you mean ‘_PyObject_GetAttrId’? [-Werror=implicit-function-declaration] (diff) 51 | if (_PyObject_LookupAttrId(base, &PyId___mro_entries__, &meth) < 0) { (diff) | ^~~~~~~~~~~~~~~~~~~~~~ (diff) | _PyObject_GetAttrId (diff) ``` --- mypyc/lib-rt/pythonsupport.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index c5423ab0fab5..2d18e19b2c56 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -48,7 +48,7 @@ update_bases(PyObject *bases) } continue; } - if (_PyObject_LookupAttrId(base, &PyId___mro_entries__, &meth) < 0) { + if (PyObject_GetOptionalAttrString(base, PyId___mro_entries__.string, &meth) < 0) { goto error; } if (!meth) { @@ -374,7 +374,7 @@ _CPyDictView_New(PyObject *dict, PyTypeObject *type) static int _CPyObject_HasAttrId(PyObject *v, _Py_Identifier *name) { PyObject *tmp = NULL; - int result = _PyObject_LookupAttrId(v, name, &tmp); + int result = PyObject_GetOptionalAttrString(v, name->string, &tmp); if (tmp) { Py_DECREF(tmp); } From 966d6d36595f9b029afb04271dc90fa5fd106005 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 8 Jul 2024 01:00:01 +0200 Subject: [PATCH 087/247] Fix `PyUnicode` functions for Python 3.13 (#17504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace `_PyUnicode_EqualToASCIIString` with `PyUnicode_EqualToUTF8`. https://docs.python.org/dev/c-api/unicode.html#c.PyUnicode_EqualToUTF8 https://github.com/python/cpython/issues/110289 Fixes ```cpp /home/runner/work/mypy/mypy/mypyc/lib-rt/getargs.c: In function ‘vgetargskeywords’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:310:45: error: implicit declaration of function ‘_PyUnicode_EqualToASCIIString’; did you mean ‘CPyUnicode_EqualToASCIIString’? [-Werror=implicit-function-declaration] (diff) 310 | #define CPyUnicode_EqualToASCIIString(x, y) _PyUnicode_EqualToASCIIString(x, y) (diff) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/getargs.c:398:21: note: in expansion of macro ‘CPyUnicode_EqualToASCIIString’ (diff) 398 | if (CPyUnicode_EqualToASCIIString(key, kwlist[i])) { (diff) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ (diff) ``` --- mypyc/lib-rt/getargs.c | 2 +- mypyc/lib-rt/pythonsupport.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/mypyc/lib-rt/getargs.c b/mypyc/lib-rt/getargs.c index 3c8b528f8048..1bc2f5b02ba8 100644 --- a/mypyc/lib-rt/getargs.c +++ b/mypyc/lib-rt/getargs.c @@ -395,7 +395,7 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, goto latefail; } for (i = pos; i < len; i++) { - if (CPyUnicode_EqualToASCIIString(key, kwlist[i])) { + if (PyUnicode_EqualToUTF8(key, kwlist[i])) { match = 1; break; } diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 2d18e19b2c56..85eb6348eb81 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -307,8 +307,6 @@ list_count(PyListObject *self, PyObject *value) return CPyTagged_ShortFromSsize_t(count); } -#define CPyUnicode_EqualToASCIIString(x, y) _PyUnicode_EqualToASCIIString(x, y) - // Adapted from genobject.c in Python 3.7.2 // Copied because it wasn't in 3.5.2 and it is undocumented anyways. /* From 4e3346ee1dee83868adc2411c4a0f4050cf2f95a Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 8 Jul 2024 01:00:31 +0200 Subject: [PATCH 088/247] Add additional includes for Python 3.13 (#17506) Define `Py_BUILD_CORE` required by `internal/...` header files. Include additional headers for moved private functions. ```cpp /opt/hostedtoolcache/Python/3.13.0-beta.3/x64/include/python3.13/internal/pycore_frame.h:8:4: error: #error "this header requires Py_BUILD_CORE define" (diff) 8 | # error "this header requires Py_BUILD_CORE define" (diff) ``` --- mypyc/lib-rt/pythonsupport.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 85eb6348eb81..69ff120dd40d 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -13,6 +13,18 @@ #include #include "mypyc_util.h" +#if CPY_3_13_FEATURES +#ifndef Py_BUILD_CORE +#define Py_BUILD_CORE +#endif +#include "internal/pycore_bytesobject.h" // _PyBytes_Join +#include "internal/pycore_call.h" // _PyObject_CallMethodIdNoArgs, _PyObject_CallMethodIdObjArgs, _PyObject_CallMethodIdOneArg +#include "internal/pycore_genobject.h" // _PyGen_FetchStopIterationValue +#include "internal/pycore_object.h" // _PyType_CalculateMetaclass +#include "internal/pycore_pyerrors.h" // _PyErr_FormatFromCause, _PyErr_SetKeyError +#include "internal/pycore_unicodeobject.h" // _PyUnicode_EQ, _PyUnicode_FastCopyCharacters +#endif + #if CPY_3_12_FEATURES #include "internal/pycore_frame.h" #endif From acc65b5a1ec6065ed06b5be5c2c55ef9ff90d610 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 8 Jul 2024 17:07:33 +0200 Subject: [PATCH 089/247] Update mypyc test output for Python 3.13 (#17508) --- mypyc/test-data/run-exceptions.test | 86 ++++++++++++++++++++++++++++ mypyc/test-data/run-loops.test | 35 +++++++++++ mypyc/test-data/run-misc.test | 5 +- mypyc/test-data/run-multimodule.test | 17 ++++++ 4 files changed, 142 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/run-exceptions.test b/mypyc/test-data/run-exceptions.test index c591fc1d8c15..1b180b933197 100644 --- a/mypyc/test-data/run-exceptions.test +++ b/mypyc/test-data/run-exceptions.test @@ -80,6 +80,43 @@ Traceback (most recent call last): File "native.py", line 23, in __init__ raise Exception Exception +[out version>=3.13] +Traceback (most recent call last): + File "driver.py", line 4, in + f([]) + ~^^^^ + File "native.py", line 3, in f + g(x) + File "native.py", line 6, in g + x[5] = 2 +IndexError: list assignment index out of range +Traceback (most recent call last): + File "driver.py", line 8, in + r1() + ~~^^ + File "native.py", line 10, in r1 + q1() + File "native.py", line 13, in q1 + raise Exception("test") +Exception: test +Traceback (most recent call last): + File "driver.py", line 12, in + r2() + ~~^^ + File "native.py", line 16, in r2 + q2() + File "native.py", line 19, in q2 + raise Exception +Exception +Traceback (most recent call last): + File "driver.py", line 16, in + hey() + ~~~^^ + File "native.py", line 26, in hey + A() + File "native.py", line 23, in __init__ + raise Exception +Exception [case testTryExcept] from typing import Any, Iterator @@ -264,6 +301,55 @@ attr! -- 'object' object has no attribute 'lol' out! == l == key! -- 0 +[out version>=3.13] +== i == + +Traceback (most recent call last): + File "driver.py", line 6, in + i() + ~^^ + File "native.py", line 44, in i + r(0) + File "native.py", line 15, in r + [0][1] +IndexError: list index out of range +== k == +Traceback (most recent call last): + File "native.py", line 59, in k + r(1) + File "native.py", line 17, in r + raise Exception('hi') +Exception: hi + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "driver.py", line 12, in + k() + ~^^ + File "native.py", line 61, in k + r(0) + File "native.py", line 15, in r + [0][1] +IndexError: list index out of range +== g == +caught! +caught! +== f == +hi +None +list index out of range +None +== h == +gonna break +None +== j == +lookup! +lookup! +attr! -- 'object' object has no attribute 'lol' +out! +== l == +key! -- 0 [case testTryFinally] from typing import Any diff --git a/mypyc/test-data/run-loops.test b/mypyc/test-data/run-loops.test index 6f7d79059a6d..95b79af1a411 100644 --- a/mypyc/test-data/run-loops.test +++ b/mypyc/test-data/run-loops.test @@ -381,6 +381,41 @@ RuntimeError: dictionary changed size during iteration 1 2 3 +[out version>=3.13] +Traceback (most recent call last): + File "driver.py", line 16, in + iterate_over_any(5) + ~~~~~~~~~~~~~~~~^^^ + File "native.py", line 6, in iterate_over_any + for element in a: +TypeError: 'int' object is not iterable +Traceback (most recent call last): + File "driver.py", line 20, in + iterate_over_iterable(broken_generator(5)) + ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ + File "native.py", line 10, in iterate_over_iterable + for element in iterable: + File "driver.py", line 8, in broken_generator + raise Exception('Exception Manually Raised') +Exception: Exception Manually Raised +Traceback (most recent call last): + File "driver.py", line 24, in + iterate_and_delete(d) + ~~~~~~~~~~~~~~~~~~^^^ + File "native.py", line 14, in iterate_and_delete + for key in d: +RuntimeError: dictionary changed size during iteration +15 +6 +3 +0 +1 +2 +3 +4 +1 +2 +3 [case testContinueFor] def f() -> None: diff --git a/mypyc/test-data/run-misc.test b/mypyc/test-data/run-misc.test index 14bb5be979ae..f07ac51dae6c 100644 --- a/mypyc/test-data/run-misc.test +++ b/mypyc/test-data/run-misc.test @@ -968,7 +968,10 @@ print(z) [case testCheckVersion] import sys -if sys.version_info[:2] == (3, 12): +if sys.version_info[:2] == (3, 13): + def version() -> int: + return 13 +elif sys.version_info[:2] == (3, 12): def version() -> int: return 12 elif sys.version_info[:2] == (3, 11): diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 70c73dc2088b..5edd5688140e 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -291,6 +291,23 @@ Traceback (most recent call last): File "other.py", line 3, in fail2 x[2] = 2 IndexError: list assignment index out of range +[out version>=3.13] +Traceback (most recent call last): + File "driver.py", line 6, in + other.fail2() + ~~~~~~~~~~~^^ + File "other.py", line 3, in fail2 + x[2] = 2 +IndexError: list assignment index out of range +Traceback (most recent call last): + File "driver.py", line 12, in + native.fail() + ~~~~~~~~~~~^^ + File "native.py", line 4, in fail + fail2() + File "other.py", line 3, in fail2 + x[2] = 2 +IndexError: list assignment index out of range [case testMultiModuleCycle] if False: From 78d1dfe9db3e2385c66b80c5e88dfca1655ae0e3 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 8 Jul 2024 17:08:00 +0200 Subject: [PATCH 090/247] Fix ManagedDict functions for Python 3.13 (#17507) `PyObject_VisitManagedDict` and `PyObject_ClearManagedDict` were made public in https://github.com/python/cpython/pull/108763. Both are available from `pythoncapi_compat.h`. --- mypyc/codegen/emitclass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 8dcf7212b694..ad95a1b0f323 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -734,7 +734,7 @@ def generate_traverse_for_class(cl: ClassIR, func_name: str, emitter: Emitter) - for attr, rtype in base.attributes.items(): emitter.emit_gc_visit(f"self->{emitter.attr(attr)}", rtype) if has_managed_dict(cl, emitter): - emitter.emit_line("_PyObject_VisitManagedDict((PyObject *)self, visit, arg);") + emitter.emit_line("PyObject_VisitManagedDict((PyObject *)self, visit, arg);") elif cl.has_dict: struct_name = cl.struct_name(emitter.names) # __dict__ lives right after the struct and __weakref__ lives right after that @@ -757,7 +757,7 @@ def generate_clear_for_class(cl: ClassIR, func_name: str, emitter: Emitter) -> N for attr, rtype in base.attributes.items(): emitter.emit_gc_clear(f"self->{emitter.attr(attr)}", rtype) if has_managed_dict(cl, emitter): - emitter.emit_line("_PyObject_ClearManagedDict((PyObject *)self);") + emitter.emit_line("PyObject_ClearManagedDict((PyObject *)self);") elif cl.has_dict: struct_name = cl.struct_name(emitter.names) # __dict__ lives right after the struct and __weakref__ lives right after that From 3c9f69487d3f01b7c24cf4dbda2460c99094d1f3 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 8 Jul 2024 17:08:22 +0200 Subject: [PATCH 091/247] Add another include for Python 3.13 (#17509) Include `internal/pycore_setobject.h` necessary for `_PySet_Update`. --- mypyc/lib-rt/pythonsupport.h | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 69ff120dd40d..8edc9abcf9f8 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -22,6 +22,7 @@ #include "internal/pycore_genobject.h" // _PyGen_FetchStopIterationValue #include "internal/pycore_object.h" // _PyType_CalculateMetaclass #include "internal/pycore_pyerrors.h" // _PyErr_FormatFromCause, _PyErr_SetKeyError +#include "internal/pycore_setobject.h" // _PySet_Update #include "internal/pycore_unicodeobject.h" // _PyUnicode_EQ, _PyUnicode_FastCopyCharacters #endif From 45bb91ed37f94140132742ab63a9a0d01f4f1b07 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 14 Jul 2024 13:02:31 +0100 Subject: [PATCH 092/247] [mypyc] Don't use _PyUnicode_EQ on 3.13, as it's no longer exported (#17523) Work on mypyc/mypyc#1056. --- mypyc/lib-rt/getargsfast.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mypyc/lib-rt/getargsfast.c b/mypyc/lib-rt/getargsfast.c index 387deed4399b..62d0dfed0a6d 100644 --- a/mypyc/lib-rt/getargsfast.c +++ b/mypyc/lib-rt/getargsfast.c @@ -271,9 +271,16 @@ find_keyword(PyObject *kwnames, PyObject *const *kwstack, PyObject *key) for (i = 0; i < nkwargs; i++) { PyObject *kwname = PyTuple_GET_ITEM(kwnames, i); assert(PyUnicode_Check(kwname)); +#if CPY_3_13_FEATURES + if (_PyUnicode_Equal(kwname, key)) { + return kwstack[i]; + } +#else if (_PyUnicode_EQ(kwname, key)) { return kwstack[i]; } +#endif + } return NULL; } From 42337a0ca0923ce9cf7492c6d847a945a69ad7a6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 14 Jul 2024 13:02:43 +0100 Subject: [PATCH 093/247] [mypyc] Don't use _PyUnicode_FastCopyCharacters on 3.13 (#17524) Work on mypyc/mypyc#1056. --- mypyc/lib-rt/str_ops.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 90b19001f8f0..4ba181bcce85 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -117,7 +117,11 @@ PyObject *CPyStr_Build(Py_ssize_t len, ...) { PyObject *item = va_arg(args, PyObject *); Py_ssize_t itemlen = PyUnicode_GET_LENGTH(item); if (itemlen != 0) { +#if CPY_3_13_FEATURES + PyUnicode_CopyCharacters(res, res_offset, item, 0, itemlen); +#else _PyUnicode_FastCopyCharacters(res, res_offset, item, 0, itemlen); +#endif res_offset += itemlen; } } From 6a0657e5959ba1777c4d427f8f355d499035d145 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 14 Jul 2024 13:02:51 +0100 Subject: [PATCH 094/247] [mypyc] Don't rely on _PyType_CalculateMetaclass on 3.13 (#17525) Copy the implementation from CPython (with minor changes), as it's no longer exported. Work on mypyc/mypyc#1056. --- mypyc/lib-rt/CPy.h | 5 +---- mypyc/lib-rt/misc_ops.c | 48 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 2ec04e4c5b5c..833b1bd2e76a 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -846,10 +846,7 @@ static inline bool CPy_TypeCheck(PyObject *o, PyObject *type) { return PyObject_TypeCheck(o, (PyTypeObject *)type); } -static inline PyObject *CPy_CalculateMetaclass(PyObject *type, PyObject *o) { - return (PyObject *)_PyType_CalculateMetaclass((PyTypeObject *)type, o); -} - +PyObject *CPy_CalculateMetaclass(PyObject *type, PyObject *o); PyObject *CPy_GetCoro(PyObject *obj); PyObject *CPyIter_Send(PyObject *iter, PyObject *val); int CPy_YieldFromErrorHandle(PyObject *iter, PyObject **outp); diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 803123d436a2..1572c4496e30 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -131,6 +131,52 @@ static bool _CPy_IsSafeMetaClass(PyTypeObject *metaclass) { return matches; } +#if CPY_3_13_FEATURES + +// Adapted from CPython 3.13.0b3 +/* Determine the most derived metatype. */ +PyObject *CPy_CalculateMetaclass(PyObject *metatype, PyObject *bases) +{ + Py_ssize_t i, nbases; + PyTypeObject *winner; + PyObject *tmp; + PyTypeObject *tmptype; + + /* Determine the proper metatype to deal with this, + and check for metatype conflicts while we're at it. + Note that if some other metatype wins to contract, + it's possible that its instances are not types. */ + + nbases = PyTuple_GET_SIZE(bases); + winner = (PyTypeObject *)metatype; + for (i = 0; i < nbases; i++) { + tmp = PyTuple_GET_ITEM(bases, i); + tmptype = Py_TYPE(tmp); + if (PyType_IsSubtype(winner, tmptype)) + continue; + if (PyType_IsSubtype(tmptype, winner)) { + winner = tmptype; + continue; + } + /* else: */ + PyErr_SetString(PyExc_TypeError, + "metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases"); + return NULL; + } + return (PyObject *)winner; +} + +#else + +PyObject *CPy_CalculateMetaclass(PyObject *metatype, PyObject *bases) { + return (PyObject *)_PyType_CalculateMetaclass((PyTypeObject *)metatype, bases); +} + +#endif + // Create a heap type based on a template non-heap type. // This is super hacky and maybe we should suck it up and use PyType_FromSpec instead. // We allow bases to be NULL to represent just inheriting from object. @@ -163,7 +209,7 @@ PyObject *CPyType_FromTemplate(PyObject *template, // Find the appropriate metaclass from our base classes. We // care about this because Generic uses a metaclass prior to // Python 3.7. - metaclass = _PyType_CalculateMetaclass(metaclass, bases); + metaclass = (PyTypeObject *)CPy_CalculateMetaclass((PyObject *)metaclass, bases); if (!metaclass) goto error; From e062793d17a67766e6a44fc4095eb72e55e50d7c Mon Sep 17 00:00:00 2001 From: Max Murin Date: Fri, 19 Jul 2024 05:07:41 -0700 Subject: [PATCH 095/247] CHANGELOG.md update for 1.11 (#17539) Add a changelog for the 1.11 release. --- CHANGELOG.md | 168 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d5919cafe33..196a75992c24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,174 @@ ## Next release +## Mypy 1.11 + +We’ve just uploaded mypy 1.11 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +#### Additional support for PEP 695 + +Mypy now has experimental support for the new type parameter syntax introduced in Python 3.12 ([PEP 695](https://peps.python.org/pep-0695/)). +This feature is still experimental and must be enabled with the `--enable-incomplete-feature=NewGenericSyntax` flag. + +This example demonstrates the new syntax: +```python + +def f[T](x: T) -> T: ... + +reveal_type(f(1)) # Revealed type is 'int' +``` + +This feature was contributed by Jukka Lehtosalo (PR [17233](https://github.com/python/mypy/pull/17233)). + + +#### Support for `functools.partial` + +Mypy now typechecks uses of `functools.partial`, which previous mypy would always accept. +This example would previously pass: + +```python +from functools import partial + +def f(a: int, b: str) -> None: ... + +g = partial(f, 1) +g(1) # error: Argument 1 to "f" has incompatible type "int"; expected "str" [arg-type] +``` + +This feature was contributed by Shantanu (PR [16939](https://github.com/python/mypy/pull/16939)). + + +#### Changes to stubtest + * Stubtest: ignore `_ios_support` (Alex Waygood, PR [17270](https://github.com/python/mypy/pull/17270)) + * stubtest: changes for py313 (Shantanu, PR [17261](https://github.com/python/mypy/pull/17261)) + + +#### Changes to stubgen + * stubgen: Gracefully handle invalid `Optional` and recognize aliases to PEP 604 unions (Ali Hamdan, PR [17386](https://github.com/python/mypy/pull/17386)) + * Fix stubgen for Python 3.13 (Jelle Zijlstra, PR [17290](https://github.com/python/mypy/pull/17290)) + * stubgen: preserve enum value initialisers (Shantanu, PR [17125](https://github.com/python/mypy/pull/17125)) + + +#### Changes to mypyc + * [mypyc] Sync pythoncapi_compat.h (Jukka Lehtosalo, PR [17390](https://github.com/python/mypy/pull/17390)) + * [mypyc] Support Python 3.12 type alias syntax (PEP 695) (Jukka Lehtosalo, PR [17384](https://github.com/python/mypy/pull/17384)) + * [mypyc] Support new syntax for generic functions and classes (PEP 695) (Jukka Lehtosalo, PR [17357](https://github.com/python/mypy/pull/17357)) + * [mypyc] Fix ParamSpec (Shantanu, PR [17309](https://github.com/python/mypy/pull/17309)) + * [mypyc] Inline fast paths of integer unboxing operations (Jukka Lehtosalo, PR [17266](https://github.com/python/mypy/pull/17266)) + * [mypyc] Inline tagged integer arithmetic and bitwise operations (Jukka Lehtosalo, PR [17265](https://github.com/python/mypy/pull/17265)) + * [mypyc] Allow specifying primitives as pure (Jukka Lehtosalo, PR [17263](https://github.com/python/mypy/pull/17263)) + + +#### Changes to error reporting + * Do not report plugin-generated methods with `explicit-override` (sobolevn, PR [17433](https://github.com/python/mypy/pull/17433)) + * Fix explicit type for partial (Ivan Levkivskyi, PR [17424](https://github.com/python/mypy/pull/17424)) + * Re-work overload overlap logic (Ivan Levkivskyi, PR [17392](https://github.com/python/mypy/pull/17392)) + * Use namespaces for function type variables (Ivan Levkivskyi, PR [17311](https://github.com/python/mypy/pull/17311)) + * Fix false positive for Final local scope variable in Protocol (GiorgosPapoutsakis, PR [17308](https://github.com/python/mypy/pull/17308)) + * Use Never in more messages, use ambiguous in join (Shantanu, PR [17304](https://github.com/python/mypy/pull/17304)) + * Log full path to config file in verbose output (dexterkennedy, PR [17180](https://github.com/python/mypy/pull/17180)) + * Added [prop-decorator] code for unsupported property decorators (#14461) (Christopher Barber, PR [16571](https://github.com/python/mypy/pull/16571)) + * Suppress second error message with `:=` and `[truthy-bool]` (Nikita Sobolev, PR [15941](https://github.com/python/mypy/pull/15941)) + * Error for assignment of functional Enum to variable of different name (Shantanu, PR [16805](https://github.com/python/mypy/pull/16805)) + * Add Error format support, and JSON output option (Tushar Sadhwani, PR [11396](https://github.com/python/mypy/pull/11396)) + + +#### Fixes for crashes + * Fix daemon crash on invalid type in TypedDict (Ivan Levkivskyi, PR [17495](https://github.com/python/mypy/pull/17495)) + * Some cleanup in partial plugin (Ivan Levkivskyi, PR [17423](https://github.com/python/mypy/pull/17423)) + * Fix crash when overriding with unpacked TypedDict (Ivan Levkivskyi, PR [17359](https://github.com/python/mypy/pull/17359)) + * Fix crash on TypedDict unpacking for ParamSpec (Ivan Levkivskyi, PR [17358](https://github.com/python/mypy/pull/17358)) + * Fix crash involving recursive union of tuples (Ivan Levkivskyi, PR [17353](https://github.com/python/mypy/pull/17353)) + * Fix crash on invalid callable property override (Ivan Levkivskyi, PR [17352](https://github.com/python/mypy/pull/17352)) + * Fix crash on unpacking self in NamedTuple (Ivan Levkivskyi, PR [17351](https://github.com/python/mypy/pull/17351)) + * Fix crash on recursive alias with an optional type (Ivan Levkivskyi, PR [17350](https://github.com/python/mypy/pull/17350)) + * Fix type comments crash inside generic definitions (Bénédikt Tran, PR [16849](https://github.com/python/mypy/pull/16849)) + + +#### Changes to documentation + * Mention --enable-incomplete-feature=NewGenericSyntax (Shantanu, PR [17462](https://github.com/python/mypy/pull/17462)) + * Use inline config in the optional error codes docs (Shantanu, PR [17374](https://github.com/python/mypy/pull/17374)) + * docs: Use lower-case generics (Seo Sanghyeon, PR [17176](https://github.com/python/mypy/pull/17176)) + * Add documentation for show-error-code-links (GiorgosPapoutsakis, PR [17144](https://github.com/python/mypy/pull/17144)) + * Update CONTRIBUTING.md to include commands for Windows (GiorgosPapoutsakis, PR [17142](https://github.com/python/mypy/pull/17142)) + + +#### Other notable contributions + * Fix ParamSpec inference against TypeVarTuple (Ivan Levkivskyi, PR [17431](https://github.com/python/mypy/pull/17431)) + * Always allow lambda calls (Ivan Levkivskyi, PR [17430](https://github.com/python/mypy/pull/17430)) + * Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) + * Fix isinstance checks with PEP 604 unions containing None (Shantanu, PR [17415](https://github.com/python/mypy/pull/17415)) + * Use (simplified) unions instead of joins for tuple fallbacks (Ivan Levkivskyi, PR [17408](https://github.com/python/mypy/pull/17408)) + * Fix self-referential upper bound in new-style type variables (Ivan Levkivskyi, PR [17407](https://github.com/python/mypy/pull/17407)) + * Consider overlap between instances and callables (Ivan Levkivskyi, PR [17389](https://github.com/python/mypy/pull/17389)) + * Support `enum.member` for python3.11+ (Nikita Sobolev, PR [17382](https://github.com/python/mypy/pull/17382)) + * Allow new-style self-types in classmethods (Ivan Levkivskyi, PR [17381](https://github.com/python/mypy/pull/17381)) + * Support `enum.nonmember` for python3.11+ (Nikita Sobolev, PR [17376](https://github.com/python/mypy/pull/17376)) + * Fix isinstance with type aliases to PEP 604 unions (Shantanu, PR [17371](https://github.com/python/mypy/pull/17371)) + * Properly handle unpacks in overlap checks (Ivan Levkivskyi, PR [17356](https://github.com/python/mypy/pull/17356)) + * Fix type application for classes with generic constructors (Ivan Levkivskyi, PR [17354](https://github.com/python/mypy/pull/17354)) + * Use polymorphic inference in unification (Ivan Levkivskyi, PR [17348](https://github.com/python/mypy/pull/17348)) + * Update 'typing_extensions' to >=4.6.0 to fix python 3.12 error (Ben Brown, PR [17312](https://github.com/python/mypy/pull/17312)) + * Avoid does not return error in lambda (Shantanu, PR [17294](https://github.com/python/mypy/pull/17294)) + * Fix for bug with descriptors in non-strict-optional (Max Murin, PR [17293](https://github.com/python/mypy/pull/17293)) + * Don’t leak unreachability from lambda body to surrounding scope (Anders Kaseorg, PR [17287](https://github.com/python/mypy/pull/17287)) + * Validate more about overrides on untyped methods (Steven Troxler, PR [17276](https://github.com/python/mypy/pull/17276)) + * Fix case involving non-ASCII chars on Windows (Alexander Leopold Shon, PR [17275](https://github.com/python/mypy/pull/17275)) + * Support namedtuple.__replace__ in Python 3.13 (Shantanu, PR [17259](https://github.com/python/mypy/pull/17259)) + * Fix for type narrowing of negative integer literals (gilesgc, PR [17256](https://github.com/python/mypy/pull/17256)) + * Support rename=True in collections.namedtuple (Jelle Zijlstra, PR [17247](https://github.com/python/mypy/pull/17247)) + * [dmypy] sort list of files for update by extension (Valentin Stanciu, PR [17245](https://github.com/python/mypy/pull/17245)) + * fix #16935 fix type of tuple[X,Y] expression (urnest, PR [17235](https://github.com/python/mypy/pull/17235)) + * Do not forget that a `TypedDict` was wrapped in `Unpack` after a `name-defined` error occurred. (Christoph Tyralla, PR [17226](https://github.com/python/mypy/pull/17226)) + * fix: annotated argument's `var` node type is explicit, not inferred (bzoracler, PR [17217](https://github.com/python/mypy/pull/17217)) + * Enum private attributes are not enum members (Ali Hamdan, PR [17182](https://github.com/python/mypy/pull/17182)) + * Fix Literal strings containing pipe characters (Jelle Zijlstra, PR [17148](https://github.com/python/mypy/pull/17148)) + * Add support for __spec__ (Shantanu, PR [14739](https://github.com/python/mypy/pull/14739)) + + +#### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=6dda799d8ad1d89e0f8aad7ac41d2d34bd838ace+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + + +#### Acknowledgements +Thanks to all mypy contributors who contributed to this release: + +- Alex Waygood +- Alexander Leopold Shon +- Ali Hamdan +- Anders Kaseorg +- Ben Brown +- Bénédikt Tran +- bzoracler +- Christoph Tyralla +- Christopher Barber +- dexterkennedy +- gilesgc +- GiorgosPapoutsakis +- Ivan Levkivskyi +- Jelle Zijlstra +- Jukka Lehtosalo +- Marc Mueller +- Matthieu Devlin +- Michael R. Crusoe +- Nikita Sobolev +- Seo Sanghyeon +- Shantanu +- sobolevn +- Steven Troxler +- Tadeu Manoel +- Tamir Duberstein +- Tushar Sadhwani +- urnest +- Valentin Stanciu + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 1.10 From 0753e2a82dad35034e000609b6e8daa37238bfaa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 19 Jul 2024 16:49:09 +0100 Subject: [PATCH 096/247] Update CHANGELOG for mypy 1.11 (#17540) Added additional sections for major features and did various other updates. --- CHANGELOG.md | 188 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 131 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 196a75992c24..b544e05ee573 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,26 +11,40 @@ We’ve just uploaded mypy 1.11 to the Python Package Index ([PyPI](https://pypi You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Additional support for PEP 695 +#### Support Python 3.12 Syntax for Generics (PEP 695) -Mypy now has experimental support for the new type parameter syntax introduced in Python 3.12 ([PEP 695](https://peps.python.org/pep-0695/)). -This feature is still experimental and must be enabled with the `--enable-incomplete-feature=NewGenericSyntax` flag. +Mypy now supports the new type parameter syntax introduced in Python 3.12 ([PEP 695](https://peps.python.org/pep-0695/)). +This feature is still experimental and must be enabled with the `--enable-incomplete-feature=NewGenericSyntax` flag, or with `enable_incomplete_feature = NewGenericSyntax` in the mypy configuration file. +We plan to enable this by default in the next mypy feature release. This example demonstrates the new syntax: -```python +```python +# Generic function def f[T](x: T) -> T: ... reveal_type(f(1)) # Revealed type is 'int' + +# Generic class +class C[T]: + def __init__(self, x: T) -> None: + self.x = x + +c = C('a') +reveal_type(c.x) # Revealed type is 'str' + +# Type alias +type A[T] = C[list[T]] ``` -This feature was contributed by Jukka Lehtosalo (PR [17233](https://github.com/python/mypy/pull/17233)). +This feature was contributed by Jukka Lehtosalo. #### Support for `functools.partial` -Mypy now typechecks uses of `functools.partial`, which previous mypy would always accept. -This example would previously pass: +Mypy now type checks uses of `functools.partial`. Previously mypy would accept arbitrary arguments. + +This example will now produce an error: ```python from functools import partial @@ -38,98 +52,158 @@ from functools import partial def f(a: int, b: str) -> None: ... g = partial(f, 1) -g(1) # error: Argument 1 to "f" has incompatible type "int"; expected "str" [arg-type] + +# Argument has incompatible type "int"; expected "str" +g(11) ``` This feature was contributed by Shantanu (PR [16939](https://github.com/python/mypy/pull/16939)). -#### Changes to stubtest - * Stubtest: ignore `_ios_support` (Alex Waygood, PR [17270](https://github.com/python/mypy/pull/17270)) - * stubtest: changes for py313 (Shantanu, PR [17261](https://github.com/python/mypy/pull/17261)) +#### Stricter Checks for Untyped Overrides + +Past mypy versions didn't check if untyped methods were compatible with overridden methods. This would result in false negatives. Now mypy performs these checks when using `--check-untyped-defs`. + +For example, this now generates an error if using `--check-untyped-defs`: + +```python +class Base: + def f(self, x: int = 0) -> None: ... + +class Derived(Base): + # Signature incompatible with "Base" + def f(self): ... +``` + +This feature was contributed by Steven Troxler (PR [17276](https://github.com/python/mypy/pull/17276)). + + +#### Type Inference Improvements + +The new polymorphic inference algorithm introduced in mypy 1.5 is now used in more situations. This improves type inference involving generic higher-order functions, in particular. + +This feature was contributed by Ivan Levkivskyi (PR [17348](https://github.com/python/mypy/pull/17348)). + +Mypy now uses unions of tuple item types in certain contexts to enable more precise inferred types. Example: + +```python +for x in (1, 'x'): + # Previously inferred as 'object' + reveal_type(x) # Revealed type is 'int | str' +``` + +This was also contributed by Ivan Levkivskyi (PR [17408](https://github.com/python/mypy/pull/17408)). + +#### Improvements to Detection of Overlapping Overloads -#### Changes to stubgen - * stubgen: Gracefully handle invalid `Optional` and recognize aliases to PEP 604 unions (Ali Hamdan, PR [17386](https://github.com/python/mypy/pull/17386)) - * Fix stubgen for Python 3.13 (Jelle Zijlstra, PR [17290](https://github.com/python/mypy/pull/17290)) - * stubgen: preserve enum value initialisers (Shantanu, PR [17125](https://github.com/python/mypy/pull/17125)) +The details of how mypy checks if two `@overload` signatures are unsafely overlapping were overhauled. This both fixes some false positives, and allows mypy to detect additional unsafe signatures. +This feature was contributed by Ivan Levkivskyi (PR [17392](https://github.com/python/mypy/pull/17392)). -#### Changes to mypyc - * [mypyc] Sync pythoncapi_compat.h (Jukka Lehtosalo, PR [17390](https://github.com/python/mypy/pull/17390)) - * [mypyc] Support Python 3.12 type alias syntax (PEP 695) (Jukka Lehtosalo, PR [17384](https://github.com/python/mypy/pull/17384)) - * [mypyc] Support new syntax for generic functions and classes (PEP 695) (Jukka Lehtosalo, PR [17357](https://github.com/python/mypy/pull/17357)) - * [mypyc] Fix ParamSpec (Shantanu, PR [17309](https://github.com/python/mypy/pull/17309)) - * [mypyc] Inline fast paths of integer unboxing operations (Jukka Lehtosalo, PR [17266](https://github.com/python/mypy/pull/17266)) - * [mypyc] Inline tagged integer arithmetic and bitwise operations (Jukka Lehtosalo, PR [17265](https://github.com/python/mypy/pull/17265)) - * [mypyc] Allow specifying primitives as pure (Jukka Lehtosalo, PR [17263](https://github.com/python/mypy/pull/17263)) +#### Better Support for Type Hints in Expressions -#### Changes to error reporting +Mypy now allows more expressions that evaluate to valid type annotations in all expression contexts. The inferred types of these expressions are also sometimes more precise. Previously they were often `object`. + +This example uses a union type that includes a callable type as an expression, and it no longer generates an error: + +```python +from typing import Callable + +print(Callable[[], int] | None) # No error +``` + +This feature was contributed by Jukka Lehtosalo (PR [17404](https://github.com/python/mypy/pull/17404)). + + +#### Mypyc Improvements + +Mypyc now supports the new syntax for generics introduced in Python 3.12 (see above). Another notable improvement is signficantly faster basic operations on `int` values. + + * Support Python 3.12 syntax for generic functions and classes (Jukka Lehtosalo, PR [17357](https://github.com/python/mypy/pull/17357)) + * Support Python 3.12 type alias syntax (Jukka Lehtosalo, PR [17384](https://github.com/python/mypy/pull/17384)) + * Fix ParamSpec (Shantanu, PR [17309](https://github.com/python/mypy/pull/17309)) + * Inline fast paths of integer unboxing operations (Jukka Lehtosalo, PR [17266](https://github.com/python/mypy/pull/17266)) + * Inline tagged integer arithmetic and bitwise operations (Jukka Lehtosalo, PR [17265](https://github.com/python/mypy/pull/17265)) + * Allow specifying primitives as pure (Jukka Lehtosalo, PR [17263](https://github.com/python/mypy/pull/17263)) + + +#### Changes to Stubtest + * Ignore `_ios_support` (Alex Waygood, PR [17270](https://github.com/python/mypy/pull/17270)) + * Improve support for Python 3.13 (Shantanu, PR [17261](https://github.com/python/mypy/pull/17261)) + + +#### Changes to Stubgen + * Gracefully handle invalid `Optional` and recognize aliases to PEP 604 unions (Ali Hamdan, PR [17386](https://github.com/python/mypy/pull/17386)) + * Fix for Python 3.13 (Jelle Zijlstra, PR [17290](https://github.com/python/mypy/pull/17290)) + * Preserve enum value initialisers (Shantanu, PR [17125](https://github.com/python/mypy/pull/17125)) + + +#### Miscellaneous New Features + * Add error format support and JSON output option via `--output json` (Tushar Sadhwani, PR [11396](https://github.com/python/mypy/pull/11396)) + * Support `enum.member` in Python 3.11+ (Nikita Sobolev, PR [17382](https://github.com/python/mypy/pull/17382)) + * Support `enum.nonmember` in Python 3.11+ (Nikita Sobolev, PR [17376](https://github.com/python/mypy/pull/17376)) + * Support `namedtuple.__replace__` in Python 3.13 (Shantanu, PR [17259](https://github.com/python/mypy/pull/17259)) + * Support `rename=True` in collections.namedtuple (Jelle Zijlstra, PR [17247](https://github.com/python/mypy/pull/17247)) + * Add support for `__spec__` (Shantanu, PR [14739](https://github.com/python/mypy/pull/14739)) + + +#### Changes to Error Reporting + * Mention `--enable-incomplete-feature=NewGenericSyntax` in messages (Shantanu, PR [17462](https://github.com/python/mypy/pull/17462)) * Do not report plugin-generated methods with `explicit-override` (sobolevn, PR [17433](https://github.com/python/mypy/pull/17433)) - * Fix explicit type for partial (Ivan Levkivskyi, PR [17424](https://github.com/python/mypy/pull/17424)) - * Re-work overload overlap logic (Ivan Levkivskyi, PR [17392](https://github.com/python/mypy/pull/17392)) - * Use namespaces for function type variables (Ivan Levkivskyi, PR [17311](https://github.com/python/mypy/pull/17311)) + * Use and display namespaces for function type variables (Ivan Levkivskyi, PR [17311](https://github.com/python/mypy/pull/17311)) * Fix false positive for Final local scope variable in Protocol (GiorgosPapoutsakis, PR [17308](https://github.com/python/mypy/pull/17308)) * Use Never in more messages, use ambiguous in join (Shantanu, PR [17304](https://github.com/python/mypy/pull/17304)) * Log full path to config file in verbose output (dexterkennedy, PR [17180](https://github.com/python/mypy/pull/17180)) - * Added [prop-decorator] code for unsupported property decorators (#14461) (Christopher Barber, PR [16571](https://github.com/python/mypy/pull/16571)) + * Added `[prop-decorator]` code for unsupported property decorators (#14461) (Christopher Barber, PR [16571](https://github.com/python/mypy/pull/16571)) * Suppress second error message with `:=` and `[truthy-bool]` (Nikita Sobolev, PR [15941](https://github.com/python/mypy/pull/15941)) - * Error for assignment of functional Enum to variable of different name (Shantanu, PR [16805](https://github.com/python/mypy/pull/16805)) - * Add Error format support, and JSON output option (Tushar Sadhwani, PR [11396](https://github.com/python/mypy/pull/11396)) + * Generate error for assignment of functional Enum to variable of different name (Shantanu, PR [16805](https://github.com/python/mypy/pull/16805)) + * Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) -#### Fixes for crashes +#### Fixes for Crashes * Fix daemon crash on invalid type in TypedDict (Ivan Levkivskyi, PR [17495](https://github.com/python/mypy/pull/17495)) - * Some cleanup in partial plugin (Ivan Levkivskyi, PR [17423](https://github.com/python/mypy/pull/17423)) + * Fix crash and bugs related to `partial()` (Ivan Levkivskyi, PR [17423](https://github.com/python/mypy/pull/17423)) * Fix crash when overriding with unpacked TypedDict (Ivan Levkivskyi, PR [17359](https://github.com/python/mypy/pull/17359)) * Fix crash on TypedDict unpacking for ParamSpec (Ivan Levkivskyi, PR [17358](https://github.com/python/mypy/pull/17358)) * Fix crash involving recursive union of tuples (Ivan Levkivskyi, PR [17353](https://github.com/python/mypy/pull/17353)) * Fix crash on invalid callable property override (Ivan Levkivskyi, PR [17352](https://github.com/python/mypy/pull/17352)) * Fix crash on unpacking self in NamedTuple (Ivan Levkivskyi, PR [17351](https://github.com/python/mypy/pull/17351)) * Fix crash on recursive alias with an optional type (Ivan Levkivskyi, PR [17350](https://github.com/python/mypy/pull/17350)) - * Fix type comments crash inside generic definitions (Bénédikt Tran, PR [16849](https://github.com/python/mypy/pull/16849)) + * Fix crash on type comment inside generic definitions (Bénédikt Tran, PR [16849](https://github.com/python/mypy/pull/16849)) -#### Changes to documentation - * Mention --enable-incomplete-feature=NewGenericSyntax (Shantanu, PR [17462](https://github.com/python/mypy/pull/17462)) - * Use inline config in the optional error codes docs (Shantanu, PR [17374](https://github.com/python/mypy/pull/17374)) - * docs: Use lower-case generics (Seo Sanghyeon, PR [17176](https://github.com/python/mypy/pull/17176)) +#### Changes to Documentation + * Use inline config in documentation for optional error codes (Shantanu, PR [17374](https://github.com/python/mypy/pull/17374)) + * Use lower-case generics in documentation (Seo Sanghyeon, PR [17176](https://github.com/python/mypy/pull/17176)) * Add documentation for show-error-code-links (GiorgosPapoutsakis, PR [17144](https://github.com/python/mypy/pull/17144)) * Update CONTRIBUTING.md to include commands for Windows (GiorgosPapoutsakis, PR [17142](https://github.com/python/mypy/pull/17142)) -#### Other notable contributions +#### Other Notable Improvements and Fixes * Fix ParamSpec inference against TypeVarTuple (Ivan Levkivskyi, PR [17431](https://github.com/python/mypy/pull/17431)) + * Fix explicit type for `partial` (Ivan Levkivskyi, PR [17424](https://github.com/python/mypy/pull/17424)) * Always allow lambda calls (Ivan Levkivskyi, PR [17430](https://github.com/python/mypy/pull/17430)) - * Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) * Fix isinstance checks with PEP 604 unions containing None (Shantanu, PR [17415](https://github.com/python/mypy/pull/17415)) - * Use (simplified) unions instead of joins for tuple fallbacks (Ivan Levkivskyi, PR [17408](https://github.com/python/mypy/pull/17408)) * Fix self-referential upper bound in new-style type variables (Ivan Levkivskyi, PR [17407](https://github.com/python/mypy/pull/17407)) * Consider overlap between instances and callables (Ivan Levkivskyi, PR [17389](https://github.com/python/mypy/pull/17389)) - * Support `enum.member` for python3.11+ (Nikita Sobolev, PR [17382](https://github.com/python/mypy/pull/17382)) * Allow new-style self-types in classmethods (Ivan Levkivskyi, PR [17381](https://github.com/python/mypy/pull/17381)) - * Support `enum.nonmember` for python3.11+ (Nikita Sobolev, PR [17376](https://github.com/python/mypy/pull/17376)) * Fix isinstance with type aliases to PEP 604 unions (Shantanu, PR [17371](https://github.com/python/mypy/pull/17371)) * Properly handle unpacks in overlap checks (Ivan Levkivskyi, PR [17356](https://github.com/python/mypy/pull/17356)) * Fix type application for classes with generic constructors (Ivan Levkivskyi, PR [17354](https://github.com/python/mypy/pull/17354)) - * Use polymorphic inference in unification (Ivan Levkivskyi, PR [17348](https://github.com/python/mypy/pull/17348)) - * Update 'typing_extensions' to >=4.6.0 to fix python 3.12 error (Ben Brown, PR [17312](https://github.com/python/mypy/pull/17312)) - * Avoid does not return error in lambda (Shantanu, PR [17294](https://github.com/python/mypy/pull/17294)) - * Fix for bug with descriptors in non-strict-optional (Max Murin, PR [17293](https://github.com/python/mypy/pull/17293)) + * Update `typing_extensions` to >=4.6.0 to fix Python 3.12 error (Ben Brown, PR [17312](https://github.com/python/mypy/pull/17312)) + * Avoid "does not return" error in lambda (Shantanu, PR [17294](https://github.com/python/mypy/pull/17294)) + * Fix bug with descriptors in non-strict-optional mode (Max Murin, PR [17293](https://github.com/python/mypy/pull/17293)) * Don’t leak unreachability from lambda body to surrounding scope (Anders Kaseorg, PR [17287](https://github.com/python/mypy/pull/17287)) - * Validate more about overrides on untyped methods (Steven Troxler, PR [17276](https://github.com/python/mypy/pull/17276)) - * Fix case involving non-ASCII chars on Windows (Alexander Leopold Shon, PR [17275](https://github.com/python/mypy/pull/17275)) - * Support namedtuple.__replace__ in Python 3.13 (Shantanu, PR [17259](https://github.com/python/mypy/pull/17259)) + * Fix issues with non-ASCII characters on Windows (Alexander Leopold Shon, PR [17275](https://github.com/python/mypy/pull/17275)) * Fix for type narrowing of negative integer literals (gilesgc, PR [17256](https://github.com/python/mypy/pull/17256)) - * Support rename=True in collections.namedtuple (Jelle Zijlstra, PR [17247](https://github.com/python/mypy/pull/17247)) - * [dmypy] sort list of files for update by extension (Valentin Stanciu, PR [17245](https://github.com/python/mypy/pull/17245)) - * fix #16935 fix type of tuple[X,Y] expression (urnest, PR [17235](https://github.com/python/mypy/pull/17235)) - * Do not forget that a `TypedDict` was wrapped in `Unpack` after a `name-defined` error occurred. (Christoph Tyralla, PR [17226](https://github.com/python/mypy/pull/17226)) - * fix: annotated argument's `var` node type is explicit, not inferred (bzoracler, PR [17217](https://github.com/python/mypy/pull/17217)) - * Enum private attributes are not enum members (Ali Hamdan, PR [17182](https://github.com/python/mypy/pull/17182)) + * Fix confusion between .py and .pyi files in mypy daemon (Valentin Stanciu, PR [17245](https://github.com/python/mypy/pull/17245)) + * Fix type of `tuple[X, Y]` expression (urnest, PR [17235](https://github.com/python/mypy/pull/17235)) + * Don't forget that a `TypedDict` was wrapped in `Unpack` after a `name-defined` error occurred (Christoph Tyralla, PR [17226](https://github.com/python/mypy/pull/17226)) + * Mark annotated argument as having an explicit, not inferred type (bzoracler, PR [17217](https://github.com/python/mypy/pull/17217)) + * Don't consider Enum private attributes as enum members (Ali Hamdan, PR [17182](https://github.com/python/mypy/pull/17182)) * Fix Literal strings containing pipe characters (Jelle Zijlstra, PR [17148](https://github.com/python/mypy/pull/17148)) - * Add support for __spec__ (Shantanu, PR [14739](https://github.com/python/mypy/pull/14739)) #### Typeshed Updates @@ -331,7 +405,7 @@ This feature was contributed by Shantanu (PR [16756](https://github.com/python/m #### Typeshed Updates -Please see [git log](https://github.com/python/typeshed/commits/main?after=7c8e82fe483a40ec4cb0a2505cfdb0f3e7cc81d9+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. +Please see [git log](https://github.com/python/typeshed/commits/main?after=6dda799d8ad1d89e0f8aad7ac41d2d34bd838ace+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. #### Mypy 1.10.1 From 0a040dd25dc1d3847994574152932112369c0840 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 19 Jul 2024 23:22:53 -0400 Subject: [PATCH 097/247] Fix types.GenericAlias lookup crash (#17543) Fixes #17542 --- mypy/checkexpr.py | 2 +- test-data/unit/check-functions.test | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 3532e18b93b2..2e8e6e2db9d5 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4341,7 +4341,7 @@ def visit_index_with_type( elif isinstance(left_type, FunctionLike) and left_type.is_type_obj(): if left_type.type_object().is_enum: return self.visit_enum_index_expr(left_type.type_object(), e.index, e) - elif left_type.type_object().type_vars: + elif left_type.type_object().type_vars and self.chk.options.python_version >= (3, 9): return self.named_type("types.GenericAlias") elif ( left_type.type_object().fullname == "builtins.type" diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 6c895c86e899..ef5a66b6ecde 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1779,6 +1779,7 @@ def Arg(x, y): pass F = Callable[[Arg(int, 'x')], int] # E: Invalid argument constructor "__main__.Arg" [case testCallableParsingFromExpr] +# flags: --python-version 3.9 from typing import Callable, List from mypy_extensions import Arg, VarArg, KwArg import mypy_extensions @@ -1799,10 +1800,23 @@ L = Callable[[Arg(name='x', type=int)], int] # ok # I have commented out the following test because I don't know how to expect the "defined here" note part of the error. # M = Callable[[Arg(gnome='x', type=int)], int] E: Invalid type alias: expression is not a valid type E: Unexpected keyword argument "gnome" for "Arg" N = Callable[[Arg(name=None, type=int)], int] # ok -O = Callable[[List[Arg(int)]], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: Type expected within [...] +O = Callable[[List[Arg(int)]], int] # E: Invalid type alias: expression is not a valid type \ + # E: Value of type "int" is not indexable \ + # E: Type expected within [...] P = Callable[[mypy_extensions.VarArg(int)], int] # ok -Q = Callable[[Arg(int, type=int)], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "type" -R = Callable[[Arg(int, 'x', name='y')], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "name" +Q = Callable[[Arg(int, type=int)], int] # E: Invalid type alias: expression is not a valid type \ + # E: Value of type "int" is not indexable \ + # E: "Arg" gets multiple values for keyword argument "type" +R = Callable[[Arg(int, 'x', name='y')], int] # E: Invalid type alias: expression is not a valid type \ + # E: Value of type "int" is not indexable \ + # E: "Arg" gets multiple values for keyword argument "name" + + + + + + + [builtins fixtures/dict.pyi] [case testCallableParsing] From 0b09116baae650441e4038b9cd486c7583d47f2a Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 20 Jul 2024 02:39:32 -0400 Subject: [PATCH 098/247] Indexing a type also produces a GenericAlias (#17546) Mentioned by Jelle in https://github.com/python/mypy/pull/17543#issuecomment-2240829583 --- mypy/checkexpr.py | 11 ++++------- test-data/unit/pythoneval.test | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 2e8e6e2db9d5..ad1b416a0227 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4341,14 +4341,11 @@ def visit_index_with_type( elif isinstance(left_type, FunctionLike) and left_type.is_type_obj(): if left_type.type_object().is_enum: return self.visit_enum_index_expr(left_type.type_object(), e.index, e) - elif left_type.type_object().type_vars and self.chk.options.python_version >= (3, 9): - return self.named_type("types.GenericAlias") - elif ( - left_type.type_object().fullname == "builtins.type" - and self.chk.options.python_version >= (3, 9) + elif self.chk.options.python_version >= (3, 9) and ( + left_type.type_object().type_vars + or left_type.type_object().fullname == "builtins.type" ): - # builtins.type is special: it's not generic in stubs, but it supports indexing - return self.named_type("typing._SpecialForm") + return self.named_type("types.GenericAlias") if isinstance(left_type, TypeVarType) and not self.has_member( left_type.upper_bound, "__getitem__" diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 222430c3ef55..832e55f333de 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1781,7 +1781,7 @@ C = str | int D: TypeAlias = str | int [out] _testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: Invalid type alias: expression is not a valid type -_testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: Unsupported left operand type for | ("") +_testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: Unsupported left operand type for | ("GenericAlias") _testTypeAliasNotSupportedWithNewStyleUnion.py:4: error: Invalid type alias: expression is not a valid type _testTypeAliasNotSupportedWithNewStyleUnion.py:4: error: Unsupported left operand type for | ("Type[str]") _testTypeAliasNotSupportedWithNewStyleUnion.py:5: error: Invalid type alias: expression is not a valid type From 6aa46f097cf30f3143c5ee4bae7f0d2e7ce914bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jul 2024 15:00:33 -0700 Subject: [PATCH 099/247] Bump setuptools from 68.2.2 to 70.0.0 (#17533) --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 75a970c5bf0e..f4fb4a20cce7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -57,5 +57,5 @@ typing-extensions==4.12.2 # via -r mypy-requirements.txt # The following packages are considered to be unsafe in a requirements file: -setuptools==68.2.2 +setuptools==70.0.0 # via -r test-requirements.in From b202f3069a7c8c2f64b6bcc6a1f63794878d6c9d Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 22 Jul 2024 22:44:37 +0300 Subject: [PATCH 100/247] Fix `typing.TypeAliasType` being undefined on python < 3.12 (#17558) Closes #17554 CC @JukkaL Refs https://github.com/python/mypy/pull/17320 --- mypy/checkexpr.py | 10 ++++++++-- test-data/unit/check-type-aliases.test | 11 ++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index ad1b416a0227..9dee743ad406 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4679,7 +4679,7 @@ def visit_type_application(self, tapp: TypeApplication) -> Type: """ if isinstance(tapp.expr, RefExpr) and isinstance(tapp.expr.node, TypeAlias): if tapp.expr.node.python_3_12_type_alias: - return self.named_type("typing.TypeAliasType") + return self.type_alias_type_type() # Subscription of a (generic) alias in runtime context, expand the alias. item = instantiate_type_alias( tapp.expr.node, @@ -4743,7 +4743,7 @@ class LongName(Generic[T]): ... y = cast(A, ...) """ if alias.python_3_12_type_alias: - return self.named_type("typing.TypeAliasType") + return self.type_alias_type_type() if isinstance(alias.target, Instance) and alias.target.invalid: # type: ignore[misc] # An invalid alias, error already has been reported return AnyType(TypeOfAny.from_error) @@ -5863,6 +5863,12 @@ def named_type(self, name: str) -> Instance: """ return self.chk.named_type(name) + def type_alias_type_type(self) -> Instance: + """Returns a `typing.TypeAliasType` or `typing_extensions.TypeAliasType`.""" + if self.chk.options.python_version >= (3, 12): + return self.named_type("typing.TypeAliasType") + return self.named_type("typing_extensions.TypeAliasType") + def is_valid_var_arg(self, typ: Type) -> bool: """Is a type valid as a *args argument?""" typ = get_proper_type(typ) diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 6f9e9eda1d02..c7b9694a9188 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -1074,7 +1074,7 @@ x: TestType = 42 y: TestType = 'a' z: TestType = object() # E: Incompatible types in assignment (expression has type "object", variable has type "Union[int, str]") -reveal_type(TestType) # N: Revealed type is "typing.TypeAliasType" +reveal_type(TestType) # N: Revealed type is "typing_extensions.TypeAliasType" TestType() # E: "TypeAliasType" not callable class A: @@ -1084,6 +1084,15 @@ yc: A.ClassAlias = "" # E: Incompatible types in assignment (expression has typ [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] +[case testTypeAliasTypePython311] +# flags: --python-version 3.11 +# Pinning to 3.11, because 3.12 has `TypeAliasType` +from typing_extensions import TypeAliasType + +TestType = TypeAliasType("TestType", int) +x: TestType = 1 +[builtins fixtures/tuple.pyi] + [case testTypeAliasTypeInvalid] from typing_extensions import TypeAliasType From f2cee90937e96603b7b28083de966f587fba757d Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 24 Jul 2024 00:27:56 +0300 Subject: [PATCH 101/247] PEP 695: Add detection and error reporting for the use of incorrect expressions within the scope of a type parameter and a type alias (#17560) This fixes part of #15238: > Add detection and error reporting for use of assignment expressions, yield, yield from, and await expressions within the type parameter scope; see [this section of PEP](https://peps.python.org/pep-0695/#type-parameter-scopes) for details --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: sobolevn --- mypy/fastparse.py | 47 +++++++++++++++++++++++++++++ mypy/message_registry.py | 24 +++++++++++++++ test-data/unit/check-python312.test | 46 ++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 75c4bd46550c..363fc8375259 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -181,10 +181,12 @@ def ast3_parse( if sys.version_info >= (3, 12): ast_TypeAlias = ast3.TypeAlias ast_ParamSpec = ast3.ParamSpec + ast_TypeVar = ast3.TypeVar ast_TypeVarTuple = ast3.TypeVarTuple else: ast_TypeAlias = Any ast_ParamSpec = Any + ast_TypeVar = Any ast_TypeVarTuple = Any N = TypeVar("N", bound=Node) @@ -345,6 +347,15 @@ def is_no_type_check_decorator(expr: ast3.expr) -> bool: return False +def find_disallowed_expression_in_annotation_scope(expr: ast3.expr | None) -> ast3.expr | None: + if expr is None: + return None + for node in ast3.walk(expr): + if isinstance(node, (ast3.Yield, ast3.YieldFrom, ast3.NamedExpr, ast3.Await)): + return node + return None + + class ASTConverter: def __init__( self, @@ -1180,6 +1191,29 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: self.class_and_function_stack.pop() return cdef + def validate_type_param(self, type_param: ast_TypeVar) -> None: + incorrect_expr = find_disallowed_expression_in_annotation_scope(type_param.bound) + if incorrect_expr is None: + return + if isinstance(incorrect_expr, (ast3.Yield, ast3.YieldFrom)): + self.fail( + message_registry.TYPE_VAR_YIELD_EXPRESSION_IN_BOUND, + type_param.lineno, + type_param.col_offset, + ) + if isinstance(incorrect_expr, ast3.NamedExpr): + self.fail( + message_registry.TYPE_VAR_NAMED_EXPRESSION_IN_BOUND, + type_param.lineno, + type_param.col_offset, + ) + if isinstance(incorrect_expr, ast3.Await): + self.fail( + message_registry.TYPE_VAR_AWAIT_EXPRESSION_IN_BOUND, + type_param.lineno, + type_param.col_offset, + ) + def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: explicit_type_params = [] for p in type_params: @@ -1202,6 +1236,7 @@ def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: conv = TypeConverter(self.errors, line=p.lineno) values = [conv.visit(t) for t in p.bound.elts] elif p.bound is not None: + self.validate_type_param(p) bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) explicit_type_params.append(TypeParam(p.name, TYPE_VAR_KIND, bound, values)) return explicit_type_params @@ -1791,11 +1826,23 @@ def visit_MatchOr(self, n: MatchOr) -> OrPattern: node = OrPattern([self.visit(pattern) for pattern in n.patterns]) return self.set_line(node, n) + def validate_type_alias(self, n: ast_TypeAlias) -> None: + incorrect_expr = find_disallowed_expression_in_annotation_scope(n.value) + if incorrect_expr is None: + return + if isinstance(incorrect_expr, (ast3.Yield, ast3.YieldFrom)): + self.fail(message_registry.TYPE_ALIAS_WITH_YIELD_EXPRESSION, n.lineno, n.col_offset) + if isinstance(incorrect_expr, ast3.NamedExpr): + self.fail(message_registry.TYPE_ALIAS_WITH_NAMED_EXPRESSION, n.lineno, n.col_offset) + if isinstance(incorrect_expr, ast3.Await): + self.fail(message_registry.TYPE_ALIAS_WITH_AWAIT_EXPRESSION, n.lineno, n.col_offset) + # TypeAlias(identifier name, type_param* type_params, expr value) def visit_TypeAlias(self, n: ast_TypeAlias) -> TypeAliasStmt | AssignmentStmt: node: TypeAliasStmt | AssignmentStmt if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: type_params = self.translate_type_params(n.type_params) + self.validate_type_alias(n) value = self.visit(n.value) # Since the value is evaluated lazily, wrap the value inside a lambda. # This helps mypyc. diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 06199e70d6b4..29d539faaed6 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -338,3 +338,27 @@ def with_additional_msg(self, info: str) -> ErrorMessage: TYPE_VAR_TOO_FEW_CONSTRAINED_TYPES: Final = ErrorMessage( "Type variable must have at least two constrained types", codes.MISC ) + +TYPE_VAR_YIELD_EXPRESSION_IN_BOUND: Final = ErrorMessage( + "Yield expression cannot be used as a type variable bound", codes.SYNTAX +) + +TYPE_VAR_NAMED_EXPRESSION_IN_BOUND: Final = ErrorMessage( + "Named expression cannot be used as a type variable bound", codes.SYNTAX +) + +TYPE_VAR_AWAIT_EXPRESSION_IN_BOUND: Final = ErrorMessage( + "Await expression cannot be used as a type variable bound", codes.SYNTAX +) + +TYPE_ALIAS_WITH_YIELD_EXPRESSION: Final = ErrorMessage( + "Yield expression cannot be used within a type alias", codes.SYNTAX +) + +TYPE_ALIAS_WITH_NAMED_EXPRESSION: Final = ErrorMessage( + "Named expression cannot be used within a type alias", codes.SYNTAX +) + +TYPE_ALIAS_WITH_AWAIT_EXPRESSION: Final = ErrorMessage( + "Await expression cannot be used within a type alias", codes.SYNTAX +) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 073ef7f4bdec..a3f4c87120cd 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1667,3 +1667,49 @@ if x["other"] is not None: type Y[T] = {"item": T, **Y[T]} # E: Overwriting TypedDict field "item" while merging [builtins fixtures/dict.pyi] [typing fixtures/typing-full.pyi] + +[case testPEP695UsingIncorrectExpressionsInTypeVariableBound] +# flags: --enable-incomplete-feature=NewGenericSyntax + +type X[T: (yield 1)] = Any # E: Yield expression cannot be used as a type variable bound +type Y[T: (yield from [])] = Any # E: Yield expression cannot be used as a type variable bound +type Z[T: (a := 1)] = Any # E: Named expression cannot be used as a type variable bound +type K[T: (await 1)] = Any # E: Await expression cannot be used as a type variable bound + +type XNested[T: (1 + (yield 1))] = Any # E: Yield expression cannot be used as a type variable bound +type YNested[T: (1 + (yield from []))] = Any # E: Yield expression cannot be used as a type variable bound +type ZNested[T: (1 + (a := 1))] = Any # E: Named expression cannot be used as a type variable bound +type KNested[T: (1 + (await 1))] = Any # E: Await expression cannot be used as a type variable bound + +class FooX[T: (yield 1)]: pass # E: Yield expression cannot be used as a type variable bound +class FooY[T: (yield from [])]: pass # E: Yield expression cannot be used as a type variable bound +class FooZ[T: (a := 1)]: pass # E: Named expression cannot be used as a type variable bound +class FooK[T: (await 1)]: pass # E: Await expression cannot be used as a type variable bound + +class FooXNested[T: (1 + (yield 1))]: pass # E: Yield expression cannot be used as a type variable bound +class FooYNested[T: (1 + (yield from []))]: pass # E: Yield expression cannot be used as a type variable bound +class FooZNested[T: (1 + (a := 1))]: pass # E: Named expression cannot be used as a type variable bound +class FooKNested[T: (1 + (await 1))]: pass # E: Await expression cannot be used as a type variable bound + +def foox[T: (yield 1)](): pass # E: Yield expression cannot be used as a type variable bound +def fooy[T: (yield from [])](): pass # E: Yield expression cannot be used as a type variable bound +def fooz[T: (a := 1)](): pass # E: Named expression cannot be used as a type variable bound +def fook[T: (await 1)](): pass # E: Await expression cannot be used as a type variable bound + +def foox_nested[T: (1 + (yield 1))](): pass # E: Yield expression cannot be used as a type variable bound +def fooy_nested[T: (1 + (yield from []))](): pass # E: Yield expression cannot be used as a type variable bound +def fooz_nested[T: (1 + (a := 1))](): pass # E: Named expression cannot be used as a type variable bound +def fook_nested[T: (1 +(await 1))](): pass # E: Await expression cannot be used as a type variable bound + +[case testPEP695UsingIncorrectExpressionsInTypeAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax + +type X = (yield 1) # E: Yield expression cannot be used within a type alias +type Y = (yield from []) # E: Yield expression cannot be used within a type alias +type Z = (a := 1) # E: Named expression cannot be used within a type alias +type K = (await 1) # E: Await expression cannot be used within a type alias + +type XNested = (1 + (yield 1)) # E: Yield expression cannot be used within a type alias +type YNested = (1 + (yield from [])) # E: Yield expression cannot be used within a type alias +type ZNested = (1 + (a := 1)) # E: Named expression cannot be used within a type alias +type KNested = (1 + (await 1)) # E: Await expression cannot be used within a type alias From 18965d6315236e2a3e381ce8789638000661d40a Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 24 Jul 2024 01:09:24 +0100 Subject: [PATCH 102/247] Remove the explicit setting of a pygments theme (#17571) This allows the underlying theme being used to provide the pygments theme instead which looks better (subjectively). --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index fa76734054ac..5934c7474536 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -92,7 +92,7 @@ # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" +# pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] From ed0cd4acba02ed19b1cf18ac0ac416dc251d7714 Mon Sep 17 00:00:00 2001 From: Jordandev678 Date: Wed, 24 Jul 2024 07:09:09 +0100 Subject: [PATCH 103/247] Narrow based on collection containment (#17344) Enables the narrowing of variable types when checking a variable is "in" a collection, and the collection type is a subtype of the variable type. Fixes #3229 This PR updates the type narrowing for the "in" operator and allows it to narrow the type of a variable to the type of the collection's items - if the collection item type is a subtype of the variable (as defined by is_subtype). Examples ```python def foobar(foo: Union[str, float]): if foo in ['a', 'b']: reveal_type(foo) # N: Revealed type is "builtins.str" else: reveal_type(foo) # N: Revealed type is "Union[builtins.str, builtins.float]" ``` ```python typ: List[Literal['a', 'b']] = ['a', 'b'] x: str = "hi!" if x in typ: reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" else: reveal_type(x) # N: Revealed type is "builtins.str" ``` One existing test was updated, which compared `Optional[A]` with "in" to `(None,)`. Piror to this change that resulted in `Union[__main__.A, None]`, which now narrows to `None`. Test cases have been added for "in", "not in", Sets, Lists, and Tuples. I did add to the existing narrowing.pyi fixture for the test cases. A search of the *.test files shows it was only used in the narrowing tests, so there shouldn't be any speed impact in other areas. --------- Co-authored-by: Jordandev678 <20153053+Jordandev678@users.noreply.github.com> --- mypy/checker.py | 15 ++-- test-data/unit/check-narrowing.test | 112 +++++++++++++++++++++++++- test-data/unit/fixtures/narrowing.pyi | 9 ++- 3 files changed, 128 insertions(+), 8 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 0ae499916ec6..59de599006a8 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6011,11 +6011,16 @@ def has_no_custom_eq_checks(t: Type) -> bool: if_map, else_map = {}, {} if left_index in narrowable_operand_index_to_hash: - # We only try and narrow away 'None' for now - if is_overlapping_none(item_type): - collection_item_type = get_proper_type( - builtin_item_type(iterable_type) - ) + collection_item_type = get_proper_type(builtin_item_type(iterable_type)) + # Narrow if the collection is a subtype + if ( + collection_item_type is not None + and collection_item_type != item_type + and is_subtype(collection_item_type, item_type) + ): + if_map[operands[left_index]] = collection_item_type + # Try and narrow away 'None' + elif is_overlapping_none(item_type): if ( collection_item_type is not None and not is_overlapping_none(collection_item_type) diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 8612df9bc663..e142fdd5d060 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -1376,13 +1376,13 @@ else: reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" if val in (None,): - reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" + reveal_type(val) # N: Revealed type is "None" else: reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" if val not in (None,): reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" else: - reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" + reveal_type(val) # N: Revealed type is "None" [builtins fixtures/primitives.pyi] [case testNarrowingWithTupleOfTypes] @@ -2114,3 +2114,111 @@ else: [typing fixtures/typing-medium.pyi] [builtins fixtures/ops.pyi] + + +[case testTypeNarrowingStringInLiteralUnion] +from typing import Literal, Tuple +typ: Tuple[Literal['a', 'b'], ...] = ('a', 'b') +x: str = "hi!" +if x in typ: + reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" +else: + reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] + +[case testTypeNarrowingStringInLiteralUnionSubset] +from typing import Literal, Tuple +typeAlpha: Tuple[Literal['a', 'b', 'c'], ...] = ('a', 'b') +strIn: str = "b" +strOut: str = "c" +if strIn in typeAlpha: + reveal_type(strIn) # N: Revealed type is "Union[Literal['a'], Literal['b'], Literal['c']]" +else: + reveal_type(strIn) # N: Revealed type is "builtins.str" +if strOut in typeAlpha: + reveal_type(strOut) # N: Revealed type is "Union[Literal['a'], Literal['b'], Literal['c']]" +else: + reveal_type(strOut) # N: Revealed type is "builtins.str" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testNarrowingStringNotInLiteralUnion] +from typing import Literal, Tuple +typeAlpha: Tuple[Literal['a', 'b', 'c'],...] = ('a', 'b', 'c') +strIn: str = "c" +strOut: str = "d" +if strIn not in typeAlpha: + reveal_type(strIn) # N: Revealed type is "builtins.str" +else: + reveal_type(strIn) # N: Revealed type is "Union[Literal['a'], Literal['b'], Literal['c']]" +if strOut in typeAlpha: + reveal_type(strOut) # N: Revealed type is "Union[Literal['a'], Literal['b'], Literal['c']]" +else: + reveal_type(strOut) # N: Revealed type is "builtins.str" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testNarrowingStringInLiteralUnionDontExpand] +from typing import Literal, Tuple +typeAlpha: Tuple[Literal['a', 'b', 'c'], ...] = ('a', 'b', 'c') +strIn: Literal['c'] = "c" +reveal_type(strIn) # N: Revealed type is "Literal['c']" +#Check we don't expand a Literal into the Union type +if strIn not in typeAlpha: + reveal_type(strIn) # N: Revealed type is "Literal['c']" +else: + reveal_type(strIn) # N: Revealed type is "Literal['c']" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] + +[case testTypeNarrowingStringInMixedUnion] +from typing import Literal, Tuple +typ: Tuple[Literal['a', 'b'], ...] = ('a', 'b') +x: str = "hi!" +if x in typ: + reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" +else: + reveal_type(x) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-medium.pyi] + +[case testTypeNarrowingStringInSet] +from typing import Literal, Set +typ: Set[Literal['a', 'b']] = {'a', 'b'} +x: str = "hi!" +if x in typ: + reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" +else: + reveal_type(x) # N: Revealed type is "builtins.str" +if x not in typ: + reveal_type(x) # N: Revealed type is "builtins.str" +else: + reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" +[builtins fixtures/narrowing.pyi] +[typing fixtures/typing-medium.pyi] + +[case testTypeNarrowingStringInList] +from typing import Literal, List +typ: List[Literal['a', 'b']] = ['a', 'b'] +x: str = "hi!" +if x in typ: + reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" +else: + reveal_type(x) # N: Revealed type is "builtins.str" +if x not in typ: + reveal_type(x) # N: Revealed type is "builtins.str" +else: + reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" +[builtins fixtures/narrowing.pyi] +[typing fixtures/typing-medium.pyi] + +[case testTypeNarrowingUnionStringFloat] +from typing import Union +def foobar(foo: Union[str, float]): + if foo in ['a', 'b']: + reveal_type(foo) # N: Revealed type is "builtins.str" + else: + reveal_type(foo) # N: Revealed type is "Union[builtins.str, builtins.float]" +[builtins fixtures/primitives.pyi] +[typing fixtures/typing-medium.pyi] diff --git a/test-data/unit/fixtures/narrowing.pyi b/test-data/unit/fixtures/narrowing.pyi index 89ee011c1c80..a36ac7f29bd2 100644 --- a/test-data/unit/fixtures/narrowing.pyi +++ b/test-data/unit/fixtures/narrowing.pyi @@ -1,5 +1,5 @@ # Builtins stub used in check-narrowing test cases. -from typing import Generic, Sequence, Tuple, Type, TypeVar, Union +from typing import Generic, Sequence, Tuple, Type, TypeVar, Union, Iterable Tco = TypeVar('Tco', covariant=True) @@ -15,6 +15,13 @@ class function: pass class ellipsis: pass class int: pass class str: pass +class float: pass class dict(Generic[KT, VT]): pass def isinstance(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass + +class list(Sequence[Tco]): + def __contains__(self, other: object) -> bool: pass +class set(Iterable[Tco], Generic[Tco]): + def __init__(self, iterable: Iterable[Tco] = ...) -> None: ... + def __contains__(self, item: object) -> bool: pass From 312694fda959357802d6425397accd8cb4b58a73 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 24 Jul 2024 02:10:16 -0400 Subject: [PATCH 104/247] Fix PEP 604 isinstance caching (#17563) Mentioned by ngnpope --- mypy/types.py | 11 +++++++++-- test-data/unit/check-incremental.test | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index 91b40536f1cf..52b3121f9fb3 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2900,12 +2900,19 @@ def relevant_items(self) -> list[Type]: return [i for i in self.items if not isinstance(get_proper_type(i), NoneType)] def serialize(self) -> JsonDict: - return {".class": "UnionType", "items": [t.serialize() for t in self.items]} + return { + ".class": "UnionType", + "items": [t.serialize() for t in self.items], + "uses_pep604_syntax": self.uses_pep604_syntax, + } @classmethod def deserialize(cls, data: JsonDict) -> UnionType: assert data[".class"] == "UnionType" - return UnionType([deserialize_type(t) for t in data["items"]]) + return UnionType( + [deserialize_type(t) for t in data["items"]], + uses_pep604_syntax=data["uses_pep604_syntax"], + ) class PartialType(ProperType): diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 24292bce3e21..173265e48e6f 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6726,3 +6726,20 @@ from typing_extensions import TypeIs def guard(x: object) -> TypeIs[int]: pass [builtins fixtures/tuple.pyi] + +[case testStartUsingPEP604Union] +# flags: --python-version 3.10 +import a +[file a.py] +import lib + +[file a.py.2] +from lib import IntOrStr +assert isinstance(1, IntOrStr) + +[file lib.py] +from typing_extensions import TypeAlias + +IntOrStr: TypeAlias = int | str +assert isinstance(1, IntOrStr) +[builtins fixtures/type.pyi] From db9837fa240918fa90f2d9a3952314bd3207ed20 Mon Sep 17 00:00:00 2001 From: bzoracler <50305397+bzoracler@users.noreply.github.com> Date: Wed, 24 Jul 2024 20:10:42 +1200 Subject: [PATCH 105/247] fix: Mismatched signature between checker plugin API and implementation (#17343) This PR changes the `fail` method's signature to be positional-only for the first two parameters, due to a mismatch between the [`CheckerPluginInterface` signature](https://github.com/python/mypy/blob/8dd268ffd84ccf549b3aa9105dd35766a899b2bd/mypy/plugin.py#L242-L244) and the implementation class ([`TypeChecker`](https://github.com/python/mypy/blob/8dd268ffd84ccf549b3aa9105dd35766a899b2bd/mypy/checker.py#L7116-L7118)). ```python class CheckerPluginInterface: ... @abstractmethod def fail( self, msg: str | ErrorMessage, ctx: Context, *, code: ErrorCode | None = None ) -> None: ... ``` ```python class TypeChecker(NodeVisitor[None], CheckerPluginInterface): ... def fail( self, msg: str | ErrorMessage, context: Context, *, code: ErrorCode | None = None ) -> None: ``` An alternative fix would be to change `TypeChecker.fail`'s parameter name to `ctx`, but that has a greater disruption potential. Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- mypy/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/plugin.py b/mypy/plugin.py index 858795addb7f..a1af7fa76350 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -240,7 +240,7 @@ def type_context(self) -> list[Type | None]: @abstractmethod def fail( - self, msg: str | ErrorMessage, ctx: Context, *, code: ErrorCode | None = None + self, msg: str | ErrorMessage, ctx: Context, /, *, code: ErrorCode | None = None ) -> None: """Emit an error message at given location.""" raise NotImplementedError From e67decb91ea5aacbb1e126463b02d78ded680a90 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 25 Jul 2024 19:55:23 +0200 Subject: [PATCH 106/247] Sync typeshed (#17586) Source commit: https://github.com/python/typeshed/commit/9eae1ae1a5b47745c7d289bb2092f2b4340e7622 --- mypy/typeshed/stdlib/VERSIONS | 4 + mypy/typeshed/stdlib/_collections_abc.pyi | 4 + mypy/typeshed/stdlib/_csv.pyi | 14 +- mypy/typeshed/stdlib/_ctypes.pyi | 17 +- mypy/typeshed/stdlib/_interpchannels.pyi | 8 +- mypy/typeshed/stdlib/_interpqueues.pyi | 16 ++ mypy/typeshed/stdlib/_interpreters.pyi | 50 ++++ mypy/typeshed/stdlib/_stat.pyi | 166 +++++------ mypy/typeshed/stdlib/_thread.pyi | 8 +- mypy/typeshed/stdlib/_tkinter.pyi | 20 +- mypy/typeshed/stdlib/_winapi.pyi | 264 +++++++++--------- mypy/typeshed/stdlib/abc.pyi | 6 +- mypy/typeshed/stdlib/argparse.pyi | 12 +- mypy/typeshed/stdlib/asyncio/base_events.pyi | 72 ++++- mypy/typeshed/stdlib/asyncio/base_futures.pyi | 8 +- mypy/typeshed/stdlib/asyncio/constants.pyi | 16 +- mypy/typeshed/stdlib/asyncio/events.pyi | 75 ++++- .../stdlib/asyncio/format_helpers.pyi | 17 +- mypy/typeshed/stdlib/asyncio/queues.pyi | 13 +- mypy/typeshed/stdlib/asyncio/sslproto.pyi | 10 +- mypy/typeshed/stdlib/asyncio/streams.pyi | 6 +- mypy/typeshed/stdlib/asyncio/unix_events.pyi | 83 ++++-- .../stdlib/asyncio/windows_events.pyi | 40 ++- .../typeshed/stdlib/asyncio/windows_utils.pyi | 4 +- mypy/typeshed/stdlib/bdb.pyi | 18 +- mypy/typeshed/stdlib/binhex.pyi | 8 +- mypy/typeshed/stdlib/builtins.pyi | 28 +- mypy/typeshed/stdlib/cmd.pyi | 4 +- mypy/typeshed/stdlib/codecs.pyi | 42 +-- .../stdlib/concurrent/futures/__init__.pyi | 47 +++- .../stdlib/concurrent/futures/_base.pyi | 18 +- mypy/typeshed/stdlib/configparser.pyi | 6 +- mypy/typeshed/stdlib/copy.pyi | 14 +- mypy/typeshed/stdlib/datetime.pyi | 6 +- mypy/typeshed/stdlib/dbm/gnu.pyi | 3 + mypy/typeshed/stdlib/dbm/ndbm.pyi | 3 + mypy/typeshed/stdlib/dbm/sqlite3.pyi | 29 ++ mypy/typeshed/stdlib/dis.pyi | 89 +++++- mypy/typeshed/stdlib/distutils/ccompiler.pyi | 11 +- mypy/typeshed/stdlib/distutils/cmd.pyi | 21 +- .../stdlib/distutils/command/bdist.pyi | 18 +- .../stdlib/distutils/command/bdist_dumb.pyi | 8 +- .../stdlib/distutils/command/bdist_msi.pyi | 6 +- .../stdlib/distutils/command/bdist_rpm.pyi | 8 +- .../distutils/command/bdist_wininst.pyi | 4 +- .../stdlib/distutils/command/build.pyi | 7 +- .../stdlib/distutils/command/build_clib.pyi | 10 +- .../stdlib/distutils/command/build_ext.pyi | 10 +- .../stdlib/distutils/command/build_py.pyi | 8 +- .../distutils/command/build_scripts.pyi | 6 +- .../stdlib/distutils/command/check.pyi | 6 +- .../stdlib/distutils/command/clean.pyi | 6 +- .../stdlib/distutils/command/config.pyi | 4 +- .../stdlib/distutils/command/install.pyi | 6 +- .../stdlib/distutils/command/install_data.pyi | 6 +- .../distutils/command/install_egg_info.pyi | 2 +- .../distutils/command/install_headers.pyi | 6 +- .../stdlib/distutils/command/install_lib.pyi | 8 +- .../distutils/command/install_scripts.pyi | 6 +- .../stdlib/distutils/command/sdist.pyi | 11 +- mypy/typeshed/stdlib/distutils/dist.pyi | 6 +- mypy/typeshed/stdlib/distutils/util.pyi | 7 +- mypy/typeshed/stdlib/email/utils.pyi | 17 +- mypy/typeshed/stdlib/fcntl.pyi | 39 ++- mypy/typeshed/stdlib/filecmp.pyi | 4 +- mypy/typeshed/stdlib/ftplib.pyi | 12 +- mypy/typeshed/stdlib/gc.pyi | 14 +- mypy/typeshed/stdlib/http/cookiejar.pyi | 2 +- .../stdlib/importlib/metadata/__init__.pyi | 15 +- .../stdlib/importlib/metadata/_meta.pyi | 20 +- .../stdlib/importlib/metadata/diagnose.pyi | 2 + mypy/typeshed/stdlib/inspect.pyi | 64 +++-- mypy/typeshed/stdlib/io.pyi | 22 +- mypy/typeshed/stdlib/ipaddress.pyi | 6 +- mypy/typeshed/stdlib/itertools.pyi | 6 +- mypy/typeshed/stdlib/lzma.pyi | 54 ++-- mypy/typeshed/stdlib/mailbox.pyi | 8 + mypy/typeshed/stdlib/mimetypes.pyi | 1 + mypy/typeshed/stdlib/mmap.pyi | 16 +- mypy/typeshed/stdlib/msvcrt.pyi | 12 +- mypy/typeshed/stdlib/multiprocessing/pool.pyi | 10 +- .../stdlib/multiprocessing/reduction.pyi | 7 +- mypy/typeshed/stdlib/nntplib.pyi | 6 +- mypy/typeshed/stdlib/os/__init__.pyi | 24 +- mypy/typeshed/stdlib/pathlib.pyi | 10 +- mypy/typeshed/stdlib/pdb.pyi | 29 +- mypy/typeshed/stdlib/poplib.pyi | 12 +- mypy/typeshed/stdlib/posix.pyi | 22 +- mypy/typeshed/stdlib/pty.pyi | 10 +- mypy/typeshed/stdlib/pydoc.pyi | 37 ++- mypy/typeshed/stdlib/readline.pyi | 4 + mypy/typeshed/stdlib/site.pyi | 9 + mypy/typeshed/stdlib/sre_constants.pyi | 3 +- mypy/typeshed/stdlib/stat.pyi | 4 +- mypy/typeshed/stdlib/symtable.pyi | 31 +- mypy/typeshed/stdlib/sys/__init__.pyi | 6 +- mypy/typeshed/stdlib/syslog.pyi | 82 +++--- mypy/typeshed/stdlib/tempfile.pyi | 2 + mypy/typeshed/stdlib/threading.pyi | 2 +- mypy/typeshed/stdlib/tkinter/__init__.pyi | 71 ++++- mypy/typeshed/stdlib/tkinter/constants.pyi | 144 +++++----- mypy/typeshed/stdlib/tkinter/font.pyi | 10 +- mypy/typeshed/stdlib/tkinter/tix.pyi | 62 ++-- mypy/typeshed/stdlib/trace.pyi | 13 +- mypy/typeshed/stdlib/turtle.pyi | 14 +- mypy/typeshed/stdlib/types.pyi | 21 +- mypy/typeshed/stdlib/typing.pyi | 12 +- mypy/typeshed/stdlib/typing_extensions.pyi | 1 + mypy/typeshed/stdlib/unittest/__init__.pyi | 16 +- mypy/typeshed/stdlib/unittest/async_case.pyi | 4 + mypy/typeshed/stdlib/unittest/loader.pyi | 36 +-- mypy/typeshed/stdlib/unittest/main.pyi | 6 +- mypy/typeshed/stdlib/unittest/mock.pyi | 72 +++-- mypy/typeshed/stdlib/warnings.pyi | 6 +- mypy/typeshed/stdlib/wave.pyi | 4 +- mypy/typeshed/stdlib/webbrowser.pyi | 7 +- mypy/typeshed/stdlib/winsound.pyi | 32 +-- mypy/typeshed/stdlib/xml/dom/pulldom.pyi | 18 +- .../typeshed/stdlib/xml/etree/ElementTree.pyi | 12 +- mypy/typeshed/stdlib/zipfile/__init__.pyi | 3 + mypy/typeshed/stdlib/zipfile/_path.pyi | 7 +- mypy/typeshed/stdlib/zlib.pyi | 38 +-- test-data/unit/pythoneval.test | 14 +- 123 files changed, 1750 insertions(+), 936 deletions(-) create mode 100644 mypy/typeshed/stdlib/_interpqueues.pyi create mode 100644 mypy/typeshed/stdlib/_interpreters.pyi create mode 100644 mypy/typeshed/stdlib/dbm/sqlite3.pyi create mode 100644 mypy/typeshed/stdlib/importlib/metadata/diagnose.pyi diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index 89754f65f3fa..641f951ce3c0 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -35,6 +35,8 @@ _dummy_threading: 3.0-3.8 _heapq: 3.0- _imp: 3.0- _interpchannels: 3.13- +_interpqueues: 3.13- +_interpreters: 3.13- _json: 3.0- _locale: 3.0- _lsprof: 3.0- @@ -112,6 +114,7 @@ curses: 3.0- dataclasses: 3.7- datetime: 3.0- dbm: 3.0- +dbm.sqlite3: 3.13- decimal: 3.0- difflib: 3.0- dis: 3.0- @@ -155,6 +158,7 @@ importlib: 3.0- importlib._abc: 3.10- importlib.metadata: 3.8- importlib.metadata._meta: 3.10- +importlib.metadata.diagnose: 3.13- importlib.readers: 3.10- importlib.resources: 3.7- importlib.resources.abc: 3.11- diff --git a/mypy/typeshed/stdlib/_collections_abc.pyi b/mypy/typeshed/stdlib/_collections_abc.pyi index e467d626e8a8..127488ee382c 100644 --- a/mypy/typeshed/stdlib/_collections_abc.pyi +++ b/mypy/typeshed/stdlib/_collections_abc.pyi @@ -70,6 +70,8 @@ _VT_co = TypeVar("_VT_co", covariant=True) # Value type covariant containers. @final class dict_keys(KeysView[_KT_co], Generic[_KT_co, _VT_co]): # undocumented def __eq__(self, value: object, /) -> bool: ... + if sys.version_info >= (3, 13): + def isdisjoint(self, other: Iterable[_KT_co], /) -> bool: ... if sys.version_info >= (3, 10): @property def mapping(self) -> MappingProxyType[_KT_co, _VT_co]: ... @@ -83,6 +85,8 @@ class dict_values(ValuesView[_VT_co], Generic[_KT_co, _VT_co]): # undocumented @final class dict_items(ItemsView[_KT_co, _VT_co]): # undocumented def __eq__(self, value: object, /) -> bool: ... + if sys.version_info >= (3, 13): + def isdisjoint(self, other: Iterable[tuple[_KT_co, _VT_co]], /) -> bool: ... if sys.version_info >= (3, 10): @property def mapping(self) -> MappingProxyType[_KT_co, _VT_co]: ... diff --git a/mypy/typeshed/stdlib/_csv.pyi b/mypy/typeshed/stdlib/_csv.pyi index 19f2dc9664b1..9bb5d27f6e35 100644 --- a/mypy/typeshed/stdlib/_csv.pyi +++ b/mypy/typeshed/stdlib/_csv.pyi @@ -1,18 +1,18 @@ import sys from _typeshed import SupportsWrite from collections.abc import Iterable, Iterator -from typing import Any, Final, Literal +from typing import Any, Final from typing_extensions import TypeAlias __version__: Final[str] -QUOTE_ALL: Literal[1] -QUOTE_MINIMAL: Literal[0] -QUOTE_NONE: Literal[3] -QUOTE_NONNUMERIC: Literal[2] +QUOTE_ALL: Final = 1 +QUOTE_MINIMAL: Final = 0 +QUOTE_NONE: Final = 3 +QUOTE_NONNUMERIC: Final = 2 if sys.version_info >= (3, 12): - QUOTE_STRINGS: Literal[4] - QUOTE_NOTNULL: Literal[5] + QUOTE_STRINGS: Final = 4 + QUOTE_NOTNULL: Final = 5 # Ideally this would be `QUOTE_ALL | QUOTE_MINIMAL | QUOTE_NONE | QUOTE_NONNUMERIC` # However, using literals in situations like these can cause false-positives (see #7258) diff --git a/mypy/typeshed/stdlib/_ctypes.pyi b/mypy/typeshed/stdlib/_ctypes.pyi index a5f20dfd30e7..f7da03a67ead 100644 --- a/mypy/typeshed/stdlib/_ctypes.pyi +++ b/mypy/typeshed/stdlib/_ctypes.pyi @@ -64,7 +64,6 @@ class _CData(metaclass=_CDataMeta): # Structure.from_buffer(...) # valid at runtime # Structure(...).from_buffer(...) # invalid at runtime # - @classmethod def from_buffer(cls, source: WriteableBuffer, offset: int = ...) -> Self: ... @classmethod @@ -72,7 +71,7 @@ class _CData(metaclass=_CDataMeta): @classmethod def from_address(cls, address: int) -> Self: ... @classmethod - def from_param(cls, obj: Any) -> Self | _CArgObject: ... + def from_param(cls, value: Any, /) -> Self | _CArgObject: ... @classmethod def in_dll(cls, library: CDLL, name: str) -> Self: ... def __buffer__(self, flags: int, /) -> memoryview: ... @@ -100,8 +99,8 @@ class _Pointer(_PointerLike, _CData, Generic[_CT]): def __getitem__(self, key: slice, /) -> list[Any]: ... def __setitem__(self, key: int, value: Any, /) -> None: ... -def POINTER(type: type[_CT]) -> type[_Pointer[_CT]]: ... -def pointer(arg: _CT, /) -> _Pointer[_CT]: ... +def POINTER(type: type[_CT], /) -> type[_Pointer[_CT]]: ... +def pointer(obj: _CT, /) -> _Pointer[_CT]: ... class _CArgObject: ... @@ -199,9 +198,9 @@ class Array(_CData, Generic[_CT]): if sys.version_info >= (3, 9): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... -def addressof(obj: _CData) -> int: ... -def alignment(obj_or_type: _CData | type[_CData]) -> int: ... +def addressof(obj: _CData, /) -> int: ... +def alignment(obj_or_type: _CData | type[_CData], /) -> int: ... def get_errno() -> int: ... -def resize(obj: _CData, size: int) -> None: ... -def set_errno(value: int) -> int: ... -def sizeof(obj_or_type: _CData | type[_CData]) -> int: ... +def resize(obj: _CData, size: int, /) -> None: ... +def set_errno(value: int, /) -> int: ... +def sizeof(obj_or_type: _CData | type[_CData], /) -> int: ... diff --git a/mypy/typeshed/stdlib/_interpchannels.pyi b/mypy/typeshed/stdlib/_interpchannels.pyi index b77fe321a071..c03496044df0 100644 --- a/mypy/typeshed/stdlib/_interpchannels.pyi +++ b/mypy/typeshed/stdlib/_interpchannels.pyi @@ -1,5 +1,5 @@ from _typeshed import structseq -from typing import Final, Literal, SupportsIndex, final +from typing import Any, Final, Literal, SupportsIndex, final from typing_extensions import Buffer, Self class ChannelError(RuntimeError): ... @@ -72,13 +72,15 @@ class ChannelInfo(structseq[int], tuple[bool, bool, bool, int, int, int, int, in @property def send_released(self) -> bool: ... -def create() -> ChannelID: ... +def create(unboundop: Literal[1, 2, 3]) -> ChannelID: ... def destroy(cid: SupportsIndex) -> None: ... def list_all() -> list[ChannelID]: ... def list_interpreters(cid: SupportsIndex, *, send: bool) -> list[int]: ... def send(cid: SupportsIndex, obj: object, *, blocking: bool = True, timeout: float | None = None) -> None: ... def send_buffer(cid: SupportsIndex, obj: Buffer, *, blocking: bool = True, timeout: float | None = None) -> None: ... -def recv(cid: SupportsIndex, default: object = ...) -> object: ... +def recv(cid: SupportsIndex, default: object = ...) -> tuple[Any, Literal[1, 2, 3]]: ... def close(cid: SupportsIndex, *, send: bool = False, recv: bool = False) -> None: ... +def get_count(cid: SupportsIndex) -> int: ... def get_info(cid: SupportsIndex) -> ChannelInfo: ... +def get_channel_defaults(cid: SupportsIndex) -> Literal[1, 2, 3]: ... def release(cid: SupportsIndex, *, send: bool = False, recv: bool = False, force: bool = False) -> None: ... diff --git a/mypy/typeshed/stdlib/_interpqueues.pyi b/mypy/typeshed/stdlib/_interpqueues.pyi new file mode 100644 index 000000000000..db5e4cff5068 --- /dev/null +++ b/mypy/typeshed/stdlib/_interpqueues.pyi @@ -0,0 +1,16 @@ +from typing import Any, SupportsIndex + +class QueueError(RuntimeError): ... +class QueueNotFoundError(QueueError): ... + +def bind(qid: SupportsIndex) -> None: ... +def create(maxsize: SupportsIndex, fmt: SupportsIndex) -> int: ... +def destroy(qid: SupportsIndex) -> None: ... +def get(qid: SupportsIndex) -> tuple[Any, int]: ... +def get_count(qid: SupportsIndex) -> int: ... +def get_maxsize(qid: SupportsIndex) -> int: ... +def get_queue_defaults(qid: SupportsIndex) -> tuple[int]: ... +def is_full(qid: SupportsIndex) -> bool: ... +def list_all() -> list[tuple[int, int]]: ... +def put(qid: SupportsIndex, obj: Any, fmt: SupportsIndex) -> None: ... +def release(qid: SupportsIndex) -> None: ... diff --git a/mypy/typeshed/stdlib/_interpreters.pyi b/mypy/typeshed/stdlib/_interpreters.pyi new file mode 100644 index 000000000000..75f661a7e8e1 --- /dev/null +++ b/mypy/typeshed/stdlib/_interpreters.pyi @@ -0,0 +1,50 @@ +import types +from collections.abc import Callable, Mapping +from typing import Final, Literal, SupportsIndex +from typing_extensions import TypeAlias + +_Configs: TypeAlias = Literal["default", "isolated", "legacy", "empty", ""] + +class InterpreterError(Exception): ... +class InterpreterNotFoundError(InterpreterError): ... +class NotShareableError(Exception): ... + +class CrossInterpreterBufferView: + def __buffer__(self, flags: int, /) -> memoryview: ... + +def new_config(name: _Configs = "isolated", /, **overides: object) -> types.SimpleNamespace: ... +def create(config: types.SimpleNamespace | _Configs | None = "isolated", *, reqrefs: bool = False) -> int: ... +def destroy(id: SupportsIndex, *, restrict: bool = False) -> None: ... +def list_all(*, require_ready: bool) -> list[tuple[int, int]]: ... +def get_current() -> tuple[int, int]: ... +def get_main() -> tuple[int, int]: ... +def is_running(id: SupportsIndex, *, restrict: bool = False) -> bool: ... +def get_config(id: SupportsIndex, *, restrict: bool = False) -> types.SimpleNamespace: ... +def whence(id: SupportsIndex) -> int: ... +def exec(id: SupportsIndex, code: str, shared: bool | None = None, *, restrict: bool = False) -> None: ... +def call( + id: SupportsIndex, + callable: Callable[..., object], + args: tuple[object, ...] | None = None, + kwargs: dict[str, object] | None = None, + *, + restrict: bool = False, +) -> object: ... +def run_string( + id: SupportsIndex, script: str | types.CodeType | Callable[[], object], shared: bool | None = None, *, restrict: bool = False +) -> None: ... +def run_func( + id: SupportsIndex, func: types.CodeType | Callable[[], object], shared: bool | None = None, *, restrict: bool = False +) -> None: ... +def set___main___attrs(id: SupportsIndex, updates: Mapping[str, object], *, restrict: bool = False) -> None: ... +def incref(id: SupportsIndex, *, implieslink: bool = False, restrict: bool = False) -> None: ... +def decref(id: SupportsIndex, *, restrict: bool = False) -> None: ... +def is_shareable(obj: object) -> bool: ... +def capture_exception(exc: BaseException | None = None) -> types.SimpleNamespace: ... + +WHENCE_UNKNOWN: Final = 0 +WHENCE_RUNTIME: Final = 1 +WHENCE_LEGACY_CAPI: Final = 2 +WHENCE_CAPI: Final = 3 +WHENCE_XI: Final = 4 +WHENCE_STDLIB: Final = 5 diff --git a/mypy/typeshed/stdlib/_stat.pyi b/mypy/typeshed/stdlib/_stat.pyi index c4e918d8b57f..903571a64bca 100644 --- a/mypy/typeshed/stdlib/_stat.pyi +++ b/mypy/typeshed/stdlib/_stat.pyi @@ -1,30 +1,30 @@ import sys -from typing import Literal - -SF_APPEND: Literal[0x00040000] -SF_ARCHIVED: Literal[0x00010000] -SF_IMMUTABLE: Literal[0x00020000] -SF_NOUNLINK: Literal[0x00100000] -SF_SNAPSHOT: Literal[0x00200000] - -ST_MODE: Literal[0] -ST_INO: Literal[1] -ST_DEV: Literal[2] -ST_NLINK: Literal[3] -ST_UID: Literal[4] -ST_GID: Literal[5] -ST_SIZE: Literal[6] -ST_ATIME: Literal[7] -ST_MTIME: Literal[8] -ST_CTIME: Literal[9] - -S_IFIFO: Literal[0o010000] -S_IFLNK: Literal[0o120000] -S_IFREG: Literal[0o100000] -S_IFSOCK: Literal[0o140000] -S_IFBLK: Literal[0o060000] -S_IFCHR: Literal[0o020000] -S_IFDIR: Literal[0o040000] +from typing import Final + +SF_APPEND: Final = 0x00040000 +SF_ARCHIVED: Final = 0x00010000 +SF_IMMUTABLE: Final = 0x00020000 +SF_NOUNLINK: Final = 0x00100000 +SF_SNAPSHOT: Final = 0x00200000 + +ST_MODE: Final = 0 +ST_INO: Final = 1 +ST_DEV: Final = 2 +ST_NLINK: Final = 3 +ST_UID: Final = 4 +ST_GID: Final = 5 +ST_SIZE: Final = 6 +ST_ATIME: Final = 7 +ST_MTIME: Final = 8 +ST_CTIME: Final = 9 + +S_IFIFO: Final = 0o010000 +S_IFLNK: Final = 0o120000 +S_IFREG: Final = 0o100000 +S_IFSOCK: Final = 0o140000 +S_IFBLK: Final = 0o060000 +S_IFCHR: Final = 0o020000 +S_IFDIR: Final = 0o040000 # These are 0 on systems that don't support the specific kind of file. # Example: Linux doesn't support door files, so S_IFDOOR is 0 on linux. @@ -32,37 +32,37 @@ S_IFDOOR: int S_IFPORT: int S_IFWHT: int -S_ISUID: Literal[0o4000] -S_ISGID: Literal[0o2000] -S_ISVTX: Literal[0o1000] - -S_IRWXU: Literal[0o0700] -S_IRUSR: Literal[0o0400] -S_IWUSR: Literal[0o0200] -S_IXUSR: Literal[0o0100] - -S_IRWXG: Literal[0o0070] -S_IRGRP: Literal[0o0040] -S_IWGRP: Literal[0o0020] -S_IXGRP: Literal[0o0010] - -S_IRWXO: Literal[0o0007] -S_IROTH: Literal[0o0004] -S_IWOTH: Literal[0o0002] -S_IXOTH: Literal[0o0001] - -S_ENFMT: Literal[0o2000] -S_IREAD: Literal[0o0400] -S_IWRITE: Literal[0o0200] -S_IEXEC: Literal[0o0100] - -UF_APPEND: Literal[0x00000004] -UF_COMPRESSED: Literal[0x00000020] # OS X 10.6+ only -UF_HIDDEN: Literal[0x00008000] # OX X 10.5+ only -UF_IMMUTABLE: Literal[0x00000002] -UF_NODUMP: Literal[0x00000001] -UF_NOUNLINK: Literal[0x00000010] -UF_OPAQUE: Literal[0x00000008] +S_ISUID: Final = 0o4000 +S_ISGID: Final = 0o2000 +S_ISVTX: Final = 0o1000 + +S_IRWXU: Final = 0o0700 +S_IRUSR: Final = 0o0400 +S_IWUSR: Final = 0o0200 +S_IXUSR: Final = 0o0100 + +S_IRWXG: Final = 0o0070 +S_IRGRP: Final = 0o0040 +S_IWGRP: Final = 0o0020 +S_IXGRP: Final = 0o0010 + +S_IRWXO: Final = 0o0007 +S_IROTH: Final = 0o0004 +S_IWOTH: Final = 0o0002 +S_IXOTH: Final = 0o0001 + +S_ENFMT: Final = 0o2000 +S_IREAD: Final = 0o0400 +S_IWRITE: Final = 0o0200 +S_IEXEC: Final = 0o0100 + +UF_APPEND: Final = 0x00000004 +UF_COMPRESSED: Final = 0x00000020 # OS X 10.6+ only +UF_HIDDEN: Final = 0x00008000 # OX X 10.5+ only +UF_IMMUTABLE: Final = 0x00000002 +UF_NODUMP: Final = 0x00000001 +UF_NOUNLINK: Final = 0x00000010 +UF_OPAQUE: Final = 0x00000008 def S_IMODE(mode: int, /) -> int: ... def S_IFMT(mode: int, /) -> int: ... @@ -84,34 +84,36 @@ if sys.platform == "win32": IO_REPARSE_TAG_APPEXECLINK: int if sys.platform == "win32": - FILE_ATTRIBUTE_ARCHIVE: Literal[32] - FILE_ATTRIBUTE_COMPRESSED: Literal[2048] - FILE_ATTRIBUTE_DEVICE: Literal[64] - FILE_ATTRIBUTE_DIRECTORY: Literal[16] - FILE_ATTRIBUTE_ENCRYPTED: Literal[16384] - FILE_ATTRIBUTE_HIDDEN: Literal[2] - FILE_ATTRIBUTE_INTEGRITY_STREAM: Literal[32768] - FILE_ATTRIBUTE_NORMAL: Literal[128] - FILE_ATTRIBUTE_NOT_CONTENT_INDEXED: Literal[8192] - FILE_ATTRIBUTE_NO_SCRUB_DATA: Literal[131072] - FILE_ATTRIBUTE_OFFLINE: Literal[4096] - FILE_ATTRIBUTE_READONLY: Literal[1] - FILE_ATTRIBUTE_REPARSE_POINT: Literal[1024] - FILE_ATTRIBUTE_SPARSE_FILE: Literal[512] - FILE_ATTRIBUTE_SYSTEM: Literal[4] - FILE_ATTRIBUTE_TEMPORARY: Literal[256] - FILE_ATTRIBUTE_VIRTUAL: Literal[65536] + FILE_ATTRIBUTE_ARCHIVE: Final = 32 + FILE_ATTRIBUTE_COMPRESSED: Final = 2048 + FILE_ATTRIBUTE_DEVICE: Final = 64 + FILE_ATTRIBUTE_DIRECTORY: Final = 16 + FILE_ATTRIBUTE_ENCRYPTED: Final = 16384 + FILE_ATTRIBUTE_HIDDEN: Final = 2 + FILE_ATTRIBUTE_INTEGRITY_STREAM: Final = 32768 + FILE_ATTRIBUTE_NORMAL: Final = 128 + FILE_ATTRIBUTE_NOT_CONTENT_INDEXED: Final = 8192 + FILE_ATTRIBUTE_NO_SCRUB_DATA: Final = 131072 + FILE_ATTRIBUTE_OFFLINE: Final = 4096 + FILE_ATTRIBUTE_READONLY: Final = 1 + FILE_ATTRIBUTE_REPARSE_POINT: Final = 1024 + FILE_ATTRIBUTE_SPARSE_FILE: Final = 512 + FILE_ATTRIBUTE_SYSTEM: Final = 4 + FILE_ATTRIBUTE_TEMPORARY: Final = 256 + FILE_ATTRIBUTE_VIRTUAL: Final = 65536 if sys.version_info >= (3, 13): - SF_SETTABLE: Literal[0x3FFF0000] + # Varies by platform. + SF_SETTABLE: Final[int] # https://github.com/python/cpython/issues/114081#issuecomment-2119017790 # SF_RESTRICTED: Literal[0x00080000] - SF_FIRMLINK: Literal[0x00800000] - SF_DATALESS: Literal[0x40000000] + SF_FIRMLINK: Final = 0x00800000 + SF_DATALESS: Final = 0x40000000 - SF_SUPPORTED: Literal[0x9F0000] - SF_SYNTHETIC: Literal[0xC0000000] + if sys.platform == "darwin": + SF_SUPPORTED: Final = 0x9F0000 + SF_SYNTHETIC: Final = 0xC0000000 - UF_TRACKED: Literal[0x00000040] - UF_DATAVAULT: Literal[0x00000080] - UF_SETTABLE: Literal[0x0000FFFF] + UF_TRACKED: Final = 0x00000040 + UF_DATAVAULT: Final = 0x00000080 + UF_SETTABLE: Final = 0x0000FFFF diff --git a/mypy/typeshed/stdlib/_thread.pyi b/mypy/typeshed/stdlib/_thread.pyi index 4ea9aa0609e5..304cb79ec96b 100644 --- a/mypy/typeshed/stdlib/_thread.pyi +++ b/mypy/typeshed/stdlib/_thread.pyi @@ -13,7 +13,7 @@ error = RuntimeError def _count() -> int: ... @final class LockType: - def acquire(self, blocking: bool = ..., timeout: float = ...) -> bool: ... + def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: ... def release(self) -> None: ... def locked(self) -> bool: ... def __enter__(self) -> bool: ... @@ -22,14 +22,14 @@ class LockType: ) -> None: ... @overload -def start_new_thread(function: Callable[[Unpack[_Ts]], object], args: tuple[Unpack[_Ts]]) -> int: ... +def start_new_thread(function: Callable[[Unpack[_Ts]], object], args: tuple[Unpack[_Ts]], /) -> int: ... @overload -def start_new_thread(function: Callable[..., object], args: tuple[Any, ...], kwargs: dict[str, Any]) -> int: ... +def start_new_thread(function: Callable[..., object], args: tuple[Any, ...], kwargs: dict[str, Any], /) -> int: ... def interrupt_main() -> None: ... def exit() -> NoReturn: ... def allocate_lock() -> LockType: ... def get_ident() -> int: ... -def stack_size(size: int = ...) -> int: ... +def stack_size(size: int = 0, /) -> int: ... TIMEOUT_MAX: float diff --git a/mypy/typeshed/stdlib/_tkinter.pyi b/mypy/typeshed/stdlib/_tkinter.pyi index aea74c8be279..a7293054d293 100644 --- a/mypy/typeshed/stdlib/_tkinter.pyi +++ b/mypy/typeshed/stdlib/_tkinter.pyi @@ -1,6 +1,6 @@ import sys from collections.abc import Callable -from typing import Any, ClassVar, Literal, final +from typing import Any, ClassVar, Final, final from typing_extensions import TypeAlias # _tkinter is meant to be only used internally by tkinter, but some tkinter @@ -95,16 +95,16 @@ class TkappType: def settrace(self, func: _TkinterTraceFunc | None, /) -> None: ... # These should be kept in sync with tkinter.tix constants, except ALL_EVENTS which doesn't match TCL_ALL_EVENTS -ALL_EVENTS: Literal[-3] -FILE_EVENTS: Literal[8] -IDLE_EVENTS: Literal[32] -TIMER_EVENTS: Literal[16] -WINDOW_EVENTS: Literal[4] +ALL_EVENTS: Final = -3 +FILE_EVENTS: Final = 8 +IDLE_EVENTS: Final = 32 +TIMER_EVENTS: Final = 16 +WINDOW_EVENTS: Final = 4 -DONT_WAIT: Literal[2] -EXCEPTION: Literal[8] -READABLE: Literal[2] -WRITABLE: Literal[4] +DONT_WAIT: Final = 2 +EXCEPTION: Final = 8 +READABLE: Final = 2 +WRITABLE: Final = 4 TCL_VERSION: str TK_VERSION: str diff --git a/mypy/typeshed/stdlib/_winapi.pyi b/mypy/typeshed/stdlib/_winapi.pyi index c6fb0484df8e..62ea124045cc 100644 --- a/mypy/typeshed/stdlib/_winapi.pyi +++ b/mypy/typeshed/stdlib/_winapi.pyi @@ -1,117 +1,117 @@ import sys from _typeshed import ReadableBuffer from collections.abc import Sequence -from typing import Any, Literal, NoReturn, final, overload +from typing import Any, Final, Literal, NoReturn, final, overload if sys.platform == "win32": - ABOVE_NORMAL_PRIORITY_CLASS: Literal[0x8000] - BELOW_NORMAL_PRIORITY_CLASS: Literal[0x4000] - - CREATE_BREAKAWAY_FROM_JOB: Literal[0x1000000] - CREATE_DEFAULT_ERROR_MODE: Literal[0x4000000] - CREATE_NO_WINDOW: Literal[0x8000000] - CREATE_NEW_CONSOLE: Literal[0x10] - CREATE_NEW_PROCESS_GROUP: Literal[0x200] - - DETACHED_PROCESS: Literal[8] - DUPLICATE_CLOSE_SOURCE: Literal[1] - DUPLICATE_SAME_ACCESS: Literal[2] - - ERROR_ALREADY_EXISTS: Literal[183] - ERROR_BROKEN_PIPE: Literal[109] - ERROR_IO_PENDING: Literal[997] - ERROR_MORE_DATA: Literal[234] - ERROR_NETNAME_DELETED: Literal[64] - ERROR_NO_DATA: Literal[232] - ERROR_NO_SYSTEM_RESOURCES: Literal[1450] - ERROR_OPERATION_ABORTED: Literal[995] - ERROR_PIPE_BUSY: Literal[231] - ERROR_PIPE_CONNECTED: Literal[535] - ERROR_SEM_TIMEOUT: Literal[121] - - FILE_FLAG_FIRST_PIPE_INSTANCE: Literal[0x80000] - FILE_FLAG_OVERLAPPED: Literal[0x40000000] - - FILE_GENERIC_READ: Literal[1179785] - FILE_GENERIC_WRITE: Literal[1179926] - - FILE_MAP_ALL_ACCESS: Literal[983071] - FILE_MAP_COPY: Literal[1] - FILE_MAP_EXECUTE: Literal[32] - FILE_MAP_READ: Literal[4] - FILE_MAP_WRITE: Literal[2] - - FILE_TYPE_CHAR: Literal[2] - FILE_TYPE_DISK: Literal[1] - FILE_TYPE_PIPE: Literal[3] - FILE_TYPE_REMOTE: Literal[32768] - FILE_TYPE_UNKNOWN: Literal[0] - - GENERIC_READ: Literal[0x80000000] - GENERIC_WRITE: Literal[0x40000000] - HIGH_PRIORITY_CLASS: Literal[0x80] - INFINITE: Literal[0xFFFFFFFF] + ABOVE_NORMAL_PRIORITY_CLASS: Final = 0x8000 + BELOW_NORMAL_PRIORITY_CLASS: Final = 0x4000 + + CREATE_BREAKAWAY_FROM_JOB: Final = 0x1000000 + CREATE_DEFAULT_ERROR_MODE: Final = 0x4000000 + CREATE_NO_WINDOW: Final = 0x8000000 + CREATE_NEW_CONSOLE: Final = 0x10 + CREATE_NEW_PROCESS_GROUP: Final = 0x200 + + DETACHED_PROCESS: Final = 8 + DUPLICATE_CLOSE_SOURCE: Final = 1 + DUPLICATE_SAME_ACCESS: Final = 2 + + ERROR_ALREADY_EXISTS: Final = 183 + ERROR_BROKEN_PIPE: Final = 109 + ERROR_IO_PENDING: Final = 997 + ERROR_MORE_DATA: Final = 234 + ERROR_NETNAME_DELETED: Final = 64 + ERROR_NO_DATA: Final = 232 + ERROR_NO_SYSTEM_RESOURCES: Final = 1450 + ERROR_OPERATION_ABORTED: Final = 995 + ERROR_PIPE_BUSY: Final = 231 + ERROR_PIPE_CONNECTED: Final = 535 + ERROR_SEM_TIMEOUT: Final = 121 + + FILE_FLAG_FIRST_PIPE_INSTANCE: Final = 0x80000 + FILE_FLAG_OVERLAPPED: Final = 0x40000000 + + FILE_GENERIC_READ: Final = 1179785 + FILE_GENERIC_WRITE: Final = 1179926 + + FILE_MAP_ALL_ACCESS: Final = 983071 + FILE_MAP_COPY: Final = 1 + FILE_MAP_EXECUTE: Final = 32 + FILE_MAP_READ: Final = 4 + FILE_MAP_WRITE: Final = 2 + + FILE_TYPE_CHAR: Final = 2 + FILE_TYPE_DISK: Final = 1 + FILE_TYPE_PIPE: Final = 3 + FILE_TYPE_REMOTE: Final = 32768 + FILE_TYPE_UNKNOWN: Final = 0 + + GENERIC_READ: Final = 0x80000000 + GENERIC_WRITE: Final = 0x40000000 + HIGH_PRIORITY_CLASS: Final = 0x80 + INFINITE: Final = 0xFFFFFFFF # Ignore the Flake8 error -- flake8-pyi assumes # most numbers this long will be implementation details, # but here we can see that it's a power of 2 - INVALID_HANDLE_VALUE: Literal[0xFFFFFFFFFFFFFFFF] # noqa: Y054 - IDLE_PRIORITY_CLASS: Literal[0x40] - NORMAL_PRIORITY_CLASS: Literal[0x20] - REALTIME_PRIORITY_CLASS: Literal[0x100] - NMPWAIT_WAIT_FOREVER: Literal[0xFFFFFFFF] - - MEM_COMMIT: Literal[0x1000] - MEM_FREE: Literal[0x10000] - MEM_IMAGE: Literal[0x1000000] - MEM_MAPPED: Literal[0x40000] - MEM_PRIVATE: Literal[0x20000] - MEM_RESERVE: Literal[0x2000] - - NULL: Literal[0] - OPEN_EXISTING: Literal[3] - - PIPE_ACCESS_DUPLEX: Literal[3] - PIPE_ACCESS_INBOUND: Literal[1] - PIPE_READMODE_MESSAGE: Literal[2] - PIPE_TYPE_MESSAGE: Literal[4] - PIPE_UNLIMITED_INSTANCES: Literal[255] - PIPE_WAIT: Literal[0] - - PAGE_EXECUTE: Literal[0x10] - PAGE_EXECUTE_READ: Literal[0x20] - PAGE_EXECUTE_READWRITE: Literal[0x40] - PAGE_EXECUTE_WRITECOPY: Literal[0x80] - PAGE_GUARD: Literal[0x100] - PAGE_NOACCESS: Literal[0x1] - PAGE_NOCACHE: Literal[0x200] - PAGE_READONLY: Literal[0x2] - PAGE_READWRITE: Literal[0x4] - PAGE_WRITECOMBINE: Literal[0x400] - PAGE_WRITECOPY: Literal[0x8] - - PROCESS_ALL_ACCESS: Literal[0x1FFFFF] - PROCESS_DUP_HANDLE: Literal[0x40] - - SEC_COMMIT: Literal[0x8000000] - SEC_IMAGE: Literal[0x1000000] - SEC_LARGE_PAGES: Literal[0x80000000] - SEC_NOCACHE: Literal[0x10000000] - SEC_RESERVE: Literal[0x4000000] - SEC_WRITECOMBINE: Literal[0x40000000] - - STARTF_USESHOWWINDOW: Literal[0x1] - STARTF_USESTDHANDLES: Literal[0x100] - - STD_ERROR_HANDLE: Literal[0xFFFFFFF4] - STD_OUTPUT_HANDLE: Literal[0xFFFFFFF5] - STD_INPUT_HANDLE: Literal[0xFFFFFFF6] - - STILL_ACTIVE: Literal[259] - SW_HIDE: Literal[0] - SYNCHRONIZE: Literal[0x100000] - WAIT_ABANDONED_0: Literal[128] - WAIT_OBJECT_0: Literal[0] - WAIT_TIMEOUT: Literal[258] + INVALID_HANDLE_VALUE: Final = 0xFFFFFFFFFFFFFFFF # noqa: Y054 + IDLE_PRIORITY_CLASS: Final = 0x40 + NORMAL_PRIORITY_CLASS: Final = 0x20 + REALTIME_PRIORITY_CLASS: Final = 0x100 + NMPWAIT_WAIT_FOREVER: Final = 0xFFFFFFFF + + MEM_COMMIT: Final = 0x1000 + MEM_FREE: Final = 0x10000 + MEM_IMAGE: Final = 0x1000000 + MEM_MAPPED: Final = 0x40000 + MEM_PRIVATE: Final = 0x20000 + MEM_RESERVE: Final = 0x2000 + + NULL: Final = 0 + OPEN_EXISTING: Final = 3 + + PIPE_ACCESS_DUPLEX: Final = 3 + PIPE_ACCESS_INBOUND: Final = 1 + PIPE_READMODE_MESSAGE: Final = 2 + PIPE_TYPE_MESSAGE: Final = 4 + PIPE_UNLIMITED_INSTANCES: Final = 255 + PIPE_WAIT: Final = 0 + + PAGE_EXECUTE: Final = 0x10 + PAGE_EXECUTE_READ: Final = 0x20 + PAGE_EXECUTE_READWRITE: Final = 0x40 + PAGE_EXECUTE_WRITECOPY: Final = 0x80 + PAGE_GUARD: Final = 0x100 + PAGE_NOACCESS: Final = 0x1 + PAGE_NOCACHE: Final = 0x200 + PAGE_READONLY: Final = 0x2 + PAGE_READWRITE: Final = 0x4 + PAGE_WRITECOMBINE: Final = 0x400 + PAGE_WRITECOPY: Final = 0x8 + + PROCESS_ALL_ACCESS: Final = 0x1FFFFF + PROCESS_DUP_HANDLE: Final = 0x40 + + SEC_COMMIT: Final = 0x8000000 + SEC_IMAGE: Final = 0x1000000 + SEC_LARGE_PAGES: Final = 0x80000000 + SEC_NOCACHE: Final = 0x10000000 + SEC_RESERVE: Final = 0x4000000 + SEC_WRITECOMBINE: Final = 0x40000000 + + STARTF_USESHOWWINDOW: Final = 0x1 + STARTF_USESTDHANDLES: Final = 0x100 + + STD_ERROR_HANDLE: Final = 0xFFFFFFF4 + STD_OUTPUT_HANDLE: Final = 0xFFFFFFF5 + STD_INPUT_HANDLE: Final = 0xFFFFFFF6 + + STILL_ACTIVE: Final = 259 + SW_HIDE: Final = 0 + SYNCHRONIZE: Final = 0x100000 + WAIT_ABANDONED_0: Final = 128 + WAIT_OBJECT_0: Final = 0 + WAIT_TIMEOUT: Final = 258 if sys.version_info >= (3, 10): LOCALE_NAME_INVARIANT: str @@ -131,32 +131,32 @@ if sys.platform == "win32": LCMAP_UPPERCASE: int if sys.version_info >= (3, 12): - COPYFILE2_CALLBACK_CHUNK_STARTED: Literal[1] - COPYFILE2_CALLBACK_CHUNK_FINISHED: Literal[2] - COPYFILE2_CALLBACK_STREAM_STARTED: Literal[3] - COPYFILE2_CALLBACK_STREAM_FINISHED: Literal[4] - COPYFILE2_CALLBACK_POLL_CONTINUE: Literal[5] - COPYFILE2_CALLBACK_ERROR: Literal[6] - - COPYFILE2_PROGRESS_CONTINUE: Literal[0] - COPYFILE2_PROGRESS_CANCEL: Literal[1] - COPYFILE2_PROGRESS_STOP: Literal[2] - COPYFILE2_PROGRESS_QUIET: Literal[3] - COPYFILE2_PROGRESS_PAUSE: Literal[4] - - COPY_FILE_FAIL_IF_EXISTS: Literal[0x1] - COPY_FILE_RESTARTABLE: Literal[0x2] - COPY_FILE_OPEN_SOURCE_FOR_WRITE: Literal[0x4] - COPY_FILE_ALLOW_DECRYPTED_DESTINATION: Literal[0x8] - COPY_FILE_COPY_SYMLINK: Literal[0x800] - COPY_FILE_NO_BUFFERING: Literal[0x1000] - COPY_FILE_REQUEST_SECURITY_PRIVILEGES: Literal[0x2000] - COPY_FILE_RESUME_FROM_PAUSE: Literal[0x4000] - COPY_FILE_NO_OFFLOAD: Literal[0x40000] - COPY_FILE_REQUEST_COMPRESSED_TRAFFIC: Literal[0x10000000] - - ERROR_ACCESS_DENIED: Literal[5] - ERROR_PRIVILEGE_NOT_HELD: Literal[1314] + COPYFILE2_CALLBACK_CHUNK_STARTED: Final = 1 + COPYFILE2_CALLBACK_CHUNK_FINISHED: Final = 2 + COPYFILE2_CALLBACK_STREAM_STARTED: Final = 3 + COPYFILE2_CALLBACK_STREAM_FINISHED: Final = 4 + COPYFILE2_CALLBACK_POLL_CONTINUE: Final = 5 + COPYFILE2_CALLBACK_ERROR: Final = 6 + + COPYFILE2_PROGRESS_CONTINUE: Final = 0 + COPYFILE2_PROGRESS_CANCEL: Final = 1 + COPYFILE2_PROGRESS_STOP: Final = 2 + COPYFILE2_PROGRESS_QUIET: Final = 3 + COPYFILE2_PROGRESS_PAUSE: Final = 4 + + COPY_FILE_FAIL_IF_EXISTS: Final = 0x1 + COPY_FILE_RESTARTABLE: Final = 0x2 + COPY_FILE_OPEN_SOURCE_FOR_WRITE: Final = 0x4 + COPY_FILE_ALLOW_DECRYPTED_DESTINATION: Final = 0x8 + COPY_FILE_COPY_SYMLINK: Final = 0x800 + COPY_FILE_NO_BUFFERING: Final = 0x1000 + COPY_FILE_REQUEST_SECURITY_PRIVILEGES: Final = 0x2000 + COPY_FILE_RESUME_FROM_PAUSE: Final = 0x4000 + COPY_FILE_NO_OFFLOAD: Final = 0x40000 + COPY_FILE_REQUEST_COMPRESSED_TRAFFIC: Final = 0x10000000 + + ERROR_ACCESS_DENIED: Final = 5 + ERROR_PRIVILEGE_NOT_HELD: Final = 1314 def CloseHandle(handle: int, /) -> None: ... @overload diff --git a/mypy/typeshed/stdlib/abc.pyi b/mypy/typeshed/stdlib/abc.pyi index 6bf7821f1c1b..fdca48ac7aaf 100644 --- a/mypy/typeshed/stdlib/abc.pyi +++ b/mypy/typeshed/stdlib/abc.pyi @@ -28,17 +28,17 @@ class ABCMeta(type): def register(cls: ABCMeta, subclass: type[_T]) -> type[_T]: ... def abstractmethod(funcobj: _FuncT) -> _FuncT: ... -@deprecated("Deprecated, use 'classmethod' with 'abstractmethod' instead") +@deprecated("Use 'classmethod' with 'abstractmethod' instead") class abstractclassmethod(classmethod[_T, _P, _R_co]): __isabstractmethod__: Literal[True] def __init__(self, callable: Callable[Concatenate[type[_T], _P], _R_co]) -> None: ... -@deprecated("Deprecated, use 'staticmethod' with 'abstractmethod' instead") +@deprecated("Use 'staticmethod' with 'abstractmethod' instead") class abstractstaticmethod(staticmethod[_P, _R_co]): __isabstractmethod__: Literal[True] def __init__(self, callable: Callable[_P, _R_co]) -> None: ... -@deprecated("Deprecated, use 'property' with 'abstractmethod' instead") +@deprecated("Use 'property' with 'abstractmethod' instead") class abstractproperty(property): __isabstractmethod__: Literal[True] diff --git a/mypy/typeshed/stdlib/argparse.pyi b/mypy/typeshed/stdlib/argparse.pyi index bc781ec8e61d..d878f3ebd6a2 100644 --- a/mypy/typeshed/stdlib/argparse.pyi +++ b/mypy/typeshed/stdlib/argparse.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import sentinel from collections.abc import Callable, Generator, Iterable, Sequence from re import Pattern -from typing import IO, Any, Generic, Literal, NewType, NoReturn, Protocol, TypeVar, overload +from typing import IO, Any, Final, Generic, NewType, NoReturn, Protocol, TypeVar, overload from typing_extensions import Self, TypeAlias, deprecated __all__ = [ @@ -43,14 +43,14 @@ _ActionStr: TypeAlias = str # callers that don't use a literal argument _NArgsStr: TypeAlias = str -ONE_OR_MORE: Literal["+"] -OPTIONAL: Literal["?"] -PARSER: Literal["A..."] -REMAINDER: Literal["..."] +ONE_OR_MORE: Final = "+" +OPTIONAL: Final = "?" +PARSER: Final = "A..." +REMAINDER: Final = "..." _SUPPRESS_T = NewType("_SUPPRESS_T", str) SUPPRESS: _SUPPRESS_T | str # not using Literal because argparse sometimes compares SUPPRESS with is # the | str is there so that foo = argparse.SUPPRESS; foo = "test" checks out in mypy -ZERO_OR_MORE: Literal["*"] +ZERO_OR_MORE: Final = "*" _UNRECOGNIZED_ARGS_ATTR: str # undocumented class ArgumentError(Exception): diff --git a/mypy/typeshed/stdlib/asyncio/base_events.pyi b/mypy/typeshed/stdlib/asyncio/base_events.pyi index 112cfeefa8f2..cba2c7799528 100644 --- a/mypy/typeshed/stdlib/asyncio/base_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/base_events.pyi @@ -49,6 +49,10 @@ class Server(AbstractServer): ssl_handshake_timeout: float | None, ) -> None: ... + if sys.version_info >= (3, 13): + def close_clients(self) -> None: ... + def abort_clients(self) -> None: ... + def get_loop(self) -> AbstractEventLoop: ... def is_serving(self) -> bool: ... async def start_serving(self) -> None: ... @@ -222,7 +226,9 @@ class BaseEventLoop(AbstractEventLoop): happy_eyeballs_delay: float | None = None, interleave: int | None = None, ) -> tuple[Transport, _ProtocolT]: ... - if sys.version_info >= (3, 11): + + if sys.version_info >= (3, 13): + # 3.13 added `keep_alive`. @overload async def create_server( self, @@ -237,6 +243,7 @@ class BaseEventLoop(AbstractEventLoop): ssl: _SSLContext = None, reuse_address: bool | None = None, reuse_port: bool | None = None, + keep_alive: bool | None = None, ssl_handshake_timeout: float | None = None, ssl_shutdown_timeout: float | None = None, start_serving: bool = True, @@ -255,30 +262,48 @@ class BaseEventLoop(AbstractEventLoop): ssl: _SSLContext = None, reuse_address: bool | None = None, reuse_port: bool | None = None, + keep_alive: bool | None = None, ssl_handshake_timeout: float | None = None, ssl_shutdown_timeout: float | None = None, start_serving: bool = True, ) -> Server: ... - async def start_tls( + elif sys.version_info >= (3, 11): + @overload + async def create_server( self, - transport: BaseTransport, - protocol: BaseProtocol, - sslcontext: ssl.SSLContext, + protocol_factory: _ProtocolFactory, + host: str | Sequence[str] | None = None, + port: int = ..., *, - server_side: bool = False, - server_hostname: str | None = None, + family: int = ..., + flags: int = ..., + sock: None = None, + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, ssl_handshake_timeout: float | None = None, ssl_shutdown_timeout: float | None = None, - ) -> Transport | None: ... - async def connect_accepted_socket( + start_serving: bool = True, + ) -> Server: ... + @overload + async def create_server( self, - protocol_factory: Callable[[], _ProtocolT], - sock: socket, + protocol_factory: _ProtocolFactory, + host: None = None, + port: None = None, *, + family: int = ..., + flags: int = ..., + sock: socket = ..., + backlog: int = 100, ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, ssl_handshake_timeout: float | None = None, ssl_shutdown_timeout: float | None = None, - ) -> tuple[Transport, _ProtocolT]: ... + start_serving: bool = True, + ) -> Server: ... else: @overload async def create_server( @@ -314,6 +339,29 @@ class BaseEventLoop(AbstractEventLoop): ssl_handshake_timeout: float | None = None, start_serving: bool = True, ) -> Server: ... + + if sys.version_info >= (3, 11): + async def start_tls( + self, + transport: BaseTransport, + protocol: BaseProtocol, + sslcontext: ssl.SSLContext, + *, + server_side: bool = False, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + ) -> Transport | None: ... + async def connect_accepted_socket( + self, + protocol_factory: Callable[[], _ProtocolT], + sock: socket, + *, + ssl: _SSLContext = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + ) -> tuple[Transport, _ProtocolT]: ... + else: async def start_tls( self, transport: BaseTransport, diff --git a/mypy/typeshed/stdlib/asyncio/base_futures.pyi b/mypy/typeshed/stdlib/asyncio/base_futures.pyi index 231766200934..55d2fbdbdb62 100644 --- a/mypy/typeshed/stdlib/asyncio/base_futures.pyi +++ b/mypy/typeshed/stdlib/asyncio/base_futures.pyi @@ -1,6 +1,6 @@ from collections.abc import Callable, Sequence from contextvars import Context -from typing import Any, Literal +from typing import Any, Final from . import futures @@ -11,9 +11,9 @@ __all__ = () # That's why the import order is reversed. from .futures import isfuture as isfuture -_PENDING: Literal["PENDING"] # undocumented -_CANCELLED: Literal["CANCELLED"] # undocumented -_FINISHED: Literal["FINISHED"] # undocumented +_PENDING: Final = "PENDING" # undocumented +_CANCELLED: Final = "CANCELLED" # undocumented +_FINISHED: Final = "FINISHED" # undocumented def _format_callbacks(cb: Sequence[tuple[Callable[[futures.Future[Any]], None], Context]]) -> str: ... # undocumented def _future_repr_info(future: futures.Future[Any]) -> list[str]: ... # undocumented diff --git a/mypy/typeshed/stdlib/asyncio/constants.pyi b/mypy/typeshed/stdlib/asyncio/constants.pyi index 7759a2844953..5c6456b0e9c0 100644 --- a/mypy/typeshed/stdlib/asyncio/constants.pyi +++ b/mypy/typeshed/stdlib/asyncio/constants.pyi @@ -1,18 +1,18 @@ import enum import sys -from typing import Literal +from typing import Final -LOG_THRESHOLD_FOR_CONNLOST_WRITES: Literal[5] -ACCEPT_RETRY_DELAY: Literal[1] -DEBUG_STACK_DEPTH: Literal[10] +LOG_THRESHOLD_FOR_CONNLOST_WRITES: Final = 5 +ACCEPT_RETRY_DELAY: Final = 1 +DEBUG_STACK_DEPTH: Final = 10 SSL_HANDSHAKE_TIMEOUT: float -SENDFILE_FALLBACK_READBUFFER_SIZE: Literal[262144] +SENDFILE_FALLBACK_READBUFFER_SIZE: Final = 262144 if sys.version_info >= (3, 11): SSL_SHUTDOWN_TIMEOUT: float - FLOW_CONTROL_HIGH_WATER_SSL_READ: Literal[256] - FLOW_CONTROL_HIGH_WATER_SSL_WRITE: Literal[512] + FLOW_CONTROL_HIGH_WATER_SSL_READ: Final = 256 + FLOW_CONTROL_HIGH_WATER_SSL_WRITE: Final = 512 if sys.version_info >= (3, 12): - THREAD_JOIN_TIMEOUT: Literal[300] + THREAD_JOIN_TIMEOUT: Final = 300 class _SendfileMode(enum.Enum): UNSUPPORTED = 1 diff --git a/mypy/typeshed/stdlib/asyncio/events.pyi b/mypy/typeshed/stdlib/asyncio/events.pyi index 8c2664666835..eed688fc792a 100644 --- a/mypy/typeshed/stdlib/asyncio/events.pyi +++ b/mypy/typeshed/stdlib/asyncio/events.pyi @@ -94,6 +94,12 @@ class TimerHandle(Handle): class AbstractServer: @abstractmethod def close(self) -> None: ... + if sys.version_info >= (3, 13): + @abstractmethod + def close_clients(self) -> None: ... + @abstractmethod + def abort_clients(self) -> None: ... + async def __aenter__(self) -> Self: ... async def __aexit__(self, *exc: Unused) -> None: ... @abstractmethod @@ -272,7 +278,9 @@ class AbstractEventLoop: happy_eyeballs_delay: float | None = None, interleave: int | None = None, ) -> tuple[Transport, _ProtocolT]: ... - if sys.version_info >= (3, 11): + + if sys.version_info >= (3, 13): + # 3.13 added `keep_alive`. @overload @abstractmethod async def create_server( @@ -288,6 +296,7 @@ class AbstractEventLoop: ssl: _SSLContext = None, reuse_address: bool | None = None, reuse_port: bool | None = None, + keep_alive: bool | None = None, ssl_handshake_timeout: float | None = None, ssl_shutdown_timeout: float | None = None, start_serving: bool = True, @@ -307,30 +316,46 @@ class AbstractEventLoop: ssl: _SSLContext = None, reuse_address: bool | None = None, reuse_port: bool | None = None, + keep_alive: bool | None = None, ssl_handshake_timeout: float | None = None, ssl_shutdown_timeout: float | None = None, start_serving: bool = True, ) -> Server: ... + elif sys.version_info >= (3, 11): + @overload @abstractmethod - async def start_tls( + async def create_server( self, - transport: WriteTransport, - protocol: BaseProtocol, - sslcontext: ssl.SSLContext, + protocol_factory: _ProtocolFactory, + host: str | Sequence[str] | None = None, + port: int = ..., *, - server_side: bool = False, - server_hostname: str | None = None, + family: int = ..., + flags: int = ..., + sock: None = None, + backlog: int = 100, + ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, ssl_handshake_timeout: float | None = None, ssl_shutdown_timeout: float | None = None, - ) -> Transport | None: ... - async def create_unix_server( + start_serving: bool = True, + ) -> Server: ... + @overload + @abstractmethod + async def create_server( self, protocol_factory: _ProtocolFactory, - path: StrPath | None = None, + host: None = None, + port: None = None, *, - sock: socket | None = None, + family: int = ..., + flags: int = ..., + sock: socket = ..., backlog: int = 100, ssl: _SSLContext = None, + reuse_address: bool | None = None, + reuse_port: bool | None = None, ssl_handshake_timeout: float | None = None, ssl_shutdown_timeout: float | None = None, start_serving: bool = True, @@ -372,6 +397,33 @@ class AbstractEventLoop: ssl_handshake_timeout: float | None = None, start_serving: bool = True, ) -> Server: ... + + if sys.version_info >= (3, 11): + @abstractmethod + async def start_tls( + self, + transport: WriteTransport, + protocol: BaseProtocol, + sslcontext: ssl.SSLContext, + *, + server_side: bool = False, + server_hostname: str | None = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + ) -> Transport | None: ... + async def create_unix_server( + self, + protocol_factory: _ProtocolFactory, + path: StrPath | None = None, + *, + sock: socket | None = None, + backlog: int = 100, + ssl: _SSLContext = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + ) -> Server: ... + else: @abstractmethod async def start_tls( self, @@ -394,6 +446,7 @@ class AbstractEventLoop: ssl_handshake_timeout: float | None = None, start_serving: bool = True, ) -> Server: ... + if sys.version_info >= (3, 11): async def connect_accepted_socket( self, diff --git a/mypy/typeshed/stdlib/asyncio/format_helpers.pyi b/mypy/typeshed/stdlib/asyncio/format_helpers.pyi index 1c78dff3948a..41505b14cd08 100644 --- a/mypy/typeshed/stdlib/asyncio/format_helpers.pyi +++ b/mypy/typeshed/stdlib/asyncio/format_helpers.pyi @@ -1,4 +1,5 @@ import functools +import sys import traceback from collections.abc import Iterable from types import FrameType, FunctionType @@ -14,7 +15,17 @@ _FuncType: TypeAlias = FunctionType | _HasWrapper | functools.partial[Any] | fun def _get_function_source(func: _FuncType) -> tuple[str, int]: ... @overload def _get_function_source(func: object) -> tuple[str, int] | None: ... -def _format_callback_source(func: object, args: Iterable[Any]) -> str: ... -def _format_args_and_kwargs(args: Iterable[Any], kwargs: dict[str, Any]) -> str: ... -def _format_callback(func: object, args: Iterable[Any], kwargs: dict[str, Any], suffix: str = "") -> str: ... + +if sys.version_info >= (3, 13): + def _format_callback_source(func: object, args: Iterable[Any], *, debug: bool = False) -> str: ... + def _format_args_and_kwargs(args: Iterable[Any], kwargs: dict[str, Any], *, debug: bool = False) -> str: ... + def _format_callback( + func: object, args: Iterable[Any], kwargs: dict[str, Any], *, debug: bool = False, suffix: str = "" + ) -> str: ... + +else: + def _format_callback_source(func: object, args: Iterable[Any]) -> str: ... + def _format_args_and_kwargs(args: Iterable[Any], kwargs: dict[str, Any]) -> str: ... + def _format_callback(func: object, args: Iterable[Any], kwargs: dict[str, Any], suffix: str = "") -> str: ... + def extract_stack(f: FrameType | None = None, limit: int | None = None) -> traceback.StackSummary: ... diff --git a/mypy/typeshed/stdlib/asyncio/queues.pyi b/mypy/typeshed/stdlib/asyncio/queues.pyi index 1d8f80f4c388..895205aa9519 100644 --- a/mypy/typeshed/stdlib/asyncio/queues.pyi +++ b/mypy/typeshed/stdlib/asyncio/queues.pyi @@ -10,13 +10,20 @@ if sys.version_info >= (3, 10): else: _LoopBoundMixin = object -__all__ = ("Queue", "PriorityQueue", "LifoQueue", "QueueFull", "QueueEmpty") - class QueueEmpty(Exception): ... class QueueFull(Exception): ... +if sys.version_info >= (3, 13): + __all__ = ("Queue", "PriorityQueue", "LifoQueue", "QueueFull", "QueueEmpty", "QueueShutDown") + +else: + __all__ = ("Queue", "PriorityQueue", "LifoQueue", "QueueFull", "QueueEmpty") + _T = TypeVar("_T") +if sys.version_info >= (3, 13): + class QueueShutDown(Exception): ... + # If Generic[_T] is last and _LoopBoundMixin is object, pyright is unhappy. # We can remove the noqa pragma when dropping 3.9 support. class Queue(Generic[_T], _LoopBoundMixin): # noqa: Y059 @@ -42,6 +49,8 @@ class Queue(Generic[_T], _LoopBoundMixin): # noqa: Y059 def task_done(self) -> None: ... if sys.version_info >= (3, 9): def __class_getitem__(cls, type: Any, /) -> GenericAlias: ... + if sys.version_info >= (3, 13): + def shutdown(self, immediate: bool = False) -> None: ... class PriorityQueue(Queue[_T]): ... class LifoQueue(Queue[_T]): ... diff --git a/mypy/typeshed/stdlib/asyncio/sslproto.pyi b/mypy/typeshed/stdlib/asyncio/sslproto.pyi index e904d7395cdc..ded1933dd659 100644 --- a/mypy/typeshed/stdlib/asyncio/sslproto.pyi +++ b/mypy/typeshed/stdlib/asyncio/sslproto.pyi @@ -3,7 +3,7 @@ import sys from collections import deque from collections.abc import Callable from enum import Enum -from typing import Any, ClassVar, Literal +from typing import Any, ClassVar, Final, Literal from typing_extensions import TypeAlias from . import constants, events, futures, protocols, transports @@ -29,10 +29,10 @@ if sys.version_info >= (3, 11): def add_flowcontrol_defaults(high: int | None, low: int | None, kb: int) -> tuple[int, int]: ... else: - _UNWRAPPED: Literal["UNWRAPPED"] - _DO_HANDSHAKE: Literal["DO_HANDSHAKE"] - _WRAPPED: Literal["WRAPPED"] - _SHUTDOWN: Literal["SHUTDOWN"] + _UNWRAPPED: Final = "UNWRAPPED" + _DO_HANDSHAKE: Final = "DO_HANDSHAKE" + _WRAPPED: Final = "WRAPPED" + _SHUTDOWN: Final = "SHUTDOWN" if sys.version_info < (3, 11): class _SSLPipe: diff --git a/mypy/typeshed/stdlib/asyncio/streams.pyi b/mypy/typeshed/stdlib/asyncio/streams.pyi index c3cc7b8c9e5a..0be5249e2169 100644 --- a/mypy/typeshed/stdlib/asyncio/streams.pyi +++ b/mypy/typeshed/stdlib/asyncio/streams.pyi @@ -2,6 +2,7 @@ import ssl import sys from _typeshed import ReadableBuffer, StrPath from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Sequence, Sized +from types import ModuleType from typing import Any, Protocol, SupportsIndex from typing_extensions import Self, TypeAlias @@ -130,7 +131,10 @@ class StreamWriter: async def start_tls( self, sslcontext: ssl.SSLContext, *, server_hostname: str | None = None, ssl_handshake_timeout: float | None = None ) -> None: ... - if sys.version_info >= (3, 11): + + if sys.version_info >= (3, 13): + def __del__(self, warnings: ModuleType = ...) -> None: ... + elif sys.version_info >= (3, 11): def __del__(self) -> None: ... class StreamReader(AsyncIterator[bytes]): diff --git a/mypy/typeshed/stdlib/asyncio/unix_events.pyi b/mypy/typeshed/stdlib/asyncio/unix_events.pyi index 3a2c62646121..5dd3831f9a0a 100644 --- a/mypy/typeshed/stdlib/asyncio/unix_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/unix_events.pyi @@ -1,15 +1,55 @@ import sys import types +from _typeshed import StrPath from abc import ABCMeta, abstractmethod from collections.abc import Callable +from socket import socket from typing import Literal from typing_extensions import Self, TypeVarTuple, Unpack, deprecated +from .base_events import Server, _ProtocolFactory, _SSLContext from .events import AbstractEventLoop, BaseDefaultEventLoopPolicy from .selector_events import BaseSelectorEventLoop _Ts = TypeVarTuple("_Ts") +if sys.platform != "win32": + if sys.version_info >= (3, 14): + __all__ = ("SelectorEventLoop", "DefaultEventLoopPolicy", "EventLoop") + elif sys.version_info >= (3, 13): + __all__ = ( + "SelectorEventLoop", + "AbstractChildWatcher", + "SafeChildWatcher", + "FastChildWatcher", + "PidfdChildWatcher", + "MultiLoopChildWatcher", + "ThreadedChildWatcher", + "DefaultEventLoopPolicy", + "EventLoop", + ) + elif sys.version_info >= (3, 9): + __all__ = ( + "SelectorEventLoop", + "AbstractChildWatcher", + "SafeChildWatcher", + "FastChildWatcher", + "PidfdChildWatcher", + "MultiLoopChildWatcher", + "ThreadedChildWatcher", + "DefaultEventLoopPolicy", + ) + else: + __all__ = ( + "SelectorEventLoop", + "AbstractChildWatcher", + "SafeChildWatcher", + "FastChildWatcher", + "MultiLoopChildWatcher", + "ThreadedChildWatcher", + "DefaultEventLoopPolicy", + ) + # This is also technically not available on Win, # but other parts of typeshed need this definition. # So, it is special cased. @@ -58,30 +98,6 @@ if sys.version_info < (3, 14): def is_active(self) -> bool: ... if sys.platform != "win32": - if sys.version_info >= (3, 14): - __all__ = ("SelectorEventLoop", "DefaultEventLoopPolicy") - elif sys.version_info >= (3, 9): - __all__ = ( - "SelectorEventLoop", - "AbstractChildWatcher", - "SafeChildWatcher", - "FastChildWatcher", - "PidfdChildWatcher", - "MultiLoopChildWatcher", - "ThreadedChildWatcher", - "DefaultEventLoopPolicy", - ) - else: - __all__ = ( - "SelectorEventLoop", - "AbstractChildWatcher", - "SafeChildWatcher", - "FastChildWatcher", - "MultiLoopChildWatcher", - "ThreadedChildWatcher", - "DefaultEventLoopPolicy", - ) - if sys.version_info < (3, 14): if sys.version_info >= (3, 12): # Doesn't actually have ABCMeta metaclass at runtime, but mypy complains if we don't have it in the stub. @@ -141,7 +157,21 @@ if sys.platform != "win32": ) -> None: ... def remove_child_handler(self, pid: int) -> bool: ... - class _UnixSelectorEventLoop(BaseSelectorEventLoop): ... + class _UnixSelectorEventLoop(BaseSelectorEventLoop): + if sys.version_info >= (3, 13): + async def create_unix_server( # type: ignore[override] + self, + protocol_factory: _ProtocolFactory, + path: StrPath | None = None, + *, + sock: socket | None = None, + backlog: int = 100, + ssl: _SSLContext = None, + ssl_handshake_timeout: float | None = None, + ssl_shutdown_timeout: float | None = None, + start_serving: bool = True, + cleanup_socket: bool = True, + ) -> Server: ... class _UnixDefaultEventLoopPolicy(BaseDefaultEventLoopPolicy): if sys.version_info < (3, 14): @@ -158,6 +188,9 @@ if sys.platform != "win32": DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy + if sys.version_info >= (3, 13): + EventLoop = SelectorEventLoop + if sys.version_info < (3, 14): if sys.version_info >= (3, 12): @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") diff --git a/mypy/typeshed/stdlib/asyncio/windows_events.pyi b/mypy/typeshed/stdlib/asyncio/windows_events.pyi index 97aa52ff8b9a..e5205ba4dcb0 100644 --- a/mypy/typeshed/stdlib/asyncio/windows_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/windows_events.pyi @@ -2,24 +2,36 @@ import socket import sys from _typeshed import Incomplete, ReadableBuffer, WriteableBuffer from collections.abc import Callable -from typing import IO, Any, ClassVar, Literal, NoReturn +from typing import IO, Any, ClassVar, Final, NoReturn from . import events, futures, proactor_events, selector_events, streams, windows_utils if sys.platform == "win32": - __all__ = ( - "SelectorEventLoop", - "ProactorEventLoop", - "IocpProactor", - "DefaultEventLoopPolicy", - "WindowsSelectorEventLoopPolicy", - "WindowsProactorEventLoopPolicy", - ) + if sys.version_info >= (3, 13): + # 3.13 added `EventLoop`. + __all__ = ( + "SelectorEventLoop", + "ProactorEventLoop", + "IocpProactor", + "DefaultEventLoopPolicy", + "WindowsSelectorEventLoopPolicy", + "WindowsProactorEventLoopPolicy", + "EventLoop", + ) + else: + __all__ = ( + "SelectorEventLoop", + "ProactorEventLoop", + "IocpProactor", + "DefaultEventLoopPolicy", + "WindowsSelectorEventLoopPolicy", + "WindowsProactorEventLoopPolicy", + ) - NULL: Literal[0] - INFINITE: Literal[0xFFFFFFFF] - ERROR_CONNECTION_REFUSED: Literal[1225] - ERROR_CONNECTION_ABORTED: Literal[1236] + NULL: Final = 0 + INFINITE: Final = 0xFFFFFFFF + ERROR_CONNECTION_REFUSED: Final = 1225 + ERROR_CONNECTION_ABORTED: Final = 1236 CONNECT_PIPE_INIT_DELAY: float CONNECT_PIPE_MAX_DELAY: float @@ -84,3 +96,5 @@ if sys.platform == "win32": def set_child_watcher(self, watcher: Any) -> NoReturn: ... DefaultEventLoopPolicy = WindowsSelectorEventLoopPolicy + if sys.version_info >= (3, 13): + EventLoop = ProactorEventLoop diff --git a/mypy/typeshed/stdlib/asyncio/windows_utils.pyi b/mypy/typeshed/stdlib/asyncio/windows_utils.pyi index 6b3589adc3cb..4fa014532376 100644 --- a/mypy/typeshed/stdlib/asyncio/windows_utils.pyi +++ b/mypy/typeshed/stdlib/asyncio/windows_utils.pyi @@ -2,13 +2,13 @@ import subprocess import sys from collections.abc import Callable from types import TracebackType -from typing import Any, AnyStr, Literal +from typing import Any, AnyStr, Final from typing_extensions import Self if sys.platform == "win32": __all__ = ("pipe", "Popen", "PIPE", "PipeHandle") - BUFSIZE: Literal[8192] + BUFSIZE: Final = 8192 PIPE = subprocess.PIPE STDOUT = subprocess.STDOUT def pipe(*, duplex: bool = False, overlapped: tuple[bool, bool] = (True, True), bufsize: int = 8192) -> tuple[int, int]: ... diff --git a/mypy/typeshed/stdlib/bdb.pyi b/mypy/typeshed/stdlib/bdb.pyi index a72e986728a7..75bfa91cc379 100644 --- a/mypy/typeshed/stdlib/bdb.pyi +++ b/mypy/typeshed/stdlib/bdb.pyi @@ -1,8 +1,8 @@ import sys -from _typeshed import ExcInfo, TraceFunction +from _typeshed import ExcInfo, TraceFunction, Unused from collections.abc import Callable, Iterable, Mapping from types import CodeType, FrameType, TracebackType -from typing import IO, Any, Literal, SupportsInt, TypeVar +from typing import IO, Any, Final, SupportsInt, TypeVar from typing_extensions import ParamSpec __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -10,7 +10,10 @@ __all__ = ["BdbQuit", "Bdb", "Breakpoint"] _T = TypeVar("_T") _P = ParamSpec("_P") -GENERATOR_AND_COROUTINE_FLAGS: Literal[672] +# A union of code-object flags at runtime. +# The exact values of code-object flags are implementation details, +# so we don't include the value of this constant in the stubs. +GENERATOR_AND_COROUTINE_FLAGS: Final[int] class BdbQuit(Exception): ... @@ -32,6 +35,9 @@ class Bdb: def dispatch_call(self, frame: FrameType, arg: None) -> TraceFunction: ... def dispatch_return(self, frame: FrameType, arg: Any) -> TraceFunction: ... def dispatch_exception(self, frame: FrameType, arg: ExcInfo) -> TraceFunction: ... + if sys.version_info >= (3, 13): + def dispatch_opcode(self, frame: FrameType, arg: Unused) -> Callable[[FrameType, str, Any], TraceFunction]: ... + def is_skipped_module(self, module_name: str) -> bool: ... def stop_here(self, frame: FrameType) -> bool: ... def break_here(self, frame: FrameType) -> bool: ... @@ -42,7 +48,13 @@ class Bdb: def user_return(self, frame: FrameType, return_value: Any) -> None: ... def user_exception(self, frame: FrameType, exc_info: ExcInfo) -> None: ... def set_until(self, frame: FrameType, lineno: int | None = None) -> None: ... + if sys.version_info >= (3, 13): + def user_opcode(self, frame: FrameType) -> None: ... # undocumented + def set_step(self) -> None: ... + if sys.version_info >= (3, 13): + def set_stepinstr(self) -> None: ... # undocumented + def set_next(self, frame: FrameType) -> None: ... def set_return(self, frame: FrameType) -> None: ... def set_trace(self, frame: FrameType | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/binhex.pyi b/mypy/typeshed/stdlib/binhex.pyi index d514be3b9b26..bdead928468f 100644 --- a/mypy/typeshed/stdlib/binhex.pyi +++ b/mypy/typeshed/stdlib/binhex.pyi @@ -1,14 +1,14 @@ from _typeshed import SizedBuffer -from typing import IO, Any, Literal +from typing import IO, Any, Final from typing_extensions import TypeAlias __all__ = ["binhex", "hexbin", "Error"] class Error(Exception): ... -REASONABLY_LARGE: Literal[32768] -LINELEN: Literal[64] -RUNCHAR: Literal[b"\x90"] +REASONABLY_LARGE: Final = 32768 +LINELEN: Final = 64 +RUNCHAR: Final = b"\x90" class FInfo: Type: str diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index 28b0b11a8e5c..07d2f1989558 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -74,6 +74,7 @@ if sys.version_info >= (3, 9): from types import GenericAlias _T = TypeVar("_T") +_I = TypeVar("_I", default=int) _T_co = TypeVar("_T_co", covariant=True) _T_contra = TypeVar("_T_contra", contravariant=True) _R_co = TypeVar("_R_co", covariant=True) @@ -728,8 +729,12 @@ class bytearray(MutableSequence[int]): def __buffer__(self, flags: int, /) -> memoryview: ... def __release_buffer__(self, buffer: memoryview, /) -> None: ... +_IntegerFormats: TypeAlias = Literal[ + "b", "B", "@b", "@B", "h", "H", "@h", "@H", "i", "I", "@i", "@I", "l", "L", "@l", "@L", "q", "Q", "@q", "@Q", "P", "@P" +] + @final -class memoryview(Sequence[int]): +class memoryview(Sequence[_I]): @property def format(self) -> str: ... @property @@ -759,13 +764,20 @@ class memoryview(Sequence[int]): def __exit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None, / ) -> None: ... - def cast(self, format: str, shape: list[int] | tuple[int, ...] = ...) -> memoryview: ... @overload - def __getitem__(self, key: SupportsIndex | tuple[SupportsIndex, ...], /) -> int: ... + def cast(self, format: Literal["c", "@c"], shape: list[int] | tuple[int, ...] = ...) -> memoryview[bytes]: ... + @overload + def cast(self, format: Literal["f", "@f", "d", "@d"], shape: list[int] | tuple[int, ...] = ...) -> memoryview[float]: ... + @overload + def cast(self, format: Literal["?"], shape: list[int] | tuple[int, ...] = ...) -> memoryview[bool]: ... @overload - def __getitem__(self, key: slice, /) -> memoryview: ... + def cast(self, format: _IntegerFormats, shape: list[int] | tuple[int, ...] = ...) -> memoryview: ... + @overload + def __getitem__(self, key: SupportsIndex | tuple[SupportsIndex, ...], /) -> _I: ... + @overload + def __getitem__(self, key: slice, /) -> memoryview[_I]: ... def __contains__(self, x: object, /) -> bool: ... - def __iter__(self) -> Iterator[int]: ... + def __iter__(self) -> Iterator[_I]: ... def __len__(self) -> int: ... def __eq__(self, value: object, /) -> bool: ... def __hash__(self) -> int: ... @@ -1761,6 +1773,7 @@ class BaseException: __suppress_context__: bool __traceback__: TracebackType | None def __init__(self, *args: object) -> None: ... + def __new__(cls, *args: Any, **kwds: Any) -> Self: ... def __setstate__(self, state: dict[str, Any] | None, /) -> None: ... def with_traceback(self, tb: TracebackType | None, /) -> Self: ... if sys.version_info >= (3, 11): @@ -1911,9 +1924,9 @@ if sys.version_info >= (3, 10): class EncodingWarning(Warning): ... if sys.version_info >= (3, 11): - _BaseExceptionT_co = TypeVar("_BaseExceptionT_co", bound=BaseException, covariant=True) + _BaseExceptionT_co = TypeVar("_BaseExceptionT_co", bound=BaseException, covariant=True, default=BaseException) _BaseExceptionT = TypeVar("_BaseExceptionT", bound=BaseException) - _ExceptionT_co = TypeVar("_ExceptionT_co", bound=Exception, covariant=True) + _ExceptionT_co = TypeVar("_ExceptionT_co", bound=Exception, covariant=True, default=Exception) _ExceptionT = TypeVar("_ExceptionT", bound=Exception) # See `check_exception_group.py` for use-cases and comments. @@ -1977,5 +1990,4 @@ if sys.version_info >= (3, 11): ) -> tuple[ExceptionGroup[_ExceptionT_co] | None, ExceptionGroup[_ExceptionT_co] | None]: ... if sys.version_info >= (3, 13): - class IncompleteInputError(SyntaxError): ... class PythonFinalizationError(RuntimeError): ... diff --git a/mypy/typeshed/stdlib/cmd.pyi b/mypy/typeshed/stdlib/cmd.pyi index 9499847fb153..0733857433be 100644 --- a/mypy/typeshed/stdlib/cmd.pyi +++ b/mypy/typeshed/stdlib/cmd.pyi @@ -1,9 +1,9 @@ from collections.abc import Callable -from typing import IO, Any, Literal +from typing import IO, Any, Final __all__ = ["Cmd"] -PROMPT: Literal["(Cmd) "] +PROMPT: Final = "(Cmd) " IDENTCHARS: str # Too big to be `Literal` class Cmd: diff --git a/mypy/typeshed/stdlib/codecs.pyi b/mypy/typeshed/stdlib/codecs.pyi index 6e53b780c473..9bc098dbc6d7 100644 --- a/mypy/typeshed/stdlib/codecs.pyi +++ b/mypy/typeshed/stdlib/codecs.pyi @@ -3,7 +3,7 @@ from _codecs import * from _typeshed import ReadableBuffer from abc import abstractmethod from collections.abc import Callable, Generator, Iterable -from typing import Any, BinaryIO, Literal, Protocol, TextIO +from typing import Any, BinaryIO, Final, Literal, Protocol, TextIO from typing_extensions import Self __all__ = [ @@ -53,10 +53,10 @@ __all__ = [ "lookup_error", ] -BOM32_BE: Literal[b"\xfe\xff"] -BOM32_LE: Literal[b"\xff\xfe"] -BOM64_BE: Literal[b"\x00\x00\xfe\xff"] -BOM64_LE: Literal[b"\xff\xfe\x00\x00"] +BOM32_BE: Final = b"\xfe\xff" +BOM32_LE: Final = b"\xff\xfe" +BOM64_BE: Final = b"\x00\x00\xfe\xff" +BOM64_LE: Final = b"\xff\xfe\x00\x00" class _WritableStream(Protocol): def write(self, data: bytes, /) -> object: ... @@ -135,23 +135,23 @@ def EncodedFile(file: _Stream, data_encoding: str, file_encoding: str | None = N def iterencode(iterator: Iterable[str], encoding: str, errors: str = "strict") -> Generator[bytes, None, None]: ... def iterdecode(iterator: Iterable[bytes], encoding: str, errors: str = "strict") -> Generator[str, None, None]: ... -BOM: Literal[b"\xff\xfe", b"\xfe\xff"] # depends on `sys.byteorder` -BOM_BE: Literal[b"\xfe\xff"] -BOM_LE: Literal[b"\xff\xfe"] -BOM_UTF8: Literal[b"\xef\xbb\xbf"] -BOM_UTF16: Literal[b"\xff\xfe", b"\xfe\xff"] # depends on `sys.byteorder` -BOM_UTF16_BE: Literal[b"\xfe\xff"] -BOM_UTF16_LE: Literal[b"\xff\xfe"] -BOM_UTF32: Literal[b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff"] # depends on `sys.byteorder` -BOM_UTF32_BE: Literal[b"\x00\x00\xfe\xff"] -BOM_UTF32_LE: Literal[b"\xff\xfe\x00\x00"] +BOM: Final[Literal[b"\xff\xfe", b"\xfe\xff"]] # depends on `sys.byteorder` +BOM_BE: Final = b"\xfe\xff" +BOM_LE: Final = b"\xff\xfe" +BOM_UTF8: Final = b"\xef\xbb\xbf" +BOM_UTF16: Final[Literal[b"\xff\xfe", b"\xfe\xff"]] # depends on `sys.byteorder` +BOM_UTF16_BE: Final = b"\xfe\xff" +BOM_UTF16_LE: Final = b"\xff\xfe" +BOM_UTF32: Final[Literal[b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff"]] # depends on `sys.byteorder` +BOM_UTF32_BE: Final = b"\x00\x00\xfe\xff" +BOM_UTF32_LE: Final = b"\xff\xfe\x00\x00" -def strict_errors(exception: UnicodeError) -> tuple[str | bytes, int]: ... -def replace_errors(exception: UnicodeError) -> tuple[str | bytes, int]: ... -def ignore_errors(exception: UnicodeError) -> tuple[str | bytes, int]: ... -def xmlcharrefreplace_errors(exception: UnicodeError) -> tuple[str | bytes, int]: ... -def backslashreplace_errors(exception: UnicodeError) -> tuple[str | bytes, int]: ... -def namereplace_errors(exception: UnicodeError) -> tuple[str | bytes, int]: ... +def strict_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... +def replace_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... +def ignore_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... +def xmlcharrefreplace_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... +def backslashreplace_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... +def namereplace_errors(exception: UnicodeError, /) -> tuple[str | bytes, int]: ... class Codec: # These are sort of @abstractmethod but sort of not. diff --git a/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi b/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi index 07314ce9d402..68fd0bc5acb4 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/__init__.pyi @@ -1,3 +1,5 @@ +import sys + from ._base import ( ALL_COMPLETED as ALL_COMPLETED, FIRST_COMPLETED as FIRST_COMPLETED, @@ -14,19 +16,36 @@ from ._base import ( from .process import ProcessPoolExecutor as ProcessPoolExecutor from .thread import ThreadPoolExecutor as ThreadPoolExecutor -__all__ = ( - "FIRST_COMPLETED", - "FIRST_EXCEPTION", - "ALL_COMPLETED", - "CancelledError", - "TimeoutError", - "BrokenExecutor", - "Future", - "Executor", - "wait", - "as_completed", - "ProcessPoolExecutor", - "ThreadPoolExecutor", -) +if sys.version_info >= (3, 13): + __all__ = ( + "FIRST_COMPLETED", + "FIRST_EXCEPTION", + "ALL_COMPLETED", + "CancelledError", + "TimeoutError", + "InvalidStateError", + "BrokenExecutor", + "Future", + "Executor", + "wait", + "as_completed", + "ProcessPoolExecutor", + "ThreadPoolExecutor", + ) +else: + __all__ = ( + "FIRST_COMPLETED", + "FIRST_EXCEPTION", + "ALL_COMPLETED", + "CancelledError", + "TimeoutError", + "BrokenExecutor", + "Future", + "Executor", + "wait", + "as_completed", + "ProcessPoolExecutor", + "ThreadPoolExecutor", + ) def __dir__() -> tuple[str, ...]: ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/_base.pyi b/mypy/typeshed/stdlib/concurrent/futures/_base.pyi index 3d5eccfc048d..0c019457902b 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/_base.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/_base.pyi @@ -4,20 +4,20 @@ from _typeshed import Unused from collections.abc import Callable, Collection, Iterable, Iterator from logging import Logger from types import TracebackType -from typing import Any, Generic, Literal, NamedTuple, Protocol, TypeVar +from typing import Any, Final, Generic, NamedTuple, Protocol, TypeVar from typing_extensions import ParamSpec, Self if sys.version_info >= (3, 9): from types import GenericAlias -FIRST_COMPLETED: Literal["FIRST_COMPLETED"] -FIRST_EXCEPTION: Literal["FIRST_EXCEPTION"] -ALL_COMPLETED: Literal["ALL_COMPLETED"] -PENDING: Literal["PENDING"] -RUNNING: Literal["RUNNING"] -CANCELLED: Literal["CANCELLED"] -CANCELLED_AND_NOTIFIED: Literal["CANCELLED_AND_NOTIFIED"] -FINISHED: Literal["FINISHED"] +FIRST_COMPLETED: Final = "FIRST_COMPLETED" +FIRST_EXCEPTION: Final = "FIRST_EXCEPTION" +ALL_COMPLETED: Final = "ALL_COMPLETED" +PENDING: Final = "PENDING" +RUNNING: Final = "RUNNING" +CANCELLED: Final = "CANCELLED" +CANCELLED_AND_NOTIFIED: Final = "CANCELLED_AND_NOTIFIED" +FINISHED: Final = "FINISHED" _FUTURE_STATES: list[str] _STATE_TO_DESCRIPTION_MAP: dict[str, str] LOGGER: Logger diff --git a/mypy/typeshed/stdlib/configparser.pyi b/mypy/typeshed/stdlib/configparser.pyi index f38bb1de674d..ee5000196e0e 100644 --- a/mypy/typeshed/stdlib/configparser.pyi +++ b/mypy/typeshed/stdlib/configparser.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import StrOrBytesPath, SupportsWrite from collections.abc import Callable, ItemsView, Iterable, Iterator, Mapping, MutableMapping, Sequence from re import Pattern -from typing import Any, ClassVar, Literal, TypeVar, overload +from typing import Any, ClassVar, Final, Literal, TypeVar, overload from typing_extensions import TypeAlias if sys.version_info >= (3, 13): @@ -83,8 +83,8 @@ _ConverterCallback: TypeAlias = Callable[[str], Any] _ConvertersMap: TypeAlias = dict[str, _ConverterCallback] _T = TypeVar("_T") -DEFAULTSECT: Literal["DEFAULT"] -MAX_INTERPOLATION_DEPTH: Literal[10] +DEFAULTSECT: Final = "DEFAULT" +MAX_INTERPOLATION_DEPTH: Final = 10 class Interpolation: def before_get(self, parser: _Parser, section: str, option: str, value: str, defaults: _Section) -> str: ... diff --git a/mypy/typeshed/stdlib/copy.pyi b/mypy/typeshed/stdlib/copy.pyi index 8a2dcc508e5d..020ce6c31b58 100644 --- a/mypy/typeshed/stdlib/copy.pyi +++ b/mypy/typeshed/stdlib/copy.pyi @@ -1,8 +1,16 @@ -from typing import Any, TypeVar +import sys +from typing import Any, Protocol, TypeVar +from typing_extensions import ParamSpec, Self __all__ = ["Error", "copy", "deepcopy"] _T = TypeVar("_T") +_SR = TypeVar("_SR", bound=_SupportsReplace[Any]) +_P = ParamSpec("_P") + +class _SupportsReplace(Protocol[_P]): + # In reality doesn't support args, but there's no other great way to express this. + def __replace__(self, *args: _P.args, **kwargs: _P.kwargs) -> Self: ... # None in CPython but non-None in Jython PyStringMap: Any @@ -11,6 +19,10 @@ PyStringMap: Any def deepcopy(x: _T, memo: dict[int, Any] | None = None, _nil: Any = []) -> _T: ... def copy(x: _T) -> _T: ... +if sys.version_info >= (3, 13): + __all__ += ["replace"] + def replace(obj: _SR, /, **changes: Any) -> _SR: ... + class Error(Exception): ... error = Error diff --git a/mypy/typeshed/stdlib/datetime.pyi b/mypy/typeshed/stdlib/datetime.pyi index 71522a59d4df..38d5ac4c0819 100644 --- a/mypy/typeshed/stdlib/datetime.pyi +++ b/mypy/typeshed/stdlib/datetime.pyi @@ -1,7 +1,7 @@ import sys from abc import abstractmethod from time import struct_time -from typing import ClassVar, Literal, NamedTuple, NoReturn, SupportsIndex, final, overload +from typing import ClassVar, Final, NamedTuple, NoReturn, SupportsIndex, final, overload from typing_extensions import Self, TypeAlias, deprecated if sys.version_info >= (3, 11): @@ -9,8 +9,8 @@ if sys.version_info >= (3, 11): elif sys.version_info >= (3, 9): __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR") -MINYEAR: Literal[1] -MAXYEAR: Literal[9999] +MINYEAR: Final = 1 +MAXYEAR: Final = 9999 class tzinfo: @abstractmethod diff --git a/mypy/typeshed/stdlib/dbm/gnu.pyi b/mypy/typeshed/stdlib/dbm/gnu.pyi index e80441cbb25b..1d1d541f5477 100644 --- a/mypy/typeshed/stdlib/dbm/gnu.pyi +++ b/mypy/typeshed/stdlib/dbm/gnu.pyi @@ -19,6 +19,9 @@ if sys.platform != "win32": def reorganize(self) -> None: ... def sync(self) -> None: ... def close(self) -> None: ... + if sys.version_info >= (3, 13): + def clear(self) -> None: ... + def __getitem__(self, item: _KeyType) -> bytes: ... def __setitem__(self, key: _KeyType, value: _ValueType) -> None: ... def __delitem__(self, key: _KeyType) -> None: ... diff --git a/mypy/typeshed/stdlib/dbm/ndbm.pyi b/mypy/typeshed/stdlib/dbm/ndbm.pyi index 02bf23ec181c..4113a7e3ffb9 100644 --- a/mypy/typeshed/stdlib/dbm/ndbm.pyi +++ b/mypy/typeshed/stdlib/dbm/ndbm.pyi @@ -15,6 +15,9 @@ if sys.platform != "win32": # Actual typename dbm, not exposed by the implementation class _dbm: def close(self) -> None: ... + if sys.version_info >= (3, 13): + def clear(self) -> None: ... + def __getitem__(self, item: _KeyType) -> bytes: ... def __setitem__(self, key: _KeyType, value: _ValueType) -> None: ... def __delitem__(self, key: _KeyType) -> None: ... diff --git a/mypy/typeshed/stdlib/dbm/sqlite3.pyi b/mypy/typeshed/stdlib/dbm/sqlite3.pyi new file mode 100644 index 000000000000..446a0cf155fa --- /dev/null +++ b/mypy/typeshed/stdlib/dbm/sqlite3.pyi @@ -0,0 +1,29 @@ +from _typeshed import ReadableBuffer, StrOrBytesPath, Unused +from collections.abc import Generator, MutableMapping +from typing import Final, Literal +from typing_extensions import LiteralString, Self, TypeAlias + +BUILD_TABLE: Final[LiteralString] +GET_SIZE: Final[LiteralString] +LOOKUP_KEY: Final[LiteralString] +STORE_KV: Final[LiteralString] +DELETE_KEY: Final[LiteralString] +ITER_KEYS: Final[LiteralString] + +_SqliteData: TypeAlias = str | ReadableBuffer | int | float + +class error(OSError): ... + +class _Database(MutableMapping[bytes, bytes]): + def __init__(self, path: StrOrBytesPath, /, *, flag: Literal["r", "w", "c", "n"], mode: int) -> None: ... + def __len__(self) -> int: ... + def __getitem__(self, key: _SqliteData) -> bytes: ... + def __setitem__(self, key: _SqliteData, value: _SqliteData) -> None: ... + def __delitem__(self, key: _SqliteData) -> None: ... + def __iter__(self) -> Generator[bytes]: ... + def close(self) -> None: ... + def keys(self) -> list[bytes]: ... # type: ignore[override] + def __enter__(self) -> Self: ... + def __exit__(self, *args: Unused) -> None: ... + +def open(filename: StrOrBytesPath, /, flag: Literal["r", "w,", "c", "n"] = "r", mode: int = 0o666) -> _Database: ... diff --git a/mypy/typeshed/stdlib/dis.pyi b/mypy/typeshed/stdlib/dis.pyi index 47c63cc8b3d3..cb69eac89c92 100644 --- a/mypy/typeshed/stdlib/dis.pyi +++ b/mypy/typeshed/stdlib/dis.pyi @@ -31,6 +31,9 @@ __all__ = [ "EXTENDED_ARG", "stack_effect", ] +if sys.version_info >= (3, 13): + __all__ += ["hasjump"] + if sys.version_info >= (3, 12): __all__ += ["hasarg", "hasexc"] else: @@ -86,12 +89,41 @@ else: is_jump_target: bool class Instruction(_Instruction): - def _disassemble(self, lineno_width: int = 3, mark_as_current: bool = False, offset_width: int = 4) -> str: ... + if sys.version_info < (3, 13): + def _disassemble(self, lineno_width: int = 3, mark_as_current: bool = False, offset_width: int = 4) -> str: ... + if sys.version_info >= (3, 13): + @property + def oparg(self) -> int: ... + @property + def baseopcode(self) -> int: ... + @property + def baseopname(self) -> str: ... + @property + def cache_offset(self) -> int: ... + @property + def end_offset(self) -> int: ... + @property + def jump_target(self) -> int: ... + @property + def is_jump_target(self) -> bool: ... class Bytecode: codeobj: types.CodeType first_line: int - if sys.version_info >= (3, 11): + if sys.version_info >= (3, 13): + show_offsets: bool + # 3.13 added `show_offsets` + def __init__( + self, + x: _HaveCodeType | str, + *, + first_line: int | None = None, + current_offset: int | None = None, + show_caches: bool = False, + adaptive: bool = False, + show_offsets: bool = False, + ) -> None: ... + elif sys.version_info >= (3, 11): def __init__( self, x: _HaveCodeType | str, @@ -101,12 +133,15 @@ class Bytecode: show_caches: bool = False, adaptive: bool = False, ) -> None: ... - @classmethod - def from_traceback(cls, tb: types.TracebackType, *, show_caches: bool = False, adaptive: bool = False) -> Self: ... else: def __init__( self, x: _HaveCodeType | str, *, first_line: int | None = None, current_offset: int | None = None ) -> None: ... + + if sys.version_info >= (3, 11): + @classmethod + def from_traceback(cls, tb: types.TracebackType, *, show_caches: bool = False, adaptive: bool = False) -> Self: ... + else: @classmethod def from_traceback(cls, tb: types.TracebackType) -> Self: ... @@ -121,7 +156,8 @@ def findlinestarts(code: _HaveCodeType) -> Iterator[tuple[int, int]]: ... def pretty_flags(flags: int) -> str: ... def code_info(x: _HaveCodeType | str) -> str: ... -if sys.version_info >= (3, 11): +if sys.version_info >= (3, 13): + # 3.13 added `show_offsets` def dis( x: _HaveCodeType | str | bytes | bytearray | None = None, *, @@ -129,20 +165,43 @@ if sys.version_info >= (3, 11): depth: int | None = None, show_caches: bool = False, adaptive: bool = False, + show_offsets: bool = False, + ) -> None: ... + def disassemble( + co: _HaveCodeType, + lasti: int = -1, + *, + file: IO[str] | None = None, + show_caches: bool = False, + adaptive: bool = False, + show_offsets: bool = False, + ) -> None: ... + def distb( + tb: types.TracebackType | None = None, + *, + file: IO[str] | None = None, + show_caches: bool = False, + adaptive: bool = False, + show_offsets: bool = False, ) -> None: ... + # 3.13 made `show_cache` `None` by default + def get_instructions( + x: _HaveCodeType, *, first_line: int | None = None, show_caches: bool | None = None, adaptive: bool = False + ) -> Iterator[Instruction]: ... -else: +elif sys.version_info >= (3, 11): + # 3.11 added `show_caches` and `adaptive` def dis( - x: _HaveCodeType | str | bytes | bytearray | None = None, *, file: IO[str] | None = None, depth: int | None = None + x: _HaveCodeType | str | bytes | bytearray | None = None, + *, + file: IO[str] | None = None, + depth: int | None = None, + show_caches: bool = False, + adaptive: bool = False, ) -> None: ... - -if sys.version_info >= (3, 11): def disassemble( co: _HaveCodeType, lasti: int = -1, *, file: IO[str] | None = None, show_caches: bool = False, adaptive: bool = False ) -> None: ... - def disco( - co: _HaveCodeType, lasti: int = -1, *, file: IO[str] | None = None, show_caches: bool = False, adaptive: bool = False - ) -> None: ... def distb( tb: types.TracebackType | None = None, *, file: IO[str] | None = None, show_caches: bool = False, adaptive: bool = False ) -> None: ... @@ -151,9 +210,13 @@ if sys.version_info >= (3, 11): ) -> Iterator[Instruction]: ... else: + def dis( + x: _HaveCodeType | str | bytes | bytearray | None = None, *, file: IO[str] | None = None, depth: int | None = None + ) -> None: ... def disassemble(co: _HaveCodeType, lasti: int = -1, *, file: IO[str] | None = None) -> None: ... - def disco(co: _HaveCodeType, lasti: int = -1, *, file: IO[str] | None = None) -> None: ... def distb(tb: types.TracebackType | None = None, *, file: IO[str] | None = None) -> None: ... def get_instructions(x: _HaveCodeType, *, first_line: int | None = None) -> Iterator[Instruction]: ... def show_code(co: _HaveCodeType, *, file: IO[str] | None = None) -> None: ... + +disco = disassemble diff --git a/mypy/typeshed/stdlib/distutils/ccompiler.pyi b/mypy/typeshed/stdlib/distutils/ccompiler.pyi index cd6efee0a210..e0f33f430e5a 100644 --- a/mypy/typeshed/stdlib/distutils/ccompiler.pyi +++ b/mypy/typeshed/stdlib/distutils/ccompiler.pyi @@ -1,10 +1,11 @@ -from _typeshed import BytesPath, StrPath +from _typeshed import BytesPath, StrPath, Unused from collections.abc import Callable, Iterable from distutils.file_util import _BytesPathT, _StrPathT -from typing import Any, Literal, overload -from typing_extensions import TypeAlias +from typing import Literal, overload +from typing_extensions import TypeAlias, TypeVarTuple, Unpack _Macro: TypeAlias = tuple[str] | tuple[str, str | None] +_Ts = TypeVarTuple("_Ts") def gen_lib_options( compiler: CCompiler, library_dirs: list[str], runtime_library_dirs: list[str], libraries: list[str] @@ -161,7 +162,9 @@ class CCompiler: def shared_object_filename(self, basename: str, strip_dir: Literal[0, False] = 0, output_dir: StrPath = "") -> str: ... @overload def shared_object_filename(self, basename: StrPath, strip_dir: Literal[1, True], output_dir: StrPath = "") -> str: ... - def execute(self, func: Callable[..., object], args: tuple[Any, ...], msg: str | None = None, level: int = 1) -> None: ... + def execute( + self, func: Callable[[Unpack[_Ts]], Unused], args: tuple[Unpack[_Ts]], msg: str | None = None, level: int = 1 + ) -> None: ... def spawn(self, cmd: list[str]) -> None: ... def mkpath(self, name: str, mode: int = 0o777) -> None: ... @overload diff --git a/mypy/typeshed/stdlib/distutils/cmd.pyi b/mypy/typeshed/stdlib/distutils/cmd.pyi index defea50e78dc..ca4fb3265324 100644 --- a/mypy/typeshed/stdlib/distutils/cmd.pyi +++ b/mypy/typeshed/stdlib/distutils/cmd.pyi @@ -3,7 +3,11 @@ from abc import abstractmethod from collections.abc import Callable, Iterable from distutils.dist import Distribution from distutils.file_util import _BytesPathT, _StrPathT -from typing import Any, ClassVar, Literal, overload +from typing import Any, ClassVar, Literal, TypeVar, overload +from typing_extensions import TypeVarTuple, Unpack + +_CommandT = TypeVar("_CommandT", bound=Command) +_Ts = TypeVarTuple("_Ts") class Command: distribution: Distribution @@ -19,17 +23,22 @@ class Command: def announce(self, msg: str, level: int = 1) -> None: ... def debug_print(self, msg: str) -> None: ... def ensure_string(self, option: str, default: str | None = None) -> None: ... - def ensure_string_list(self, option: str | list[str]) -> None: ... + def ensure_string_list(self, option: str) -> None: ... def ensure_filename(self, option: str) -> None: ... def ensure_dirname(self, option: str) -> None: ... def get_command_name(self) -> str: ... def set_undefined_options(self, src_cmd: str, *option_pairs: tuple[str, str]) -> None: ... def get_finalized_command(self, command: str, create: bool | Literal[0, 1] = 1) -> Command: ... - def reinitialize_command(self, command: Command | str, reinit_subcommands: bool | Literal[0, 1] = 0) -> Command: ... + @overload + def reinitialize_command(self, command: str, reinit_subcommands: bool | Literal[0, 1] = 0) -> Command: ... + @overload + def reinitialize_command(self, command: _CommandT, reinit_subcommands: bool | Literal[0, 1] = 0) -> _CommandT: ... def run_command(self, command: str) -> None: ... def get_sub_commands(self) -> list[str]: ... def warn(self, msg: str) -> None: ... - def execute(self, func: Callable[..., object], args: Iterable[Any], msg: str | None = None, level: int = 1) -> None: ... + def execute( + self, func: Callable[[Unpack[_Ts]], Unused], args: tuple[Unpack[_Ts]], msg: str | None = None, level: int = 1 + ) -> None: ... def mkpath(self, name: str, mode: int = 0o777) -> None: ... @overload def copy_file( @@ -89,8 +98,8 @@ class Command: self, infiles: str | list[str] | tuple[str, ...], outfile: StrOrBytesPath, - func: Callable[..., object], - args: list[Any], + func: Callable[[Unpack[_Ts]], Unused], + args: tuple[Unpack[_Ts]], exec_msg: str | None = None, skip_msg: str | None = None, level: Unused = 1, diff --git a/mypy/typeshed/stdlib/distutils/command/bdist.pyi b/mypy/typeshed/stdlib/distutils/command/bdist.pyi index e1f141d3a40f..43d77087f7d8 100644 --- a/mypy/typeshed/stdlib/distutils/command/bdist.pyi +++ b/mypy/typeshed/stdlib/distutils/command/bdist.pyi @@ -1,4 +1,6 @@ -from typing import Any +from _typeshed import Unused +from collections.abc import Callable +from typing import Any, ClassVar from ..cmd import Command @@ -6,13 +8,13 @@ def show_formats() -> None: ... class bdist(Command): description: str - user_options: Any - boolean_options: Any - help_options: Any - no_format_option: Any - default_format: Any - format_commands: Any - format_command: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] + no_format_option: ClassVar[tuple[str, ...]] + default_format: ClassVar[dict[str, str]] + format_commands: ClassVar[list[str]] + format_command: ClassVar[dict[str, tuple[str, str]]] bdist_base: Any plat_name: Any formats: Any diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi index 74cca4d13cd0..19997882dd53 100644 --- a/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi +++ b/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi @@ -1,12 +1,12 @@ -from typing import Any +from typing import Any, ClassVar from ..cmd import Command class bdist_dumb(Command): description: str - user_options: Any - boolean_options: Any - default_format: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + default_format: ClassVar[dict[str, str]] bdist_dir: Any plat_name: Any format: Any diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi index d1eb374ff52b..d0eac1a3be5b 100644 --- a/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi +++ b/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi @@ -1,5 +1,5 @@ import sys -from typing import Any, Literal +from typing import Any, ClassVar, Literal from ..cmd import Command @@ -16,8 +16,8 @@ if sys.platform == "win32": class bdist_msi(Command): description: str - user_options: Any - boolean_options: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] all_versions: Any other_version: str if sys.version_info >= (3, 9): diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi index 76691310b599..89c43e1b974c 100644 --- a/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi +++ b/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi @@ -1,12 +1,12 @@ -from typing import Any +from typing import Any, ClassVar from ..cmd import Command class bdist_rpm(Command): description: str - user_options: Any - boolean_options: Any - negative_opt: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + negative_opt: ClassVar[dict[str, str]] bdist_base: Any rpm_base: Any dist_dir: Any diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_wininst.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_wininst.pyi index 8491d3126200..cf333bc5400d 100644 --- a/mypy/typeshed/stdlib/distutils/command/bdist_wininst.pyi +++ b/mypy/typeshed/stdlib/distutils/command/bdist_wininst.pyi @@ -1,10 +1,10 @@ from _typeshed import StrOrBytesPath from distutils.cmd import Command -from typing import Any, ClassVar +from typing import ClassVar class bdist_wininst(Command): description: ClassVar[str] - user_options: ClassVar[list[tuple[Any, ...]]] + user_options: ClassVar[list[tuple[str, str | None, str]]] boolean_options: ClassVar[list[str]] def initialize_options(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/build.pyi b/mypy/typeshed/stdlib/distutils/command/build.pyi index 31fc036d4f97..78ba6b7042dc 100644 --- a/mypy/typeshed/stdlib/distutils/command/build.pyi +++ b/mypy/typeshed/stdlib/distutils/command/build.pyi @@ -1,3 +1,4 @@ +from _typeshed import Unused from collections.abc import Callable from typing import Any, ClassVar @@ -7,9 +8,9 @@ def show_compilers() -> None: ... class build(Command): description: str - user_options: Any - boolean_options: Any - help_options: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] build_base: str build_purelib: Any build_platlib: Any diff --git a/mypy/typeshed/stdlib/distutils/command/build_clib.pyi b/mypy/typeshed/stdlib/distutils/command/build_clib.pyi index 32ab182b30d0..1f66e2efc20c 100644 --- a/mypy/typeshed/stdlib/distutils/command/build_clib.pyi +++ b/mypy/typeshed/stdlib/distutils/command/build_clib.pyi @@ -1,4 +1,6 @@ -from typing import Any +from _typeshed import Unused +from collections.abc import Callable +from typing import Any, ClassVar from ..cmd import Command @@ -6,9 +8,9 @@ def show_compilers() -> None: ... class build_clib(Command): description: str - user_options: Any - boolean_options: Any - help_options: Any + user_options: ClassVar[list[tuple[str, str, str]]] + boolean_options: ClassVar[list[str]] + help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] build_clib: Any build_temp: Any libraries: Any diff --git a/mypy/typeshed/stdlib/distutils/command/build_ext.pyi b/mypy/typeshed/stdlib/distutils/command/build_ext.pyi index 5eb541fb9101..a0813c314021 100644 --- a/mypy/typeshed/stdlib/distutils/command/build_ext.pyi +++ b/mypy/typeshed/stdlib/distutils/command/build_ext.pyi @@ -1,4 +1,6 @@ -from typing import Any +from _typeshed import Unused +from collections.abc import Callable +from typing import Any, ClassVar from ..cmd import Command @@ -9,9 +11,9 @@ def show_compilers() -> None: ... class build_ext(Command): description: str sep_by: Any - user_options: Any - boolean_options: Any - help_options: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] extensions: Any build_lib: Any plat_name: Any diff --git a/mypy/typeshed/stdlib/distutils/command/build_py.pyi b/mypy/typeshed/stdlib/distutils/command/build_py.pyi index 4c607c6dabe9..90f06751416a 100644 --- a/mypy/typeshed/stdlib/distutils/command/build_py.pyi +++ b/mypy/typeshed/stdlib/distutils/command/build_py.pyi @@ -1,13 +1,13 @@ -from typing import Any, Literal +from typing import Any, ClassVar, Literal from ..cmd import Command from ..util import Mixin2to3 as Mixin2to3 class build_py(Command): description: str - user_options: Any - boolean_options: Any - negative_opt: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + negative_opt: ClassVar[dict[str, str]] build_lib: Any py_modules: Any package: Any diff --git a/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi b/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi index 42135eceafef..7871bb8a5719 100644 --- a/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi +++ b/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, ClassVar from ..cmd import Command from ..util import Mixin2to3 as Mixin2to3 @@ -7,8 +7,8 @@ first_line_re: Any class build_scripts(Command): description: str - user_options: Any - boolean_options: Any + user_options: ClassVar[list[tuple[str, str, str]]] + boolean_options: ClassVar[list[str]] build_dir: Any scripts: Any force: Any diff --git a/mypy/typeshed/stdlib/distutils/command/check.pyi b/mypy/typeshed/stdlib/distutils/command/check.pyi index da041d82587d..c67e4cbfdfe0 100644 --- a/mypy/typeshed/stdlib/distutils/command/check.pyi +++ b/mypy/typeshed/stdlib/distutils/command/check.pyi @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Any, ClassVar, Literal from typing_extensions import TypeAlias from ..cmd import Command @@ -26,8 +26,8 @@ HAS_DOCUTILS: bool class check(Command): description: str - user_options: Any - boolean_options: Any + user_options: ClassVar[list[tuple[str, str, str]]] + boolean_options: ClassVar[list[str]] restructuredtext: int metadata: int strict: int diff --git a/mypy/typeshed/stdlib/distutils/command/clean.pyi b/mypy/typeshed/stdlib/distutils/command/clean.pyi index 99560aa8a716..55f0a0eeaf10 100644 --- a/mypy/typeshed/stdlib/distutils/command/clean.pyi +++ b/mypy/typeshed/stdlib/distutils/command/clean.pyi @@ -1,11 +1,11 @@ -from typing import Any +from typing import Any, ClassVar from ..cmd import Command class clean(Command): description: str - user_options: Any - boolean_options: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] build_base: Any build_lib: Any build_temp: Any diff --git a/mypy/typeshed/stdlib/distutils/command/config.pyi b/mypy/typeshed/stdlib/distutils/command/config.pyi index 391f5a862038..2f528c2c290b 100644 --- a/mypy/typeshed/stdlib/distutils/command/config.pyi +++ b/mypy/typeshed/stdlib/distutils/command/config.pyi @@ -1,7 +1,7 @@ from _typeshed import StrOrBytesPath from collections.abc import Sequence from re import Pattern -from typing import Any, Literal +from typing import Any, ClassVar, Literal from ..ccompiler import CCompiler from ..cmd import Command @@ -11,7 +11,7 @@ LANG_EXT: dict[str, str] class config(Command): description: str # Tuple is full name, short name, description - user_options: Sequence[tuple[str, str | None, str]] + user_options: ClassVar[list[tuple[str, str | None, str]]] compiler: str | CCompiler cc: str | None include_dirs: Sequence[str] | None diff --git a/mypy/typeshed/stdlib/distutils/command/install.pyi b/mypy/typeshed/stdlib/distutils/command/install.pyi index 8b2295d7a3c7..b0a5a82fc3f6 100644 --- a/mypy/typeshed/stdlib/distutils/command/install.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install.pyi @@ -9,9 +9,9 @@ INSTALL_SCHEMES: dict[str, dict[Any, Any]] class install(Command): description: str - user_options: Any - boolean_options: Any - negative_opt: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + negative_opt: ClassVar[dict[str, str]] prefix: str | None exec_prefix: Any home: str | None diff --git a/mypy/typeshed/stdlib/distutils/command/install_data.pyi b/mypy/typeshed/stdlib/distutils/command/install_data.pyi index 6cc9b528ac9d..342c7a7ccca4 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_data.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_data.pyi @@ -1,11 +1,11 @@ -from typing import Any +from typing import Any, ClassVar from ..cmd import Command class install_data(Command): description: str - user_options: Any - boolean_options: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] install_dir: Any outfiles: Any root: Any diff --git a/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi b/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi index 776eafc1de09..3fd54989d14f 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi @@ -4,7 +4,7 @@ from ..cmd import Command class install_egg_info(Command): description: ClassVar[str] - user_options: ClassVar[list[tuple[str, str | None, str]]] + user_options: ClassVar[list[tuple[str, str, str]]] install_dir: Any def initialize_options(self) -> None: ... target: Any diff --git a/mypy/typeshed/stdlib/distutils/command/install_headers.pyi b/mypy/typeshed/stdlib/distutils/command/install_headers.pyi index 795bd1cf8356..7854d2393a98 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_headers.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_headers.pyi @@ -1,11 +1,11 @@ -from typing import Any +from typing import Any, ClassVar from ..cmd import Command class install_headers(Command): description: str - user_options: Any - boolean_options: Any + user_options: ClassVar[list[tuple[str, str, str]]] + boolean_options: ClassVar[list[str]] install_dir: Any force: int outfiles: Any diff --git a/mypy/typeshed/stdlib/distutils/command/install_lib.pyi b/mypy/typeshed/stdlib/distutils/command/install_lib.pyi index a6a5e4e73f4c..718d082b7b07 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_lib.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_lib.pyi @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, ClassVar from ..cmd import Command @@ -6,9 +6,9 @@ PYTHON_SOURCE_EXTENSION: str class install_lib(Command): description: str - user_options: Any - boolean_options: Any - negative_opt: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + negative_opt: ClassVar[dict[str, str]] install_dir: Any build_dir: Any force: int diff --git a/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi b/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi index 92728a16a747..5ee5589ad33d 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi @@ -1,11 +1,11 @@ -from typing import Any +from typing import Any, ClassVar from ..cmd import Command class install_scripts(Command): description: str - user_options: Any - boolean_options: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] install_dir: Any force: int build_dir: Any diff --git a/mypy/typeshed/stdlib/distutils/command/sdist.pyi b/mypy/typeshed/stdlib/distutils/command/sdist.pyi index db303f77a463..5b7fe2419551 100644 --- a/mypy/typeshed/stdlib/distutils/command/sdist.pyi +++ b/mypy/typeshed/stdlib/distutils/command/sdist.pyi @@ -1,3 +1,4 @@ +from _typeshed import Unused from collections.abc import Callable from typing import Any, ClassVar @@ -8,13 +9,13 @@ def show_formats() -> None: ... class sdist(Command): description: str def checking_metadata(self): ... - user_options: Any - boolean_options: Any - help_options: Any - negative_opt: Any + user_options: ClassVar[list[tuple[str, str | None, str]]] + boolean_options: ClassVar[list[str]] + help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] + negative_opt: ClassVar[dict[str, str]] # Any to work around variance issues sub_commands: ClassVar[list[tuple[str, Callable[[Any], bool] | None]]] - READMES: Any + READMES: ClassVar[tuple[str, ...]] template: Any manifest: Any use_defaults: int diff --git a/mypy/typeshed/stdlib/distutils/dist.pyi b/mypy/typeshed/stdlib/distutils/dist.pyi index 4094df903325..21ddbc425918 100644 --- a/mypy/typeshed/stdlib/distutils/dist.pyi +++ b/mypy/typeshed/stdlib/distutils/dist.pyi @@ -1,8 +1,8 @@ from _typeshed import Incomplete, StrOrBytesPath, StrPath, SupportsWrite -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, MutableMapping from distutils.cmd import Command from re import Pattern -from typing import IO, Any, ClassVar, Literal, TypeVar, overload +from typing import IO, ClassVar, Literal, TypeVar, overload from typing_extensions import TypeAlias command_re: Pattern[str] @@ -60,7 +60,7 @@ class DistributionMetadata: class Distribution: cmdclass: dict[str, type[Command]] metadata: DistributionMetadata - def __init__(self, attrs: Mapping[str, Any] | None = None) -> None: ... + def __init__(self, attrs: MutableMapping[str, Incomplete] | None = None) -> None: ... def get_option_dict(self, command: str) -> dict[str, tuple[str, str]]: ... def parse_config_files(self, filenames: Iterable[str] | None = None) -> None: ... @overload diff --git a/mypy/typeshed/stdlib/distutils/util.pyi b/mypy/typeshed/stdlib/distutils/util.pyi index 515b5b2b86d9..0e1bb4165d99 100644 --- a/mypy/typeshed/stdlib/distutils/util.pyi +++ b/mypy/typeshed/stdlib/distutils/util.pyi @@ -1,6 +1,9 @@ from _typeshed import StrPath, Unused from collections.abc import Callable, Container, Iterable, Mapping from typing import Any, Literal +from typing_extensions import TypeVarTuple, Unpack + +_Ts = TypeVarTuple("_Ts") def get_host_platform() -> str: ... def get_platform() -> str: ... @@ -10,8 +13,8 @@ def check_environ() -> None: ... def subst_vars(s: str, local_vars: Mapping[str, str]) -> None: ... def split_quoted(s: str) -> list[str]: ... def execute( - func: Callable[..., object], - args: tuple[Any, ...], + func: Callable[[Unpack[_Ts]], Unused], + args: tuple[Unpack[_Ts]], msg: str | None = None, verbose: bool | Literal[0, 1] = 0, dry_run: bool | Literal[0, 1] = 0, diff --git a/mypy/typeshed/stdlib/email/utils.pyi b/mypy/typeshed/stdlib/email/utils.pyi index 0b62647532db..2724dbf6ec2f 100644 --- a/mypy/typeshed/stdlib/email/utils.pyi +++ b/mypy/typeshed/stdlib/email/utils.pyi @@ -1,6 +1,7 @@ import datetime import sys from _typeshed import Unused +from collections.abc import Iterable from email import _ParamType from email.charset import Charset from typing import overload @@ -28,9 +29,21 @@ _PDTZ: TypeAlias = tuple[int, int, int, int, int, int, int, int, int, int | None def quote(str: str) -> str: ... def unquote(str: str) -> str: ... -def parseaddr(addr: str | None) -> tuple[str, str]: ... + +if sys.version_info >= (3, 13): + def parseaddr(addr: str | list[str], *, strict: bool = True) -> tuple[str, str]: ... + +else: + def parseaddr(addr: str) -> tuple[str, str]: ... + def formataddr(pair: tuple[str | None, str], charset: str | Charset = "utf-8") -> str: ... -def getaddresses(fieldvalues: list[str]) -> list[tuple[str, str]]: ... + +if sys.version_info >= (3, 13): + def getaddresses(fieldvalues: Iterable[str], *, strict: bool = True) -> list[tuple[str, str]]: ... + +else: + def getaddresses(fieldvalues: Iterable[str]) -> list[tuple[str, str]]: ... + @overload def parsedate(data: None) -> None: ... @overload diff --git a/mypy/typeshed/stdlib/fcntl.pyi b/mypy/typeshed/stdlib/fcntl.pyi index ccf638205bbe..376611f166b8 100644 --- a/mypy/typeshed/stdlib/fcntl.pyi +++ b/mypy/typeshed/stdlib/fcntl.pyi @@ -1,6 +1,6 @@ import sys from _typeshed import FileDescriptorLike, ReadOnlyBuffer, WriteableBuffer -from typing import Any, Literal, overload +from typing import Any, Final, Literal, overload from typing_extensions import Buffer if sys.platform != "win32": @@ -44,9 +44,10 @@ if sys.platform != "win32": F_SEAL_SHRINK: int F_SEAL_WRITE: int if sys.version_info >= (3, 9): - F_OFD_GETLK: int - F_OFD_SETLK: int - F_OFD_SETLKW: int + F_OFD_GETLK: Final[int] + F_OFD_SETLK: Final[int] + F_OFD_SETLKW: Final[int] + if sys.version_info >= (3, 10): F_GETPIPE_SZ: int F_SETPIPE_SZ: int @@ -105,6 +106,36 @@ if sys.platform != "win32": FICLONE: int FICLONERANGE: int + if sys.version_info >= (3, 13) and sys.platform == "linux": + F_OWNER_TID: Final = 0 + F_OWNER_PID: Final = 1 + F_OWNER_PGRP: Final = 2 + F_SETOWN_EX: Final = 15 + F_GETOWN_EX: Final = 16 + F_SEAL_FUTURE_WRITE: Final = 16 + F_GET_RW_HINT: Final = 1035 + F_SET_RW_HINT: Final = 1036 + F_GET_FILE_RW_HINT: Final = 1037 + F_SET_FILE_RW_HINT: Final = 1038 + RWH_WRITE_LIFE_NOT_SET: Final = 0 + RWH_WRITE_LIFE_NONE: Final = 1 + RWH_WRITE_LIFE_SHORT: Final = 2 + RWH_WRITE_LIFE_MEDIUM: Final = 3 + RWH_WRITE_LIFE_LONG: Final = 4 + RWH_WRITE_LIFE_EXTREME: Final = 5 + + if sys.version_info >= (3, 11) and sys.platform == "darwin": + F_OFD_SETLK: Final = 90 + F_OFD_SETLKW: Final = 91 + F_OFD_GETLK: Final = 92 + + if sys.version_info >= (3, 13) and sys.platform != "linux": + # OSx and NetBSD + F_GETNOSIGPIPE: Final[int] + F_SETNOSIGPIPE: Final[int] + # OSx and FreeBSD + F_RDAHEAD: Final[int] + @overload def fcntl(fd: FileDescriptorLike, cmd: int, arg: int = 0, /) -> int: ... @overload diff --git a/mypy/typeshed/stdlib/filecmp.pyi b/mypy/typeshed/stdlib/filecmp.pyi index 5c8232d800d5..dfec2da72344 100644 --- a/mypy/typeshed/stdlib/filecmp.pyi +++ b/mypy/typeshed/stdlib/filecmp.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import GenericPath, StrOrBytesPath from collections.abc import Callable, Iterable, Sequence -from typing import Any, AnyStr, Generic, Literal +from typing import Any, AnyStr, Final, Generic, Literal if sys.version_info >= (3, 9): from types import GenericAlias @@ -9,7 +9,7 @@ if sys.version_info >= (3, 9): __all__ = ["clear_cache", "cmp", "dircmp", "cmpfiles", "DEFAULT_IGNORES"] DEFAULT_IGNORES: list[str] -BUFSIZE: Literal[8192] +BUFSIZE: Final = 8192 def cmp(f1: StrOrBytesPath, f2: StrOrBytesPath, shallow: bool | Literal[0, 1] = True) -> bool: ... def cmpfiles( diff --git a/mypy/typeshed/stdlib/ftplib.pyi b/mypy/typeshed/stdlib/ftplib.pyi index 9e7097ddc56e..1b96e0d504b7 100644 --- a/mypy/typeshed/stdlib/ftplib.pyi +++ b/mypy/typeshed/stdlib/ftplib.pyi @@ -4,16 +4,16 @@ from collections.abc import Callable, Iterable, Iterator from socket import socket from ssl import SSLContext from types import TracebackType -from typing import Any, Literal, TextIO +from typing import Any, Final, Literal, TextIO from typing_extensions import Self __all__ = ["FTP", "error_reply", "error_temp", "error_perm", "error_proto", "all_errors", "FTP_TLS"] -MSG_OOB: Literal[1] -FTP_PORT: Literal[21] -MAXLINE: Literal[8192] -CRLF: Literal["\r\n"] -B_CRLF: Literal[b"\r\n"] +MSG_OOB: Final = 1 +FTP_PORT: Final = 21 +MAXLINE: Final = 8192 +CRLF: Final = "\r\n" +B_CRLF: Final = b"\r\n" class Error(Exception): ... class error_reply(Error): ... diff --git a/mypy/typeshed/stdlib/gc.pyi b/mypy/typeshed/stdlib/gc.pyi index 31179add314c..9d34e0d6213a 100644 --- a/mypy/typeshed/stdlib/gc.pyi +++ b/mypy/typeshed/stdlib/gc.pyi @@ -1,13 +1,13 @@ import sys from collections.abc import Callable -from typing import Any, Literal +from typing import Any, Final, Literal from typing_extensions import TypeAlias -DEBUG_COLLECTABLE: Literal[2] -DEBUG_LEAK: Literal[38] -DEBUG_SAVEALL: Literal[32] -DEBUG_STATS: Literal[1] -DEBUG_UNCOLLECTABLE: Literal[4] +DEBUG_COLLECTABLE: Final = 2 +DEBUG_LEAK: Final = 38 +DEBUG_SAVEALL: Final = 32 +DEBUG_STATS: Final = 1 +DEBUG_UNCOLLECTABLE: Final = 4 _CallbackType: TypeAlias = Callable[[Literal["start", "stop"], dict[str, int]], object] @@ -34,4 +34,4 @@ if sys.version_info >= (3, 9): def isenabled() -> bool: ... def set_debug(flags: int, /) -> None: ... -def set_threshold(threshold0: int, threshold1: int = ..., threshold2: int = ...) -> None: ... +def set_threshold(threshold0: int, threshold1: int = ..., threshold2: int = ..., /) -> None: ... diff --git a/mypy/typeshed/stdlib/http/cookiejar.pyi b/mypy/typeshed/stdlib/http/cookiejar.pyi index faac20d13125..56097f163afd 100644 --- a/mypy/typeshed/stdlib/http/cookiejar.pyi +++ b/mypy/typeshed/stdlib/http/cookiejar.pyi @@ -42,7 +42,7 @@ class CookieJar(Iterable[Cookie]): def __len__(self) -> int: ... class FileCookieJar(CookieJar): - filename: str + filename: str | None delayload: bool def __init__(self, filename: StrPath | None = None, delayload: bool = False, policy: CookiePolicy | None = None) -> None: ... def save(self, filename: str | None = None, ignore_discard: bool = False, ignore_expires: bool = False) -> None: ... diff --git a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi index 56ee20523950..37b9a3882179 100644 --- a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi +++ b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi @@ -1,6 +1,7 @@ import abc import pathlib import sys +import types from _collections_abc import dict_keys, dict_values from _typeshed import StrPath from collections.abc import Iterable, Iterator, Mapping @@ -36,11 +37,8 @@ if sys.version_info >= (3, 10): from importlib.metadata._meta import PackageMetadata as PackageMetadata, SimplePath def packages_distributions() -> Mapping[str, list[str]]: ... - if sys.version_info >= (3, 12): - # It's generic but shouldn't be - _SimplePath: TypeAlias = SimplePath[Any] - else: - _SimplePath: TypeAlias = SimplePath + _SimplePath: TypeAlias = SimplePath + else: _SimplePath: TypeAlias = Path @@ -48,7 +46,9 @@ class PackageNotFoundError(ModuleNotFoundError): @property def name(self) -> str: ... # type: ignore[override] -if sys.version_info >= (3, 11): +if sys.version_info >= (3, 13): + _EntryPointBase = object +elif sys.version_info >= (3, 11): class DeprecatedTuple: def __getitem__(self, item: int) -> str: ... @@ -226,6 +226,9 @@ class Distribution(_distribution_parent): if sys.version_info >= (3, 10): @property def name(self) -> str: ... + if sys.version_info >= (3, 13): + @property + def origin(self) -> types.SimpleNamespace: ... class DistributionFinder(MetaPathFinder): class Context: diff --git a/mypy/typeshed/stdlib/importlib/metadata/_meta.pyi b/mypy/typeshed/stdlib/importlib/metadata/_meta.pyi index 3eac226b7065..9f791dab254f 100644 --- a/mypy/typeshed/stdlib/importlib/metadata/_meta.pyi +++ b/mypy/typeshed/stdlib/importlib/metadata/_meta.pyi @@ -1,9 +1,12 @@ import sys +from _typeshed import StrPath from collections.abc import Iterator -from typing import Any, Protocol, TypeVar, overload +from os import PathLike +from typing import Any, Protocol, overload +from typing_extensions import TypeVar _T = TypeVar("_T") -_T_co = TypeVar("_T_co", covariant=True) +_T_co = TypeVar("_T_co", covariant=True, default=Any) class PackageMetadata(Protocol): def __len__(self) -> int: ... @@ -22,7 +25,18 @@ class PackageMetadata(Protocol): @overload def get(self, name: str, failobj: _T) -> _T | str: ... -if sys.version_info >= (3, 12): +if sys.version_info >= (3, 13): + class SimplePath(Protocol): + def joinpath(self, other: StrPath, /) -> SimplePath: ... + def __truediv__(self, other: StrPath, /) -> SimplePath: ... + # Incorrect at runtime + @property + def parent(self) -> PathLike[str]: ... + def read_text(self, encoding: str | None = None) -> str: ... + def read_bytes(self) -> bytes: ... + def exists(self) -> bool: ... + +elif sys.version_info >= (3, 12): class SimplePath(Protocol[_T_co]): # At runtime this is defined as taking `str | _T`, but that causes trouble. # See #11436. diff --git a/mypy/typeshed/stdlib/importlib/metadata/diagnose.pyi b/mypy/typeshed/stdlib/importlib/metadata/diagnose.pyi new file mode 100644 index 000000000000..565872fd976f --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/metadata/diagnose.pyi @@ -0,0 +1,2 @@ +def inspect(path: str) -> None: ... +def run() -> None: ... diff --git a/mypy/typeshed/stdlib/inspect.pyi b/mypy/typeshed/stdlib/inspect.pyi index 23e0663d0d60..1eb9fc502e12 100644 --- a/mypy/typeshed/stdlib/inspect.pyi +++ b/mypy/typeshed/stdlib/inspect.pyi @@ -25,7 +25,7 @@ from types import ( TracebackType, WrapperDescriptorType, ) -from typing import Any, ClassVar, Literal, NamedTuple, Protocol, TypeVar, overload +from typing import Any, ClassVar, Final, Literal, NamedTuple, Protocol, TypeVar, overload from typing_extensions import ParamSpec, Self, TypeAlias, TypeGuard, TypeIs if sys.version_info >= (3, 11): @@ -161,35 +161,39 @@ class BlockFinder: last: int def tokeneater(self, type: int, token: str, srowcol: tuple[int, int], erowcol: tuple[int, int], line: str) -> None: ... -CO_OPTIMIZED: Literal[1] -CO_NEWLOCALS: Literal[2] -CO_VARARGS: Literal[4] -CO_VARKEYWORDS: Literal[8] -CO_NESTED: Literal[16] -CO_GENERATOR: Literal[32] -CO_NOFREE: Literal[64] -CO_COROUTINE: Literal[128] -CO_ITERABLE_COROUTINE: Literal[256] -CO_ASYNC_GENERATOR: Literal[512] -TPFLAGS_IS_ABSTRACT: Literal[1048576] +CO_OPTIMIZED: Final = 1 +CO_NEWLOCALS: Final = 2 +CO_VARARGS: Final = 4 +CO_VARKEYWORDS: Final = 8 +CO_NESTED: Final = 16 +CO_GENERATOR: Final = 32 +CO_NOFREE: Final = 64 +CO_COROUTINE: Final = 128 +CO_ITERABLE_COROUTINE: Final = 256 +CO_ASYNC_GENERATOR: Final = 512 +TPFLAGS_IS_ABSTRACT: Final = 1048576 modulesbyfile: dict[str, Any] _GetMembersPredicateTypeGuard: TypeAlias = Callable[[Any], TypeGuard[_T]] +_GetMembersPredicateTypeIs: TypeAlias = Callable[[Any], TypeIs[_T]] _GetMembersPredicate: TypeAlias = Callable[[Any], bool] -_GetMembersReturnTypeGuard: TypeAlias = list[tuple[str, _T]] -_GetMembersReturn: TypeAlias = list[tuple[str, Any]] +_GetMembersReturn: TypeAlias = list[tuple[str, _T]] @overload -def getmembers(object: object, predicate: _GetMembersPredicateTypeGuard[_T]) -> _GetMembersReturnTypeGuard[_T]: ... +def getmembers(object: object, predicate: _GetMembersPredicateTypeGuard[_T]) -> _GetMembersReturn[_T]: ... @overload -def getmembers(object: object, predicate: _GetMembersPredicate | None = None) -> _GetMembersReturn: ... +def getmembers(object: object, predicate: _GetMembersPredicateTypeIs[_T]) -> _GetMembersReturn[_T]: ... +@overload +def getmembers(object: object, predicate: _GetMembersPredicate | None = None) -> _GetMembersReturn[Any]: ... if sys.version_info >= (3, 11): @overload - def getmembers_static(object: object, predicate: _GetMembersPredicateTypeGuard[_T]) -> _GetMembersReturnTypeGuard[_T]: ... + def getmembers_static(object: object, predicate: _GetMembersPredicateTypeGuard[_T]) -> _GetMembersReturn[_T]: ... + @overload + def getmembers_static(object: object, predicate: _GetMembersPredicateTypeIs[_T]) -> _GetMembersReturn[_T]: ... @overload - def getmembers_static(object: object, predicate: _GetMembersPredicate | None = None) -> _GetMembersReturn: ... + def getmembers_static(object: object, predicate: _GetMembersPredicate | None = None) -> _GetMembersReturn[Any]: ... def getmodulename(path: StrPath) -> str | None: ... def ismodule(object: object) -> TypeIs[ModuleType]: ... @@ -360,10 +364,10 @@ class _ParameterKind(enum.IntEnum): def description(self) -> str: ... if sys.version_info >= (3, 12): - AGEN_CREATED: Literal["AGEN_CREATED"] - AGEN_RUNNING: Literal["AGEN_RUNNING"] - AGEN_SUSPENDED: Literal["AGEN_SUSPENDED"] - AGEN_CLOSED: Literal["AGEN_CLOSED"] + AGEN_CREATED: Final = "AGEN_CREATED" + AGEN_RUNNING: Final = "AGEN_RUNNING" + AGEN_SUSPENDED: Final = "AGEN_SUSPENDED" + AGEN_CLOSED: Final = "AGEN_CLOSED" def getasyncgenstate( agen: AsyncGenerator[Any, Any] @@ -580,19 +584,19 @@ def getattr_static(obj: object, attr: str, default: Any | None = ...) -> Any: .. # Current State of Generators and Coroutines # -GEN_CREATED: Literal["GEN_CREATED"] -GEN_RUNNING: Literal["GEN_RUNNING"] -GEN_SUSPENDED: Literal["GEN_SUSPENDED"] -GEN_CLOSED: Literal["GEN_CLOSED"] +GEN_CREATED: Final = "GEN_CREATED" +GEN_RUNNING: Final = "GEN_RUNNING" +GEN_SUSPENDED: Final = "GEN_SUSPENDED" +GEN_CLOSED: Final = "GEN_CLOSED" def getgeneratorstate( generator: Generator[Any, Any, Any] ) -> Literal["GEN_CREATED", "GEN_RUNNING", "GEN_SUSPENDED", "GEN_CLOSED"]: ... -CORO_CREATED: Literal["CORO_CREATED"] -CORO_RUNNING: Literal["CORO_RUNNING"] -CORO_SUSPENDED: Literal["CORO_SUSPENDED"] -CORO_CLOSED: Literal["CORO_CLOSED"] +CORO_CREATED: Final = "CORO_CREATED" +CORO_RUNNING: Final = "CORO_RUNNING" +CORO_SUSPENDED: Final = "CORO_SUSPENDED" +CORO_CLOSED: Final = "CORO_CLOSED" def getcoroutinestate( coroutine: Coroutine[Any, Any, Any] diff --git a/mypy/typeshed/stdlib/io.pyi b/mypy/typeshed/stdlib/io.pyi index 01f3bfc06a27..a386a914ddcd 100644 --- a/mypy/typeshed/stdlib/io.pyi +++ b/mypy/typeshed/stdlib/io.pyi @@ -6,7 +6,7 @@ from _typeshed import FileDescriptorOrPath, ReadableBuffer, WriteableBuffer from collections.abc import Callable, Iterable, Iterator from os import _Opener from types import TracebackType -from typing import IO, Any, BinaryIO, Literal, Protocol, TextIO, TypeVar, overload, type_check_only +from typing import IO, Any, BinaryIO, Final, Generic, Literal, Protocol, TextIO, TypeVar, overload, type_check_only from typing_extensions import Self __all__ = [ @@ -36,11 +36,11 @@ if sys.version_info >= (3, 11): _T = TypeVar("_T") -DEFAULT_BUFFER_SIZE: Literal[8192] +DEFAULT_BUFFER_SIZE: Final = 8192 -SEEK_SET: Literal[0] -SEEK_CUR: Literal[1] -SEEK_END: Literal[2] +SEEK_SET: Final = 0 +SEEK_CUR: Final = 1 +SEEK_END: Final = 2 open = builtins.open @@ -173,12 +173,12 @@ class _WrappedBuffer(Protocol): # def seek(self, offset: Literal[0], whence: Literal[2]) -> int: ... # def tell(self) -> int: ... -# TODO: Should be generic over the buffer type, but needs to wait for -# TypeVar defaults. -class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible definitions of write in the base classes +_BufferT_co = TypeVar("_BufferT_co", bound=_WrappedBuffer, default=_WrappedBuffer, covariant=True) + +class TextIOWrapper(TextIOBase, TextIO, Generic[_BufferT_co]): # type: ignore[misc] # incompatible definitions of write in the base classes def __init__( self, - buffer: _WrappedBuffer, + buffer: _BufferT_co, encoding: str | None = None, errors: str | None = None, newline: str | None = None, @@ -187,7 +187,7 @@ class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible d ) -> None: ... # Equals the "buffer" argument passed in to the constructor. @property - def buffer(self) -> BinaryIO: ... + def buffer(self) -> _BufferT_co: ... # type: ignore[override] @property def closed(self) -> bool: ... @property @@ -211,7 +211,7 @@ class TextIOWrapper(TextIOBase, TextIO): # type: ignore[misc] # incompatible d def readline(self, size: int = -1, /) -> str: ... # type: ignore[override] def readlines(self, hint: int = -1, /) -> list[str]: ... # type: ignore[override] # Equals the "buffer" argument passed in to the constructor. - def detach(self) -> BinaryIO: ... + def detach(self) -> _BufferT_co: ... # type: ignore[override] # TextIOWrapper's version of seek only supports a limited subset of # operations. def seek(self, cookie: int, whence: int = 0, /) -> int: ... diff --git a/mypy/typeshed/stdlib/ipaddress.pyi b/mypy/typeshed/stdlib/ipaddress.pyi index 03decc74e65e..f51ea87dcfcf 100644 --- a/mypy/typeshed/stdlib/ipaddress.pyi +++ b/mypy/typeshed/stdlib/ipaddress.pyi @@ -1,11 +1,11 @@ import sys from collections.abc import Iterable, Iterator -from typing import Any, Generic, Literal, SupportsInt, TypeVar, overload +from typing import Any, Final, Generic, Literal, SupportsInt, TypeVar, overload from typing_extensions import Self, TypeAlias # Undocumented length constants -IPV4LENGTH: Literal[32] -IPV6LENGTH: Literal[128] +IPV4LENGTH: Final = 32 +IPV6LENGTH: Final = 128 _A = TypeVar("_A", IPv4Address, IPv6Address) _N = TypeVar("_N", IPv4Network, IPv6Network) diff --git a/mypy/typeshed/stdlib/itertools.pyi b/mypy/typeshed/stdlib/itertools.pyi index 16e04829c6cf..1635b6a0a072 100644 --- a/mypy/typeshed/stdlib/itertools.pyi +++ b/mypy/typeshed/stdlib/itertools.pyi @@ -326,6 +326,10 @@ if sys.version_info >= (3, 10): if sys.version_info >= (3, 12): class batched(Iterator[tuple[_T_co, ...]], Generic[_T_co]): - def __new__(cls, iterable: Iterable[_T_co], n: int) -> Self: ... + if sys.version_info >= (3, 13): + def __new__(cls, iterable: Iterable[_T_co], n: int, *, strict: bool = False) -> Self: ... + else: + def __new__(cls, iterable: Iterable[_T_co], n: int) -> Self: ... + def __iter__(self) -> Self: ... def __next__(self) -> tuple[_T_co, ...]: ... diff --git a/mypy/typeshed/stdlib/lzma.pyi b/mypy/typeshed/stdlib/lzma.pyi index c05e46a02aeb..2df2b9a8bd6a 100644 --- a/mypy/typeshed/stdlib/lzma.pyi +++ b/mypy/typeshed/stdlib/lzma.pyi @@ -1,7 +1,7 @@ from _compression import BaseStream from _typeshed import ReadableBuffer, StrOrBytesPath from collections.abc import Mapping, Sequence -from typing import IO, Any, Literal, TextIO, final, overload +from typing import IO, Any, Final, Literal, TextIO, final, overload from typing_extensions import Self, TypeAlias __all__ = [ @@ -50,33 +50,33 @@ _PathOrFile: TypeAlias = StrOrBytesPath | IO[bytes] _FilterChain: TypeAlias = Sequence[Mapping[str, Any]] -FORMAT_AUTO: Literal[0] -FORMAT_XZ: Literal[1] -FORMAT_ALONE: Literal[2] -FORMAT_RAW: Literal[3] -CHECK_NONE: Literal[0] -CHECK_CRC32: Literal[1] -CHECK_CRC64: Literal[4] -CHECK_SHA256: Literal[10] -CHECK_ID_MAX: Literal[15] -CHECK_UNKNOWN: Literal[16] +FORMAT_AUTO: Final = 0 +FORMAT_XZ: Final = 1 +FORMAT_ALONE: Final = 2 +FORMAT_RAW: Final = 3 +CHECK_NONE: Final = 0 +CHECK_CRC32: Final = 1 +CHECK_CRC64: Final = 4 +CHECK_SHA256: Final = 10 +CHECK_ID_MAX: Final = 15 +CHECK_UNKNOWN: Final = 16 FILTER_LZMA1: int # v big number -FILTER_LZMA2: Literal[33] -FILTER_DELTA: Literal[3] -FILTER_X86: Literal[4] -FILTER_IA64: Literal[6] -FILTER_ARM: Literal[7] -FILTER_ARMTHUMB: Literal[8] -FILTER_SPARC: Literal[9] -FILTER_POWERPC: Literal[5] -MF_HC3: Literal[3] -MF_HC4: Literal[4] -MF_BT2: Literal[18] -MF_BT3: Literal[19] -MF_BT4: Literal[20] -MODE_FAST: Literal[1] -MODE_NORMAL: Literal[2] -PRESET_DEFAULT: Literal[6] +FILTER_LZMA2: Final = 33 +FILTER_DELTA: Final = 3 +FILTER_X86: Final = 4 +FILTER_IA64: Final = 6 +FILTER_ARM: Final = 7 +FILTER_ARMTHUMB: Final = 8 +FILTER_SPARC: Final = 9 +FILTER_POWERPC: Final = 5 +MF_HC3: Final = 3 +MF_HC4: Final = 4 +MF_BT2: Final = 18 +MF_BT3: Final = 19 +MF_BT4: Final = 20 +MODE_FAST: Final = 1 +MODE_NORMAL: Final = 2 +PRESET_DEFAULT: Final = 6 PRESET_EXTREME: int # v big number # from _lzma.c diff --git a/mypy/typeshed/stdlib/mailbox.pyi b/mypy/typeshed/stdlib/mailbox.pyi index 2f43f9552652..a98a00a42853 100644 --- a/mypy/typeshed/stdlib/mailbox.pyi +++ b/mypy/typeshed/stdlib/mailbox.pyi @@ -115,6 +115,14 @@ class Maildir(Mailbox[MaildirMessage]): def get_message(self, key: str) -> MaildirMessage: ... def get_bytes(self, key: str) -> bytes: ... def get_file(self, key: str) -> _ProxyFile[bytes]: ... + if sys.version_info >= (3, 13): + def get_info(self, key: str) -> str: ... + def set_info(self, key: str, info: str) -> None: ... + def get_flags(self, key: str) -> str: ... + def set_flags(self, key: str, flags: str) -> None: ... + def add_flag(self, key: str, flag: str) -> None: ... + def remove_flag(self, key: str, flag: str) -> None: ... + def iterkeys(self) -> Iterator[str]: ... def __contains__(self, key: str) -> bool: ... def __len__(self) -> int: ... diff --git a/mypy/typeshed/stdlib/mimetypes.pyi b/mypy/typeshed/stdlib/mimetypes.pyi index 517193e3516f..9914a34a2d6a 100644 --- a/mypy/typeshed/stdlib/mimetypes.pyi +++ b/mypy/typeshed/stdlib/mimetypes.pyi @@ -45,6 +45,7 @@ class MimeTypes: types_map: tuple[dict[str, str], dict[str, str]] types_map_inv: tuple[dict[str, str], dict[str, str]] def __init__(self, filenames: tuple[str, ...] = (), strict: bool = True) -> None: ... + def add_type(self, type: str, ext: str, strict: bool = True) -> None: ... def guess_extension(self, type: str, strict: bool = True) -> str | None: ... def guess_type(self, url: StrPath, strict: bool = True) -> tuple[str | None, str | None]: ... def guess_all_extensions(self, type: str, strict: bool = True) -> list[str]: ... diff --git a/mypy/typeshed/stdlib/mmap.pyi b/mypy/typeshed/stdlib/mmap.pyi index 7688970e5786..a0c150d6e7e8 100644 --- a/mypy/typeshed/stdlib/mmap.pyi +++ b/mypy/typeshed/stdlib/mmap.pyi @@ -1,7 +1,7 @@ import sys from _typeshed import ReadableBuffer, Unused from collections.abc import Iterable, Iterator, Sized -from typing import Final, NoReturn, overload +from typing import Final, Literal, NoReturn, overload from typing_extensions import Self ACCESS_DEFAULT: int @@ -77,7 +77,7 @@ class mmap(Iterable[int], Sized): def __buffer__(self, flags: int, /) -> memoryview: ... def __release_buffer__(self, buffer: memoryview, /) -> None: ... if sys.version_info >= (3, 13): - def seekable(self) -> bool: ... + def seekable(self) -> Literal[True]: ... if sys.platform != "win32": MADV_NORMAL: int @@ -118,4 +118,16 @@ if sys.version_info >= (3, 13) and sys.platform != "win32": MAP_32BIT: Final = 32768 if sys.version_info >= (3, 13) and sys.platform == "darwin": + MAP_NORESERVE: Final = 64 + MAP_NOEXTEND: Final = 256 + MAP_HASSEMAPHORE: Final = 512 + MAP_NOCACHE: Final = 1024 + MAP_JIT: Final = 2048 + MAP_RESILIENT_CODESIGN: Final = 8192 + MAP_RESILIENT_MEDIA: Final = 16384 + MAP_TRANSLATED_ALLOW_EXECUTE: Final = 131072 + MAP_UNIX03: Final = 262144 MAP_TPRO: Final = 524288 + +if sys.version_info >= (3, 13) and sys.platform == "linux": + MAP_NORESERVE: Final = 16384 diff --git a/mypy/typeshed/stdlib/msvcrt.pyi b/mypy/typeshed/stdlib/msvcrt.pyi index 54b3674a3a46..403a5d933522 100644 --- a/mypy/typeshed/stdlib/msvcrt.pyi +++ b/mypy/typeshed/stdlib/msvcrt.pyi @@ -1,14 +1,14 @@ import sys -from typing import Final, Literal +from typing import Final # This module is only available on Windows if sys.platform == "win32": CRT_ASSEMBLY_VERSION: Final[str] - LK_UNLCK: Literal[0] - LK_LOCK: Literal[1] - LK_NBLCK: Literal[2] - LK_RLCK: Literal[3] - LK_NBRLCK: Literal[4] + LK_UNLCK: Final = 0 + LK_LOCK: Final = 1 + LK_NBLCK: Final = 2 + LK_RLCK: Final = 3 + LK_NBRLCK: Final = 4 SEM_FAILCRITICALERRORS: int SEM_NOALIGNMENTFAULTEXCEPT: int SEM_NOGPFAULTERRORBOX: int diff --git a/mypy/typeshed/stdlib/multiprocessing/pool.pyi b/mypy/typeshed/stdlib/multiprocessing/pool.pyi index d2d611e3ca62..950ed1d8c56b 100644 --- a/mypy/typeshed/stdlib/multiprocessing/pool.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/pool.pyi @@ -1,7 +1,7 @@ import sys from collections.abc import Callable, Iterable, Iterator, Mapping from types import TracebackType -from typing import Any, Generic, Literal, TypeVar +from typing import Any, Final, Generic, TypeVar from typing_extensions import Self if sys.version_info >= (3, 9): @@ -97,7 +97,7 @@ class ThreadPool(Pool): ) -> None: ... # undocumented -INIT: Literal["INIT"] -RUN: Literal["RUN"] -CLOSE: Literal["CLOSE"] -TERMINATE: Literal["TERMINATE"] +INIT: Final = "INIT" +RUN: Final = "RUN" +CLOSE: Final = "CLOSE" +TERMINATE: Final = "TERMINATE" diff --git a/mypy/typeshed/stdlib/multiprocessing/reduction.pyi b/mypy/typeshed/stdlib/multiprocessing/reduction.pyi index 91532633e1b9..6d23e20e6981 100644 --- a/mypy/typeshed/stdlib/multiprocessing/reduction.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/reduction.pyi @@ -8,7 +8,7 @@ from copyreg import _DispatchTableType from multiprocessing import connection from pickle import _ReducedType from socket import socket -from typing import Any, Literal +from typing import Any, Final if sys.platform == "win32": __all__ = ["send_handle", "recv_handle", "ForkingPickler", "register", "dump", "DupHandle", "duplicate", "steal_handle"] @@ -43,10 +43,7 @@ if sys.platform == "win32": def detach(self) -> int: ... else: - if sys.platform == "darwin": - ACKNOWLEDGE: Literal[True] - else: - ACKNOWLEDGE: Literal[False] + ACKNOWLEDGE: Final[bool] def recvfds(sock: socket, size: int) -> list[int]: ... def send_handle(conn: HasFileno, handle: int, destination_pid: Unused) -> None: ... diff --git a/mypy/typeshed/stdlib/nntplib.pyi b/mypy/typeshed/stdlib/nntplib.pyi index 969c657e9aab..85dfbff1cb50 100644 --- a/mypy/typeshed/stdlib/nntplib.pyi +++ b/mypy/typeshed/stdlib/nntplib.pyi @@ -5,7 +5,7 @@ import sys from _typeshed import Unused from builtins import list as _list # conflicts with a method named "list" from collections.abc import Iterable -from typing import IO, Any, Literal, NamedTuple +from typing import IO, Any, Final, NamedTuple from typing_extensions import Self, TypeAlias __all__ = [ @@ -31,8 +31,8 @@ class NNTPPermanentError(NNTPError): ... class NNTPProtocolError(NNTPError): ... class NNTPDataError(NNTPError): ... -NNTP_PORT: Literal[119] -NNTP_SSL_PORT: Literal[563] +NNTP_PORT: Final = 119 +NNTP_SSL_PORT: Final = 563 class GroupInfo(NamedTuple): group: str diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index 9b00117a5599..e2d272cb4112 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -971,7 +971,8 @@ else: def spawnvp(mode: int, file: StrOrBytesPath, args: _ExecVArgs) -> int: ... def spawnvpe(mode: int, file: StrOrBytesPath, args: _ExecVArgs, env: _ExecEnv) -> int: ... def wait() -> tuple[int, int]: ... # Unix only - if sys.platform != "darwin": + # Added to MacOS in 3.13 + if sys.platform != "darwin" or sys.version_info >= (3, 13): @final class waitid_result(structseq[int], tuple[int, int, int, int, int]): if sys.version_info >= (3, 10): @@ -1155,3 +1156,24 @@ if sys.version_info >= (3, 12) and sys.platform == "linux": CLONE_VM: int def unshare(flags: int) -> None: ... def setns(fd: FileDescriptorLike, nstype: int = 0) -> None: ... + +if sys.version_info >= (3, 13) and sys.platform != "win32": + def posix_openpt(oflag: int, /) -> int: ... + def grantpt(fd: FileDescriptorLike, /) -> None: ... + def unlockpt(fd: FileDescriptorLike, /) -> None: ... + def ptsname(fd: FileDescriptorLike, /) -> str: ... + +if sys.version_info >= (3, 13) and sys.platform == "linux": + TFD_TIMER_ABSTIME: Final = 1 + TFD_TIMER_CANCEL_ON_SET: Final = 2 + TFD_NONBLOCK: Final[int] + TFD_CLOEXEC: Final[int] + POSIX_SPAWN_CLOSEFROM: Final[int] + + def timerfd_create(clockid: int, /, *, flags: int = 0) -> int: ... + def timerfd_settime( + fd: FileDescriptor, /, *, flags: int = 0, initial: float = 0.0, interval: float = 0.0 + ) -> tuple[float, float]: ... + def timerfd_settime_ns(fd: FileDescriptor, /, *, flags: int = 0, initial: int = 0, interval: int = 0) -> tuple[int, int]: ... + def timerfd_gettime(fd: FileDescriptor, /) -> tuple[float, float]: ... + def timerfd_gettime_ns(fd: FileDescriptor, /) -> tuple[int, int]: ... diff --git a/mypy/typeshed/stdlib/pathlib.pyi b/mypy/typeshed/stdlib/pathlib.pyi index c8c8dde0f33e..116bf6431831 100644 --- a/mypy/typeshed/stdlib/pathlib.pyi +++ b/mypy/typeshed/stdlib/pathlib.pyi @@ -49,7 +49,7 @@ class PurePath(PathLike[str]): def stem(self) -> str: ... if sys.version_info >= (3, 12): def __new__(cls, *args: StrPath, **kwargs: Unused) -> Self: ... - def __init__(self, *args: StrPath) -> None: ... + def __init__(self, *args: StrPath) -> None: ... # pyright: ignore[reportInconsistentConstructor] else: def __new__(cls, *args: StrPath) -> Self: ... @@ -101,7 +101,11 @@ class PurePosixPath(PurePath): ... class PureWindowsPath(PurePath): ... class Path(PurePath): - def __new__(cls, *args: StrPath, **kwargs: Any) -> Self: ... + if sys.version_info >= (3, 12): + def __new__(cls, *args: StrPath, **kwargs: Unused) -> Self: ... # pyright: ignore[reportInconsistentConstructor] + else: + def __new__(cls, *args: StrPath, **kwargs: Unused) -> Self: ... + @classmethod def cwd(cls) -> Self: ... if sys.version_info >= (3, 10): @@ -113,7 +117,7 @@ class Path(PurePath): if sys.version_info >= (3, 13): @classmethod - def from_uri(cls, uri: str) -> Path: ... + def from_uri(cls, uri: str) -> Self: ... def is_dir(self, *, follow_symlinks: bool = True) -> bool: ... def is_file(self, *, follow_symlinks: bool = True) -> bool: ... def read_text(self, encoding: str | None = None, errors: str | None = None, newline: str | None = None) -> str: ... diff --git a/mypy/typeshed/stdlib/pdb.pyi b/mypy/typeshed/stdlib/pdb.pyi index 487adddd04bf..d49315427813 100644 --- a/mypy/typeshed/stdlib/pdb.pyi +++ b/mypy/typeshed/stdlib/pdb.pyi @@ -5,7 +5,7 @@ from cmd import Cmd from collections.abc import Callable, Iterable, Mapping, Sequence from inspect import _SourceObjectType from types import CodeType, FrameType, TracebackType -from typing import IO, Any, ClassVar, TypeVar +from typing import IO, Any, ClassVar, Final, TypeVar from typing_extensions import ParamSpec, Self __all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace", "post_mortem", "help"] @@ -30,6 +30,9 @@ class Pdb(Bdb, Cmd): commands_resuming: ClassVar[list[str]] + if sys.version_info >= (3, 13): + MAX_CHAINED_EXCEPTION_DEPTH: Final = 999 + aliases: dict[str, str] mainpyfile: str _wait_for_mainpyfile: bool @@ -58,8 +61,16 @@ class Pdb(Bdb, Cmd): if sys.version_info < (3, 11): def execRcLines(self) -> None: ... + if sys.version_info >= (3, 13): + user_opcode = Bdb.user_line + def bp_commands(self, frame: FrameType) -> bool: ... - def interaction(self, frame: FrameType | None, traceback: TracebackType | None) -> None: ... + + if sys.version_info >= (3, 13): + def interaction(self, frame: FrameType | None, tb_or_exc: TracebackType | BaseException | None) -> None: ... + else: + def interaction(self, frame: FrameType | None, traceback: TracebackType | None) -> None: ... + def displayhook(self, obj: object) -> None: ... def handle_command_def(self, line: str) -> bool: ... def defaultFile(self) -> str: ... @@ -72,6 +83,9 @@ class Pdb(Bdb, Cmd): if sys.version_info < (3, 11): def _runscript(self, filename: str) -> None: ... + if sys.version_info >= (3, 13): + def completedefault(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... # type: ignore[override] + def do_commands(self, arg: str) -> bool | None: ... def do_break(self, arg: str, temporary: bool = ...) -> bool | None: ... def do_tbreak(self, arg: str) -> bool | None: ... @@ -81,6 +95,9 @@ class Pdb(Bdb, Cmd): def do_ignore(self, arg: str) -> bool | None: ... def do_clear(self, arg: str) -> bool | None: ... def do_where(self, arg: str) -> bool | None: ... + if sys.version_info >= (3, 13): + def do_exceptions(self, arg: str) -> bool | None: ... + def do_up(self, arg: str) -> bool | None: ... def do_down(self, arg: str) -> bool | None: ... def do_until(self, arg: str) -> bool | None: ... @@ -125,8 +142,14 @@ class Pdb(Bdb, Cmd): def help_exec(self) -> None: ... def help_pdb(self) -> None: ... def sigint_handler(self, signum: signal.Signals, frame: FrameType) -> None: ... - def message(self, msg: str) -> None: ... + if sys.version_info >= (3, 13): + def message(self, msg: str, end: str = "\n") -> None: ... + else: + def message(self, msg: str) -> None: ... + def error(self, msg: str) -> None: ... + if sys.version_info >= (3, 13): + def completenames(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... # type: ignore[override] if sys.version_info >= (3, 12): def set_convenience_variable(self, frame: FrameType, name: str, value: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/poplib.pyi b/mypy/typeshed/stdlib/poplib.pyi index 12f1d16a0d6f..7476f2991978 100644 --- a/mypy/typeshed/stdlib/poplib.pyi +++ b/mypy/typeshed/stdlib/poplib.pyi @@ -3,7 +3,7 @@ import ssl import sys from builtins import list as _list # conflicts with a method named "list" from re import Pattern -from typing import Any, BinaryIO, Literal, NoReturn, overload +from typing import Any, BinaryIO, Final, NoReturn, overload from typing_extensions import TypeAlias __all__ = ["POP3", "error_proto", "POP3_SSL"] @@ -12,11 +12,11 @@ _LongResp: TypeAlias = tuple[bytes, list[bytes], int] class error_proto(Exception): ... -POP3_PORT: Literal[110] -POP3_SSL_PORT: Literal[995] -CR: Literal[b"\r"] -LF: Literal[b"\n"] -CRLF: Literal[b"\r\n"] +POP3_PORT: Final = 110 +POP3_SSL_PORT: Final = 995 +CR: Final = b"\r" +LF: Final = b"\n" +CRLF: Final = b"\r\n" HAVE_SSL: bool class POP3: diff --git a/mypy/typeshed/stdlib/posix.pyi b/mypy/typeshed/stdlib/posix.pyi index b31b8f3d3524..1a4f22af82cf 100644 --- a/mypy/typeshed/stdlib/posix.pyi +++ b/mypy/typeshed/stdlib/posix.pyi @@ -236,6 +236,23 @@ if sys.platform != "win32": if sys.version_info >= (3, 11): from os import login_tty as login_tty + if sys.version_info >= (3, 13): + from os import grantpt as grantpt, posix_openpt as posix_openpt, ptsname as ptsname, unlockpt as unlockpt + + if sys.version_info >= (3, 13) and sys.platform == "linux": + from os import ( + POSIX_SPAWN_CLOSEFROM as POSIX_SPAWN_CLOSEFROM, + TFD_CLOEXEC as TFD_CLOEXEC, + TFD_NONBLOCK as TFD_NONBLOCK, + TFD_TIMER_ABSTIME as TFD_TIMER_ABSTIME, + TFD_TIMER_CANCEL_ON_SET as TFD_TIMER_CANCEL_ON_SET, + timerfd_create as timerfd_create, + timerfd_gettime as timerfd_gettime, + timerfd_gettime_ns as timerfd_gettime_ns, + timerfd_settime as timerfd_settime, + timerfd_settime_ns as timerfd_settime_ns, + ) + if sys.platform != "linux": from os import chflags as chflags, lchflags as lchflags, lchmod as lchmod @@ -269,13 +286,14 @@ if sys.platform != "win32": sched_setscheduler as sched_setscheduler, setresgid as setresgid, setresuid as setresuid, - waitid as waitid, - waitid_result as waitid_result, ) if sys.version_info >= (3, 10): from os import RWF_APPEND as RWF_APPEND + if sys.platform != "darwin" or sys.version_info >= (3, 13): + from os import waitid as waitid, waitid_result as waitid_result + if sys.platform == "linux": from os import ( GRND_NONBLOCK as GRND_NONBLOCK, diff --git a/mypy/typeshed/stdlib/pty.pyi b/mypy/typeshed/stdlib/pty.pyi index 022b08046c54..4c9e42b4ec5e 100644 --- a/mypy/typeshed/stdlib/pty.pyi +++ b/mypy/typeshed/stdlib/pty.pyi @@ -1,17 +1,17 @@ import sys from collections.abc import Callable, Iterable -from typing import Literal +from typing import Final from typing_extensions import TypeAlias if sys.platform != "win32": __all__ = ["openpty", "fork", "spawn"] _Reader: TypeAlias = Callable[[int], bytes] - STDIN_FILENO: Literal[0] - STDOUT_FILENO: Literal[1] - STDERR_FILENO: Literal[2] + STDIN_FILENO: Final = 0 + STDOUT_FILENO: Final = 1 + STDERR_FILENO: Final = 2 - CHILD: Literal[0] + CHILD: Final = 0 def openpty() -> tuple[int, int]: ... def master_open() -> tuple[int, str]: ... # deprecated, use openpty() def slave_open(tty_name: str) -> int: ... # deprecated, use openpty() diff --git a/mypy/typeshed/stdlib/pydoc.pyi b/mypy/typeshed/stdlib/pydoc.pyi index 1a90eb30efca..144f782acad5 100644 --- a/mypy/typeshed/stdlib/pydoc.pyi +++ b/mypy/typeshed/stdlib/pydoc.pyi @@ -5,7 +5,7 @@ from builtins import list as _list # "list" conflicts with method name from collections.abc import Callable, Container, Mapping, MutableMapping from reprlib import Repr from types import MethodType, ModuleType, TracebackType -from typing import IO, Any, AnyStr, Final, NoReturn, TypeVar +from typing import IO, Any, AnyStr, Final, NoReturn, Protocol, TypeVar from typing_extensions import TypeGuard __all__ = ["help"] @@ -17,6 +17,9 @@ __date__: Final[str] __version__: Final[str] __credits__: Final[str] +class _Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: ... + def pathdirs() -> list[str]: ... def getdoc(object: object) -> str: ... def splitdoc(doc: AnyStr) -> tuple[AnyStr, AnyStr]: ... @@ -229,16 +232,36 @@ class TextDoc(Doc): doc: Any | None = None, ) -> str: ... -def pager(text: str) -> None: ... -def getpager() -> Callable[[str], None]: ... +if sys.version_info >= (3, 13): + def pager(text: str, title: str = "") -> None: ... + +else: + def pager(text: str) -> None: ... + def plain(text: str) -> str: ... -def pipepager(text: str, cmd: str) -> None: ... -def tempfilepager(text: str, cmd: str) -> None: ... -def ttypager(text: str) -> None: ... -def plainpager(text: str) -> None: ... def describe(thing: Any) -> str: ... def locate(path: str, forceload: bool = ...) -> object: ... +if sys.version_info >= (3, 13): + def get_pager() -> _Pager: ... + def pipe_pager(text: str, cmd: str, title: str = "") -> None: ... + def tempfile_pager(text: str, cmd: str, title: str = "") -> None: ... + def tty_pager(text: str, title: str = "") -> None: ... + def plain_pager(text: str, title: str = "") -> None: ... + + # For backwards compatibility. + getpager = get_pager + pipepager = pipe_pager + tempfilepager = tempfile_pager + ttypager = tty_pager + plainpager = plain_pager +else: + def getpager() -> Callable[[str], None]: ... + def pipepager(text: str, cmd: str) -> None: ... + def tempfilepager(text: str, cmd: str) -> None: ... + def ttypager(text: str) -> None: ... + def plainpager(text: str) -> None: ... + text: TextDoc html: HTMLDoc diff --git a/mypy/typeshed/stdlib/readline.pyi b/mypy/typeshed/stdlib/readline.pyi index 688ae48d9f92..7325c267b32c 100644 --- a/mypy/typeshed/stdlib/readline.pyi +++ b/mypy/typeshed/stdlib/readline.pyi @@ -1,6 +1,7 @@ import sys from _typeshed import StrOrBytesPath from collections.abc import Callable, Sequence +from typing import Literal from typing_extensions import TypeAlias if sys.platform != "win32": @@ -34,3 +35,6 @@ if sys.platform != "win32": def set_completer_delims(string: str, /) -> None: ... def get_completer_delims() -> str: ... def set_completion_display_matches_hook(function: _CompDisp | None = None, /) -> None: ... + + if sys.version_info >= (3, 13): + backend: Literal["readline", "editline"] diff --git a/mypy/typeshed/stdlib/site.pyi b/mypy/typeshed/stdlib/site.pyi index a8c6bcb417f4..6e39677aaea0 100644 --- a/mypy/typeshed/stdlib/site.pyi +++ b/mypy/typeshed/stdlib/site.pyi @@ -1,3 +1,4 @@ +import sys from _typeshed import StrPath from collections.abc import Iterable @@ -13,7 +14,15 @@ def addsitedir(sitedir: str, known_paths: set[str] | None = None) -> None: ... def addsitepackages(known_paths: set[str] | None, prefixes: Iterable[str] | None = None) -> set[str] | None: ... # undocumented def addusersitepackages(known_paths: set[str] | None) -> set[str] | None: ... # undocumented def check_enableusersite() -> bool | None: ... # undocumented + +if sys.version_info >= (3, 13): + def gethistoryfile() -> str: ... # undocumented + def enablerlcompleter() -> None: ... # undocumented + +if sys.version_info >= (3, 13): + def register_readline() -> None: ... # undocumented + def execsitecustomize() -> None: ... # undocumented def execusercustomize() -> None: ... # undocumented def getsitepackages(prefixes: Iterable[str] | None = None) -> list[str]: ... diff --git a/mypy/typeshed/stdlib/sre_constants.pyi b/mypy/typeshed/stdlib/sre_constants.pyi index d522372c438c..0c1e484bb07e 100644 --- a/mypy/typeshed/stdlib/sre_constants.pyi +++ b/mypy/typeshed/stdlib/sre_constants.pyi @@ -30,7 +30,8 @@ AT_LOCALE: dict[_NamedIntConstant, _NamedIntConstant] AT_UNICODE: dict[_NamedIntConstant, _NamedIntConstant] CH_LOCALE: dict[_NamedIntConstant, _NamedIntConstant] CH_UNICODE: dict[_NamedIntConstant, _NamedIntConstant] -SRE_FLAG_TEMPLATE: int +if sys.version_info < (3, 13): + SRE_FLAG_TEMPLATE: int SRE_FLAG_IGNORECASE: int SRE_FLAG_LOCALE: int SRE_FLAG_MULTILINE: int diff --git a/mypy/typeshed/stdlib/stat.pyi b/mypy/typeshed/stdlib/stat.pyi index f3bdd92c1068..face28ab0cbb 100644 --- a/mypy/typeshed/stdlib/stat.pyi +++ b/mypy/typeshed/stdlib/stat.pyi @@ -1,7 +1,7 @@ import sys from _stat import * -from typing import Literal +from typing import Final if sys.version_info >= (3, 13): # https://github.com/python/cpython/issues/114081#issuecomment-2119017790 - SF_RESTRICTED: Literal[0x00080000] + SF_RESTRICTED: Final = 0x00080000 diff --git a/mypy/typeshed/stdlib/symtable.pyi b/mypy/typeshed/stdlib/symtable.pyi index 0f080954ba2c..5481d4d1dd4a 100644 --- a/mypy/typeshed/stdlib/symtable.pyi +++ b/mypy/typeshed/stdlib/symtable.pyi @@ -5,11 +5,30 @@ from typing import Any __all__ = ["symtable", "SymbolTable", "Class", "Function", "Symbol"] +if sys.version_info >= (3, 13): + __all__ += ["SymbolTableType"] + def symtable(code: str, filename: str, compile_type: str) -> SymbolTable: ... +if sys.version_info >= (3, 13): + from enum import StrEnum + + class SymbolTableType(StrEnum): + MODULE = "module" + FUNCTION = "function" + CLASS = "class" + ANNOTATION = "annotation" + TYPE_ALIAS = "type alias" + TYPE_PARAMETERS = "type parameters" + TYPE_VARIABLE = "type variable" + class SymbolTable: def __init__(self, raw_table: Any, filename: str) -> None: ... - def get_type(self) -> str: ... + if sys.version_info >= (3, 13): + def get_type(self) -> SymbolTableType: ... + else: + def get_type(self) -> str: ... + def get_id(self) -> int: ... def get_name(self) -> str: ... def get_lineno(self) -> int: ... @@ -42,13 +61,23 @@ class Symbol: def get_name(self) -> str: ... def is_referenced(self) -> bool: ... def is_parameter(self) -> bool: ... + if sys.version_info >= (3, 14): + def is_type_parameter(self) -> bool: ... + def is_global(self) -> bool: ... def is_declared_global(self) -> bool: ... def is_local(self) -> bool: ... def is_annotated(self) -> bool: ... def is_free(self) -> bool: ... + if sys.version_info >= (3, 14): + def is_free_class(self) -> bool: ... + def is_imported(self) -> bool: ... def is_assigned(self) -> bool: ... + if sys.version_info >= (3, 14): + def is_comp_iter(self) -> bool: ... + def is_comp_cell(self) -> bool: ... + def is_namespace(self) -> bool: ... def get_namespaces(self) -> Sequence[SymbolTable]: ... def get_namespace(self) -> SymbolTable: ... diff --git a/mypy/typeshed/stdlib/sys/__init__.pyi b/mypy/typeshed/stdlib/sys/__init__.pyi index 9989a27b2bc1..d65ddfe3825d 100644 --- a/mypy/typeshed/stdlib/sys/__init__.pyi +++ b/mypy/typeshed/stdlib/sys/__init__.pyi @@ -355,7 +355,11 @@ def set_int_max_str_digits(maxdigits: int) -> None: ... def get_int_max_str_digits() -> int: ... if sys.version_info >= (3, 12): - def getunicodeinternedsize() -> int: ... + if sys.version_info >= (3, 13): + def getunicodeinternedsize(*, _only_immortal: bool = False) -> int: ... + else: + def getunicodeinternedsize() -> int: ... + def deactivate_stack_trampoline() -> None: ... def is_stack_trampoline_active() -> bool: ... # It always exists, but raises on non-linux platforms: diff --git a/mypy/typeshed/stdlib/syslog.pyi b/mypy/typeshed/stdlib/syslog.pyi index d539dd5e4579..1e0d0d383902 100644 --- a/mypy/typeshed/stdlib/syslog.pyi +++ b/mypy/typeshed/stdlib/syslog.pyi @@ -1,48 +1,50 @@ import sys -from typing import Literal, overload +from typing import Final, overload if sys.platform != "win32": - LOG_ALERT: Literal[1] - LOG_AUTH: Literal[32] - LOG_AUTHPRIV: Literal[80] - LOG_CONS: Literal[2] - LOG_CRIT: Literal[2] - LOG_CRON: Literal[72] - LOG_DAEMON: Literal[24] - LOG_DEBUG: Literal[7] - LOG_EMERG: Literal[0] - LOG_ERR: Literal[3] - LOG_INFO: Literal[6] - LOG_KERN: Literal[0] - LOG_LOCAL0: Literal[128] - LOG_LOCAL1: Literal[136] - LOG_LOCAL2: Literal[144] - LOG_LOCAL3: Literal[152] - LOG_LOCAL4: Literal[160] - LOG_LOCAL5: Literal[168] - LOG_LOCAL6: Literal[176] - LOG_LOCAL7: Literal[184] - LOG_LPR: Literal[48] - LOG_MAIL: Literal[16] - LOG_NDELAY: Literal[8] - LOG_NEWS: Literal[56] - LOG_NOTICE: Literal[5] - LOG_NOWAIT: Literal[16] - LOG_ODELAY: Literal[4] - LOG_PERROR: Literal[32] - LOG_PID: Literal[1] - LOG_SYSLOG: Literal[40] - LOG_USER: Literal[8] - LOG_UUCP: Literal[64] - LOG_WARNING: Literal[4] + LOG_ALERT: Final = 1 + LOG_AUTH: Final = 32 + LOG_AUTHPRIV: Final = 80 + LOG_CONS: Final = 2 + LOG_CRIT: Final = 2 + LOG_CRON: Final = 72 + LOG_DAEMON: Final = 24 + LOG_DEBUG: Final = 7 + LOG_EMERG: Final = 0 + LOG_ERR: Final = 3 + LOG_INFO: Final = 6 + LOG_KERN: Final = 0 + LOG_LOCAL0: Final = 128 + LOG_LOCAL1: Final = 136 + LOG_LOCAL2: Final = 144 + LOG_LOCAL3: Final = 152 + LOG_LOCAL4: Final = 160 + LOG_LOCAL5: Final = 168 + LOG_LOCAL6: Final = 176 + LOG_LOCAL7: Final = 184 + LOG_LPR: Final = 48 + LOG_MAIL: Final = 16 + LOG_NDELAY: Final = 8 + LOG_NEWS: Final = 56 + LOG_NOTICE: Final = 5 + LOG_NOWAIT: Final = 16 + LOG_ODELAY: Final = 4 + LOG_PERROR: Final = 32 + LOG_PID: Final = 1 + LOG_SYSLOG: Final = 40 + LOG_USER: Final = 8 + LOG_UUCP: Final = 64 + LOG_WARNING: Final = 4 if sys.version_info >= (3, 13): - LOG_FTP: Literal[88] - LOG_INSTALL: Literal[112] - LOG_LAUNCHD: Literal[192] - LOG_NETINFO: Literal[96] - LOG_RAS: Literal[120] - LOG_REMOTEAUTH: Literal[104] + LOG_FTP: Final = 88 + + if sys.platform == "darwin": + LOG_INSTALL: Final = 112 + LOG_LAUNCHD: Final = 192 + LOG_NETINFO: Final = 96 + LOG_RAS: Final = 120 + LOG_REMOTEAUTH: Final = 104 def LOG_MASK(pri: int, /) -> int: ... def LOG_UPTO(pri: int, /) -> int: ... diff --git a/mypy/typeshed/stdlib/tempfile.pyi b/mypy/typeshed/stdlib/tempfile.pyi index 3ae8cca39f77..d31fd74d3482 100644 --- a/mypy/typeshed/stdlib/tempfile.pyi +++ b/mypy/typeshed/stdlib/tempfile.pyi @@ -264,6 +264,8 @@ class _TemporaryFileWrapper(IO[AnyStr]): def writelines(self: _TemporaryFileWrapper[bytes], lines: Iterable[ReadableBuffer]) -> None: ... @overload def writelines(self, lines: Iterable[AnyStr]) -> None: ... + @property + def closed(self) -> bool: ... if sys.version_info >= (3, 11): _SpooledTemporaryFileBase = io.IOBase diff --git a/mypy/typeshed/stdlib/threading.pyi b/mypy/typeshed/stdlib/threading.pyi index 1ecadef508d0..c441a04681e2 100644 --- a/mypy/typeshed/stdlib/threading.pyi +++ b/mypy/typeshed/stdlib/threading.pyi @@ -61,7 +61,7 @@ if sys.version_info >= (3, 10): def gettrace() -> TraceFunction | None: ... def getprofile() -> ProfileFunction | None: ... -def stack_size(size: int = ...) -> int: ... +def stack_size(size: int = 0, /) -> int: ... TIMEOUT_MAX: float diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index d8ce17535eab..77953525bebe 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -1,7 +1,7 @@ import _tkinter import sys from _typeshed import Incomplete, StrEnum, StrOrBytesPath -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from tkinter.constants import * from tkinter.font import _FontDescription from types import TracebackType @@ -3331,9 +3331,33 @@ class PhotoImage(Image, _PhotoImageLike): def blank(self) -> None: ... def cget(self, option: str) -> str: ... def __getitem__(self, key: str) -> str: ... # always string: image['height'] can be '0' - def copy(self) -> PhotoImage: ... - def zoom(self, x: int, y: int | Literal[""] = "") -> PhotoImage: ... - def subsample(self, x: int, y: int | Literal[""] = "") -> PhotoImage: ... + if sys.version_info >= (3, 13): + def copy( + self, + *, + from_coords: Iterable[int] | None = None, + zoom: int | tuple[int, int] | list[int] | None = None, + subsample: int | tuple[int, int] | list[int] | None = None, + ) -> PhotoImage: ... + def subsample(self, x: int, y: Literal[""] = "", *, from_coords: Iterable[int] | None = None) -> PhotoImage: ... + def zoom(self, x: int, y: Literal[""] = "", *, from_coords: Iterable[int] | None = None) -> PhotoImage: ... + def copy_replace( + self, + sourceImage: PhotoImage | str, + *, + from_coords: Iterable[int] | None = None, + to: Iterable[int] | None = None, + shrink: bool = False, + zoom: int | tuple[int, int] | list[int] | None = None, + subsample: int | tuple[int, int] | list[int] | None = None, + # `None` defaults to overlay. + compositingrule: Literal["overlay", "set"] | None = None, + ) -> None: ... + else: + def copy(self) -> PhotoImage: ... + def zoom(self, x: int, y: int | Literal[""] = "") -> PhotoImage: ... + def subsample(self, x: int, y: int | Literal[""] = "") -> PhotoImage: ... + def get(self, x: int, y: int) -> tuple[int, int, int]: ... def put( self, @@ -3348,7 +3372,44 @@ class PhotoImage(Image, _PhotoImageLike): ), to: tuple[int, int] | None = None, ) -> None: ... - def write(self, filename: StrOrBytesPath, format: str | None = None, from_coords: tuple[int, int] | None = None) -> None: ... + if sys.version_info >= (3, 13): + def read( + self, + filename: StrOrBytesPath, + format: str | None = None, + *, + from_coords: Iterable[int] | None = None, + to: Iterable[int] | None = None, + shrink: bool = False, + ) -> None: ... + def write( + self, + filename: StrOrBytesPath, + format: str | None = None, + from_coords: Iterable[int] | None = None, + *, + background: str | None = None, + grayscale: bool = False, + ) -> None: ... + @overload + def data( + self, format: str, *, from_coords: Iterable[int] | None = None, background: str | None = None, grayscale: bool = False + ) -> bytes: ... + @overload + def data( + self, + format: None = None, + *, + from_coords: Iterable[int] | None = None, + background: str | None = None, + grayscale: bool = False, + ) -> tuple[str, ...]: ... + + else: + def write( + self, filename: StrOrBytesPath, format: str | None = None, from_coords: tuple[int, int] | None = None + ) -> None: ... + def transparency_get(self, x: int, y: int) -> bool: ... def transparency_set(self, x: int, y: int, boolean: bool) -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/constants.pyi b/mypy/typeshed/stdlib/tkinter/constants.pyi index 74fa72acb0bf..0b497f3a42e4 100644 --- a/mypy/typeshed/stdlib/tkinter/constants.pyi +++ b/mypy/typeshed/stdlib/tkinter/constants.pyi @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Final # These are not actually bools. See #4669 NO: bool @@ -7,74 +7,74 @@ TRUE: bool FALSE: bool ON: bool OFF: bool -N: Literal["n"] -S: Literal["s"] -W: Literal["w"] -E: Literal["e"] -NW: Literal["nw"] -SW: Literal["sw"] -NE: Literal["ne"] -SE: Literal["se"] -NS: Literal["ns"] -EW: Literal["ew"] -NSEW: Literal["nsew"] -CENTER: Literal["center"] -NONE: Literal["none"] -X: Literal["x"] -Y: Literal["y"] -BOTH: Literal["both"] -LEFT: Literal["left"] -TOP: Literal["top"] -RIGHT: Literal["right"] -BOTTOM: Literal["bottom"] -RAISED: Literal["raised"] -SUNKEN: Literal["sunken"] -FLAT: Literal["flat"] -RIDGE: Literal["ridge"] -GROOVE: Literal["groove"] -SOLID: Literal["solid"] -HORIZONTAL: Literal["horizontal"] -VERTICAL: Literal["vertical"] -NUMERIC: Literal["numeric"] -CHAR: Literal["char"] -WORD: Literal["word"] -BASELINE: Literal["baseline"] -INSIDE: Literal["inside"] -OUTSIDE: Literal["outside"] -SEL: Literal["sel"] -SEL_FIRST: Literal["sel.first"] -SEL_LAST: Literal["sel.last"] -END: Literal["end"] -INSERT: Literal["insert"] -CURRENT: Literal["current"] -ANCHOR: Literal["anchor"] -ALL: Literal["all"] -NORMAL: Literal["normal"] -DISABLED: Literal["disabled"] -ACTIVE: Literal["active"] -HIDDEN: Literal["hidden"] -CASCADE: Literal["cascade"] -CHECKBUTTON: Literal["checkbutton"] -COMMAND: Literal["command"] -RADIOBUTTON: Literal["radiobutton"] -SEPARATOR: Literal["separator"] -SINGLE: Literal["single"] -BROWSE: Literal["browse"] -MULTIPLE: Literal["multiple"] -EXTENDED: Literal["extended"] -DOTBOX: Literal["dotbox"] -UNDERLINE: Literal["underline"] -PIESLICE: Literal["pieslice"] -CHORD: Literal["chord"] -ARC: Literal["arc"] -FIRST: Literal["first"] -LAST: Literal["last"] -BUTT: Literal["butt"] -PROJECTING: Literal["projecting"] -ROUND: Literal["round"] -BEVEL: Literal["bevel"] -MITER: Literal["miter"] -MOVETO: Literal["moveto"] -SCROLL: Literal["scroll"] -UNITS: Literal["units"] -PAGES: Literal["pages"] +N: Final = "n" +S: Final = "s" +W: Final = "w" +E: Final = "e" +NW: Final = "nw" +SW: Final = "sw" +NE: Final = "ne" +SE: Final = "se" +NS: Final = "ns" +EW: Final = "ew" +NSEW: Final = "nsew" +CENTER: Final = "center" +NONE: Final = "none" +X: Final = "x" +Y: Final = "y" +BOTH: Final = "both" +LEFT: Final = "left" +TOP: Final = "top" +RIGHT: Final = "right" +BOTTOM: Final = "bottom" +RAISED: Final = "raised" +SUNKEN: Final = "sunken" +FLAT: Final = "flat" +RIDGE: Final = "ridge" +GROOVE: Final = "groove" +SOLID: Final = "solid" +HORIZONTAL: Final = "horizontal" +VERTICAL: Final = "vertical" +NUMERIC: Final = "numeric" +CHAR: Final = "char" +WORD: Final = "word" +BASELINE: Final = "baseline" +INSIDE: Final = "inside" +OUTSIDE: Final = "outside" +SEL: Final = "sel" +SEL_FIRST: Final = "sel.first" +SEL_LAST: Final = "sel.last" +END: Final = "end" +INSERT: Final = "insert" +CURRENT: Final = "current" +ANCHOR: Final = "anchor" +ALL: Final = "all" +NORMAL: Final = "normal" +DISABLED: Final = "disabled" +ACTIVE: Final = "active" +HIDDEN: Final = "hidden" +CASCADE: Final = "cascade" +CHECKBUTTON: Final = "checkbutton" +COMMAND: Final = "command" +RADIOBUTTON: Final = "radiobutton" +SEPARATOR: Final = "separator" +SINGLE: Final = "single" +BROWSE: Final = "browse" +MULTIPLE: Final = "multiple" +EXTENDED: Final = "extended" +DOTBOX: Final = "dotbox" +UNDERLINE: Final = "underline" +PIESLICE: Final = "pieslice" +CHORD: Final = "chord" +ARC: Final = "arc" +FIRST: Final = "first" +LAST: Final = "last" +BUTT: Final = "butt" +PROJECTING: Final = "projecting" +ROUND: Final = "round" +BEVEL: Final = "bevel" +MITER: Final = "miter" +MOVETO: Final = "moveto" +SCROLL: Final = "scroll" +UNITS: Final = "units" +PAGES: Final = "pages" diff --git a/mypy/typeshed/stdlib/tkinter/font.pyi b/mypy/typeshed/stdlib/tkinter/font.pyi index 46625014d4ac..317f3068be63 100644 --- a/mypy/typeshed/stdlib/tkinter/font.pyi +++ b/mypy/typeshed/stdlib/tkinter/font.pyi @@ -1,16 +1,16 @@ import _tkinter import sys import tkinter -from typing import Any, Literal, TypedDict, overload +from typing import Any, Final, Literal, TypedDict, overload from typing_extensions import TypeAlias if sys.version_info >= (3, 9): __all__ = ["NORMAL", "ROMAN", "BOLD", "ITALIC", "nametofont", "Font", "families", "names"] -NORMAL: Literal["normal"] -ROMAN: Literal["roman"] -BOLD: Literal["bold"] -ITALIC: Literal["italic"] +NORMAL: Final = "normal" +ROMAN: Final = "roman" +BOLD: Final = "bold" +ITALIC: Final = "italic" _FontDescription: TypeAlias = ( str # "Helvetica 12" diff --git a/mypy/typeshed/stdlib/tkinter/tix.pyi b/mypy/typeshed/stdlib/tkinter/tix.pyi index 73649de427e8..7891364fa02c 100644 --- a/mypy/typeshed/stdlib/tkinter/tix.pyi +++ b/mypy/typeshed/stdlib/tkinter/tix.pyi @@ -1,38 +1,38 @@ import tkinter from _typeshed import Incomplete -from typing import Any, Literal - -WINDOW: Literal["window"] -TEXT: Literal["text"] -STATUS: Literal["status"] -IMMEDIATE: Literal["immediate"] -IMAGE: Literal["image"] -IMAGETEXT: Literal["imagetext"] -BALLOON: Literal["balloon"] -AUTO: Literal["auto"] -ACROSSTOP: Literal["acrosstop"] - -ASCII: Literal["ascii"] -CELL: Literal["cell"] -COLUMN: Literal["column"] -DECREASING: Literal["decreasing"] -INCREASING: Literal["increasing"] -INTEGER: Literal["integer"] -MAIN: Literal["main"] -MAX: Literal["max"] -REAL: Literal["real"] -ROW: Literal["row"] -S_REGION: Literal["s-region"] -X_REGION: Literal["x-region"] -Y_REGION: Literal["y-region"] +from typing import Any, Final + +WINDOW: Final = "window" +TEXT: Final = "text" +STATUS: Final = "status" +IMMEDIATE: Final = "immediate" +IMAGE: Final = "image" +IMAGETEXT: Final = "imagetext" +BALLOON: Final = "balloon" +AUTO: Final = "auto" +ACROSSTOP: Final = "acrosstop" + +ASCII: Final = "ascii" +CELL: Final = "cell" +COLUMN: Final = "column" +DECREASING: Final = "decreasing" +INCREASING: Final = "increasing" +INTEGER: Final = "integer" +MAIN: Final = "main" +MAX: Final = "max" +REAL: Final = "real" +ROW: Final = "row" +S_REGION: Final = "s-region" +X_REGION: Final = "x-region" +Y_REGION: Final = "y-region" # These should be kept in sync with _tkinter constants, except TCL_ALL_EVENTS which doesn't match ALL_EVENTS -TCL_DONT_WAIT: Literal[2] -TCL_WINDOW_EVENTS: Literal[4] -TCL_FILE_EVENTS: Literal[8] -TCL_TIMER_EVENTS: Literal[16] -TCL_IDLE_EVENTS: Literal[32] -TCL_ALL_EVENTS: Literal[0] +TCL_DONT_WAIT: Final = 2 +TCL_WINDOW_EVENTS: Final = 4 +TCL_FILE_EVENTS: Final = 8 +TCL_TIMER_EVENTS: Final = 16 +TCL_IDLE_EVENTS: Final = 32 +TCL_ALL_EVENTS: Final = 0 class tixCommand: def tix_addbitmapdir(self, directory: str) -> None: ... diff --git a/mypy/typeshed/stdlib/trace.pyi b/mypy/typeshed/stdlib/trace.pyi index d32647a55cb5..04390f119195 100644 --- a/mypy/typeshed/stdlib/trace.pyi +++ b/mypy/typeshed/stdlib/trace.pyi @@ -27,7 +27,18 @@ class CoverageResults: outfile: StrPath | None = None, ) -> None: ... # undocumented def update(self, other: CoverageResults) -> None: ... - def write_results(self, show_missing: bool = True, summary: bool = False, coverdir: StrPath | None = None) -> None: ... + if sys.version_info >= (3, 13): + def write_results( + self, + show_missing: bool = True, + summary: bool = False, + coverdir: StrPath | None = None, + *, + ignore_missing_files: bool = False, + ) -> None: ... + else: + def write_results(self, show_missing: bool = True, summary: bool = False, coverdir: StrPath | None = None) -> None: ... + def write_results_file( self, path: StrPath, lines: Sequence[str], lnotab: Any, lines_hit: Mapping[int, int], encoding: str | None = None ) -> tuple[int, int]: ... diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index fd0723fd73ed..199feee746cb 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -101,7 +101,6 @@ __all__ = [ "setheading", "setpos", "setposition", - "settiltangle", "setundobuffer", "setx", "sety", @@ -132,6 +131,9 @@ __all__ = [ if sys.version_info >= (3, 12): __all__ += ["teleport"] +if sys.version_info < (3, 13): + __all__ += ["settiltangle"] + # Note: '_Color' is the alias we use for arguments and _AnyColor is the # alias we use for return types. Really, these two aliases should be the # same, but as per the "no union returns" typeshed policy, we'll return @@ -399,7 +401,10 @@ class RawTurtle(TPen, TNavigator): self, t11: float | None = None, t12: float | None = None, t21: float | None = None, t22: float | None = None ) -> None: ... def get_shapepoly(self) -> _PolygonCoords | None: ... - def settiltangle(self, angle: float) -> None: ... + + if sys.version_info < (3, 13): + def settiltangle(self, angle: float) -> None: ... + @overload def tiltangle(self, angle: None = None) -> float: ... @overload @@ -672,7 +677,10 @@ def shapetransform( t11: float | None = None, t12: float | None = None, t21: float | None = None, t22: float | None = None ) -> None: ... def get_shapepoly() -> _PolygonCoords | None: ... -def settiltangle(angle: float) -> None: ... + +if sys.version_info < (3, 13): + def settiltangle(angle: float) -> None: ... + @overload def tiltangle(angle: None = None) -> float: ... @overload diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 9e9dc56b8529..1e3eacd9f1fa 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -245,7 +245,7 @@ class CodeType: co_qualname: str = ..., co_linetable: bytes = ..., co_exceptiontable: bytes = ..., - ) -> CodeType: ... + ) -> Self: ... elif sys.version_info >= (3, 10): def replace( self, @@ -266,7 +266,7 @@ class CodeType: co_filename: str = ..., co_name: str = ..., co_linetable: bytes = ..., - ) -> CodeType: ... + ) -> Self: ... else: def replace( self, @@ -287,7 +287,10 @@ class CodeType: co_filename: str = ..., co_name: str = ..., co_lnotab: bytes = ..., - ) -> CodeType: ... + ) -> Self: ... + + if sys.version_info >= (3, 13): + __replace__ = replace @final class MappingProxyType(Mapping[_KT, _VT_co]): @@ -301,6 +304,10 @@ class MappingProxyType(Mapping[_KT, _VT_co]): def keys(self) -> KeysView[_KT]: ... def values(self) -> ValuesView[_VT_co]: ... def items(self) -> ItemsView[_KT, _VT_co]: ... + @overload + def get(self, key: _KT, /) -> _VT_co | None: ... # type: ignore[override] + @overload + def get(self, key: _KT, default: _VT_co | _T2, /) -> _VT_co | _T2: ... # type: ignore[override] if sys.version_info >= (3, 9): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... def __reversed__(self) -> Iterator[_KT]: ... @@ -309,11 +316,17 @@ class MappingProxyType(Mapping[_KT, _VT_co]): class SimpleNamespace: __hash__: ClassVar[None] # type: ignore[assignment] - def __init__(self, **kwargs: Any) -> None: ... + if sys.version_info >= (3, 13): + def __init__(self, mapping_or_iterable: Mapping[str, Any] | Iterable[tuple[str, Any]] = (), /, **kwargs: Any) -> None: ... + else: + def __init__(self, **kwargs: Any) -> None: ... + def __eq__(self, value: object, /) -> bool: ... def __getattribute__(self, name: str, /) -> Any: ... def __setattr__(self, name: str, value: Any, /) -> None: ... def __delattr__(self, name: str, /) -> None: ... + if sys.version_info >= (3, 13): + def __replace__(self, **kwargs: Any) -> Self: ... class ModuleType: __name__: str diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index 92427f91f022..f4de1fa86de5 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -542,16 +542,18 @@ class AsyncIterator(AsyncIterable[_T_co], Protocol[_T_co]): class AsyncGenerator(AsyncIterator[_YieldT_co], Generic[_YieldT_co, _SendT_contra]): def __anext__(self) -> Awaitable[_YieldT_co]: ... @abstractmethod - def asend(self, value: _SendT_contra, /) -> Awaitable[_YieldT_co]: ... + def asend(self, value: _SendT_contra, /) -> Coroutine[Any, Any, _YieldT_co]: ... @overload @abstractmethod def athrow( self, typ: type[BaseException], val: BaseException | object = None, tb: TracebackType | None = None, / - ) -> Awaitable[_YieldT_co]: ... + ) -> Coroutine[Any, Any, _YieldT_co]: ... @overload @abstractmethod - def athrow(self, typ: BaseException, val: None = None, tb: TracebackType | None = None, /) -> Awaitable[_YieldT_co]: ... - def aclose(self) -> Awaitable[None]: ... + def athrow( + self, typ: BaseException, val: None = None, tb: TracebackType | None = None, / + ) -> Coroutine[Any, Any, _YieldT_co]: ... + def aclose(self) -> Coroutine[Any, Any, None]: ... @property def ag_await(self) -> Any: ... @property @@ -1054,7 +1056,7 @@ if sys.version_info >= (3, 12): # It's writable on types, but not on instances of TypeAliasType. @property def __module__(self) -> str | None: ... # type: ignore[override] - def __getitem__(self, parameters: Any) -> Any: ... + def __getitem__(self, parameters: Any) -> GenericAlias: ... def __or__(self, right: Any) -> _SpecialForm: ... def __ror__(self, left: Any) -> _SpecialForm: ... diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index a7d2b2c2e083..1e4f90a0a722 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -403,6 +403,7 @@ else: # It's writable on types, but not on instances of TypeAliasType. @property def __module__(self) -> str | None: ... # type: ignore[override] + # Returns typing._GenericAlias, which isn't stubbed. def __getitem__(self, parameters: Any) -> Any: ... if sys.version_info >= (3, 10): def __or__(self, right: Any) -> _SpecialForm: ... diff --git a/mypy/typeshed/stdlib/unittest/__init__.pyi b/mypy/typeshed/stdlib/unittest/__init__.pyi index f2532ccf7fd8..546ea77bb4ca 100644 --- a/mypy/typeshed/stdlib/unittest/__init__.pyi +++ b/mypy/typeshed/stdlib/unittest/__init__.pyi @@ -11,13 +11,7 @@ from .case import ( skipIf as skipIf, skipUnless as skipUnless, ) -from .loader import ( - TestLoader as TestLoader, - defaultTestLoader as defaultTestLoader, - findTestCases as findTestCases, - getTestCaseNames as getTestCaseNames, - makeSuite as makeSuite, -) +from .loader import TestLoader as TestLoader, defaultTestLoader as defaultTestLoader from .main import TestProgram as TestProgram, main as main from .result import TestResult as TestResult from .runner import TextTestResult as TextTestResult, TextTestRunner as TextTestRunner @@ -52,12 +46,14 @@ __all__ = [ "registerResult", "removeResult", "removeHandler", - "getTestCaseNames", - "makeSuite", - "findTestCases", "addModuleCleanup", ] +if sys.version_info < (3, 13): + from .loader import findTestCases as findTestCases, getTestCaseNames as getTestCaseNames, makeSuite as makeSuite + + __all__ += ["getTestCaseNames", "makeSuite", "findTestCases"] + if sys.version_info >= (3, 11): __all__ += ["enterModuleContext", "doModuleCleanups"] diff --git a/mypy/typeshed/stdlib/unittest/async_case.pyi b/mypy/typeshed/stdlib/unittest/async_case.pyi index 12d6ef49e828..565dd91c0fda 100644 --- a/mypy/typeshed/stdlib/unittest/async_case.pyi +++ b/mypy/typeshed/stdlib/unittest/async_case.pyi @@ -1,4 +1,5 @@ import sys +from asyncio.events import AbstractEventLoop from collections.abc import Awaitable, Callable from typing import TypeVar from typing_extensions import ParamSpec @@ -12,6 +13,9 @@ _T = TypeVar("_T") _P = ParamSpec("_P") class IsolatedAsyncioTestCase(TestCase): + if sys.version_info >= (3, 13): + loop_factory: Callable[[], AbstractEventLoop] | None = None + async def asyncSetUp(self) -> None: ... async def asyncTearDown(self) -> None: ... def addAsyncCleanup(self, func: Callable[_P, Awaitable[object]], /, *args: _P.args, **kwargs: _P.kwargs) -> None: ... diff --git a/mypy/typeshed/stdlib/unittest/loader.pyi b/mypy/typeshed/stdlib/unittest/loader.pyi index 202309ac1d93..657f3d6dca71 100644 --- a/mypy/typeshed/stdlib/unittest/loader.pyi +++ b/mypy/typeshed/stdlib/unittest/loader.pyi @@ -5,7 +5,7 @@ from collections.abc import Callable, Sequence from re import Pattern from types import ModuleType from typing import Any -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated _SortComparisonMethod: TypeAlias = Callable[[str, str], int] _SuiteClass: TypeAlias = Callable[[list[unittest.case.TestCase]], unittest.suite.TestSuite] @@ -34,18 +34,22 @@ class TestLoader: defaultTestLoader: TestLoader -def getTestCaseNames( - testCaseClass: type[unittest.case.TestCase], - prefix: str, - sortUsing: _SortComparisonMethod = ..., - testNamePatterns: list[str] | None = None, -) -> Sequence[str]: ... -def makeSuite( - testCaseClass: type[unittest.case.TestCase], - prefix: str = "test", - sortUsing: _SortComparisonMethod = ..., - suiteClass: _SuiteClass = ..., -) -> unittest.suite.TestSuite: ... -def findTestCases( - module: ModuleType, prefix: str = "test", sortUsing: _SortComparisonMethod = ..., suiteClass: _SuiteClass = ... -) -> unittest.suite.TestSuite: ... +if sys.version_info < (3, 13): + @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") + def getTestCaseNames( + testCaseClass: type[unittest.case.TestCase], + prefix: str, + sortUsing: _SortComparisonMethod = ..., + testNamePatterns: list[str] | None = None, + ) -> Sequence[str]: ... + @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") + def makeSuite( + testCaseClass: type[unittest.case.TestCase], + prefix: str = "test", + sortUsing: _SortComparisonMethod = ..., + suiteClass: _SuiteClass = ..., + ) -> unittest.suite.TestSuite: ... + @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") + def findTestCases( + module: ModuleType, prefix: str = "test", sortUsing: _SortComparisonMethod = ..., suiteClass: _SuiteClass = ... + ) -> unittest.suite.TestSuite: ... diff --git a/mypy/typeshed/stdlib/unittest/main.pyi b/mypy/typeshed/stdlib/unittest/main.pyi index 55bc1ec741db..3eb3d1612a3c 100644 --- a/mypy/typeshed/stdlib/unittest/main.pyi +++ b/mypy/typeshed/stdlib/unittest/main.pyi @@ -6,6 +6,7 @@ import unittest.suite from collections.abc import Iterable from types import ModuleType from typing import Any, Protocol +from typing_extensions import deprecated MAIN_EXAMPLES: str MODULE_EXAMPLES: str @@ -61,7 +62,10 @@ class TestProgram: tb_locals: bool = False, ) -> None: ... - def usageExit(self, msg: Any = None) -> None: ... + if sys.version_info < (3, 13): + @deprecated("Deprecated in Python 3.11; removal scheduled for Python 3.13") + def usageExit(self, msg: Any = None) -> None: ... + def parseArgs(self, argv: list[str]) -> None: ... def createTests(self, from_discovery: bool = False, Loader: unittest.loader.TestLoader | None = None) -> None: ... def runTests(self) -> None: ... # undocumented diff --git a/mypy/typeshed/stdlib/unittest/mock.pyi b/mypy/typeshed/stdlib/unittest/mock.pyi index dd61b83a658a..84620b7f3889 100644 --- a/mypy/typeshed/stdlib/unittest/mock.pyi +++ b/mypy/typeshed/stdlib/unittest/mock.pyi @@ -12,23 +12,44 @@ _F = TypeVar("_F", bound=Callable[..., Any]) _AF = TypeVar("_AF", bound=Callable[..., Coroutine[Any, Any, Any]]) _P = ParamSpec("_P") -__all__ = ( - "Mock", - "MagicMock", - "patch", - "sentinel", - "DEFAULT", - "ANY", - "call", - "create_autospec", - "AsyncMock", - "FILTER_DIR", - "NonCallableMock", - "NonCallableMagicMock", - "mock_open", - "PropertyMock", - "seal", -) +if sys.version_info >= (3, 13): + # ThreadingMock added in 3.13 + __all__ = ( + "Mock", + "MagicMock", + "patch", + "sentinel", + "DEFAULT", + "ANY", + "call", + "create_autospec", + "ThreadingMock", + "AsyncMock", + "FILTER_DIR", + "NonCallableMock", + "NonCallableMagicMock", + "mock_open", + "PropertyMock", + "seal", + ) +else: + __all__ = ( + "Mock", + "MagicMock", + "patch", + "sentinel", + "DEFAULT", + "ANY", + "call", + "create_autospec", + "AsyncMock", + "FILTER_DIR", + "NonCallableMock", + "NonCallableMagicMock", + "mock_open", + "PropertyMock", + "seal", + ) if sys.version_info < (3, 9): __version__: Final[str] @@ -124,7 +145,6 @@ class NonCallableMock(Base, Any): def __delattr__(self, name: str) -> None: ... def __setattr__(self, name: str, value: Any) -> None: ... def __dir__(self) -> list[str]: ... - def _calls_repr(self, prefix: str = "Calls") -> str: ... def assert_called_with(self, *args: Any, **kwargs: Any) -> None: ... def assert_not_called(self) -> None: ... def assert_called_once_with(self, *args: Any, **kwargs: Any) -> None: ... @@ -150,6 +170,10 @@ class NonCallableMock(Base, Any): def _format_mock_call_signature(self, args: Any, kwargs: Any) -> str: ... def _call_matcher(self, _call: tuple[_Call, ...]) -> _Call: ... def _get_child_mock(self, **kw: Any) -> NonCallableMock: ... + if sys.version_info >= (3, 13): + def _calls_repr(self) -> str: ... + else: + def _calls_repr(self, prefix: str = "Calls") -> str: ... class CallableMixin(Base): side_effect: Any @@ -427,4 +451,16 @@ class PropertyMock(Mock): def __get__(self, obj: _T, obj_type: type[_T] | None = None) -> Self: ... def __set__(self, obj: Any, val: Any) -> None: ... +if sys.version_info >= (3, 13): + class ThreadingMixin(Base): + DEFAULT_TIMEOUT: Final[float | None] = None + + def __init__(self, /, *args: Any, timeout: float | None | _SentinelObject = ..., **kwargs: Any) -> None: ... + # Same as `NonCallableMock.reset_mock.` + def reset_mock(self, visited: Any = None, *, return_value: bool = False, side_effect: bool = False) -> None: ... + def wait_until_called(self, *, timeout: float | None | _SentinelObject = ...) -> None: ... + def wait_until_any_call_with(self, *args: Any, **kwargs: Any) -> None: ... + + class ThreadingMock(ThreadingMixin, MagicMixin, Mock): ... + def seal(mock: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/warnings.pyi b/mypy/typeshed/stdlib/warnings.pyi index 539a8f2379c1..c7ab1cb091dd 100644 --- a/mypy/typeshed/stdlib/warnings.pyi +++ b/mypy/typeshed/stdlib/warnings.pyi @@ -21,8 +21,10 @@ if sys.version_info >= (3, 13): _T = TypeVar("_T") _W = TypeVar("_W", bound=list[WarningMessage] | None) -_ActionKind: TypeAlias = Literal["default", "error", "ignore", "always", "module", "once"] - +if sys.version_info >= (3, 14): + _ActionKind: TypeAlias = Literal["default", "error", "ignore", "always", "module", "once"] +else: + _ActionKind: TypeAlias = Literal["default", "error", "ignore", "always", "all", "module", "once"] filters: Sequence[tuple[str, str | None, type[Warning], str | None, int]] # undocumented, do not mutate def showwarning( diff --git a/mypy/typeshed/stdlib/wave.pyi b/mypy/typeshed/stdlib/wave.pyi index 9137f1e47643..9319d5347c79 100644 --- a/mypy/typeshed/stdlib/wave.pyi +++ b/mypy/typeshed/stdlib/wave.pyi @@ -1,6 +1,6 @@ import sys from _typeshed import ReadableBuffer, Unused -from typing import IO, Any, BinaryIO, Literal, NamedTuple, NoReturn, overload +from typing import IO, Any, BinaryIO, Final, Literal, NamedTuple, NoReturn, overload from typing_extensions import Self, TypeAlias, deprecated if sys.version_info >= (3, 9): @@ -12,7 +12,7 @@ _File: TypeAlias = str | IO[bytes] class Error(Exception): ... -WAVE_FORMAT_PCM: Literal[1] +WAVE_FORMAT_PCM: Final = 1 class _wave_params(NamedTuple): nchannels: int diff --git a/mypy/typeshed/stdlib/webbrowser.pyi b/mypy/typeshed/stdlib/webbrowser.pyi index 2b3f978c814b..d7bf033172f6 100644 --- a/mypy/typeshed/stdlib/webbrowser.pyi +++ b/mypy/typeshed/stdlib/webbrowser.pyi @@ -2,6 +2,7 @@ import sys from abc import abstractmethod from collections.abc import Callable, Sequence from typing import Literal +from typing_extensions import deprecated __all__ = ["Error", "open", "open_new", "open_new_tab", "get", "register"] @@ -62,8 +63,10 @@ if sys.platform == "win32": def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... if sys.platform == "darwin": - class MacOSX(BaseBrowser): - def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... + if sys.version_info < (3, 13): + @deprecated("Deprecated in 3.11, to be removed in 3.13.") + class MacOSX(BaseBrowser): + def open(self, url: str, new: int = 0, autoraise: bool = True) -> bool: ... class MacOSXOSAScript(BaseBrowser): # In runtime this class does not have `name` and `basename` if sys.version_info >= (3, 11): diff --git a/mypy/typeshed/stdlib/winsound.pyi b/mypy/typeshed/stdlib/winsound.pyi index bacc5302826f..a20e81f94f98 100644 --- a/mypy/typeshed/stdlib/winsound.pyi +++ b/mypy/typeshed/stdlib/winsound.pyi @@ -1,24 +1,24 @@ import sys from _typeshed import ReadableBuffer -from typing import Literal, overload +from typing import Final, Literal, overload if sys.platform == "win32": - SND_APPLICATION: Literal[128] - SND_FILENAME: Literal[131072] - SND_ALIAS: Literal[65536] - SND_LOOP: Literal[8] - SND_MEMORY: Literal[4] - SND_PURGE: Literal[64] - SND_ASYNC: Literal[1] - SND_NODEFAULT: Literal[2] - SND_NOSTOP: Literal[16] - SND_NOWAIT: Literal[8192] + SND_APPLICATION: Final = 128 + SND_FILENAME: Final = 131072 + SND_ALIAS: Final = 65536 + SND_LOOP: Final = 8 + SND_MEMORY: Final = 4 + SND_PURGE: Final = 64 + SND_ASYNC: Final = 1 + SND_NODEFAULT: Final = 2 + SND_NOSTOP: Final = 16 + SND_NOWAIT: Final = 8192 - MB_ICONASTERISK: Literal[64] - MB_ICONEXCLAMATION: Literal[48] - MB_ICONHAND: Literal[16] - MB_ICONQUESTION: Literal[32] - MB_OK: Literal[0] + MB_ICONASTERISK: Final = 64 + MB_ICONEXCLAMATION: Final = 48 + MB_ICONHAND: Final = 16 + MB_ICONQUESTION: Final = 32 + MB_OK: Final = 0 def Beep(frequency: int, duration: int) -> None: ... # Can actually accept anything ORed with 4, and if not it's definitely str, but that's inexpressible @overload diff --git a/mypy/typeshed/stdlib/xml/dom/pulldom.pyi b/mypy/typeshed/stdlib/xml/dom/pulldom.pyi index 95436ab5dd38..50250de5cb2f 100644 --- a/mypy/typeshed/stdlib/xml/dom/pulldom.pyi +++ b/mypy/typeshed/stdlib/xml/dom/pulldom.pyi @@ -1,20 +1,20 @@ import sys from _typeshed import Incomplete, SupportsRead from collections.abc import Sequence -from typing import Literal +from typing import Final, Literal from typing_extensions import TypeAlias from xml.dom.minidom import Document, DOMImplementation, Element, Text from xml.sax.handler import ContentHandler from xml.sax.xmlreader import XMLReader -START_ELEMENT: Literal["START_ELEMENT"] -END_ELEMENT: Literal["END_ELEMENT"] -COMMENT: Literal["COMMENT"] -START_DOCUMENT: Literal["START_DOCUMENT"] -END_DOCUMENT: Literal["END_DOCUMENT"] -PROCESSING_INSTRUCTION: Literal["PROCESSING_INSTRUCTION"] -IGNORABLE_WHITESPACE: Literal["IGNORABLE_WHITESPACE"] -CHARACTERS: Literal["CHARACTERS"] +START_ELEMENT: Final = "START_ELEMENT" +END_ELEMENT: Final = "END_ELEMENT" +COMMENT: Final = "COMMENT" +START_DOCUMENT: Final = "START_DOCUMENT" +END_DOCUMENT: Final = "END_DOCUMENT" +PROCESSING_INSTRUCTION: Final = "PROCESSING_INSTRUCTION" +IGNORABLE_WHITESPACE: Final = "IGNORABLE_WHITESPACE" +CHARACTERS: Final = "CHARACTERS" _DocumentFactory: TypeAlias = DOMImplementation | None _Node: TypeAlias = Document | Element | Text diff --git a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi index 9198bd3322d9..4849b0ea1c35 100644 --- a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi +++ b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi @@ -239,9 +239,15 @@ if sys.version_info >= (3, 9): def indent(tree: Element | ElementTree, space: str = " ", level: int = 0) -> None: ... def parse(source: _FileRead, parser: XMLParser | None = None) -> ElementTree: ... -def iterparse( - source: _FileRead, events: Sequence[str] | None = None, parser: XMLParser | None = None -) -> Iterator[tuple[str, Any]]: ... + +class _IterParseIterator(Iterator[tuple[str, Any]]): + def __next__(self) -> tuple[str, Any]: ... + if sys.version_info >= (3, 13): + def close(self) -> None: ... + if sys.version_info >= (3, 11): + def __del__(self) -> None: ... + +def iterparse(source: _FileRead, events: Sequence[str] | None = None, parser: XMLParser | None = None) -> _IterParseIterator: ... class XMLPullParser: def __init__(self, events: Sequence[str] | None = None, *, _parser: XMLParser | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/zipfile/__init__.pyi b/mypy/typeshed/stdlib/zipfile/__init__.pyi index b61e07f8b90d..aa52a0b56e41 100644 --- a/mypy/typeshed/stdlib/zipfile/__init__.pyi +++ b/mypy/typeshed/stdlib/zipfile/__init__.pyi @@ -206,6 +206,9 @@ class ZipInfo: compress_size: int file_size: int orig_filename: str # undocumented + if sys.version_info >= (3, 13): + compress_level: int | None + def __init__(self, filename: str = "NoName", date_time: _DateTuple = (1980, 1, 1, 0, 0, 0)) -> None: ... @classmethod def from_file(cls, filename: StrPath, arcname: StrPath | None = None, *, strict_timestamps: bool = True) -> Self: ... diff --git a/mypy/typeshed/stdlib/zipfile/_path.pyi b/mypy/typeshed/stdlib/zipfile/_path.pyi index 0398824e1fd2..bafbbeeb0d0b 100644 --- a/mypy/typeshed/stdlib/zipfile/_path.pyi +++ b/mypy/typeshed/stdlib/zipfile/_path.pyi @@ -3,12 +3,14 @@ from _typeshed import StrPath from collections.abc import Iterator, Sequence from io import TextIOWrapper from os import PathLike -from typing import IO, Literal, overload +from typing import IO, Literal, TypeVar, overload from typing_extensions import Self, TypeAlias from zipfile import ZipFile _ReadWriteBinaryMode: TypeAlias = Literal["r", "w", "rb", "wb"] +_ZF = TypeVar("_ZF", bound=ZipFile) + if sys.version_info >= (3, 12): class InitializedState: def __init__(self, *args: object, **kwargs: object) -> None: ... @@ -23,6 +25,9 @@ if sys.version_info >= (3, 12): @overload @classmethod def make(cls, source: StrPath | IO[bytes]) -> Self: ... + if sys.version_info >= (3, 13): + @classmethod + def inject(cls, zf: _ZF) -> _ZF: ... class Path: root: CompleteDirs diff --git a/mypy/typeshed/stdlib/zlib.pyi b/mypy/typeshed/stdlib/zlib.pyi index 234770172d40..2f6c40656038 100644 --- a/mypy/typeshed/stdlib/zlib.pyi +++ b/mypy/typeshed/stdlib/zlib.pyi @@ -1,29 +1,29 @@ import sys from _typeshed import ReadableBuffer -from typing import Literal +from typing import Final -DEFLATED: Literal[8] +DEFLATED: Final = 8 DEF_MEM_LEVEL: int # can change -DEF_BUF_SIZE: Literal[16384] +DEF_BUF_SIZE: Final = 16384 MAX_WBITS: int ZLIB_VERSION: str # can change ZLIB_RUNTIME_VERSION: str # can change -Z_NO_COMPRESSION: Literal[0] -Z_PARTIAL_FLUSH: Literal[1] -Z_BEST_COMPRESSION: Literal[9] -Z_BEST_SPEED: Literal[1] -Z_BLOCK: Literal[5] -Z_DEFAULT_COMPRESSION: Literal[-1] -Z_DEFAULT_STRATEGY: Literal[0] -Z_FILTERED: Literal[1] -Z_FINISH: Literal[4] -Z_FIXED: Literal[4] -Z_FULL_FLUSH: Literal[3] -Z_HUFFMAN_ONLY: Literal[2] -Z_NO_FLUSH: Literal[0] -Z_RLE: Literal[3] -Z_SYNC_FLUSH: Literal[2] -Z_TREES: Literal[6] +Z_NO_COMPRESSION: Final = 0 +Z_PARTIAL_FLUSH: Final = 1 +Z_BEST_COMPRESSION: Final = 9 +Z_BEST_SPEED: Final = 1 +Z_BLOCK: Final = 5 +Z_DEFAULT_COMPRESSION: Final = -1 +Z_DEFAULT_STRATEGY: Final = 0 +Z_FILTERED: Final = 1 +Z_FINISH: Final = 4 +Z_FIXED: Final = 4 +Z_FULL_FLUSH: Final = 3 +Z_HUFFMAN_ONLY: Final = 2 +Z_NO_FLUSH: Final = 0 +Z_RLE: Final = 3 +Z_SYNC_FLUSH: Final = 2 +Z_TREES: Final = 6 class error(Exception): ... diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 832e55f333de..b65a4cd59f79 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -286,7 +286,7 @@ f.write(b'x') f.foobar() [out] _program.py:3: error: Argument 1 to "write" of "TextIOBase" has incompatible type "bytes"; expected "str" -_program.py:4: error: "TextIOWrapper" has no attribute "foobar" +_program.py:4: error: "TextIOWrapper[_WrappedBuffer]" has no attribute "foobar" [case testOpenReturnTypeInference] reveal_type(open('x')) @@ -295,8 +295,8 @@ reveal_type(open('x', 'rb')) mode = 'rb' reveal_type(open('x', mode)) [out] -_program.py:1: note: Revealed type is "io.TextIOWrapper" -_program.py:2: note: Revealed type is "io.TextIOWrapper" +_program.py:1: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" +_program.py:2: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" _program.py:3: note: Revealed type is "io.BufferedReader" _program.py:5: note: Revealed type is "typing.IO[Any]" @@ -319,8 +319,8 @@ reveal_type(p.open('rb')) mode = 'rb' reveal_type(p.open(mode)) [out] -_program.py:3: note: Revealed type is "io.TextIOWrapper" -_program.py:4: note: Revealed type is "io.TextIOWrapper" +_program.py:3: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" +_program.py:4: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" _program.py:5: note: Revealed type is "io.BufferedReader" _program.py:7: note: Revealed type is "typing.IO[Any]" @@ -332,8 +332,8 @@ reveal_type(p.open(errors='replace', mode='r')) mode = 'r' reveal_type(p.open(mode=mode, errors='replace')) [out] -_program.py:3: note: Revealed type is "io.TextIOWrapper" -_program.py:4: note: Revealed type is "io.TextIOWrapper" +_program.py:3: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" +_program.py:4: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" _program.py:6: note: Revealed type is "typing.IO[Any]" [case testGenericPatterns] From 8b74b5ab03742ba86348dfe14d0468a995f74ee7 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Fri, 26 Jul 2024 16:08:45 -0700 Subject: [PATCH 107/247] Fix `RawExpressionType.accept` crash with `--cache-fine-grained` (#17588) Commit 1072c78ad375b7f0511549287f54432050396717 (#17148) converted all quoted types into `RawExpressionType`, which raised an `AssertionError` when `accept`ing a `TypeTriggersVisitor`. - Fixes #17574. - Fixes #17587. Signed-off-by: Anders Kaseorg --- mypy/types.py | 2 ++ test-data/unit/check-typeddict.test | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/mypy/types.py b/mypy/types.py index 52b3121f9fb3..c29d686ce5d8 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2705,6 +2705,8 @@ def simple_name(self) -> str: return self.base_type_name.replace("builtins.", "") def accept(self, visitor: TypeVisitor[T]) -> T: + if self.node is not None: + return self.node.accept(visitor) assert isinstance(visitor, SyntheticTypeVisitor) ret: T = visitor.visit_raw_expression_type(self) return ret diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 6a5120159c2d..1ef08f825e7a 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1442,6 +1442,18 @@ reveal_type(x) # N: Revealed type is "TypedDict('__main__.X', {'a': TypedDict('_ reveal_type(x['a']['b']) # N: Revealed type is "builtins.int" [builtins fixtures/dict.pyi] +[case testTypedDictForwardReferenceCacheFineGrained] +# flags: --cache-fine-grained +from mypy_extensions import TypedDict +class A(TypedDict): + b: "B" +class B(TypedDict): + c: "C" +class C(TypedDict): + d: "D" +class D: + pass + [case testSelfRecursiveTypedDictInheriting] from mypy_extensions import TypedDict From a0dbbd5b462136914bb7a378221ae094b6844710 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:39:58 -0700 Subject: [PATCH 108/247] Sync typeshed (#17619) Source commit: https://github.com/python/typeshed/commit/4ef2d66663fc080fefa379e6ae5fc45d4f8b54eb --- mypy/typeshed/stdlib/_curses.pyi | 17 +- mypy/typeshed/stdlib/_decimal.pyi | 28 +- mypy/typeshed/stdlib/_osx_support.pyi | 10 +- mypy/typeshed/stdlib/argparse.pyi | 2 +- mypy/typeshed/stdlib/asyncio/tasks.pyi | 6 +- mypy/typeshed/stdlib/email/charset.pyi | 8 +- mypy/typeshed/stdlib/gzip.pyi | 16 +- mypy/typeshed/stdlib/io.pyi | 2 +- .../stdlib/lib2to3/fixes/fix_asserts.pyi | 4 +- .../stdlib/lib2to3/fixes/fix_idioms.pyi | 6 +- .../stdlib/lib2to3/fixes/fix_imports.pyi | 4 +- .../stdlib/lib2to3/fixes/fix_imports2.pyi | 4 +- .../stdlib/lib2to3/fixes/fix_methodattrs.pyi | 4 +- .../stdlib/lib2to3/fixes/fix_renames.pyi | 6 +- .../stdlib/lib2to3/fixes/fix_urllib.pyi | 4 +- mypy/typeshed/stdlib/lib2to3/pgen2/token.pyi | 126 +++---- mypy/typeshed/stdlib/logging/__init__.pyi | 22 +- mypy/typeshed/stdlib/logging/config.pyi | 6 +- mypy/typeshed/stdlib/logging/handlers.pyi | 14 +- mypy/typeshed/stdlib/modulefinder.pyi | 14 +- .../stdlib/multiprocessing/forkserver.pyi | 6 +- .../multiprocessing/popen_spawn_win32.pyi | 10 +- .../stdlib/multiprocessing/reduction.pyi | 2 +- .../typeshed/stdlib/multiprocessing/spawn.pyi | 6 +- mypy/typeshed/stdlib/multiprocessing/util.pyi | 18 +- mypy/typeshed/stdlib/optparse.pyi | 6 +- mypy/typeshed/stdlib/pyexpat/__init__.pyi | 5 +- mypy/typeshed/stdlib/sqlite3/dbapi2.pyi | 344 +++++++++--------- mypy/typeshed/stdlib/subprocess.pyi | 12 +- mypy/typeshed/stdlib/tty.pyi | 16 +- mypy/typeshed/stdlib/unittest/case.pyi | 17 +- mypy/typeshed/stdlib/unittest/loader.pyi | 4 +- mypy/typeshed/stdlib/unittest/main.pyi | 6 +- mypy/typeshed/stdlib/unittest/result.pyi | 6 +- mypy/typeshed/stdlib/unittest/util.pyi | 14 +- mypy/typeshed/stdlib/xmlrpc/client.pyi | 32 +- mypy/typeshed/stdlib/zipfile/__init__.pyi | 16 +- 37 files changed, 418 insertions(+), 405 deletions(-) diff --git a/mypy/typeshed/stdlib/_curses.pyi b/mypy/typeshed/stdlib/_curses.pyi index eb1d7b9bde9f..505637574af1 100644 --- a/mypy/typeshed/stdlib/_curses.pyi +++ b/mypy/typeshed/stdlib/_curses.pyi @@ -368,11 +368,7 @@ def tparm( ) -> bytes: ... def typeahead(fd: int, /) -> None: ... def unctrl(ch: _ChType, /) -> bytes: ... - -if sys.version_info < (3, 12) or sys.platform != "darwin": - # The support for macos was dropped in 3.12 - def unget_wch(ch: int | str, /) -> None: ... - +def unget_wch(ch: int | str, /) -> None: ... def ungetch(ch: _ChType, /) -> None: ... def ungetmouse(id: int, x: int, y: int, z: int, bstate: int, /) -> None: ... def update_lines_cols() -> None: ... @@ -447,13 +443,10 @@ class _CursesWindow: def getch(self) -> int: ... @overload def getch(self, y: int, x: int) -> int: ... - if sys.version_info < (3, 12) or sys.platform != "darwin": - # The support for macos was dropped in 3.12 - @overload - def get_wch(self) -> int | str: ... - @overload - def get_wch(self, y: int, x: int) -> int | str: ... - + @overload + def get_wch(self) -> int | str: ... + @overload + def get_wch(self, y: int, x: int) -> int | str: ... @overload def getkey(self) -> str: ... @overload diff --git a/mypy/typeshed/stdlib/_decimal.pyi b/mypy/typeshed/stdlib/_decimal.pyi index 90d16215c280..937a04ac3799 100644 --- a/mypy/typeshed/stdlib/_decimal.pyi +++ b/mypy/typeshed/stdlib/_decimal.pyi @@ -17,20 +17,20 @@ class DecimalTuple(NamedTuple): digits: tuple[int, ...] exponent: int | Literal["n", "N", "F"] -ROUND_DOWN: str -ROUND_HALF_UP: str -ROUND_HALF_EVEN: str -ROUND_CEILING: str -ROUND_FLOOR: str -ROUND_UP: str -ROUND_HALF_DOWN: str -ROUND_05UP: str -HAVE_CONTEXTVAR: bool -HAVE_THREADS: bool -MAX_EMAX: int -MAX_PREC: int -MIN_EMIN: int -MIN_ETINY: int +ROUND_DOWN: Final[str] +ROUND_HALF_UP: Final[str] +ROUND_HALF_EVEN: Final[str] +ROUND_CEILING: Final[str] +ROUND_FLOOR: Final[str] +ROUND_UP: Final[str] +ROUND_HALF_DOWN: Final[str] +ROUND_05UP: Final[str] +HAVE_CONTEXTVAR: Final[bool] +HAVE_THREADS: Final[bool] +MAX_EMAX: Final[int] +MAX_PREC: Final[int] +MIN_EMIN: Final[int] +MIN_ETINY: Final[int] class DecimalException(ArithmeticError): ... class Clamped(DecimalException): ... diff --git a/mypy/typeshed/stdlib/_osx_support.pyi b/mypy/typeshed/stdlib/_osx_support.pyi index 64dbdd24fd40..fb00e6986dd0 100644 --- a/mypy/typeshed/stdlib/_osx_support.pyi +++ b/mypy/typeshed/stdlib/_osx_support.pyi @@ -1,5 +1,5 @@ from collections.abc import Iterable, Sequence -from typing import TypeVar +from typing import Final, TypeVar _T = TypeVar("_T") _K = TypeVar("_K") @@ -7,15 +7,15 @@ _V = TypeVar("_V") __all__ = ["compiler_fixup", "customize_config_vars", "customize_compiler", "get_platform_osx"] -_UNIVERSAL_CONFIG_VARS: tuple[str, ...] # undocumented -_COMPILER_CONFIG_VARS: tuple[str, ...] # undocumented -_INITPRE: str # undocumented +_UNIVERSAL_CONFIG_VARS: Final[tuple[str, ...]] # undocumented +_COMPILER_CONFIG_VARS: Final[tuple[str, ...]] # undocumented +_INITPRE: Final[str] # undocumented def _find_executable(executable: str, path: str | None = None) -> str | None: ... # undocumented def _read_output(commandstring: str, capture_stderr: bool = False) -> str | None: ... # undocumented def _find_build_tool(toolname: str) -> str: ... # undocumented -_SYSTEM_VERSION: str | None # undocumented +_SYSTEM_VERSION: Final[str | None] # undocumented def _get_system_version() -> str: ... # undocumented def _remove_original_values(_config_vars: dict[str, str]) -> None: ... # undocumented diff --git a/mypy/typeshed/stdlib/argparse.pyi b/mypy/typeshed/stdlib/argparse.pyi index d878f3ebd6a2..66fa4e15291f 100644 --- a/mypy/typeshed/stdlib/argparse.pyi +++ b/mypy/typeshed/stdlib/argparse.pyi @@ -51,7 +51,7 @@ _SUPPRESS_T = NewType("_SUPPRESS_T", str) SUPPRESS: _SUPPRESS_T | str # not using Literal because argparse sometimes compares SUPPRESS with is # the | str is there so that foo = argparse.SUPPRESS; foo = "test" checks out in mypy ZERO_OR_MORE: Final = "*" -_UNRECOGNIZED_ARGS_ATTR: str # undocumented +_UNRECOGNIZED_ARGS_ATTR: Final[str] # undocumented class ArgumentError(Exception): argument_name: str | None diff --git a/mypy/typeshed/stdlib/asyncio/tasks.pyi b/mypy/typeshed/stdlib/asyncio/tasks.pyi index 4613bca70c1a..f23ecef126d6 100644 --- a/mypy/typeshed/stdlib/asyncio/tasks.pyi +++ b/mypy/typeshed/stdlib/asyncio/tasks.pyi @@ -429,7 +429,11 @@ class Task(Future[_T_co]): # type: ignore[type-var] # pyright: ignore[reportIn self, coro: _TaskCompatibleCoro[_T_co], *, loop: AbstractEventLoop = ..., name: str | None = ... ) -> None: ... - def get_coro(self) -> _TaskCompatibleCoro[_T_co]: ... + if sys.version_info >= (3, 12): + def get_coro(self) -> _TaskCompatibleCoro[_T_co] | None: ... + else: + def get_coro(self) -> _TaskCompatibleCoro[_T_co]: ... + def get_name(self) -> str: ... def set_name(self, value: object, /) -> None: ... if sys.version_info >= (3, 12): diff --git a/mypy/typeshed/stdlib/email/charset.pyi b/mypy/typeshed/stdlib/email/charset.pyi index 2d12df337207..2939192c9526 100644 --- a/mypy/typeshed/stdlib/email/charset.pyi +++ b/mypy/typeshed/stdlib/email/charset.pyi @@ -1,12 +1,12 @@ from collections.abc import Callable, Iterator from email.message import Message -from typing import overload +from typing import Final, overload __all__ = ["Charset", "add_alias", "add_charset", "add_codec"] -QP: int # undocumented -BASE64: int # undocumented -SHORTEST: int # undocumented +QP: Final[int] # undocumented +BASE64: Final[int] # undocumented +SHORTEST: Final[int] # undocumented class Charset: input_charset: str diff --git a/mypy/typeshed/stdlib/gzip.pyi b/mypy/typeshed/stdlib/gzip.pyi index 542945698bba..9b32008dcbf6 100644 --- a/mypy/typeshed/stdlib/gzip.pyi +++ b/mypy/typeshed/stdlib/gzip.pyi @@ -3,7 +3,7 @@ import sys import zlib from _typeshed import ReadableBuffer, SizedBuffer, StrOrBytesPath from io import FileIO -from typing import Literal, Protocol, TextIO, overload +from typing import Final, Literal, Protocol, TextIO, overload from typing_extensions import TypeAlias __all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"] @@ -12,14 +12,14 @@ _ReadBinaryMode: TypeAlias = Literal["r", "rb"] _WriteBinaryMode: TypeAlias = Literal["a", "ab", "w", "wb", "x", "xb"] _OpenTextMode: TypeAlias = Literal["rt", "at", "wt", "xt"] -READ: object # undocumented -WRITE: object # undocumented +READ: Final[object] # undocumented +WRITE: Final[object] # undocumented -FTEXT: int # actually Literal[1] # undocumented -FHCRC: int # actually Literal[2] # undocumented -FEXTRA: int # actually Literal[4] # undocumented -FNAME: int # actually Literal[8] # undocumented -FCOMMENT: int # actually Literal[16] # undocumented +FTEXT: Final[int] # actually Literal[1] # undocumented +FHCRC: Final[int] # actually Literal[2] # undocumented +FEXTRA: Final[int] # actually Literal[4] # undocumented +FNAME: Final[int] # actually Literal[8] # undocumented +FCOMMENT: Final[int] # actually Literal[16] # undocumented class _ReadableFileobj(Protocol): def read(self, n: int, /) -> bytes: ... diff --git a/mypy/typeshed/stdlib/io.pyi b/mypy/typeshed/stdlib/io.pyi index a386a914ddcd..2d64d261951d 100644 --- a/mypy/typeshed/stdlib/io.pyi +++ b/mypy/typeshed/stdlib/io.pyi @@ -168,7 +168,7 @@ class _WrappedBuffer(Protocol): def writable(self) -> bool: ... def truncate(self, size: int, /) -> int: ... def fileno(self) -> int: ... - def isatty(self) -> int: ... + def isatty(self) -> bool: ... # Optional: Only needs to be present if seekable() returns True. # def seek(self, offset: Literal[0], whence: Literal[2]) -> int: ... # def tell(self) -> int: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_asserts.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_asserts.pyi index fb0b472aa12a..1bf7db2f76e9 100644 --- a/mypy/typeshed/stdlib/lib2to3/fixes/fix_asserts.pyi +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_asserts.pyi @@ -1,8 +1,8 @@ -from typing import ClassVar, Literal +from typing import ClassVar, Final, Literal from ..fixer_base import BaseFix -NAMES: dict[str, str] +NAMES: Final[dict[str, str]] class FixAsserts(BaseFix): BM_compatible: ClassVar[Literal[False]] diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_idioms.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_idioms.pyi index 4595c57c7eb9..6b2723d09d43 100644 --- a/mypy/typeshed/stdlib/lib2to3/fixes/fix_idioms.pyi +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_idioms.pyi @@ -1,9 +1,9 @@ -from typing import ClassVar, Literal +from typing import ClassVar, Final, Literal from .. import fixer_base -CMP: str -TYPE: str +CMP: Final[str] +TYPE: Final[str] class FixIdioms(fixer_base.BaseFix): BM_compatible: ClassVar[Literal[False]] diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports.pyi index dd6f72dd88ac..c747af529f44 100644 --- a/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports.pyi +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports.pyi @@ -1,11 +1,11 @@ from _typeshed import StrPath from collections.abc import Generator -from typing import ClassVar, Literal +from typing import ClassVar, Final, Literal from .. import fixer_base from ..pytree import Node -MAPPING: dict[str, str] +MAPPING: Final[dict[str, str]] def alternates(members): ... def build_pattern(mapping=...) -> Generator[str, None, None]: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports2.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports2.pyi index 8d55433085dd..618ecd0424d8 100644 --- a/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports2.pyi +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_imports2.pyi @@ -1,6 +1,8 @@ +from typing import Final + from . import fix_imports -MAPPING: dict[str, str] +MAPPING: Final[dict[str, str]] class FixImports2(fix_imports.FixImports): mapping = MAPPING diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_methodattrs.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_methodattrs.pyi index 594b5e2c95c9..ca9b71e43f85 100644 --- a/mypy/typeshed/stdlib/lib2to3/fixes/fix_methodattrs.pyi +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_methodattrs.pyi @@ -1,8 +1,8 @@ -from typing import ClassVar, Literal +from typing import ClassVar, Final, Literal from .. import fixer_base -MAP: dict[str, str] +MAP: Final[dict[str, str]] class FixMethodattrs(fixer_base.BaseFix): BM_compatible: ClassVar[Literal[True]] diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_renames.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_renames.pyi index 6283f1ab7ce2..652d8f15ea1a 100644 --- a/mypy/typeshed/stdlib/lib2to3/fixes/fix_renames.pyi +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_renames.pyi @@ -1,10 +1,10 @@ from collections.abc import Generator -from typing import ClassVar, Literal +from typing import ClassVar, Final, Literal from .. import fixer_base -MAPPING: dict[str, dict[str, str]] -LOOKUP: dict[tuple[str, str], str] +MAPPING: Final[dict[str, dict[str, str]]] +LOOKUP: Final[dict[tuple[str, str], str]] def alternates(members): ... def build_pattern() -> Generator[str, None, None]: ... diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_urllib.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_urllib.pyi index 625472f609ab..abdcc0f62970 100644 --- a/mypy/typeshed/stdlib/lib2to3/fixes/fix_urllib.pyi +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_urllib.pyi @@ -1,9 +1,9 @@ from collections.abc import Generator -from typing import Literal +from typing import Final, Literal from .fix_imports import FixImports -MAPPING: dict[str, list[tuple[Literal["urllib.request", "urllib.parse", "urllib.error"], list[str]]]] +MAPPING: Final[dict[str, list[tuple[Literal["urllib.request", "urllib.parse", "urllib.error"], list[str]]]]] def build_pattern() -> Generator[str, None, None]: ... diff --git a/mypy/typeshed/stdlib/lib2to3/pgen2/token.pyi b/mypy/typeshed/stdlib/lib2to3/pgen2/token.pyi index debcb2193987..6898517acee6 100644 --- a/mypy/typeshed/stdlib/lib2to3/pgen2/token.pyi +++ b/mypy/typeshed/stdlib/lib2to3/pgen2/token.pyi @@ -1,65 +1,67 @@ -ENDMARKER: int -NAME: int -NUMBER: int -STRING: int -NEWLINE: int -INDENT: int -DEDENT: int -LPAR: int -RPAR: int -LSQB: int -RSQB: int -COLON: int -COMMA: int -SEMI: int -PLUS: int -MINUS: int -STAR: int -SLASH: int -VBAR: int -AMPER: int -LESS: int -GREATER: int -EQUAL: int -DOT: int -PERCENT: int -BACKQUOTE: int -LBRACE: int -RBRACE: int -EQEQUAL: int -NOTEQUAL: int -LESSEQUAL: int -GREATEREQUAL: int -TILDE: int -CIRCUMFLEX: int -LEFTSHIFT: int -RIGHTSHIFT: int -DOUBLESTAR: int -PLUSEQUAL: int -MINEQUAL: int -STAREQUAL: int -SLASHEQUAL: int -PERCENTEQUAL: int -AMPEREQUAL: int -VBAREQUAL: int -CIRCUMFLEXEQUAL: int -LEFTSHIFTEQUAL: int -RIGHTSHIFTEQUAL: int -DOUBLESTAREQUAL: int -DOUBLESLASH: int -DOUBLESLASHEQUAL: int -OP: int -COMMENT: int -NL: int -RARROW: int -AT: int -ATEQUAL: int -AWAIT: int -ASYNC: int -ERRORTOKEN: int -COLONEQUAL: int -N_TOKENS: int -NT_OFFSET: int +from typing import Final + +ENDMARKER: Final[int] +NAME: Final[int] +NUMBER: Final[int] +STRING: Final[int] +NEWLINE: Final[int] +INDENT: Final[int] +DEDENT: Final[int] +LPAR: Final[int] +RPAR: Final[int] +LSQB: Final[int] +RSQB: Final[int] +COLON: Final[int] +COMMA: Final[int] +SEMI: Final[int] +PLUS: Final[int] +MINUS: Final[int] +STAR: Final[int] +SLASH: Final[int] +VBAR: Final[int] +AMPER: Final[int] +LESS: Final[int] +GREATER: Final[int] +EQUAL: Final[int] +DOT: Final[int] +PERCENT: Final[int] +BACKQUOTE: Final[int] +LBRACE: Final[int] +RBRACE: Final[int] +EQEQUAL: Final[int] +NOTEQUAL: Final[int] +LESSEQUAL: Final[int] +GREATEREQUAL: Final[int] +TILDE: Final[int] +CIRCUMFLEX: Final[int] +LEFTSHIFT: Final[int] +RIGHTSHIFT: Final[int] +DOUBLESTAR: Final[int] +PLUSEQUAL: Final[int] +MINEQUAL: Final[int] +STAREQUAL: Final[int] +SLASHEQUAL: Final[int] +PERCENTEQUAL: Final[int] +AMPEREQUAL: Final[int] +VBAREQUAL: Final[int] +CIRCUMFLEXEQUAL: Final[int] +LEFTSHIFTEQUAL: Final[int] +RIGHTSHIFTEQUAL: Final[int] +DOUBLESTAREQUAL: Final[int] +DOUBLESLASH: Final[int] +DOUBLESLASHEQUAL: Final[int] +OP: Final[int] +COMMENT: Final[int] +NL: Final[int] +RARROW: Final[int] +AT: Final[int] +ATEQUAL: Final[int] +AWAIT: Final[int] +ASYNC: Final[int] +ERRORTOKEN: Final[int] +COLONEQUAL: Final[int] +N_TOKENS: Final[int] +NT_OFFSET: Final[int] tok_name: dict[int, str] def ISTERMINAL(x: int) -> bool: ... diff --git a/mypy/typeshed/stdlib/logging/__init__.pyi b/mypy/typeshed/stdlib/logging/__init__.pyi index 4c6163257236..e6e6e8f645a0 100644 --- a/mypy/typeshed/stdlib/logging/__init__.pyi +++ b/mypy/typeshed/stdlib/logging/__init__.pyi @@ -7,7 +7,7 @@ from re import Pattern from string import Template from time import struct_time from types import FrameType, TracebackType -from typing import Any, ClassVar, Generic, Literal, Protocol, TextIO, TypeVar, overload +from typing import Any, ClassVar, Final, Generic, Literal, Protocol, TextIO, TypeVar, overload from typing_extensions import Self, TypeAlias, deprecated if sys.version_info >= (3, 11): @@ -236,14 +236,14 @@ class Logger(Filterer): def hasHandlers(self) -> bool: ... def callHandlers(self, record: LogRecord) -> None: ... # undocumented -CRITICAL: int -FATAL: int -ERROR: int -WARNING: int -WARN: int -INFO: int -DEBUG: int -NOTSET: int +CRITICAL: Final = 50 +FATAL: Final = CRITICAL +ERROR: Final = 40 +WARNING: Final = 30 +WARN: Final = WARNING +INFO: Final = 20 +DEBUG: Final = 10 +NOTSET: Final = 0 class Handler(Filterer): level: int # undocumented @@ -684,6 +684,6 @@ class StrFormatStyle(PercentStyle): # undocumented class StringTemplateStyle(PercentStyle): # undocumented _tpl: Template -_STYLES: dict[str, tuple[PercentStyle, str]] +_STYLES: Final[dict[str, tuple[PercentStyle, str]]] -BASIC_FORMAT: str +BASIC_FORMAT: Final[str] diff --git a/mypy/typeshed/stdlib/logging/config.pyi b/mypy/typeshed/stdlib/logging/config.pyi index 7a26846addbb..83fe7461cb5c 100644 --- a/mypy/typeshed/stdlib/logging/config.pyi +++ b/mypy/typeshed/stdlib/logging/config.pyi @@ -4,14 +4,14 @@ from collections.abc import Callable, Hashable, Iterable, Sequence from configparser import RawConfigParser from re import Pattern from threading import Thread -from typing import IO, Any, Literal, SupportsIndex, TypedDict, overload +from typing import IO, Any, Final, Literal, SupportsIndex, TypedDict, overload from typing_extensions import Required, TypeAlias from . import Filter, Filterer, Formatter, Handler, Logger, _FilterType, _FormatStyle, _Level DEFAULT_LOGGING_CONFIG_PORT: int -RESET_ERROR: int # undocumented -IDENTIFIER: Pattern[str] # undocumented +RESET_ERROR: Final[int] # undocumented +IDENTIFIER: Final[Pattern[str]] # undocumented if sys.version_info >= (3, 11): class _RootLoggerConfiguration(TypedDict, total=False): diff --git a/mypy/typeshed/stdlib/logging/handlers.pyi b/mypy/typeshed/stdlib/logging/handlers.pyi index 4e97012abba1..91f9fe57e46f 100644 --- a/mypy/typeshed/stdlib/logging/handlers.pyi +++ b/mypy/typeshed/stdlib/logging/handlers.pyi @@ -8,16 +8,16 @@ from logging import FileHandler, Handler, LogRecord from re import Pattern from socket import SocketKind, socket from threading import Thread -from typing import Any, ClassVar, Protocol, TypeVar +from typing import Any, ClassVar, Final, Protocol, TypeVar _T = TypeVar("_T") -DEFAULT_TCP_LOGGING_PORT: int -DEFAULT_UDP_LOGGING_PORT: int -DEFAULT_HTTP_LOGGING_PORT: int -DEFAULT_SOAP_LOGGING_PORT: int -SYSLOG_UDP_PORT: int -SYSLOG_TCP_PORT: int +DEFAULT_TCP_LOGGING_PORT: Final[int] +DEFAULT_UDP_LOGGING_PORT: Final[int] +DEFAULT_HTTP_LOGGING_PORT: Final[int] +DEFAULT_SOAP_LOGGING_PORT: Final[int] +SYSLOG_UDP_PORT: Final[int] +SYSLOG_TCP_PORT: Final[int] class WatchedFileHandler(FileHandler): dev: int # undocumented diff --git a/mypy/typeshed/stdlib/modulefinder.pyi b/mypy/typeshed/stdlib/modulefinder.pyi index 132cac5f1878..2cf948ba898a 100644 --- a/mypy/typeshed/stdlib/modulefinder.pyi +++ b/mypy/typeshed/stdlib/modulefinder.pyi @@ -1,15 +1,15 @@ import sys from collections.abc import Container, Iterable, Iterator, Sequence from types import CodeType -from typing import IO, Any +from typing import IO, Any, Final if sys.version_info < (3, 11): - LOAD_CONST: int # undocumented - IMPORT_NAME: int # undocumented - STORE_NAME: int # undocumented - STORE_GLOBAL: int # undocumented - STORE_OPS: tuple[int, int] # undocumented - EXTENDED_ARG: int # undocumented + LOAD_CONST: Final[int] # undocumented + IMPORT_NAME: Final[int] # undocumented + STORE_NAME: Final[int] # undocumented + STORE_GLOBAL: Final[int] # undocumented + STORE_OPS: Final[tuple[int, int]] # undocumented + EXTENDED_ARG: Final[int] # undocumented packagePathMap: dict[str, list[str]] # undocumented diff --git a/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi b/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi index 9a15f2683b7d..31b982856355 100644 --- a/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/forkserver.pyi @@ -1,12 +1,12 @@ from _typeshed import FileDescriptorLike, Unused from collections.abc import Sequence from struct import Struct -from typing import Any +from typing import Any, Final __all__ = ["ensure_running", "get_inherited_fds", "connect_to_new_process", "set_forkserver_preload"] -MAXFDS_TO_SEND: int -SIGNED_STRUCT: Struct +MAXFDS_TO_SEND: Final = 256 +SIGNED_STRUCT: Final[Struct] class ForkServer: def set_forkserver_preload(self, modules_names: list[str]) -> None: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/popen_spawn_win32.pyi b/mypy/typeshed/stdlib/multiprocessing/popen_spawn_win32.pyi index 3dc9d5bd7332..481b9eec5a37 100644 --- a/mypy/typeshed/stdlib/multiprocessing/popen_spawn_win32.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/popen_spawn_win32.pyi @@ -1,16 +1,16 @@ import sys from multiprocessing.process import BaseProcess -from typing import ClassVar +from typing import ClassVar, Final from .util import Finalize if sys.platform == "win32": __all__ = ["Popen"] - TERMINATE: int - WINEXE: bool - WINSERVICE: bool - WINENV: bool + TERMINATE: Final[int] + WINEXE: Final[bool] + WINSERVICE: Final[bool] + WINENV: Final[bool] class Popen: finalizer: Finalize diff --git a/mypy/typeshed/stdlib/multiprocessing/reduction.pyi b/mypy/typeshed/stdlib/multiprocessing/reduction.pyi index 6d23e20e6981..a31987bcc3cb 100644 --- a/mypy/typeshed/stdlib/multiprocessing/reduction.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/reduction.pyi @@ -15,7 +15,7 @@ if sys.platform == "win32": else: __all__ = ["send_handle", "recv_handle", "ForkingPickler", "register", "dump", "DupFd", "sendfds", "recvfds"] -HAVE_SEND_HANDLE: bool +HAVE_SEND_HANDLE: Final[bool] class ForkingPickler(pickle.Pickler): dispatch_table: _DispatchTableType diff --git a/mypy/typeshed/stdlib/multiprocessing/spawn.pyi b/mypy/typeshed/stdlib/multiprocessing/spawn.pyi index 26ff165756bf..43ce2f07d996 100644 --- a/mypy/typeshed/stdlib/multiprocessing/spawn.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/spawn.pyi @@ -1,6 +1,6 @@ from collections.abc import Mapping, Sequence from types import ModuleType -from typing import Any +from typing import Any, Final __all__ = [ "_main", @@ -12,8 +12,8 @@ __all__ = [ "import_main_path", ] -WINEXE: bool -WINSERVICE: bool +WINEXE: Final[bool] +WINSERVICE: Final[bool] def set_executable(exe: str) -> None: ... def get_executable() -> str: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/util.pyi b/mypy/typeshed/stdlib/multiprocessing/util.pyi index 790d6c7467f0..d5b6384afd5e 100644 --- a/mypy/typeshed/stdlib/multiprocessing/util.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/util.pyi @@ -2,7 +2,7 @@ import threading from _typeshed import ConvertibleToInt, Incomplete, Unused from collections.abc import Callable, Iterable, Mapping, MutableMapping, Sequence from logging import Logger, _Level as _LoggingLevel -from typing import Any, Generic, TypeVar, overload +from typing import Any, Final, Generic, TypeVar, overload __all__ = [ "sub_debug", @@ -25,14 +25,14 @@ __all__ = [ _T = TypeVar("_T") _R_co = TypeVar("_R_co", default=Any, covariant=True) -NOTSET: int -SUBDEBUG: int -DEBUG: int -INFO: int -SUBWARNING: int +NOTSET: Final[int] +SUBDEBUG: Final[int] +DEBUG: Final[int] +INFO: Final[int] +SUBWARNING: Final[int] -LOGGER_NAME: str -DEFAULT_LOGGING_FORMAT: str +LOGGER_NAME: Final[str] +DEFAULT_LOGGING_FORMAT: Final[str] def sub_debug(msg: object, *args: object) -> None: ... def debug(msg: object, *args: object) -> None: ... @@ -92,7 +92,7 @@ class ForkAwareThreadLock: class ForkAwareLocal(threading.local): ... -MAXFD: int +MAXFD: Final[int] def close_all_fds_except(fds: Iterable[int]) -> None: ... def spawnv_passfds(path: bytes, args: Sequence[ConvertibleToInt], passfds: Sequence[int]) -> int: ... diff --git a/mypy/typeshed/stdlib/optparse.pyi b/mypy/typeshed/stdlib/optparse.pyi index a179c2d1bb3c..b513bb647060 100644 --- a/mypy/typeshed/stdlib/optparse.pyi +++ b/mypy/typeshed/stdlib/optparse.pyi @@ -1,7 +1,7 @@ from _typeshed import Incomplete from abc import abstractmethod from collections.abc import Callable, Iterable, Mapping, Sequence -from typing import IO, Any, AnyStr, Literal, overload +from typing import IO, Any, AnyStr, Literal, NoReturn, overload __all__ = [ "Option", @@ -231,8 +231,8 @@ class OptionParser(OptionContainer): def check_values(self, values: Values, args: list[str]) -> tuple[Values, list[str]]: ... def disable_interspersed_args(self) -> None: ... def enable_interspersed_args(self) -> None: ... - def error(self, msg: str) -> None: ... - def exit(self, status: int = 0, msg: str | None = None) -> None: ... + def error(self, msg: str) -> NoReturn: ... + def exit(self, status: int = 0, msg: str | None = None) -> NoReturn: ... def expand_prog_name(self, s: str) -> str: ... def format_epilog(self, formatter: HelpFormatter) -> str: ... def format_help(self, formatter: HelpFormatter | None = None) -> str: ... diff --git a/mypy/typeshed/stdlib/pyexpat/__init__.pyi b/mypy/typeshed/stdlib/pyexpat/__init__.pyi index 88bf9464d130..64decd56bee6 100644 --- a/mypy/typeshed/stdlib/pyexpat/__init__.pyi +++ b/mypy/typeshed/stdlib/pyexpat/__init__.pyi @@ -1,10 +1,10 @@ from _typeshed import ReadableBuffer, SupportsRead from collections.abc import Callable from pyexpat import errors as errors, model as model -from typing import Any, final +from typing import Any, Final, final from typing_extensions import TypeAlias -EXPAT_VERSION: str # undocumented +EXPAT_VERSION: Final[str] # undocumented version_info: tuple[int, int, int] # undocumented native_encoding: str # undocumented features: list[tuple[str, int]] # undocumented @@ -15,7 +15,6 @@ class ExpatError(Exception): offset: int error = ExpatError - XML_PARAM_ENTITY_PARSING_NEVER: int XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE: int XML_PARAM_ENTITY_PARSING_ALWAYS: int diff --git a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi index 3cb4b93e88fe..9e46012ee777 100644 --- a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi +++ b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi @@ -4,7 +4,7 @@ from _typeshed import ReadableBuffer, StrOrBytesPath, SupportsLenAndGetItem, Unu from collections.abc import Callable, Generator, Iterable, Iterator, Mapping from datetime import date, datetime, time from types import TracebackType -from typing import Any, Literal, Protocol, SupportsIndex, TypeVar, final, overload +from typing import Any, Final, Literal, Protocol, SupportsIndex, TypeVar, final, overload from typing_extensions import Self, TypeAlias _T = TypeVar("_T") @@ -35,186 +35,186 @@ Binary = memoryview # The remaining definitions are imported from _sqlite3. -PARSE_COLNAMES: int -PARSE_DECLTYPES: int -SQLITE_ALTER_TABLE: int -SQLITE_ANALYZE: int -SQLITE_ATTACH: int -SQLITE_CREATE_INDEX: int -SQLITE_CREATE_TABLE: int -SQLITE_CREATE_TEMP_INDEX: int -SQLITE_CREATE_TEMP_TABLE: int -SQLITE_CREATE_TEMP_TRIGGER: int -SQLITE_CREATE_TEMP_VIEW: int -SQLITE_CREATE_TRIGGER: int -SQLITE_CREATE_VIEW: int -SQLITE_CREATE_VTABLE: int -SQLITE_DELETE: int -SQLITE_DENY: int -SQLITE_DETACH: int -SQLITE_DONE: int -SQLITE_DROP_INDEX: int -SQLITE_DROP_TABLE: int -SQLITE_DROP_TEMP_INDEX: int -SQLITE_DROP_TEMP_TABLE: int -SQLITE_DROP_TEMP_TRIGGER: int -SQLITE_DROP_TEMP_VIEW: int -SQLITE_DROP_TRIGGER: int -SQLITE_DROP_VIEW: int -SQLITE_DROP_VTABLE: int -SQLITE_FUNCTION: int -SQLITE_IGNORE: int -SQLITE_INSERT: int -SQLITE_OK: int +PARSE_COLNAMES: Final[int] +PARSE_DECLTYPES: Final[int] +SQLITE_ALTER_TABLE: Final[int] +SQLITE_ANALYZE: Final[int] +SQLITE_ATTACH: Final[int] +SQLITE_CREATE_INDEX: Final[int] +SQLITE_CREATE_TABLE: Final[int] +SQLITE_CREATE_TEMP_INDEX: Final[int] +SQLITE_CREATE_TEMP_TABLE: Final[int] +SQLITE_CREATE_TEMP_TRIGGER: Final[int] +SQLITE_CREATE_TEMP_VIEW: Final[int] +SQLITE_CREATE_TRIGGER: Final[int] +SQLITE_CREATE_VIEW: Final[int] +SQLITE_CREATE_VTABLE: Final[int] +SQLITE_DELETE: Final[int] +SQLITE_DENY: Final[int] +SQLITE_DETACH: Final[int] +SQLITE_DONE: Final[int] +SQLITE_DROP_INDEX: Final[int] +SQLITE_DROP_TABLE: Final[int] +SQLITE_DROP_TEMP_INDEX: Final[int] +SQLITE_DROP_TEMP_TABLE: Final[int] +SQLITE_DROP_TEMP_TRIGGER: Final[int] +SQLITE_DROP_TEMP_VIEW: Final[int] +SQLITE_DROP_TRIGGER: Final[int] +SQLITE_DROP_VIEW: Final[int] +SQLITE_DROP_VTABLE: Final[int] +SQLITE_FUNCTION: Final[int] +SQLITE_IGNORE: Final[int] +SQLITE_INSERT: Final[int] +SQLITE_OK: Final[int] if sys.version_info >= (3, 11): - SQLITE_LIMIT_LENGTH: int - SQLITE_LIMIT_SQL_LENGTH: int - SQLITE_LIMIT_COLUMN: int - SQLITE_LIMIT_EXPR_DEPTH: int - SQLITE_LIMIT_COMPOUND_SELECT: int - SQLITE_LIMIT_VDBE_OP: int - SQLITE_LIMIT_FUNCTION_ARG: int - SQLITE_LIMIT_ATTACHED: int - SQLITE_LIMIT_LIKE_PATTERN_LENGTH: int - SQLITE_LIMIT_VARIABLE_NUMBER: int - SQLITE_LIMIT_TRIGGER_DEPTH: int - SQLITE_LIMIT_WORKER_THREADS: int -SQLITE_PRAGMA: int -SQLITE_READ: int -SQLITE_REINDEX: int -SQLITE_RECURSIVE: int -SQLITE_SAVEPOINT: int -SQLITE_SELECT: int -SQLITE_TRANSACTION: int -SQLITE_UPDATE: int + SQLITE_LIMIT_LENGTH: Final[int] + SQLITE_LIMIT_SQL_LENGTH: Final[int] + SQLITE_LIMIT_COLUMN: Final[int] + SQLITE_LIMIT_EXPR_DEPTH: Final[int] + SQLITE_LIMIT_COMPOUND_SELECT: Final[int] + SQLITE_LIMIT_VDBE_OP: Final[int] + SQLITE_LIMIT_FUNCTION_ARG: Final[int] + SQLITE_LIMIT_ATTACHED: Final[int] + SQLITE_LIMIT_LIKE_PATTERN_LENGTH: Final[int] + SQLITE_LIMIT_VARIABLE_NUMBER: Final[int] + SQLITE_LIMIT_TRIGGER_DEPTH: Final[int] + SQLITE_LIMIT_WORKER_THREADS: Final[int] +SQLITE_PRAGMA: Final[int] +SQLITE_READ: Final[int] +SQLITE_REINDEX: Final[int] +SQLITE_RECURSIVE: Final[int] +SQLITE_SAVEPOINT: Final[int] +SQLITE_SELECT: Final[int] +SQLITE_TRANSACTION: Final[int] +SQLITE_UPDATE: Final[int] adapters: dict[tuple[type[Any], type[Any]], _Adapter[Any]] converters: dict[str, _Converter] sqlite_version: str version: str if sys.version_info >= (3, 11): - SQLITE_ABORT: int - SQLITE_ABORT_ROLLBACK: int - SQLITE_AUTH: int - SQLITE_AUTH_USER: int - SQLITE_BUSY: int - SQLITE_BUSY_RECOVERY: int - SQLITE_BUSY_SNAPSHOT: int - SQLITE_BUSY_TIMEOUT: int - SQLITE_CANTOPEN: int - SQLITE_CANTOPEN_CONVPATH: int - SQLITE_CANTOPEN_DIRTYWAL: int - SQLITE_CANTOPEN_FULLPATH: int - SQLITE_CANTOPEN_ISDIR: int - SQLITE_CANTOPEN_NOTEMPDIR: int - SQLITE_CANTOPEN_SYMLINK: int - SQLITE_CONSTRAINT: int - SQLITE_CONSTRAINT_CHECK: int - SQLITE_CONSTRAINT_COMMITHOOK: int - SQLITE_CONSTRAINT_FOREIGNKEY: int - SQLITE_CONSTRAINT_FUNCTION: int - SQLITE_CONSTRAINT_NOTNULL: int - SQLITE_CONSTRAINT_PINNED: int - SQLITE_CONSTRAINT_PRIMARYKEY: int - SQLITE_CONSTRAINT_ROWID: int - SQLITE_CONSTRAINT_TRIGGER: int - SQLITE_CONSTRAINT_UNIQUE: int - SQLITE_CONSTRAINT_VTAB: int - SQLITE_CORRUPT: int - SQLITE_CORRUPT_INDEX: int - SQLITE_CORRUPT_SEQUENCE: int - SQLITE_CORRUPT_VTAB: int - SQLITE_EMPTY: int - SQLITE_ERROR: int - SQLITE_ERROR_MISSING_COLLSEQ: int - SQLITE_ERROR_RETRY: int - SQLITE_ERROR_SNAPSHOT: int - SQLITE_FORMAT: int - SQLITE_FULL: int - SQLITE_INTERNAL: int - SQLITE_INTERRUPT: int - SQLITE_IOERR: int - SQLITE_IOERR_ACCESS: int - SQLITE_IOERR_AUTH: int - SQLITE_IOERR_BEGIN_ATOMIC: int - SQLITE_IOERR_BLOCKED: int - SQLITE_IOERR_CHECKRESERVEDLOCK: int - SQLITE_IOERR_CLOSE: int - SQLITE_IOERR_COMMIT_ATOMIC: int - SQLITE_IOERR_CONVPATH: int - SQLITE_IOERR_CORRUPTFS: int - SQLITE_IOERR_DATA: int - SQLITE_IOERR_DELETE: int - SQLITE_IOERR_DELETE_NOENT: int - SQLITE_IOERR_DIR_CLOSE: int - SQLITE_IOERR_DIR_FSYNC: int - SQLITE_IOERR_FSTAT: int - SQLITE_IOERR_FSYNC: int - SQLITE_IOERR_GETTEMPPATH: int - SQLITE_IOERR_LOCK: int - SQLITE_IOERR_MMAP: int - SQLITE_IOERR_NOMEM: int - SQLITE_IOERR_RDLOCK: int - SQLITE_IOERR_READ: int - SQLITE_IOERR_ROLLBACK_ATOMIC: int - SQLITE_IOERR_SEEK: int - SQLITE_IOERR_SHMLOCK: int - SQLITE_IOERR_SHMMAP: int - SQLITE_IOERR_SHMOPEN: int - SQLITE_IOERR_SHMSIZE: int - SQLITE_IOERR_SHORT_READ: int - SQLITE_IOERR_TRUNCATE: int - SQLITE_IOERR_UNLOCK: int - SQLITE_IOERR_VNODE: int - SQLITE_IOERR_WRITE: int - SQLITE_LOCKED: int - SQLITE_LOCKED_SHAREDCACHE: int - SQLITE_LOCKED_VTAB: int - SQLITE_MISMATCH: int - SQLITE_MISUSE: int - SQLITE_NOLFS: int - SQLITE_NOMEM: int - SQLITE_NOTADB: int - SQLITE_NOTFOUND: int - SQLITE_NOTICE: int - SQLITE_NOTICE_RECOVER_ROLLBACK: int - SQLITE_NOTICE_RECOVER_WAL: int - SQLITE_OK_LOAD_PERMANENTLY: int - SQLITE_OK_SYMLINK: int - SQLITE_PERM: int - SQLITE_PROTOCOL: int - SQLITE_RANGE: int - SQLITE_READONLY: int - SQLITE_READONLY_CANTINIT: int - SQLITE_READONLY_CANTLOCK: int - SQLITE_READONLY_DBMOVED: int - SQLITE_READONLY_DIRECTORY: int - SQLITE_READONLY_RECOVERY: int - SQLITE_READONLY_ROLLBACK: int - SQLITE_ROW: int - SQLITE_SCHEMA: int - SQLITE_TOOBIG: int - SQLITE_WARNING: int - SQLITE_WARNING_AUTOINDEX: int + SQLITE_ABORT: Final[int] + SQLITE_ABORT_ROLLBACK: Final[int] + SQLITE_AUTH: Final[int] + SQLITE_AUTH_USER: Final[int] + SQLITE_BUSY: Final[int] + SQLITE_BUSY_RECOVERY: Final[int] + SQLITE_BUSY_SNAPSHOT: Final[int] + SQLITE_BUSY_TIMEOUT: Final[int] + SQLITE_CANTOPEN: Final[int] + SQLITE_CANTOPEN_CONVPATH: Final[int] + SQLITE_CANTOPEN_DIRTYWAL: Final[int] + SQLITE_CANTOPEN_FULLPATH: Final[int] + SQLITE_CANTOPEN_ISDIR: Final[int] + SQLITE_CANTOPEN_NOTEMPDIR: Final[int] + SQLITE_CANTOPEN_SYMLINK: Final[int] + SQLITE_CONSTRAINT: Final[int] + SQLITE_CONSTRAINT_CHECK: Final[int] + SQLITE_CONSTRAINT_COMMITHOOK: Final[int] + SQLITE_CONSTRAINT_FOREIGNKEY: Final[int] + SQLITE_CONSTRAINT_FUNCTION: Final[int] + SQLITE_CONSTRAINT_NOTNULL: Final[int] + SQLITE_CONSTRAINT_PINNED: Final[int] + SQLITE_CONSTRAINT_PRIMARYKEY: Final[int] + SQLITE_CONSTRAINT_ROWID: Final[int] + SQLITE_CONSTRAINT_TRIGGER: Final[int] + SQLITE_CONSTRAINT_UNIQUE: Final[int] + SQLITE_CONSTRAINT_VTAB: Final[int] + SQLITE_CORRUPT: Final[int] + SQLITE_CORRUPT_INDEX: Final[int] + SQLITE_CORRUPT_SEQUENCE: Final[int] + SQLITE_CORRUPT_VTAB: Final[int] + SQLITE_EMPTY: Final[int] + SQLITE_ERROR: Final[int] + SQLITE_ERROR_MISSING_COLLSEQ: Final[int] + SQLITE_ERROR_RETRY: Final[int] + SQLITE_ERROR_SNAPSHOT: Final[int] + SQLITE_FORMAT: Final[int] + SQLITE_FULL: Final[int] + SQLITE_INTERNAL: Final[int] + SQLITE_INTERRUPT: Final[int] + SQLITE_IOERR: Final[int] + SQLITE_IOERR_ACCESS: Final[int] + SQLITE_IOERR_AUTH: Final[int] + SQLITE_IOERR_BEGIN_ATOMIC: Final[int] + SQLITE_IOERR_BLOCKED: Final[int] + SQLITE_IOERR_CHECKRESERVEDLOCK: Final[int] + SQLITE_IOERR_CLOSE: Final[int] + SQLITE_IOERR_COMMIT_ATOMIC: Final[int] + SQLITE_IOERR_CONVPATH: Final[int] + SQLITE_IOERR_CORRUPTFS: Final[int] + SQLITE_IOERR_DATA: Final[int] + SQLITE_IOERR_DELETE: Final[int] + SQLITE_IOERR_DELETE_NOENT: Final[int] + SQLITE_IOERR_DIR_CLOSE: Final[int] + SQLITE_IOERR_DIR_FSYNC: Final[int] + SQLITE_IOERR_FSTAT: Final[int] + SQLITE_IOERR_FSYNC: Final[int] + SQLITE_IOERR_GETTEMPPATH: Final[int] + SQLITE_IOERR_LOCK: Final[int] + SQLITE_IOERR_MMAP: Final[int] + SQLITE_IOERR_NOMEM: Final[int] + SQLITE_IOERR_RDLOCK: Final[int] + SQLITE_IOERR_READ: Final[int] + SQLITE_IOERR_ROLLBACK_ATOMIC: Final[int] + SQLITE_IOERR_SEEK: Final[int] + SQLITE_IOERR_SHMLOCK: Final[int] + SQLITE_IOERR_SHMMAP: Final[int] + SQLITE_IOERR_SHMOPEN: Final[int] + SQLITE_IOERR_SHMSIZE: Final[int] + SQLITE_IOERR_SHORT_READ: Final[int] + SQLITE_IOERR_TRUNCATE: Final[int] + SQLITE_IOERR_UNLOCK: Final[int] + SQLITE_IOERR_VNODE: Final[int] + SQLITE_IOERR_WRITE: Final[int] + SQLITE_LOCKED: Final[int] + SQLITE_LOCKED_SHAREDCACHE: Final[int] + SQLITE_LOCKED_VTAB: Final[int] + SQLITE_MISMATCH: Final[int] + SQLITE_MISUSE: Final[int] + SQLITE_NOLFS: Final[int] + SQLITE_NOMEM: Final[int] + SQLITE_NOTADB: Final[int] + SQLITE_NOTFOUND: Final[int] + SQLITE_NOTICE: Final[int] + SQLITE_NOTICE_RECOVER_ROLLBACK: Final[int] + SQLITE_NOTICE_RECOVER_WAL: Final[int] + SQLITE_OK_LOAD_PERMANENTLY: Final[int] + SQLITE_OK_SYMLINK: Final[int] + SQLITE_PERM: Final[int] + SQLITE_PROTOCOL: Final[int] + SQLITE_RANGE: Final[int] + SQLITE_READONLY: Final[int] + SQLITE_READONLY_CANTINIT: Final[int] + SQLITE_READONLY_CANTLOCK: Final[int] + SQLITE_READONLY_DBMOVED: Final[int] + SQLITE_READONLY_DIRECTORY: Final[int] + SQLITE_READONLY_RECOVERY: Final[int] + SQLITE_READONLY_ROLLBACK: Final[int] + SQLITE_ROW: Final[int] + SQLITE_SCHEMA: Final[int] + SQLITE_TOOBIG: Final[int] + SQLITE_WARNING: Final[int] + SQLITE_WARNING_AUTOINDEX: Final[int] if sys.version_info >= (3, 12): - LEGACY_TRANSACTION_CONTROL: int - SQLITE_DBCONFIG_DEFENSIVE: int - SQLITE_DBCONFIG_DQS_DDL: int - SQLITE_DBCONFIG_DQS_DML: int - SQLITE_DBCONFIG_ENABLE_FKEY: int - SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: int - SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: int - SQLITE_DBCONFIG_ENABLE_QPSG: int - SQLITE_DBCONFIG_ENABLE_TRIGGER: int - SQLITE_DBCONFIG_ENABLE_VIEW: int - SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: int - SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: int - SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: int - SQLITE_DBCONFIG_RESET_DATABASE: int - SQLITE_DBCONFIG_TRIGGER_EQP: int - SQLITE_DBCONFIG_TRUSTED_SCHEMA: int - SQLITE_DBCONFIG_WRITABLE_SCHEMA: int + LEGACY_TRANSACTION_CONTROL: Final[int] + SQLITE_DBCONFIG_DEFENSIVE: Final[int] + SQLITE_DBCONFIG_DQS_DDL: Final[int] + SQLITE_DBCONFIG_DQS_DML: Final[int] + SQLITE_DBCONFIG_ENABLE_FKEY: Final[int] + SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: Final[int] + SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: Final[int] + SQLITE_DBCONFIG_ENABLE_QPSG: Final[int] + SQLITE_DBCONFIG_ENABLE_TRIGGER: Final[int] + SQLITE_DBCONFIG_ENABLE_VIEW: Final[int] + SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: Final[int] + SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: Final[int] + SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: Final[int] + SQLITE_DBCONFIG_RESET_DATABASE: Final[int] + SQLITE_DBCONFIG_TRIGGER_EQP: Final[int] + SQLITE_DBCONFIG_TRUSTED_SCHEMA: Final[int] + SQLITE_DBCONFIG_WRITABLE_SCHEMA: Final[int] # Can take or return anything depending on what's in the registry. @overload diff --git a/mypy/typeshed/stdlib/subprocess.pyi b/mypy/typeshed/stdlib/subprocess.pyi index b01bac2455ce..2a5859807b51 100644 --- a/mypy/typeshed/stdlib/subprocess.pyi +++ b/mypy/typeshed/stdlib/subprocess.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import ReadableBuffer, StrOrBytesPath from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from types import TracebackType -from typing import IO, Any, AnyStr, Generic, Literal, TypeVar, overload +from typing import IO, Any, AnyStr, Final, Generic, Literal, TypeVar, overload from typing_extensions import Self, TypeAlias if sys.version_info >= (3, 9): @@ -74,8 +74,8 @@ _T = TypeVar("_T") # These two are private but documented if sys.version_info >= (3, 11): - _USE_VFORK: bool -_USE_POSIX_SPAWN: bool + _USE_VFORK: Final[bool] +_USE_POSIX_SPAWN: Final[bool] class CompletedProcess(Generic[_T]): # morally: _CMD @@ -1810,9 +1810,9 @@ else: text: bool | None = None, ) -> Any: ... # morally: -> str | bytes -PIPE: int -STDOUT: int -DEVNULL: int +PIPE: Final[int] +STDOUT: Final[int] +DEVNULL: Final[int] class SubprocessError(Exception): ... diff --git a/mypy/typeshed/stdlib/tty.pyi b/mypy/typeshed/stdlib/tty.pyi index add0d57a8d4b..0611879cf1b2 100644 --- a/mypy/typeshed/stdlib/tty.pyi +++ b/mypy/typeshed/stdlib/tty.pyi @@ -1,6 +1,6 @@ import sys import termios -from typing import IO +from typing import IO, Final from typing_extensions import TypeAlias if sys.platform != "win32": @@ -15,13 +15,13 @@ if sys.platform != "win32": _FD: TypeAlias = int | IO[str] # XXX: Undocumented integer constants - IFLAG: int - OFLAG: int - CFLAG: int - LFLAG: int - ISPEED: int - OSPEED: int - CC: int + IFLAG: Final[int] + OFLAG: Final[int] + CFLAG: Final[int] + LFLAG: Final[int] + ISPEED: Final[int] + OSPEED: Final[int] + CC: Final[int] def setraw(fd: _FD, when: int = 2) -> _ModeSetterReturn: ... def setcbreak(fd: _FD, when: int = 2) -> _ModeSetterReturn: ... diff --git a/mypy/typeshed/stdlib/unittest/case.pyi b/mypy/typeshed/stdlib/unittest/case.pyi index b63292604ecc..a92f03f9745f 100644 --- a/mypy/typeshed/stdlib/unittest/case.pyi +++ b/mypy/typeshed/stdlib/unittest/case.pyi @@ -6,7 +6,20 @@ from collections.abc import Callable, Container, Iterable, Mapping, Sequence, Se from contextlib import AbstractContextManager from re import Pattern from types import TracebackType -from typing import Any, AnyStr, ClassVar, Generic, NamedTuple, NoReturn, Protocol, SupportsAbs, SupportsRound, TypeVar, overload +from typing import ( + Any, + AnyStr, + ClassVar, + Final, + Generic, + NamedTuple, + NoReturn, + Protocol, + SupportsAbs, + SupportsRound, + TypeVar, + overload, +) from typing_extensions import ParamSpec, Self, TypeAlias from warnings import WarningMessage @@ -22,7 +35,7 @@ _E = TypeVar("_E", bound=BaseException) _FT = TypeVar("_FT", bound=Callable[..., Any]) _P = ParamSpec("_P") -DIFF_OMITTED: str +DIFF_OMITTED: Final[str] class _BaseTestCaseContext: test_case: TestCase diff --git a/mypy/typeshed/stdlib/unittest/loader.pyi b/mypy/typeshed/stdlib/unittest/loader.pyi index 657f3d6dca71..598e3cd84a5e 100644 --- a/mypy/typeshed/stdlib/unittest/loader.pyi +++ b/mypy/typeshed/stdlib/unittest/loader.pyi @@ -4,13 +4,13 @@ import unittest.suite from collections.abc import Callable, Sequence from re import Pattern from types import ModuleType -from typing import Any +from typing import Any, Final from typing_extensions import TypeAlias, deprecated _SortComparisonMethod: TypeAlias = Callable[[str, str], int] _SuiteClass: TypeAlias = Callable[[list[unittest.case.TestCase]], unittest.suite.TestSuite] -VALID_MODULE_NAME: Pattern[str] +VALID_MODULE_NAME: Final[Pattern[str]] class TestLoader: errors: list[type[BaseException]] diff --git a/mypy/typeshed/stdlib/unittest/main.pyi b/mypy/typeshed/stdlib/unittest/main.pyi index 3eb3d1612a3c..22f2ec10634d 100644 --- a/mypy/typeshed/stdlib/unittest/main.pyi +++ b/mypy/typeshed/stdlib/unittest/main.pyi @@ -5,11 +5,11 @@ import unittest.result import unittest.suite from collections.abc import Iterable from types import ModuleType -from typing import Any, Protocol +from typing import Any, Final, Protocol from typing_extensions import deprecated -MAIN_EXAMPLES: str -MODULE_EXAMPLES: str +MAIN_EXAMPLES: Final[str] +MODULE_EXAMPLES: Final[str] class _TestRunner(Protocol): def run(self, test: unittest.suite.TestSuite | unittest.case.TestCase, /) -> unittest.result.TestResult: ... diff --git a/mypy/typeshed/stdlib/unittest/result.pyi b/mypy/typeshed/stdlib/unittest/result.pyi index 436fabf20c65..0761baaa2830 100644 --- a/mypy/typeshed/stdlib/unittest/result.pyi +++ b/mypy/typeshed/stdlib/unittest/result.pyi @@ -2,14 +2,14 @@ import sys import unittest.case from _typeshed import OptExcInfo from collections.abc import Callable -from typing import Any, TextIO, TypeVar +from typing import Any, Final, TextIO, TypeVar from typing_extensions import TypeAlias _F = TypeVar("_F", bound=Callable[..., Any]) _DurationsType: TypeAlias = list[tuple[str, float]] -STDOUT_LINE: str -STDERR_LINE: str +STDOUT_LINE: Final[str] +STDERR_LINE: Final[str] # undocumented def failfast(method: _F) -> _F: ... diff --git a/mypy/typeshed/stdlib/unittest/util.pyi b/mypy/typeshed/stdlib/unittest/util.pyi index c42d1346e4b7..945b0cecfed0 100644 --- a/mypy/typeshed/stdlib/unittest/util.pyi +++ b/mypy/typeshed/stdlib/unittest/util.pyi @@ -1,16 +1,16 @@ from collections.abc import MutableSequence, Sequence -from typing import Any, TypeVar +from typing import Any, Final, TypeVar from typing_extensions import TypeAlias _T = TypeVar("_T") _Mismatch: TypeAlias = tuple[_T, _T, int] -_MAX_LENGTH: int -_PLACEHOLDER_LEN: int -_MIN_BEGIN_LEN: int -_MIN_END_LEN: int -_MIN_COMMON_LEN: int -_MIN_DIFF_LEN: int +_MAX_LENGTH: Final[int] +_PLACEHOLDER_LEN: Final[int] +_MIN_BEGIN_LEN: Final[int] +_MIN_END_LEN: Final[int] +_MIN_COMMON_LEN: Final[int] +_MIN_DIFF_LEN: Final[int] def _shorten(s: str, prefixlen: int, suffixlen: int) -> str: ... def _common_shorten_repr(*args: str) -> tuple[str, ...]: ... diff --git a/mypy/typeshed/stdlib/xmlrpc/client.pyi b/mypy/typeshed/stdlib/xmlrpc/client.pyi index 2be5f7df2d7d..d254102acc55 100644 --- a/mypy/typeshed/stdlib/xmlrpc/client.pyi +++ b/mypy/typeshed/stdlib/xmlrpc/client.pyi @@ -6,7 +6,7 @@ from collections.abc import Callable, Iterable, Mapping from datetime import datetime from io import BytesIO from types import TracebackType -from typing import Any, Literal, Protocol, overload +from typing import Any, Final, Literal, Protocol, overload from typing_extensions import Self, TypeAlias class _SupportsTimeTuple(Protocol): @@ -34,22 +34,22 @@ _HostType: TypeAlias = tuple[str, dict[str, str]] | str def escape(s: str) -> str: ... # undocumented -MAXINT: int # undocumented -MININT: int # undocumented +MAXINT: Final[int] # undocumented +MININT: Final[int] # undocumented -PARSE_ERROR: int # undocumented -SERVER_ERROR: int # undocumented -APPLICATION_ERROR: int # undocumented -SYSTEM_ERROR: int # undocumented -TRANSPORT_ERROR: int # undocumented +PARSE_ERROR: Final[int] # undocumented +SERVER_ERROR: Final[int] # undocumented +APPLICATION_ERROR: Final[int] # undocumented +SYSTEM_ERROR: Final[int] # undocumented +TRANSPORT_ERROR: Final[int] # undocumented -NOT_WELLFORMED_ERROR: int # undocumented -UNSUPPORTED_ENCODING: int # undocumented -INVALID_ENCODING_CHAR: int # undocumented -INVALID_XMLRPC: int # undocumented -METHOD_NOT_FOUND: int # undocumented -INVALID_METHOD_PARAMS: int # undocumented -INTERNAL_ERROR: int # undocumented +NOT_WELLFORMED_ERROR: Final[int] # undocumented +UNSUPPORTED_ENCODING: Final[int] # undocumented +INVALID_ENCODING_CHAR: Final[int] # undocumented +INVALID_XMLRPC: Final[int] # undocumented +METHOD_NOT_FOUND: Final[int] # undocumented +INVALID_METHOD_PARAMS: Final[int] # undocumented +INTERNAL_ERROR: Final[int] # undocumented class Error(Exception): ... @@ -98,7 +98,7 @@ class Binary: def _binary(data: ReadableBuffer) -> Binary: ... # undocumented -WRAPPERS: tuple[type[DateTime], type[Binary]] # undocumented +WRAPPERS: Final[tuple[type[DateTime], type[Binary]]] # undocumented class ExpatParser: # undocumented def __init__(self, target: Unmarshaller) -> None: ... diff --git a/mypy/typeshed/stdlib/zipfile/__init__.pyi b/mypy/typeshed/stdlib/zipfile/__init__.pyi index aa52a0b56e41..57a8a6aaa40a 100644 --- a/mypy/typeshed/stdlib/zipfile/__init__.pyi +++ b/mypy/typeshed/stdlib/zipfile/__init__.pyi @@ -5,7 +5,7 @@ from collections.abc import Callable, Iterable, Iterator from io import TextIOWrapper from os import PathLike from types import TracebackType -from typing import IO, Literal, Protocol, overload +from typing import IO, Final, Literal, Protocol, overload from typing_extensions import Self, TypeAlias __all__ = [ @@ -300,10 +300,10 @@ else: def is_zipfile(filename: StrOrBytesPath | _SupportsReadSeekTell) -> bool: ... -ZIP_STORED: int -ZIP_DEFLATED: int -ZIP64_LIMIT: int -ZIP_FILECOUNT_LIMIT: int -ZIP_MAX_COMMENT: int -ZIP_BZIP2: int -ZIP_LZMA: int +ZIP_STORED: Final[int] +ZIP_DEFLATED: Final[int] +ZIP64_LIMIT: Final[int] +ZIP_FILECOUNT_LIMIT: Final[int] +ZIP_MAX_COMMENT: Final[int] +ZIP_BZIP2: Final[int] +ZIP_LZMA: Final[int] From 6eb3ee55a2ea2cefccb41acf0fb520d6d404d24d Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sat, 3 Aug 2024 17:51:18 +0300 Subject: [PATCH 109/247] Fix error code handling in `stubtest` with `--mypy-config-file` (#17629) This is the first PR in the series of unifing configuration parsing for different mypy tools. I would like to have the simple and focused. --- mypy/main.py | 17 +---------------- mypy/options.py | 17 +++++++++++++++++ mypy/stubtest.py | 6 ++++++ mypy/test/teststubtest.py | 13 +++++++++++++ 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 05044335ecee..49a395b478b3 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -19,7 +19,6 @@ validate_package_allow_list, ) from mypy.error_formatter import OUTPUT_CHOICES -from mypy.errorcodes import error_codes from mypy.errors import CompileError from mypy.find_sources import InvalidSourceList, create_source_list from mypy.fscache import FileSystemCache @@ -1336,21 +1335,7 @@ def set_strict_flags() -> None: validate_package_allow_list(options.untyped_calls_exclude) - # Process `--enable-error-code` and `--disable-error-code` flags - disabled_codes = set(options.disable_error_code) - enabled_codes = set(options.enable_error_code) - - valid_error_codes = set(error_codes.keys()) - - invalid_codes = (enabled_codes | disabled_codes) - valid_error_codes - if invalid_codes: - parser.error(f"Invalid error code(s): {', '.join(sorted(invalid_codes))}") - - options.disabled_error_codes |= {error_codes[code] for code in disabled_codes} - options.enabled_error_codes |= {error_codes[code] for code in enabled_codes} - - # Enabling an error code always overrides disabling - options.disabled_error_codes -= options.enabled_error_codes + options.process_error_codes(error_callback=parser.error) # Validate incomplete features. for feature in options.enable_incomplete_feature: diff --git a/mypy/options.py b/mypy/options.py index bff096d82c15..5ab397e0e156 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -416,6 +416,23 @@ def snapshot(self) -> dict[str, object]: def __repr__(self) -> str: return f"Options({pprint.pformat(self.snapshot())})" + def process_error_codes(self, *, error_callback: Callable[[str], Any]) -> None: + # Process `--enable-error-code` and `--disable-error-code` flags + disabled_codes = set(self.disable_error_code) + enabled_codes = set(self.enable_error_code) + + valid_error_codes = set(error_codes.keys()) + + invalid_codes = (enabled_codes | disabled_codes) - valid_error_codes + if invalid_codes: + error_callback(f"Invalid error code(s): {', '.join(sorted(invalid_codes))}") + + self.disabled_error_codes |= {error_codes[code] for code in disabled_codes} + self.enabled_error_codes |= {error_codes[code] for code in enabled_codes} + + # Enabling an error code always overrides disabling + self.disabled_error_codes -= self.enabled_error_codes + def apply_changes(self, changes: dict[str, object]) -> Options: # Note: effects of this method *must* be idempotent. new_options = Options() diff --git a/mypy/stubtest.py b/mypy/stubtest.py index a7cde8b8fe6c..6299f21e48e9 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1941,6 +1941,12 @@ def set_strict_flags() -> None: # not needed yet parse_config_file(options, set_strict_flags, options.config_file, sys.stdout, sys.stderr) + def error_callback(msg: str) -> typing.NoReturn: + print(_style("error:", color="red", bold=True), msg) + sys.exit(1) + + options.process_error_codes(error_callback=error_callback) + try: modules = build_stubs(modules, options, find_submodules=not args.check_typeshed) except StubtestFailure as stubtest_failure: diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 418308e2e65e..1cc6c38e6e85 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -2477,6 +2477,19 @@ def test_config_file(self) -> None: output = run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file) assert output == "Success: no issues found in 1 module\n" + def test_config_file_error_codes(self) -> None: + runtime = "temp = 5\n" + stub = "temp = SOME_GLOBAL_CONST" + output = run_stubtest(stub=stub, runtime=runtime, options=[]) + assert output == ( + "error: not checking stubs due to mypy build errors:\n" + 'test_module.pyi:1: error: Name "SOME_GLOBAL_CONST" is not defined [name-defined]\n' + ) + + config_file = "[mypy]\ndisable_error_code = name-defined\n" + output = run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file) + assert output == "Success: no issues found in 1 module\n" + def test_no_modules(self) -> None: output = io.StringIO() with contextlib.redirect_stdout(output): From fe4df128e9cb76cc694494e9b0cd129d4aaa6ebd Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 4 Aug 2024 12:18:47 +0300 Subject: [PATCH 110/247] Fix `stubgen --no-analysis/--parse-only` docs (#17632) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The option is called `--inspect-mode`, not `--inspect` It used to be: ``` » stubgen --help usage: stubgen [-h] [more options, see -h] [-m MODULE] [-p PACKAGE] [files ...] Generate draft stubs for modules. Stubs are generated in directory ./out, to avoid overriding files with manual changes. This directory is assumed to exist. positional arguments: files generate stubs for given files or directories options: --no-analysis, --parse-only don't perform semantic analysis of sources, just parse them (only applies to Python modules, might affect quality of stubs. Not compatible with --inspect) --inspect-mode import and inspect modules instead of parsing source code.This is the default behavior for c modules and pyc-only packages, but it is also useful for pure python modules with dynamically generated members. ``` --- mypy/stubgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 8478bd2135e4..4e6e3efc000b 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -1772,7 +1772,7 @@ def parse_options(args: list[str]) -> Options: action="store_true", help="don't perform semantic analysis of sources, just parse them " "(only applies to Python modules, might affect quality of stubs. " - "Not compatible with --inspect)", + "Not compatible with --inspect-mode)", ) parser.add_argument( "--inspect-mode", From 2d55435606002f446e48884d165f1d2ffe2cb5c6 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 4 Aug 2024 14:15:53 +0300 Subject: [PATCH 111/247] List all incomplete features in `--enable-incomplete-feature` docs (#17633) It was strange to see only one feature: ```rst .. option:: --enable-incomplete-feature {PreciseTupleTypes} ``` --- docs/source/command_line.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 906231dc7e42..c085b63107b0 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1008,7 +1008,7 @@ format into the specified directory. Enabling incomplete/experimental features ***************************************** -.. option:: --enable-incomplete-feature {PreciseTupleTypes} +.. option:: --enable-incomplete-feature {PreciseTupleTypes, NewGenericSyntax, InlineTypedDict} Some features may require several mypy releases to implement, for example due to their complexity, potential for backwards incompatibility, or From 41dcf1ac5be5d1d9b46d20b97bae101e30735f44 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 4 Aug 2024 12:22:29 +0100 Subject: [PATCH 112/247] Revert "Fix `RawExpressionType.accept` crash with `--cache-fine-grained`" (#17637) Reverts python/mypy#17588 --- mypy/types.py | 2 -- test-data/unit/check-typeddict.test | 12 ------------ 2 files changed, 14 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index c29d686ce5d8..52b3121f9fb3 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2705,8 +2705,6 @@ def simple_name(self) -> str: return self.base_type_name.replace("builtins.", "") def accept(self, visitor: TypeVisitor[T]) -> T: - if self.node is not None: - return self.node.accept(visitor) assert isinstance(visitor, SyntheticTypeVisitor) ret: T = visitor.visit_raw_expression_type(self) return ret diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 1ef08f825e7a..6a5120159c2d 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1442,18 +1442,6 @@ reveal_type(x) # N: Revealed type is "TypedDict('__main__.X', {'a': TypedDict('_ reveal_type(x['a']['b']) # N: Revealed type is "builtins.int" [builtins fixtures/dict.pyi] -[case testTypedDictForwardReferenceCacheFineGrained] -# flags: --cache-fine-grained -from mypy_extensions import TypedDict -class A(TypedDict): - b: "B" -class B(TypedDict): - c: "C" -class C(TypedDict): - d: "D" -class D: - pass - [case testSelfRecursiveTypedDictInheriting] from mypy_extensions import TypedDict From bc39f1714e2e5f39eb90f05f74575a7c397f068b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 4 Aug 2024 12:50:05 +0100 Subject: [PATCH 113/247] Revert "Fix Literal strings containing pipe characters" (#17638) Reverts python/mypy#17148 --- mypy/fastparse.py | 11 ++- mypy/semanal.py | 31 ++++---- mypy/server/astmerge.py | 3 +- mypy/stubutil.py | 16 +--- mypy/type_visitor.py | 4 - mypy/typeanal.py | 21 ++++-- mypy/types.py | 75 +++++++++++-------- mypy/typetraverser.py | 3 +- mypyc/irbuild/classdef.py | 9 ++- test-data/unit/check-final.test | 2 - test-data/unit/check-literal.test | 4 - test-data/unit/check-namedtuple.test | 8 +- .../unit/check-parameter-specification.test | 23 +----- test-data/unit/check-typeguard.test | 11 --- test-data/unit/check-typeis.test | 11 --- 15 files changed, 90 insertions(+), 142 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 363fc8375259..ab7eef924bd1 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -331,7 +331,14 @@ def parse_type_string( """ try: _, node = parse_type_comment(f"({expr_string})", line=line, column=column, errors=None) - return RawExpressionType(expr_string, expr_fallback_name, line, column, node=node) + if isinstance(node, UnboundType) and node.original_str_expr is None: + node.original_str_expr = expr_string + node.original_str_fallback = expr_fallback_name + return node + elif isinstance(node, UnionType): + return node + else: + return RawExpressionType(expr_string, expr_fallback_name, line, column) except (SyntaxError, ValueError): # Note: the parser will raise a `ValueError` instead of a SyntaxError if # the string happens to contain things like \x00. @@ -1057,8 +1064,6 @@ def set_type_optional(self, type: Type | None, initializer: Expression | None) - return # Indicate that type should be wrapped in an Optional if arg is initialized to None. optional = isinstance(initializer, NameExpr) and initializer.name == "None" - if isinstance(type, RawExpressionType) and type.node is not None: - type = type.node if isinstance(type, UnboundType): type.optional = optional diff --git a/mypy/semanal.py b/mypy/semanal.py index f36149076fe6..782985e3fbab 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3437,10 +3437,10 @@ def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: def analyze_lvalues(self, s: AssignmentStmt) -> None: # We cannot use s.type, because analyze_simple_literal_type() will set it. explicit = s.unanalyzed_type is not None - final_type = self.unwrap_final_type(s.unanalyzed_type) - if final_type is not None: + if self.is_final_type(s.unanalyzed_type): # We need to exclude bare Final. - if not final_type.args: + assert isinstance(s.unanalyzed_type, UnboundType) + if not s.unanalyzed_type.args: explicit = False if s.rvalue: @@ -3506,19 +3506,19 @@ def unwrap_final(self, s: AssignmentStmt) -> bool: Returns True if Final[...] was present. """ - final_type = self.unwrap_final_type(s.unanalyzed_type) - if final_type is None: + if not s.unanalyzed_type or not self.is_final_type(s.unanalyzed_type): return False - if len(final_type.args) > 1: - self.fail("Final[...] takes at most one type argument", final_type) + assert isinstance(s.unanalyzed_type, UnboundType) + if len(s.unanalyzed_type.args) > 1: + self.fail("Final[...] takes at most one type argument", s.unanalyzed_type) invalid_bare_final = False - if not final_type.args: + if not s.unanalyzed_type.args: s.type = None if isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs: invalid_bare_final = True self.fail("Type in Final[...] can only be omitted if there is an initializer", s) else: - s.type = final_type.args[0] + s.type = s.unanalyzed_type.args[0] if s.type is not None and self.is_classvar(s.type): self.fail("Variable should not be annotated with both ClassVar and Final", s) @@ -4937,18 +4937,13 @@ def is_classvar(self, typ: Type) -> bool: return False return sym.node.fullname == "typing.ClassVar" - def unwrap_final_type(self, typ: Type | None) -> UnboundType | None: - if typ is None: - return None - typ = typ.resolve_string_annotation() + def is_final_type(self, typ: Type | None) -> bool: if not isinstance(typ, UnboundType): - return None + return False sym = self.lookup_qualified(typ.name, typ) if not sym or not sym.node: - return None - if sym.node.fullname in FINAL_TYPE_NAMES: - return typ - return None + return False + return sym.node.fullname in FINAL_TYPE_NAMES def fail_invalid_classvar(self, context: Context) -> None: self.fail(message_registry.CLASS_VAR_OUTSIDE_OF_CLASS, context) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index e6648fbb4be7..174c2922c767 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -507,8 +507,7 @@ def visit_typeddict_type(self, typ: TypedDictType) -> None: typ.fallback.accept(self) def visit_raw_expression_type(self, t: RawExpressionType) -> None: - if t.node is not None: - t.node.accept(self) + pass def visit_literal_type(self, typ: LiteralType) -> None: typ.fallback.accept(self) diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 2f2db0dbbe53..04b36e149957 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -17,16 +17,7 @@ from mypy.modulefinder import ModuleNotFoundReason from mypy.moduleinspect import InspectError, ModuleInspect from mypy.stubdoc import ArgSig, FunctionSig -from mypy.types import ( - AnyType, - NoneType, - RawExpressionType, - Type, - TypeList, - TypeStrVisitor, - UnboundType, - UnionType, -) +from mypy.types import AnyType, NoneType, Type, TypeList, TypeStrVisitor, UnboundType, UnionType # Modules that may fail when imported, or that may have side effects (fully qualified). NOT_IMPORTABLE_MODULES = () @@ -302,11 +293,12 @@ def args_str(self, args: Iterable[Type]) -> str: The main difference from list_str is the preservation of quotes for string arguments """ + types = ["builtins.bytes", "builtins.str"] res = [] for arg in args: arg_str = arg.accept(self) - if isinstance(arg, RawExpressionType): - res.append(repr(arg.literal_value)) + if isinstance(arg, UnboundType) and arg.original_str_fallback in types: + res.append(f"'{arg_str}'") else: res.append(arg_str) return ", ".join(res) diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index e685c49904bc..59e13d12485c 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -382,8 +382,6 @@ def visit_typeddict_type(self, t: TypedDictType) -> T: return self.query_types(t.items.values()) def visit_raw_expression_type(self, t: RawExpressionType) -> T: - if t.node is not None: - return t.node.accept(self) return self.strategy([]) def visit_literal_type(self, t: LiteralType) -> T: @@ -524,8 +522,6 @@ def visit_typeddict_type(self, t: TypedDictType) -> bool: return self.query_types(list(t.items.values())) def visit_raw_expression_type(self, t: RawExpressionType) -> bool: - if t.node is not None: - return t.node.accept(self) return self.default def visit_literal_type(self, t: LiteralType) -> bool: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index f63aef30a09a..f88c3f91d1c6 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1112,7 +1112,6 @@ def visit_callable_type( return ret def anal_type_guard(self, t: Type) -> Type | None: - t = t.resolve_string_annotation() if isinstance(t, UnboundType): sym = self.lookup_qualified(t.name, t) if sym is not None and sym.node is not None: @@ -1131,7 +1130,6 @@ def anal_type_guard_arg(self, t: UnboundType, fullname: str) -> Type | None: return None def anal_type_is(self, t: Type) -> Type | None: - t = t.resolve_string_annotation() if isinstance(t, UnboundType): sym = self.lookup_qualified(t.name, t) if sym is not None and sym.node is not None: @@ -1149,7 +1147,6 @@ def anal_type_is_arg(self, t: UnboundType, fullname: str) -> Type | None: def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type: """Analyze signature argument type for *args and **kwargs argument.""" - t = t.resolve_string_annotation() if isinstance(t, UnboundType) and t.name and "." in t.name and not t.args: components = t.name.split(".") tvar_name = ".".join(components[:-1]) @@ -1275,8 +1272,6 @@ def visit_raw_expression_type(self, t: RawExpressionType) -> Type: # make signatures like "foo(x: 20) -> None" legal, we can change # this method so it generates and returns an actual LiteralType # instead. - if t.node is not None: - return t.node.accept(self) if self.report_invalid_types: if t.base_type_name in ("builtins.int", "builtins.bool"): @@ -1539,7 +1534,6 @@ def analyze_callable_args( invalid_unpacks: list[Type] = [] second_unpack_last = False for i, arg in enumerate(arglist.items): - arg = arg.resolve_string_annotation() if isinstance(arg, CallableArgument): args.append(arg.typ) names.append(arg.name) @@ -1620,6 +1614,18 @@ def analyze_literal_type(self, t: UnboundType) -> Type: return UnionType.make_union(output, line=t.line) def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> list[Type] | None: + # This UnboundType was originally defined as a string. + if isinstance(arg, UnboundType) and arg.original_str_expr is not None: + assert arg.original_str_fallback is not None + return [ + LiteralType( + value=arg.original_str_expr, + fallback=self.named_type(arg.original_str_fallback), + line=arg.line, + column=arg.column, + ) + ] + # If arg is an UnboundType that was *not* originally defined as # a string, try expanding it in case it's a type alias or something. if isinstance(arg, UnboundType): @@ -2605,8 +2611,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> None: self.process_types(list(t.items.values())) def visit_raw_expression_type(self, t: RawExpressionType) -> None: - if t.node is not None: - t.node.accept(self) + pass def visit_literal_type(self, t: LiteralType) -> None: pass diff --git a/mypy/types.py b/mypy/types.py index 52b3121f9fb3..72374c1555c9 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -271,9 +271,6 @@ def can_be_true_default(self) -> bool: def can_be_false_default(self) -> bool: return True - def resolve_string_annotation(self) -> Type: - return self - def accept(self, visitor: TypeVisitor[T]) -> T: raise RuntimeError("Not implemented", type(self)) @@ -906,7 +903,14 @@ def copy_modified( class UnboundType(ProperType): """Instance type that has not been bound during semantic analysis.""" - __slots__ = ("name", "args", "optional", "empty_tuple_index") + __slots__ = ( + "name", + "args", + "optional", + "empty_tuple_index", + "original_str_expr", + "original_str_fallback", + ) def __init__( self, @@ -916,6 +920,8 @@ def __init__( column: int = -1, optional: bool = False, empty_tuple_index: bool = False, + original_str_expr: str | None = None, + original_str_fallback: str | None = None, ) -> None: super().__init__(line, column) if not args: @@ -927,6 +933,21 @@ def __init__( self.optional = optional # Special case for X[()] self.empty_tuple_index = empty_tuple_index + # If this UnboundType was originally defined as a str or bytes, keep track of + # the original contents of that string-like thing. This way, if this UnboundExpr + # ever shows up inside of a LiteralType, we can determine whether that + # Literal[...] is valid or not. E.g. Literal[foo] is most likely invalid + # (unless 'foo' is an alias for another literal or something) and + # Literal["foo"] most likely is. + # + # We keep track of the entire string instead of just using a boolean flag + # so we can distinguish between things like Literal["foo"] vs + # Literal[" foo "]. + # + # We also keep track of what the original base fallback type was supposed to be + # so we don't have to try and recompute it later + self.original_str_expr = original_str_expr + self.original_str_fallback = original_str_fallback def copy_modified(self, args: Bogus[Sequence[Type] | None] = _dummy) -> UnboundType: if args is _dummy: @@ -938,19 +959,25 @@ def copy_modified(self, args: Bogus[Sequence[Type] | None] = _dummy) -> UnboundT column=self.column, optional=self.optional, empty_tuple_index=self.empty_tuple_index, + original_str_expr=self.original_str_expr, + original_str_fallback=self.original_str_fallback, ) def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_unbound_type(self) def __hash__(self) -> int: - return hash((self.name, self.optional, tuple(self.args))) + return hash((self.name, self.optional, tuple(self.args), self.original_str_expr)) def __eq__(self, other: object) -> bool: if not isinstance(other, UnboundType): return NotImplemented return ( - self.name == other.name and self.optional == other.optional and self.args == other.args + self.name == other.name + and self.optional == other.optional + and self.args == other.args + and self.original_str_expr == other.original_str_expr + and self.original_str_fallback == other.original_str_fallback ) def serialize(self) -> JsonDict: @@ -958,12 +985,19 @@ def serialize(self) -> JsonDict: ".class": "UnboundType", "name": self.name, "args": [a.serialize() for a in self.args], + "expr": self.original_str_expr, + "expr_fallback": self.original_str_fallback, } @classmethod def deserialize(cls, data: JsonDict) -> UnboundType: assert data[".class"] == "UnboundType" - return UnboundType(data["name"], [deserialize_type(a) for a in data["args"]]) + return UnboundType( + data["name"], + [deserialize_type(a) for a in data["args"]], + original_str_expr=data["expr"], + original_str_fallback=data["expr_fallback"], + ) class CallableArgument(ProperType): @@ -2646,7 +2680,7 @@ class RawExpressionType(ProperType): This synthetic type is only used at the beginning stages of semantic analysis and should be completely removing during the process for mapping UnboundTypes to - actual types: we turn it into its "node" argument, a LiteralType, or an AnyType. + actual types: we either turn it into a LiteralType or an AnyType. For example, suppose `Foo[1]` is initially represented as the following: @@ -2684,7 +2718,7 @@ class RawExpressionType(ProperType): ) """ - __slots__ = ("literal_value", "base_type_name", "note", "node") + __slots__ = ("literal_value", "base_type_name", "note") def __init__( self, @@ -2693,13 +2727,11 @@ def __init__( line: int = -1, column: int = -1, note: str | None = None, - node: Type | None = None, ) -> None: super().__init__(line, column) self.literal_value = literal_value self.base_type_name = base_type_name self.note = note - self.node = node def simple_name(self) -> str: return self.base_type_name.replace("builtins.", "") @@ -2709,21 +2741,6 @@ def accept(self, visitor: TypeVisitor[T]) -> T: ret: T = visitor.visit_raw_expression_type(self) return ret - def copy_modified(self, node: Type | None) -> RawExpressionType: - return RawExpressionType( - literal_value=self.literal_value, - base_type_name=self.base_type_name, - line=self.line, - column=self.column, - note=self.note, - node=node, - ) - - def resolve_string_annotation(self) -> Type: - if self.node is not None: - return self.node.resolve_string_annotation() - return self - def serialize(self) -> JsonDict: assert False, "Synthetic types don't serialize" @@ -2735,7 +2752,6 @@ def __eq__(self, other: object) -> bool: return ( self.base_type_name == other.base_type_name and self.literal_value == other.literal_value - and self.node == other.node ) else: return NotImplemented @@ -3411,8 +3427,6 @@ def item_str(name: str, typ: str) -> str: return f"TypedDict({prefix}{s})" def visit_raw_expression_type(self, t: RawExpressionType) -> str: - if t.node is not None: - return t.node.accept(self) return repr(t.literal_value) def visit_literal_type(self, t: LiteralType) -> str: @@ -3476,9 +3490,6 @@ def visit_ellipsis_type(self, t: EllipsisType) -> Type: return t def visit_raw_expression_type(self, t: RawExpressionType) -> Type: - if t.node is not None: - node = t.node.accept(self) - return t.copy_modified(node=node) return t def visit_type_list(self, t: TypeList) -> Type: diff --git a/mypy/typetraverser.py b/mypy/typetraverser.py index 4d740a802b55..a28bbf422b61 100644 --- a/mypy/typetraverser.py +++ b/mypy/typetraverser.py @@ -130,8 +130,7 @@ def visit_partial_type(self, t: PartialType) -> None: pass def visit_raw_expression_type(self, t: RawExpressionType) -> None: - if t.node is not None: - t.node.accept(self) + pass def visit_type_alias_type(self, t: TypeAliasType) -> None: # TODO: sometimes we want to traverse target as well diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 2152da099e81..7e0a842b1b41 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -26,7 +26,7 @@ TypeParam, is_class_var, ) -from mypy.types import ENUM_REMOVED_PROPS, Instance, RawExpressionType, get_proper_type +from mypy.types import ENUM_REMOVED_PROPS, Instance, UnboundType, get_proper_type from mypyc.common import PROPSET_PREFIX from mypyc.ir.class_ir import ClassIR, NonExtClassInfo from mypyc.ir.func_ir import FuncDecl, FuncSignature @@ -640,15 +640,16 @@ def add_non_ext_class_attr_ann( if typ is None: # FIXME: if get_type_info is not provided, don't fall back to stmt.type? ann_type = get_proper_type(stmt.type) - if isinstance(stmt.unanalyzed_type, RawExpressionType) and isinstance( - stmt.unanalyzed_type.literal_value, str + if ( + isinstance(stmt.unanalyzed_type, UnboundType) + and stmt.unanalyzed_type.original_str_expr is not None ): # Annotation is a forward reference, so don't attempt to load the actual # type and load the string instead. # # TODO: is it possible to determine whether a non-string annotation is # actually a forward reference due to the __annotations__ future? - typ = builder.load_str(stmt.unanalyzed_type.literal_value) + typ = builder.load_str(stmt.unanalyzed_type.original_str_expr) elif isinstance(ann_type, Instance): typ = load_type(builder, ann_type.type, stmt.line) else: diff --git a/test-data/unit/check-final.test b/test-data/unit/check-final.test index dadf76a283b0..763183159e94 100644 --- a/test-data/unit/check-final.test +++ b/test-data/unit/check-final.test @@ -6,13 +6,11 @@ [case testFinalDefiningModuleVar] from typing import Final -w: 'Final' = int() x: Final = int() y: Final[float] = int() z: Final[int] = int() bad: Final[str] = int() # E: Incompatible types in assignment (expression has type "int", variable has type "str") -reveal_type(w) # N: Revealed type is "builtins.int" reveal_type(x) # N: Revealed type is "builtins.int" reveal_type(y) # N: Revealed type is "builtins.float" reveal_type(z) # N: Revealed type is "builtins.int" diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index 6d76ce176aaf..e8d942f5d74d 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -12,12 +12,8 @@ reveal_type(g1) # N: Revealed type is "def (x: Literal['A['])" def f2(x: 'A B') -> None: pass # E: Invalid type comment or annotation def g2(x: Literal['A B']) -> None: pass -def h2(x: 'A|int') -> None: pass # E: Name "A" is not defined -def i2(x: Literal['A|B']) -> None: pass reveal_type(f2) # N: Revealed type is "def (x: Any)" reveal_type(g2) # N: Revealed type is "def (x: Literal['A B'])" -reveal_type(h2) # N: Revealed type is "def (x: Union[Any, builtins.int])" -reveal_type(i2) # N: Revealed type is "def (x: Literal['A|B'])" [builtins fixtures/tuple.pyi] [out] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 147270dff72e..df2c7ffc8067 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -824,20 +824,14 @@ class Fraction(Real): [builtins fixtures/tuple.pyi] [case testForwardReferenceInNamedTuple] -from typing import List, NamedTuple +from typing import NamedTuple class A(NamedTuple): b: 'B' x: int - y: List['B'] class B: pass - -def f(a: A): - reveal_type(a.b) # N: Revealed type is "__main__.B" - reveal_type(a.x) # N: Revealed type is "builtins.int" - reveal_type(a.y) # N: Revealed type is "builtins.list[__main__.B]" [builtins fixtures/tuple.pyi] [case testTypeNamedTupleClassmethod] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index e6d8cec3f0b0..c2afb61586a8 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1193,28 +1193,7 @@ def func(callback: Callable[P, str]) -> Callable[P, str]: return inner [builtins fixtures/paramspec.pyi] -[case testParamSpecArgsAndKwargsStringified] -from typing import Callable -from typing_extensions import ParamSpec - -P1 = ParamSpec("P1") - -def func(callback: Callable[P1, str]) -> Callable[P1, str]: - def inner(*args: "P1.args", **kwargs: "P1.kwargs") -> str: - return "foo" - return inner - -@func -def outer(a: int) -> str: - return "" - -outer(1) # OK -outer("x") # E: Argument 1 to "outer" has incompatible type "str"; expected "int" -outer(a=1) # OK -outer(b=1) # E: Unexpected keyword argument "b" for "outer" -[builtins fixtures/paramspec.pyi] - -[case testParamSpecArgsAndKwargsMismatch] +[case testParamSpecArgsAndKwargsMissmatch] from typing import Callable from typing_extensions import ParamSpec diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index e1b7a86aba63..27b88553fb43 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -9,17 +9,6 @@ def main(a: object) -> None: reveal_type(a) # N: Revealed type is "builtins.object" [builtins fixtures/tuple.pyi] -[case testTypeGuardStringified] -from typing_extensions import TypeGuard -class Point: pass -def is_point(a: object) -> "TypeGuard[Point]": pass -def main(a: object) -> None: - if is_point(a): - reveal_type(a) # N: Revealed type is "__main__.Point" - else: - reveal_type(a) # N: Revealed type is "builtins.object" -[builtins fixtures/tuple.pyi] - [case testTypeGuardTypeArgsNone] from typing_extensions import TypeGuard def foo(a: object) -> TypeGuard: # E: TypeGuard must have exactly one type argument diff --git a/test-data/unit/check-typeis.test b/test-data/unit/check-typeis.test index 83467d5e3683..6b96845504ab 100644 --- a/test-data/unit/check-typeis.test +++ b/test-data/unit/check-typeis.test @@ -9,17 +9,6 @@ def main(a: object) -> None: reveal_type(a) # N: Revealed type is "builtins.object" [builtins fixtures/tuple.pyi] -[case testTypeIsStringified] -from typing_extensions import TypeIs -class Point: pass -def is_point(a: object) -> "TypeIs[Point]": pass -def main(a: object) -> None: - if is_point(a): - reveal_type(a) # N: Revealed type is "__main__.Point" - else: - reveal_type(a) # N: Revealed type is "builtins.object" -[builtins fixtures/tuple.pyi] - [case testTypeIsElif] from typing_extensions import TypeIs from typing import Union From b56f357a45838c204c951deeb6cbdbf143258e0c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 4 Aug 2024 17:56:13 +0100 Subject: [PATCH 114/247] Unwrap TypedDict item types before storing (#17640) Fixes https://github.com/python/mypy/issues/17604 Fixes https://github.com/python/mypy/issues/17608 Fix is trivial, rectify an obvious omission in my original PR. --- mypy/semanal_typeddict.py | 4 +++- test-data/unit/check-typeddict.test | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index e639871364ce..832530df55e5 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -324,7 +324,9 @@ def analyze_typeddict_classdef_fields( return None, [], [], set() # Need to defer types.append(analyzed) if not has_placeholder(analyzed): - stmt.type = analyzed + stmt.type = ( + analyzed.item if isinstance(analyzed, RequiredType) else analyzed + ) # ...despite possible minor failures that allow further analysis. if stmt.type is None or hasattr(stmt, "new_syntax") and not stmt.new_syntax: self.fail(TPDICT_CLASS_ERROR, stmt) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 6a5120159c2d..bfc7b6394d52 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -2382,6 +2382,14 @@ class ForceDeferredEval: pass [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] +[case testTypedDictRequiredUnimportedAny] +# flags: --disallow-any-unimported +from typing import NotRequired, TypedDict +from nonexistent import Foo # type: ignore[import-not-found] +class Bar(TypedDict): + foo: NotRequired[Foo] # E: Type of variable becomes "Any" due to an unfollowed import +[typing fixtures/typing-typeddict.pyi] + -- Required[] [case testDoesRecognizeRequiredInTypedDictWithClass] From 6f20721ce2fa1c7d3bbef42e3eaf953b9f0868f9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 4 Aug 2024 19:10:57 +0100 Subject: [PATCH 115/247] Fix crash on a callable attribute with single unpack (#17641) Fixes https://github.com/python/mypy/issues/17518 --- mypy/typeops.py | 6 ++++-- test-data/unit/check-typeddict.test | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index 4fe187f811ca..036d782be89c 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -310,12 +310,14 @@ class B(A): pass if not func.arg_types: # Invalid method, return something. return cast(F, func) - if func.arg_kinds[0] == ARG_STAR: + if func.arg_kinds[0] in (ARG_STAR, ARG_STAR2): # The signature is of the form 'def foo(*args, ...)'. # In this case we shouldn't drop the first arg, # since func will be absorbed by the *args. - # TODO: infer bounds on the type of *args? + + # In the case of **kwargs we should probably emit an error, but + # for now we simply skip it, to avoid crashes down the line. return cast(F, func) self_param_type = get_proper_type(func.arg_types[0]) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index bfc7b6394d52..dc1929751977 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -3579,6 +3579,24 @@ class Test: run(test2, other=0, **params) # E: Argument "other" to "run" has incompatible type "int"; expected "str" [builtins fixtures/tuple.pyi] +[case testTypedDictUnpackSingleWithSubtypingNoCrash] +from typing import Callable +from typing_extensions import TypedDict, Unpack + +class Kwargs(TypedDict): + name: str + +def f(**kwargs: Unpack[Kwargs]) -> None: + pass + +class C: + d: Callable[[Unpack[Kwargs]], None] + +# TODO: it is an old question whether we should allow this, for now simply don't crash. +class D(C): + d = f +[builtins fixtures/tuple.pyi] + [case testTypedDictInlineNoOldStyleAlias] # flags: --enable-incomplete-feature=InlineTypedDict X = {"int": int, "str": str} From 50d0d047eb3dedf3b801c3731a37d56c320bad3e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 4 Aug 2024 22:27:05 +0100 Subject: [PATCH 116/247] Always reset binder when checking deferred nodes (#17643) Fixes https://github.com/python/mypy/issues/17595 --- mypy/checker.py | 9 +++---- test-data/unit/check-functions.test | 37 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 59de599006a8..82633a5b6c0e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -540,10 +540,11 @@ def check_partial(self, node: DeferredNodeType | FineGrainedDeferredNodeType) -> self.check_top_level(node) else: self.recurse_into_functions = True - if isinstance(node, LambdaExpr): - self.expr_checker.accept(node) - else: - self.accept(node) + with self.binder.top_frame_context(): + if isinstance(node, LambdaExpr): + self.expr_checker.accept(node) + else: + self.accept(node) def check_top_level(self, node: MypyFile) -> None: """Check only the top-level of a module, skipping function definitions.""" diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index ef5a66b6ecde..b681c544a6bd 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -3409,3 +3409,40 @@ for factory in ( reveal_type(factory) # N: Revealed type is "def () -> Union[builtins.str, None]" var = factory() [builtins fixtures/tuple.pyi] + +[case testLambdaInDeferredDecoratorNoCrash] +def foo(x): + pass + +class Bar: + def baz(self, x): + pass + +class Qux(Bar): + @foo(lambda x: None) + def baz(self, x) -> None: + pass +[builtins fixtures/tuple.pyi] + +[case testGeneratorInDeferredDecoratorNoCrash] +from typing import Protocol, TypeVar + +T = TypeVar("T", covariant=True) + +class SupportsNext(Protocol[T]): + def __next__(self) -> T: ... + +def next(i: SupportsNext[T]) -> T: ... + +def foo(x): + pass + +class Bar: + def baz(self, x): + pass + +class Qux(Bar): + @next(f for f in [foo]) + def baz(self, x) -> None: + pass +[builtins fixtures/tuple.pyi] From fc5e1ffa4e4137b3483e0ed6052dd445604cc7fc Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 5 Aug 2024 09:50:28 +0300 Subject: [PATCH 117/247] Fix typo in error_code_list.rst (#17645) --- docs/source/error_code_list.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index 64d9a1d03287..85c8d437a856 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -1184,7 +1184,7 @@ comment: .. code-block:: python - class MyClass + class MyClass: @special # type: ignore[prop-decorator] @property def magic(self) -> str: From 9d56820a63d5b47b53ff0ab6e96ce0951c774ae5 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 5 Aug 2024 09:50:56 +0300 Subject: [PATCH 118/247] Add `enable_incomplete_feature` validation to `stubtest` (#17635) Closes #17634 Refs #17628 Refs #17629 --- mypy/main.py | 10 ++-------- mypy/options.py | 10 ++++++++++ mypy/stubtest.py | 6 ++++++ mypy/test/teststubtest.py | 14 ++++++++++++++ 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 49a395b478b3..f177bb1c2062 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -23,7 +23,7 @@ from mypy.find_sources import InvalidSourceList, create_source_list from mypy.fscache import FileSystemCache from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, get_search_dirs, mypy_path -from mypy.options import COMPLETE_FEATURES, INCOMPLETE_FEATURES, BuildType, Options +from mypy.options import INCOMPLETE_FEATURES, BuildType, Options from mypy.split_namespace import SplitNamespace from mypy.version import __version__ @@ -1336,13 +1336,7 @@ def set_strict_flags() -> None: validate_package_allow_list(options.untyped_calls_exclude) options.process_error_codes(error_callback=parser.error) - - # Validate incomplete features. - for feature in options.enable_incomplete_feature: - if feature not in INCOMPLETE_FEATURES | COMPLETE_FEATURES: - parser.error(f"Unknown incomplete feature: {feature}") - if feature in COMPLETE_FEATURES: - print(f"Warning: {feature} is already enabled by default") + options.process_incomplete_features(error_callback=parser.error, warning_callback=print) # Compute absolute path for custom typeshed (if present). if options.custom_typeshed_dir is not None: diff --git a/mypy/options.py b/mypy/options.py index 5ab397e0e156..5e64d5e40035 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -433,6 +433,16 @@ def process_error_codes(self, *, error_callback: Callable[[str], Any]) -> None: # Enabling an error code always overrides disabling self.disabled_error_codes -= self.enabled_error_codes + def process_incomplete_features( + self, *, error_callback: Callable[[str], Any], warning_callback: Callable[[str], Any] + ) -> None: + # Validate incomplete features. + for feature in self.enable_incomplete_feature: + if feature not in INCOMPLETE_FEATURES | COMPLETE_FEATURES: + error_callback(f"Unknown incomplete feature: {feature}") + if feature in COMPLETE_FEATURES: + warning_callback(f"Warning: {feature} is already enabled by default") + def apply_changes(self, changes: dict[str, object]) -> Options: # Note: effects of this method *must* be idempotent. new_options = Options() diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 6299f21e48e9..ca17ccfe2f5b 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1945,7 +1945,13 @@ def error_callback(msg: str) -> typing.NoReturn: print(_style("error:", color="red", bold=True), msg) sys.exit(1) + def warning_callback(msg: str) -> None: + print(_style("warning:", color="yellow", bold=True), msg) + options.process_error_codes(error_callback=error_callback) + options.process_incomplete_features( + error_callback=error_callback, warning_callback=warning_callback + ) try: modules = build_stubs(modules, options, find_submodules=not args.check_typeshed) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 1cc6c38e6e85..c10c683ffdac 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -2490,6 +2490,20 @@ def test_config_file_error_codes(self) -> None: output = run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file) assert output == "Success: no issues found in 1 module\n" + def test_config_file_wrong_incomplete_feature(self) -> None: + runtime = "x = 1\n" + stub = "x: int\n" + config_file = "[mypy]\nenable_incomplete_feature = Unpack\n" + output = run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file) + assert output == ( + "warning: Warning: Unpack is already enabled by default\n" + "Success: no issues found in 1 module\n" + ) + + config_file = "[mypy]\nenable_incomplete_feature = not-a-valid-name\n" + with self.assertRaises(SystemExit): + run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file) + def test_no_modules(self) -> None: output = io.StringIO() with contextlib.redirect_stdout(output): From 3f8c6cbf2d1d174065971d478e61144ed9c7a225 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 10 Aug 2024 13:33:11 -0700 Subject: [PATCH 119/247] Resolve TypeVar upper bounds in functools.partial (#17660) Mostly fixes #17646 --- mypy/checker.py | 2 ++ test-data/unit/check-functools.test | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 82633a5b6c0e..db65660bbfbd 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -683,6 +683,8 @@ def extract_callable_type(self, inner_type: Type | None, ctx: Context) -> Callab inner_type = get_proper_type(inner_type) outer_type: CallableType | None = None if inner_type is not None and not isinstance(inner_type, AnyType): + if isinstance(inner_type, TypeVarLikeType): + inner_type = get_proper_type(inner_type.upper_bound) if isinstance(inner_type, TypeType): if isinstance(inner_type.item, Instance): inner_type = expand_type_by_instance( diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index 710d3e66dfad..9f8fbd42440b 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -541,3 +541,19 @@ p(1, "no") # E: Argument 2 to "A" has incompatible type "str"; expected "int" q: partial[A] = partial(A, 1) # OK [builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialTypeVarBound] +from typing import Callable, TypeVar, Type +import functools + +T = TypeVar("T", bound=Callable[[str, int], str]) +S = TypeVar("S", bound=Type[int]) + +def foo(f: T) -> T: + g = functools.partial(f, "foo") + return f + +def bar(f: S) -> S: + g = functools.partial(f, "foo") + return f +[builtins fixtures/primitives.pyi] From 2632ea363191502b9ddb10553c7fedd553737b06 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sun, 11 Aug 2024 00:46:20 +0200 Subject: [PATCH 120/247] stubtest: Add support for cached_property (#17626) Fixes #17625 --- mypy/stubtest.py | 3 ++ mypy/test/teststubtest.py | 100 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index ca17ccfe2f5b..c54f83f33b00 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1224,6 +1224,9 @@ def _verify_readonly_property(stub: nodes.Decorator, runtime: Any) -> Iterator[s if isinstance(runtime, property): yield from _verify_final_method(stub.func, runtime.fget, MISSING) return + if isinstance(runtime, functools.cached_property): + yield from _verify_final_method(stub.func, runtime.func, MISSING) + return if inspect.isdatadescriptor(runtime): # It's enough like a property... return diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index c10c683ffdac..c6e92258e7f4 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -893,6 +893,106 @@ class FineAndDandy: error=None, ) + @collect_cases + def test_cached_property(self) -> Iterator[Case]: + yield Case( + stub=""" + from functools import cached_property + class Good: + @cached_property + def read_only_attr(self) -> int: ... + @cached_property + def read_only_attr2(self) -> int: ... + """, + runtime=""" + import functools as ft + from functools import cached_property + class Good: + @cached_property + def read_only_attr(self): return 1 + @ft.cached_property + def read_only_attr2(self): return 1 + """, + error=None, + ) + yield Case( + stub=""" + from functools import cached_property + class Bad: + @cached_property + def f(self) -> int: ... + """, + runtime=""" + class Bad: + def f(self) -> int: return 1 + """, + error="Bad.f", + ) + yield Case( + stub=""" + from functools import cached_property + class GoodCachedAttr: + @cached_property + def f(self) -> int: ... + """, + runtime=""" + class GoodCachedAttr: + f = 1 + """, + error=None, + ) + yield Case( + stub=""" + from functools import cached_property + class BadCachedAttr: + @cached_property + def f(self) -> str: ... + """, + runtime=""" + class BadCachedAttr: + f = 1 + """, + error="BadCachedAttr.f", + ) + yield Case( + stub=""" + from functools import cached_property + from typing import final + class FinalGood: + @cached_property + @final + def attr(self) -> int: ... + """, + runtime=""" + from functools import cached_property + from typing import final + class FinalGood: + @cached_property + @final + def attr(self): + return 1 + """, + error=None, + ) + yield Case( + stub=""" + from functools import cached_property + class FinalBad: + @cached_property + def attr(self) -> int: ... + """, + runtime=""" + from functools import cached_property + from typing_extensions import final + class FinalBad: + @cached_property + @final + def attr(self): + return 1 + """, + error="FinalBad.attr", + ) + @collect_cases def test_var(self) -> Iterator[Case]: yield Case(stub="x1: int", runtime="x1 = 5", error=None) From 3da16bd6004ec9685c476f77526abd36e7632813 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 11 Aug 2024 22:26:32 +0100 Subject: [PATCH 121/247] An alternative fix for a union-like literal string (#17639) It is unfortunate to add two extra slots to a common type (and I guess this is why it was rejected in the original PR), but all other alternatives I tried are hacky and/or dangerous. So, this is a price to pay for introducing a new type syntax. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/fastparse.py | 4 +--- mypy/typeanal.py | 6 +++++- mypy/types.py | 16 +++++++++++++--- test-data/unit/check-literal.test | 4 ++++ 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index ab7eef924bd1..abcce74c6064 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -331,12 +331,10 @@ def parse_type_string( """ try: _, node = parse_type_comment(f"({expr_string})", line=line, column=column, errors=None) - if isinstance(node, UnboundType) and node.original_str_expr is None: + if isinstance(node, (UnboundType, UnionType)) and node.original_str_expr is None: node.original_str_expr = expr_string node.original_str_fallback = expr_fallback_name return node - elif isinstance(node, UnionType): - return node else: return RawExpressionType(expr_string, expr_fallback_name, line, column) except (SyntaxError, ValueError): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index f88c3f91d1c6..274b4b893a98 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1615,7 +1615,11 @@ def analyze_literal_type(self, t: UnboundType) -> Type: def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> list[Type] | None: # This UnboundType was originally defined as a string. - if isinstance(arg, UnboundType) and arg.original_str_expr is not None: + if ( + isinstance(arg, ProperType) + and isinstance(arg, (UnboundType, UnionType)) + and arg.original_str_expr is not None + ): assert arg.original_str_fallback is not None return [ LiteralType( diff --git a/mypy/types.py b/mypy/types.py index 72374c1555c9..618228c361f3 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -914,7 +914,7 @@ class UnboundType(ProperType): def __init__( self, - name: str | None, + name: str, args: Sequence[Type] | None = None, line: int = -1, column: int = -1, @@ -926,7 +926,6 @@ def __init__( super().__init__(line, column) if not args: args = [] - assert name is not None self.name = name self.args = tuple(args) # Should this type be wrapped in an Optional? @@ -2849,7 +2848,13 @@ def is_singleton_type(self) -> bool: class UnionType(ProperType): """The union type Union[T1, ..., Tn] (at least one type argument).""" - __slots__ = ("items", "is_evaluated", "uses_pep604_syntax") + __slots__ = ( + "items", + "is_evaluated", + "uses_pep604_syntax", + "original_str_expr", + "original_str_fallback", + ) def __init__( self, @@ -2868,6 +2873,11 @@ def __init__( self.is_evaluated = is_evaluated # uses_pep604_syntax is True if Union uses OR syntax (X | Y) self.uses_pep604_syntax = uses_pep604_syntax + # The meaning of these two is the same as for UnboundType. A UnionType can be + # return by type parser from a string "A|B", and we need to be able to fall back + # to plain string, when such a string appears inside a Literal[...]. + self.original_str_expr: str | None = None + self.original_str_fallback: str | None = None def can_be_true_default(self) -> bool: return any(item.can_be_true for item in self.items) diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index e8d942f5d74d..6d76ce176aaf 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -12,8 +12,12 @@ reveal_type(g1) # N: Revealed type is "def (x: Literal['A['])" def f2(x: 'A B') -> None: pass # E: Invalid type comment or annotation def g2(x: Literal['A B']) -> None: pass +def h2(x: 'A|int') -> None: pass # E: Name "A" is not defined +def i2(x: Literal['A|B']) -> None: pass reveal_type(f2) # N: Revealed type is "def (x: Any)" reveal_type(g2) # N: Revealed type is "def (x: Literal['A B'])" +reveal_type(h2) # N: Revealed type is "def (x: Union[Any, builtins.int])" +reveal_type(i2) # N: Revealed type is "def (x: Literal['A|B'])" [builtins fixtures/tuple.pyi] [out] From f0c394a8b9ed44ef9830ac3a7b2d13df2f60f084 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 12 Aug 2024 19:21:50 +0300 Subject: [PATCH 122/247] Stubgen add `--version` (#17662) --- mypy/stubgen.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 4e6e3efc000b..f4a418838161 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -54,6 +54,7 @@ import mypy.parse import mypy.traverser import mypy.util +import mypy.version from mypy.build import build from mypy.errors import CompileError, Errors from mypy.find_sources import InvalidSourceList, create_source_list @@ -1847,6 +1848,9 @@ def parse_options(args: list[str]) -> Options: dest="files", help="generate stubs for given files or directories", ) + parser.add_argument( + "--version", action="version", version="%(prog)s " + mypy.version.__version__ + ) ns = parser.parse_args(args) From 5b46f1c4baeb6271cbf3fc4ab57f8d78b0232e3f Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 12 Aug 2024 19:40:32 +0300 Subject: [PATCH 123/247] [stubgen] Fix crash on literal class-level keywords (#17663) Closes https://github.com/python/mypy/issues/17661 --- mypy/stubgen.py | 22 ++++++++++++++++++++-- test-data/unit/stubgen.test | 10 ++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index f4a418838161..14417f55545e 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -305,9 +305,26 @@ def visit_name_expr(self, node: NameExpr) -> str: def visit_member_expr(self, o: MemberExpr) -> str: return self._visit_ref_expr(o) - def visit_str_expr(self, node: StrExpr) -> str: + def _visit_literal_node( + self, node: StrExpr | BytesExpr | IntExpr | FloatExpr | ComplexExpr + ) -> str: return repr(node.value) + def visit_str_expr(self, node: StrExpr) -> str: + return self._visit_literal_node(node) + + def visit_bytes_expr(self, node: BytesExpr) -> str: + return f"b{self._visit_literal_node(node)}" + + def visit_int_expr(self, node: IntExpr) -> str: + return self._visit_literal_node(node) + + def visit_float_expr(self, node: FloatExpr) -> str: + return self._visit_literal_node(node) + + def visit_complex_expr(self, node: ComplexExpr) -> str: + return self._visit_literal_node(node) + def visit_index_expr(self, node: IndexExpr) -> str: base_fullname = self.stubgen.get_fullname(node.base) if base_fullname == "typing.Union": @@ -805,7 +822,8 @@ def get_base_types(self, cdef: ClassDef) -> list[str]: for name, value in cdef.keywords.items(): if name == "metaclass": continue # handled separately - base_types.append(f"{name}={value.accept(p)}") + processed_value = value.accept(p) or "..." # at least, don't crash + base_types.append(f"{name}={processed_value}") return base_types def get_class_decorators(self, cdef: ClassDef) -> list[str]: diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index 94d0edb2ae37..fe0538159aa3 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -4405,3 +4405,13 @@ Y = int | None Z = Incomplete W = int | str | None R = type[int | str] | None + +[case testClassInheritanceWithKeywordsConstants] +class Test(Whatever, a=1, b='b', c=True, d=1.5, e=None, f=1j, g=b'123'): ... +[out] +class Test(Whatever, a=1, b='b', c=True, d=1.5, e=None, f=1j, g=b'123'): ... + +[case testClassInheritanceWithKeywordsDynamic] +class Test(Whatever, keyword=SomeName * 2, attr=SomeName.attr): ... +[out] +class Test(Whatever, keyword=SomeName * 2, attr=SomeName.attr): ... From 8332c6e7992fa9655b79ef55c410a805ad1e6d34 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 14 Aug 2024 10:08:43 +0300 Subject: [PATCH 124/247] Add optional stderr result to `run_stubtest` in tests (#17636) This adds a test for incorrect error code in `stubtest`'s config file. Refs https://github.com/python/mypy/issues/17628 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- mypy/test/teststubtest.py | 42 +++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index c6e92258e7f4..70687b499651 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -144,9 +144,9 @@ def __invert__(self: _T) -> _T: pass """ -def run_stubtest( +def run_stubtest_with_stderr( stub: str, runtime: str, options: list[str], config_file: str | None = None -) -> str: +) -> tuple[str, str]: with use_tmp_dir(TEST_MODULE_NAME) as tmp_dir: with open("builtins.pyi", "w") as f: f.write(stubtest_builtins_stub) @@ -163,13 +163,26 @@ def run_stubtest( f.write(config_file) options = options + ["--mypy-config-file", f"{TEST_MODULE_NAME}_config.ini"] output = io.StringIO() - with contextlib.redirect_stdout(output): + outerr = io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(outerr): test_stubs(parse_options([TEST_MODULE_NAME] + options), use_builtins_fixtures=True) - return remove_color_code( - output.getvalue() - # remove cwd as it's not available from outside - .replace(os.path.realpath(tmp_dir) + os.sep, "").replace(tmp_dir + os.sep, "") - ) + filtered_output = remove_color_code( + output.getvalue() + # remove cwd as it's not available from outside + .replace(os.path.realpath(tmp_dir) + os.sep, "").replace(tmp_dir + os.sep, "") + ) + filtered_outerr = remove_color_code( + outerr.getvalue() + # remove cwd as it's not available from outside + .replace(os.path.realpath(tmp_dir) + os.sep, "").replace(tmp_dir + os.sep, "") + ) + return filtered_output, filtered_outerr + + +def run_stubtest( + stub: str, runtime: str, options: list[str], config_file: str | None = None +) -> str: + return run_stubtest_with_stderr(stub, runtime, options, config_file)[0] class Case: @@ -2590,6 +2603,19 @@ def test_config_file_error_codes(self) -> None: output = run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file) assert output == "Success: no issues found in 1 module\n" + def test_config_file_error_codes_invalid(self) -> None: + runtime = "temp = 5\n" + stub = "temp: int\n" + config_file = "[mypy]\ndisable_error_code = not-a-valid-name\n" + output, outerr = run_stubtest_with_stderr( + stub=stub, runtime=runtime, options=[], config_file=config_file + ) + assert output == "Success: no issues found in 1 module\n" + assert outerr == ( + "test_module_config.ini: [mypy]: disable_error_code: " + "Invalid error code(s): not-a-valid-name\n" + ) + def test_config_file_wrong_incomplete_feature(self) -> None: runtime = "x = 1\n" stub = "x: int\n" From fe15ee69b9225f808f8ed735671b73c31ae1bed8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 14 Aug 2024 22:34:18 +0100 Subject: [PATCH 125/247] [mypyc] Avoid uses of _PyObject_CallMethodOneArg on 3.13 (#17526) It's no longer available in CPython 3.13 betas. This fixes some dict primitives on 3.13. Work on mypyc/mypyc#1056. --- mypyc/lib-rt/dict_ops.c | 12 ++++++++++-- mypyc/lib-rt/pythonsupport.h | 2 ++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/dict_ops.c b/mypyc/lib-rt/dict_ops.c index 031df8f63c49..b33233521afd 100644 --- a/mypyc/lib-rt/dict_ops.c +++ b/mypyc/lib-rt/dict_ops.c @@ -78,7 +78,11 @@ PyObject *CPyDict_SetDefault(PyObject *dict, PyObject *key, PyObject *value) { return ret; } _Py_IDENTIFIER(setdefault); - return _PyObject_CallMethodIdObjArgs(dict, &PyId_setdefault, key, value, NULL); + PyObject *name = _PyUnicode_FromId(&PyId_setdefault); /* borrowed */ + if (name == NULL) { + return NULL; + } + return PyObject_CallMethodObjArgs(dict, name, key, value, NULL); } PyObject *CPyDict_SetDefaultWithNone(PyObject *dict, PyObject *key) { @@ -133,7 +137,11 @@ static inline int CPy_ObjectToStatus(PyObject *obj) { static int CPyDict_UpdateGeneral(PyObject *dict, PyObject *stuff) { _Py_IDENTIFIER(update); - PyObject *res = _PyObject_CallMethodIdOneArg(dict, &PyId_update, stuff); + PyObject *name = _PyUnicode_FromId(&PyId_update); /* borrowed */ + if (name == NULL) { + return -1; + } + PyObject *res = PyObject_CallMethodOneArg(dict, name, stuff); return CPy_ObjectToStatus(res); } diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 8edc9abcf9f8..bf7e5203758d 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -401,6 +401,8 @@ _CPyObject_HasAttrId(PyObject *v, _Py_Identifier *name) { _PyObject_CallMethodIdObjArgs((self), (name), NULL) #define _PyObject_CallMethodIdOneArg(self, name, arg) \ _PyObject_CallMethodIdObjArgs((self), (name), (arg), NULL) +#define PyObject_CallMethodOneArg(self, name, arg) \ + PyObject_CallMethodObjArgs((self), (name), (arg), NULL) #endif #if CPY_3_13_FEATURES From 35679e2e7e5f2c8e4408806cc8deff4c15b4647f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 30 Aug 2024 11:47:14 +0100 Subject: [PATCH 126/247] Fix another crash scenario on recursive tuple types (#17708) Fixes https://github.com/python/mypy/issues/17691 This looks quite ad-hoc, but the only clean alternative is to remove the existing recursive types optimization we have in `subtypes.py`. But the best I can get without that optimization is -7% performance penalty (on self-check). So I think it is not worth it, since it is still a corner case. --- mypy/typeops.py | 9 +++++++-- mypy/types.py | 10 ++++++++-- test-data/unit/check-recursive-types.test | 12 ++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index 036d782be89c..d22448a715e5 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -114,7 +114,11 @@ def tuple_fallback(typ: TupleType) -> Instance: else: items.append(item) return Instance( - info, [make_simplified_union(items)], extra_attrs=typ.partial_fallback.extra_attrs + info, + # Note: flattening recursive unions is dangerous, since it can fool recursive + # types optimization in subtypes.py and go into infinite recursion. + [make_simplified_union(items, handle_recursive=False)], + extra_attrs=typ.partial_fallback.extra_attrs, ) @@ -440,6 +444,7 @@ def make_simplified_union( *, keep_erased: bool = False, contract_literals: bool = True, + handle_recursive: bool = True, ) -> ProperType: """Build union type with redundant union items removed. @@ -465,7 +470,7 @@ def make_simplified_union( to_union(). """ # Step 1: expand all nested unions - items = flatten_nested_unions(items) + items = flatten_nested_unions(items, handle_recursive=handle_recursive) # Step 2: fast path for single item if len(items) == 1: diff --git a/mypy/types.py b/mypy/types.py index 618228c361f3..78244d0f9cf4 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3629,7 +3629,7 @@ def extend_args_for_prefix_and_suffix( def flatten_nested_unions( - types: Sequence[Type], handle_type_alias_type: bool = True + types: Sequence[Type], *, handle_type_alias_type: bool = True, handle_recursive: bool = True ) -> list[Type]: """Flatten nested unions in a type list.""" if not isinstance(types, list): @@ -3643,7 +3643,13 @@ def flatten_nested_unions( flat_items: list[Type] = [] for t in typelist: - tp = get_proper_type(t) if handle_type_alias_type else t + if handle_type_alias_type: + if not handle_recursive and isinstance(t, TypeAliasType) and t.is_recursive: + tp: Type = t + else: + tp = get_proper_type(t) + else: + tp = t if isinstance(tp, ProperType) and isinstance(tp, UnionType): flat_items.extend( flatten_nested_unions(tp.items, handle_type_alias_type=handle_type_alias_type) diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index d5c8acd1bc15..ac1ea0c0035a 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -994,3 +994,15 @@ class T2(Tuple[T1, "T4", "T4"]): ... class T3(Tuple[str, "T4", "T4"]): ... T4 = Union[T2, T3] [builtins fixtures/tuple.pyi] + +[case testRecursiveTupleFallback5] +from typing import Protocol, Tuple, Union + +class Proto(Protocol): + def __len__(self) -> int: ... + +A = Union[Proto, Tuple[A]] +ta: Tuple[A] +p: Proto +p = ta +[builtins fixtures/tuple.pyi] From 0a6d40b125f6b353621016d8a9022212104f7877 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:50:02 +0200 Subject: [PATCH 127/247] Sync typeshed (#17724) The automatic sync failed due to a merge conflict. Source commit: https://github.com/python/typeshed/commit/661fe27658acd6b6058addcb29375381b8cae56e --------- Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: AlexWaygood --- ...ert-sum-literal-integer-change-13961.patch | 12 +- mypy/typeshed/stdlib/_ast.pyi | 451 +++++++++++++++++- mypy/typeshed/stdlib/_collections_abc.pyi | 9 +- mypy/typeshed/stdlib/_ctypes.pyi | 4 +- mypy/typeshed/stdlib/_operator.pyi | 6 +- mypy/typeshed/stdlib/_stat.pyi | 12 +- mypy/typeshed/stdlib/_thread.pyi | 26 +- mypy/typeshed/stdlib/_tkinter.pyi | 4 +- mypy/typeshed/stdlib/argparse.pyi | 12 +- mypy/typeshed/stdlib/ast.pyi | 35 +- mypy/typeshed/stdlib/asyncio/tasks.pyi | 12 +- mypy/typeshed/stdlib/asyncio/unix_events.pyi | 2 +- mypy/typeshed/stdlib/builtins.pyi | 15 +- mypy/typeshed/stdlib/cmd.pyi | 3 +- mypy/typeshed/stdlib/collections/__init__.pyi | 9 +- mypy/typeshed/stdlib/contextlib.pyi | 2 + mypy/typeshed/stdlib/crypt.pyi | 11 +- mypy/typeshed/stdlib/ctypes/__init__.pyi | 5 + mypy/typeshed/stdlib/ctypes/_endian.pyi | 9 +- mypy/typeshed/stdlib/dataclasses.pyi | 15 +- mypy/typeshed/stdlib/datetime.pyi | 4 +- mypy/typeshed/stdlib/distutils/cmd.pyi | 120 +++++ .../stdlib/distutils/command/__init__.pyi | 48 ++ .../stdlib/distutils/command/check.pyi | 4 +- .../stdlib/distutils/command/config.pyi | 4 +- .../stdlib/distutils/command/install.pyi | 13 +- .../stdlib/distutils/command/install_lib.pyi | 4 +- mypy/typeshed/stdlib/distutils/core.pyi | 4 +- .../stdlib/distutils/cygwinccompiler.pyi | 10 +- mypy/typeshed/stdlib/distutils/debug.pyi | 4 +- mypy/typeshed/stdlib/distutils/dist.pyi | 153 +++++- .../stdlib/distutils/fancy_getopt.pyi | 12 +- mypy/typeshed/stdlib/distutils/log.pyi | 12 +- mypy/typeshed/stdlib/distutils/sysconfig.pyi | 14 +- mypy/typeshed/stdlib/email/message.pyi | 9 +- mypy/typeshed/stdlib/email/utils.pyi | 5 +- mypy/typeshed/stdlib/filecmp.pyi | 25 +- mypy/typeshed/stdlib/ftplib.pyi | 2 +- .../stdlib/importlib/metadata/__init__.pyi | 4 +- mypy/typeshed/stdlib/json/encoder.pyi | 12 +- .../stdlib/lib2to3/fixes/fix_unicode.pyi | 2 +- mypy/typeshed/stdlib/logging/__init__.pyi | 67 ++- .../stdlib/multiprocessing/sharedctypes.pyi | 8 +- mypy/typeshed/stdlib/operator.pyi | 3 + mypy/typeshed/stdlib/os/__init__.pyi | 4 +- mypy/typeshed/stdlib/pathlib.pyi | 29 +- mypy/typeshed/stdlib/pdb.pyi | 2 +- mypy/typeshed/stdlib/poplib.pyi | 3 +- mypy/typeshed/stdlib/pty.pyi | 11 +- mypy/typeshed/stdlib/pyexpat/__init__.pyi | 6 +- mypy/typeshed/stdlib/pyexpat/errors.pyi | 88 ++-- mypy/typeshed/stdlib/pyexpat/model.pyi | 22 +- mypy/typeshed/stdlib/re.pyi | 18 +- mypy/typeshed/stdlib/socketserver.pyi | 13 +- mypy/typeshed/stdlib/sqlite3/dbapi2.pyi | 10 +- mypy/typeshed/stdlib/symtable.pyi | 5 +- mypy/typeshed/stdlib/tarfile.pyi | 2 +- mypy/typeshed/stdlib/tempfile.pyi | 4 +- mypy/typeshed/stdlib/tkinter/__init__.pyi | 34 +- mypy/typeshed/stdlib/tkinter/constants.pyi | 12 +- mypy/typeshed/stdlib/tkinter/dialog.pyi | 4 +- mypy/typeshed/stdlib/tkinter/messagebox.pyi | 34 +- mypy/typeshed/stdlib/tkinter/ttk.pyi | 11 +- mypy/typeshed/stdlib/turtle.pyi | 12 +- mypy/typeshed/stdlib/types.pyi | 6 +- mypy/typeshed/stdlib/typing.pyi | 11 +- mypy/typeshed/stdlib/unittest/mock.pyi | 2 +- mypy/typeshed/stdlib/unittest/runner.pyi | 46 +- mypy/typeshed/stdlib/urllib/parse.pyi | 4 +- mypy/typeshed/stdlib/urllib/request.pyi | 1 + mypy/typeshed/stdlib/xml/dom/__init__.pyi | 44 +- .../stdlib/xml/etree/ElementInclude.pyi | 9 +- .../typeshed/stdlib/xml/etree/ElementTree.pyi | 4 +- mypy/typeshed/stdlib/zipfile/__init__.pyi | 81 +++- mypy/typeshed/stdlib/zipfile/_path.pyi | 1 + test-data/unit/pythoneval.test | 4 +- 76 files changed, 1365 insertions(+), 369 deletions(-) diff --git a/misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch b/misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch index 044e672bfda5..331628af1424 100644 --- a/misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch +++ b/misc/typeshed_patches/0001-Revert-sum-literal-integer-change-13961.patch @@ -1,4 +1,4 @@ -From 44bc98bd50e7170887f0740b53ed95a8eb04f00e Mon Sep 17 00:00:00 2001 +From 58c6a6ab863c1c38e95ccafaf13792ed9c00e499 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 29 Oct 2022 12:47:21 -0700 Subject: [PATCH] Revert sum literal integer change (#13961) @@ -19,18 +19,18 @@ within mypy, I might pursue upstreaming this in typeshed. 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi -index 99919c64c..680cd5561 100644 +index ea9f8c894..a6065cc67 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi -@@ -1596,7 +1596,7 @@ _SupportsSumNoDefaultT = TypeVar("_SupportsSumNoDefaultT", bound=_SupportsSumWit +@@ -1653,7 +1653,7 @@ _SupportsSumNoDefaultT = TypeVar("_SupportsSumNoDefaultT", bound=_SupportsSumWit # without creating many false-positive errors (see #7578). # Instead, we special-case the most common examples of this: bool and literal integers. @overload --def sum(iterable: Iterable[bool | _LiteralInteger], /, start: int = 0) -> int: ... # type: ignore[overload-overlap] -+def sum(iterable: Iterable[bool], /, start: int = 0) -> int: ... # type: ignore[overload-overlap] +-def sum(iterable: Iterable[bool | _LiteralInteger], /, start: int = 0) -> int: ... ++def sum(iterable: Iterable[bool], /, start: int = 0) -> int: ... @overload def sum(iterable: Iterable[_SupportsSumNoDefaultT], /) -> _SupportsSumNoDefaultT | Literal[0]: ... @overload -- -2.39.3 (Apple Git-146) +2.46.0 diff --git a/mypy/typeshed/stdlib/_ast.pyi b/mypy/typeshed/stdlib/_ast.pyi index d14c6d39a162..1dbceac428c1 100644 --- a/mypy/typeshed/stdlib/_ast.pyi +++ b/mypy/typeshed/stdlib/_ast.pyi @@ -1,7 +1,7 @@ import sys import typing_extensions from typing import Any, ClassVar, Generic, Literal, TypedDict, overload -from typing_extensions import Unpack +from typing_extensions import Self, Unpack PyCF_ONLY_AST: Literal[1024] PyCF_TYPE_COMMENTS: Literal[4096] @@ -34,6 +34,9 @@ class AST: if sys.version_info >= (3, 13): _field_types: ClassVar[dict[str, Any]] + if sys.version_info >= (3, 14): + def __replace__(self) -> Self: ... + class mod(AST): ... class type_ignore(AST): ... @@ -44,6 +47,9 @@ class TypeIgnore(type_ignore): tag: str def __init__(self, lineno: int, tag: str) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, lineno: int = ..., tag: str = ...) -> Self: ... + class FunctionType(mod): if sys.version_info >= (3, 10): __match_args__ = ("argtypes", "returns") @@ -57,6 +63,9 @@ class FunctionType(mod): else: def __init__(self, argtypes: list[expr], returns: expr) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, argtypes: list[expr] = ..., returns: expr = ...) -> Self: ... + class Module(mod): if sys.version_info >= (3, 10): __match_args__ = ("body", "type_ignores") @@ -67,6 +76,9 @@ class Module(mod): else: def __init__(self, body: list[stmt], type_ignores: list[TypeIgnore]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, body: list[stmt] = ..., type_ignores: list[TypeIgnore] = ...) -> Self: ... + class Interactive(mod): if sys.version_info >= (3, 10): __match_args__ = ("body",) @@ -76,12 +88,18 @@ class Interactive(mod): else: def __init__(self, body: list[stmt]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, body: list[stmt] = ...) -> Self: ... + class Expression(mod): if sys.version_info >= (3, 10): __match_args__ = ("body",) body: expr def __init__(self, body: expr) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, body: expr = ...) -> Self: ... + class stmt(AST): lineno: int col_offset: int @@ -89,6 +107,9 @@ class stmt(AST): end_col_offset: int | None def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, **kwargs: Unpack[_Attributes]) -> Self: ... + class FunctionDef(stmt): if sys.version_info >= (3, 12): __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment", "type_params") @@ -152,6 +173,19 @@ class FunctionDef(stmt): **kwargs: Unpack[_Attributes], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier = ..., + args: arguments = ..., + body: list[stmt] = ..., + decorator_list: list[expr] = ..., + returns: expr | None = ..., + type_comment: str | None = ..., + type_params: list[type_param] = ..., + ) -> Self: ... + class AsyncFunctionDef(stmt): if sys.version_info >= (3, 12): __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment", "type_params") @@ -215,6 +249,19 @@ class AsyncFunctionDef(stmt): **kwargs: Unpack[_Attributes], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier = ..., + args: arguments = ..., + body: list[stmt], + decorator_list: list[expr], + returns: expr | None, + type_comment: str | None, + type_params: list[type_param], + ) -> Self: ... + class ClassDef(stmt): if sys.version_info >= (3, 12): __match_args__ = ("name", "bases", "keywords", "body", "decorator_list", "type_params") @@ -260,12 +307,28 @@ class ClassDef(stmt): **kwargs: Unpack[_Attributes], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier, + bases: list[expr], + keywords: list[keyword], + body: list[stmt], + decorator_list: list[expr], + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> Self: ... + class Return(stmt): if sys.version_info >= (3, 10): __match_args__ = ("value",) value: expr | None def __init__(self, value: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Delete(stmt): if sys.version_info >= (3, 10): __match_args__ = ("targets",) @@ -275,6 +338,9 @@ class Delete(stmt): else: def __init__(self, targets: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, targets: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Assign(stmt): if sys.version_info >= (3, 10): __match_args__ = ("targets", "value", "type_comment") @@ -295,6 +361,11 @@ class Assign(stmt): self, targets: list[expr], value: expr, type_comment: str | None = None, **kwargs: Unpack[_Attributes] ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, targets: list[expr] = ..., value: expr = ..., type_comment: str | None = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class AugAssign(stmt): if sys.version_info >= (3, 10): __match_args__ = ("target", "op", "value") @@ -305,6 +376,16 @@ class AugAssign(stmt): self, target: Name | Attribute | Subscript, op: operator, value: expr, **kwargs: Unpack[_Attributes] ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: Name | Attribute | Subscript = ..., + op: operator = ..., + value: expr = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + class AnnAssign(stmt): if sys.version_info >= (3, 10): __match_args__ = ("target", "annotation", "value", "simple") @@ -332,6 +413,17 @@ class AnnAssign(stmt): **kwargs: Unpack[_Attributes], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: Name | Attribute | Subscript = ..., + annotation: expr = ..., + value: expr | None = ..., + simple: int = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + class For(stmt): if sys.version_info >= (3, 10): __match_args__ = ("target", "iter", "body", "orelse", "type_comment") @@ -361,6 +453,18 @@ class For(stmt): **kwargs: Unpack[_Attributes], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: expr = ..., + iter: expr = ..., + body: list[stmt] = ..., + orelse: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + class AsyncFor(stmt): if sys.version_info >= (3, 10): __match_args__ = ("target", "iter", "body", "orelse", "type_comment") @@ -390,6 +494,18 @@ class AsyncFor(stmt): **kwargs: Unpack[_Attributes], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: expr = ..., + iter: expr = ..., + body: list[stmt] = ..., + orelse: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + class While(stmt): if sys.version_info >= (3, 10): __match_args__ = ("test", "body", "orelse") @@ -403,6 +519,9 @@ class While(stmt): else: def __init__(self, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> Self: ... + class If(stmt): if sys.version_info >= (3, 10): __match_args__ = ("test", "body", "orelse") @@ -416,6 +535,11 @@ class If(stmt): else: def __init__(self, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, test: expr = ..., body: list[stmt] = ..., orelse: list[stmt] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class With(stmt): if sys.version_info >= (3, 10): __match_args__ = ("items", "body", "type_comment") @@ -435,6 +559,16 @@ class With(stmt): self, items: list[withitem], body: list[stmt], type_comment: str | None = None, **kwargs: Unpack[_Attributes] ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + items: list[withitem] = ..., + body: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + class AsyncWith(stmt): if sys.version_info >= (3, 10): __match_args__ = ("items", "body", "type_comment") @@ -454,6 +588,16 @@ class AsyncWith(stmt): self, items: list[withitem], body: list[stmt], type_comment: str | None = None, **kwargs: Unpack[_Attributes] ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + items: list[withitem] = ..., + body: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + class Raise(stmt): if sys.version_info >= (3, 10): __match_args__ = ("exc", "cause") @@ -461,6 +605,9 @@ class Raise(stmt): cause: expr | None def __init__(self, exc: expr | None = None, cause: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, exc: expr | None = ..., cause: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Try(stmt): if sys.version_info >= (3, 10): __match_args__ = ("body", "handlers", "orelse", "finalbody") @@ -487,6 +634,17 @@ class Try(stmt): **kwargs: Unpack[_Attributes], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + body: list[stmt] = ..., + handlers: list[ExceptHandler] = ..., + orelse: list[stmt] = ..., + finalbody: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + if sys.version_info >= (3, 11): class TryStar(stmt): __match_args__ = ("body", "handlers", "orelse", "finalbody") @@ -513,6 +671,17 @@ if sys.version_info >= (3, 11): **kwargs: Unpack[_Attributes], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + body: list[stmt] = ..., + handlers: list[ExceptHandler] = ..., + orelse: list[stmt] = ..., + finalbody: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + class Assert(stmt): if sys.version_info >= (3, 10): __match_args__ = ("test", "msg") @@ -520,6 +689,9 @@ class Assert(stmt): msg: expr | None def __init__(self, test: expr, msg: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, test: expr, msg: expr | None, **kwargs: Unpack[_Attributes]) -> Self: ... + class Import(stmt): if sys.version_info >= (3, 10): __match_args__ = ("names",) @@ -529,6 +701,9 @@ class Import(stmt): else: def __init__(self, names: list[alias], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, names: list[alias] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class ImportFrom(stmt): if sys.version_info >= (3, 10): __match_args__ = ("module", "names", "level") @@ -550,6 +725,11 @@ class ImportFrom(stmt): self, module: str | None = None, *, names: list[alias], level: int, **kwargs: Unpack[_Attributes] ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, module: str | None = ..., names: list[alias] = ..., level: int = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class Global(stmt): if sys.version_info >= (3, 10): __match_args__ = ("names",) @@ -559,6 +739,9 @@ class Global(stmt): else: def __init__(self, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> Self: ... + class Nonlocal(stmt): if sys.version_info >= (3, 10): __match_args__ = ("names",) @@ -568,12 +751,18 @@ class Nonlocal(stmt): else: def __init__(self, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, names: list[_Identifier] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Expr(stmt): if sys.version_info >= (3, 10): __match_args__ = ("value",) value: expr def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Pass(stmt): ... class Break(stmt): ... class Continue(stmt): ... @@ -585,6 +774,9 @@ class expr(AST): end_col_offset: int | None def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, **kwargs: Unpack[_Attributes]) -> Self: ... + class BoolOp(expr): if sys.version_info >= (3, 10): __match_args__ = ("op", "values") @@ -595,6 +787,9 @@ class BoolOp(expr): else: def __init__(self, op: boolop, values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, op: boolop = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class BinOp(expr): if sys.version_info >= (3, 10): __match_args__ = ("left", "op", "right") @@ -603,6 +798,11 @@ class BinOp(expr): right: expr def __init__(self, left: expr, op: operator, right: expr, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, left: expr = ..., op: operator = ..., right: expr = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class UnaryOp(expr): if sys.version_info >= (3, 10): __match_args__ = ("op", "operand") @@ -610,6 +810,9 @@ class UnaryOp(expr): operand: expr def __init__(self, op: unaryop, operand: expr, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, op: unaryop = ..., operand: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Lambda(expr): if sys.version_info >= (3, 10): __match_args__ = ("args", "body") @@ -617,6 +820,9 @@ class Lambda(expr): body: expr def __init__(self, args: arguments, body: expr, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, args: arguments = ..., body: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class IfExp(expr): if sys.version_info >= (3, 10): __match_args__ = ("test", "body", "orelse") @@ -625,6 +831,11 @@ class IfExp(expr): orelse: expr def __init__(self, test: expr, body: expr, orelse: expr, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, test: expr = ..., body: expr = ..., orelse: expr = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class Dict(expr): if sys.version_info >= (3, 10): __match_args__ = ("keys", "values") @@ -635,6 +846,11 @@ class Dict(expr): else: def __init__(self, keys: list[expr | None], values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, keys: list[expr | None] = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class Set(expr): if sys.version_info >= (3, 10): __match_args__ = ("elts",) @@ -644,6 +860,9 @@ class Set(expr): else: def __init__(self, elts: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, elts: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class ListComp(expr): if sys.version_info >= (3, 10): __match_args__ = ("elt", "generators") @@ -654,6 +873,11 @@ class ListComp(expr): else: def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class SetComp(expr): if sys.version_info >= (3, 10): __match_args__ = ("elt", "generators") @@ -664,6 +888,11 @@ class SetComp(expr): else: def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class DictComp(expr): if sys.version_info >= (3, 10): __match_args__ = ("key", "value", "generators") @@ -677,6 +906,11 @@ class DictComp(expr): else: def __init__(self, key: expr, value: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, key: expr = ..., value: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class GeneratorExp(expr): if sys.version_info >= (3, 10): __match_args__ = ("elt", "generators") @@ -687,24 +921,38 @@ class GeneratorExp(expr): else: def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class Await(expr): if sys.version_info >= (3, 10): __match_args__ = ("value",) value: expr def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Yield(expr): if sys.version_info >= (3, 10): __match_args__ = ("value",) value: expr | None def __init__(self, value: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class YieldFrom(expr): if sys.version_info >= (3, 10): __match_args__ = ("value",) value: expr def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Compare(expr): if sys.version_info >= (3, 10): __match_args__ = ("left", "ops", "comparators") @@ -718,6 +966,11 @@ class Compare(expr): else: def __init__(self, left: expr, ops: list[cmpop], comparators: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, left: expr = ..., ops: list[cmpop] = ..., comparators: list[expr] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class Call(expr): if sys.version_info >= (3, 10): __match_args__ = ("func", "args", "keywords") @@ -731,6 +984,11 @@ class Call(expr): else: def __init__(self, func: expr, args: list[expr], keywords: list[keyword], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, func: expr = ..., args: list[expr] = ..., keywords: list[keyword] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class FormattedValue(expr): if sys.version_info >= (3, 10): __match_args__ = ("value", "conversion", "format_spec") @@ -739,6 +997,11 @@ class FormattedValue(expr): format_spec: expr | None def __init__(self, value: expr, conversion: int, format_spec: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, value: expr = ..., conversion: int = ..., format_spec: expr | None = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class JoinedStr(expr): if sys.version_info >= (3, 10): __match_args__ = ("values",) @@ -748,16 +1011,24 @@ class JoinedStr(expr): else: def __init__(self, values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Constant(expr): if sys.version_info >= (3, 10): __match_args__ = ("value", "kind") value: Any # None, str, bytes, bool, int, float, complex, Ellipsis kind: str | None - # Aliases for value, for backwards compatibility - s: Any - n: int | float | complex + if sys.version_info < (3, 14): + # Aliases for value, for backwards compatibility + s: Any + n: int | float | complex + def __init__(self, value: Any, kind: str | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, value: Any = ..., kind: str | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class NamedExpr(expr): if sys.version_info >= (3, 10): __match_args__ = ("target", "value") @@ -765,6 +1036,9 @@ class NamedExpr(expr): value: expr def __init__(self, target: Name, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, target: Name = ..., value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Attribute(expr): if sys.version_info >= (3, 10): __match_args__ = ("value", "attr", "ctx") @@ -773,6 +1047,11 @@ class Attribute(expr): ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` def __init__(self, value: expr, attr: _Identifier, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, value: expr = ..., attr: _Identifier = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + if sys.version_info >= (3, 9): _Slice: typing_extensions.TypeAlias = expr _SliceAttributes: typing_extensions.TypeAlias = _Attributes @@ -792,6 +1071,16 @@ class Slice(_Slice): self, lower: expr | None = None, upper: expr | None = None, step: expr | None = None, **kwargs: Unpack[_SliceAttributes] ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + lower: expr | None = ..., + upper: expr | None = ..., + step: expr | None = ..., + **kwargs: Unpack[_SliceAttributes], + ) -> Self: ... + if sys.version_info < (3, 9): class ExtSlice(slice): dims: list[slice] @@ -809,6 +1098,11 @@ class Subscript(expr): ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` def __init__(self, value: expr, slice: _Slice, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, value: expr = ..., slice: _Slice = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class Starred(expr): if sys.version_info >= (3, 10): __match_args__ = ("value", "ctx") @@ -816,6 +1110,9 @@ class Starred(expr): ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` def __init__(self, value: expr, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Name(expr): if sys.version_info >= (3, 10): __match_args__ = ("id", "ctx") @@ -823,6 +1120,9 @@ class Name(expr): ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` def __init__(self, id: _Identifier, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, id: _Identifier = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class List(expr): if sys.version_info >= (3, 10): __match_args__ = ("elts", "ctx") @@ -833,6 +1133,9 @@ class List(expr): else: def __init__(self, elts: list[expr], ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class Tuple(expr): if sys.version_info >= (3, 10): __match_args__ = ("elts", "ctx") @@ -845,6 +1148,9 @@ class Tuple(expr): else: def __init__(self, elts: list[expr], ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class expr_context(AST): ... if sys.version_info < (3, 9): @@ -908,6 +1214,9 @@ class comprehension(AST): else: def __init__(self, target: expr, iter: expr, ifs: list[expr], is_async: int) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, target: expr = ..., iter: expr = ..., ifs: list[expr] = ..., is_async: int = ...) -> Self: ... + class excepthandler(AST): lineno: int col_offset: int @@ -915,6 +1224,11 @@ class excepthandler(AST): end_col_offset: int | None def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, lineno: int = ..., col_offset: int = ..., end_lineno: int | None = ..., end_col_offset: int | None = ... + ) -> Self: ... + class ExceptHandler(excepthandler): if sys.version_info >= (3, 10): __match_args__ = ("type", "name", "body") @@ -935,6 +1249,16 @@ class ExceptHandler(excepthandler): self, type: expr | None = None, name: _Identifier | None = None, *, body: list[stmt], **kwargs: Unpack[_Attributes] ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + type: expr | None = ..., + name: _Identifier | None = ..., + body: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + class arguments(AST): if sys.version_info >= (3, 10): __match_args__ = ("posonlyargs", "args", "vararg", "kwonlyargs", "kw_defaults", "kwarg", "defaults") @@ -993,6 +1317,19 @@ class arguments(AST): defaults: list[expr], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + posonlyargs: list[arg] = ..., + args: list[arg] = ..., + vararg: arg | None = ..., + kwonlyargs: list[arg] = ..., + kw_defaults: list[expr | None] = ..., + kwarg: arg | None = ..., + defaults: list[expr] = ..., + ) -> Self: ... + class arg(AST): lineno: int col_offset: int @@ -1007,6 +1344,16 @@ class arg(AST): self, arg: _Identifier, annotation: expr | None = None, type_comment: str | None = None, **kwargs: Unpack[_Attributes] ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + arg: _Identifier = ..., + annotation: expr | None = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + class keyword(AST): lineno: int col_offset: int @@ -1021,6 +1368,9 @@ class keyword(AST): @overload def __init__(self, arg: _Identifier | None = None, *, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, arg: _Identifier | None = ..., value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class alias(AST): lineno: int col_offset: int @@ -1032,6 +1382,9 @@ class alias(AST): asname: _Identifier | None def __init__(self, name: str, asname: _Identifier | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, name: str = ..., asname: _Identifier | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + class withitem(AST): if sys.version_info >= (3, 10): __match_args__ = ("context_expr", "optional_vars") @@ -1039,6 +1392,9 @@ class withitem(AST): optional_vars: expr | None def __init__(self, context_expr: expr, optional_vars: expr | None = None) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, context_expr: expr = ..., optional_vars: expr | None = ...) -> Self: ... + if sys.version_info >= (3, 10): class Match(stmt): __match_args__ = ("subject", "cases") @@ -1049,6 +1405,11 @@ if sys.version_info >= (3, 10): else: def __init__(self, subject: expr, cases: list[match_case], **kwargs: Unpack[_Attributes]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, subject: expr = ..., cases: list[match_case] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + class pattern(AST): lineno: int col_offset: int @@ -1056,6 +1417,11 @@ if sys.version_info >= (3, 10): end_col_offset: int def __init__(self, **kwargs: Unpack[_Attributes[int]]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, lineno: int = ..., col_offset: int = ..., end_lineno: int = ..., end_col_offset: int = ... + ) -> Self: ... + # Without the alias, Pyright complains variables named pattern are recursively defined _Pattern: typing_extensions.TypeAlias = pattern @@ -1072,16 +1438,25 @@ if sys.version_info >= (3, 10): @overload def __init__(self, pattern: _Pattern, guard: expr | None = None, *, body: list[stmt]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, pattern: _Pattern = ..., guard: expr | None = ..., body: list[stmt] = ...) -> Self: ... + class MatchValue(pattern): __match_args__ = ("value",) value: expr def __init__(self, value: expr, **kwargs: Unpack[_Attributes[int]]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + class MatchSingleton(pattern): __match_args__ = ("value",) value: Literal[True, False] | None def __init__(self, value: Literal[True, False] | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, value: Literal[True, False] | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + class MatchSequence(pattern): __match_args__ = ("patterns",) patterns: list[pattern] @@ -1090,11 +1465,17 @@ if sys.version_info >= (3, 10): else: def __init__(self, patterns: list[pattern], **kwargs: Unpack[_Attributes[int]]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + class MatchStar(pattern): __match_args__ = ("name",) name: _Identifier | None def __init__(self, name: _Identifier | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, name: _Identifier | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + class MatchMapping(pattern): __match_args__ = ("keys", "patterns", "rest") keys: list[expr] @@ -1117,6 +1498,16 @@ if sys.version_info >= (3, 10): **kwargs: Unpack[_Attributes[int]], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + keys: list[expr] = ..., + patterns: list[pattern] = ..., + rest: _Identifier | None = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + class MatchClass(pattern): __match_args__ = ("cls", "patterns", "kwd_attrs", "kwd_patterns") cls: expr @@ -1142,6 +1533,17 @@ if sys.version_info >= (3, 10): **kwargs: Unpack[_Attributes[int]], ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + cls: expr = ..., + patterns: list[pattern] = ..., + kwd_attrs: list[_Identifier] = ..., + kwd_patterns: list[pattern] = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + class MatchAs(pattern): __match_args__ = ("pattern", "name") pattern: _Pattern | None @@ -1150,6 +1552,11 @@ if sys.version_info >= (3, 10): self, pattern: _Pattern | None = None, name: _Identifier | None = None, **kwargs: Unpack[_Attributes[int]] ) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, pattern: _Pattern | None = ..., name: _Identifier | None = ..., **kwargs: Unpack[_Attributes[int]] + ) -> Self: ... + class MatchOr(pattern): __match_args__ = ("patterns",) patterns: list[pattern] @@ -1158,6 +1565,9 @@ if sys.version_info >= (3, 10): else: def __init__(self, patterns: list[pattern], **kwargs: Unpack[_Attributes[int]]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + if sys.version_info >= (3, 12): class type_param(AST): lineno: int @@ -1166,6 +1576,9 @@ if sys.version_info >= (3, 12): end_col_offset: int def __init__(self, **kwargs: Unpack[_Attributes[int]]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__(self, **kwargs: Unpack[_Attributes[int]]) -> Self: ... + class TypeVar(type_param): if sys.version_info >= (3, 13): __match_args__ = ("name", "bound", "default_value") @@ -1185,6 +1598,16 @@ if sys.version_info >= (3, 12): else: def __init__(self, name: _Identifier, bound: expr | None = None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier = ..., + bound: expr | None = ..., + default_value: expr | None = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + class ParamSpec(type_param): if sys.version_info >= (3, 13): __match_args__ = ("name", "default_value") @@ -1199,6 +1622,11 @@ if sys.version_info >= (3, 12): else: def __init__(self, name: _Identifier, **kwargs: Unpack[_Attributes[int]]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, name: _Identifier = ..., default_value: expr | None = ..., **kwargs: Unpack[_Attributes[int]] + ) -> Self: ... + class TypeVarTuple(type_param): if sys.version_info >= (3, 13): __match_args__ = ("name", "default_value") @@ -1213,6 +1641,11 @@ if sys.version_info >= (3, 12): else: def __init__(self, name: _Identifier, **kwargs: Unpack[_Attributes[int]]) -> None: ... + if sys.version_info >= (3, 14): + def __replace__( + self, *, name: _Identifier = ..., default_value: expr | None = ..., **kwargs: Unpack[_Attributes[int]] + ) -> Self: ... + class TypeAlias(stmt): __match_args__ = ("name", "type_params", "value") name: Name @@ -1231,3 +1664,13 @@ if sys.version_info >= (3, 12): def __init__( self, name: Name, type_params: list[type_param], value: expr, **kwargs: Unpack[_Attributes[int]] ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: Name = ..., + type_params: list[type_param] = ..., + value: expr = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... diff --git a/mypy/typeshed/stdlib/_collections_abc.pyi b/mypy/typeshed/stdlib/_collections_abc.pyi index 127488ee382c..8b1ac9c7eb8b 100644 --- a/mypy/typeshed/stdlib/_collections_abc.pyi +++ b/mypy/typeshed/stdlib/_collections_abc.pyi @@ -1,13 +1,12 @@ import sys from abc import abstractmethod from types import MappingProxyType -from typing import ( # noqa: Y022,Y038,Y057 +from typing import ( # noqa: Y022,Y038 AbstractSet as Set, AsyncGenerator as AsyncGenerator, AsyncIterable as AsyncIterable, AsyncIterator as AsyncIterator, Awaitable as Awaitable, - ByteString as ByteString, Callable as Callable, Collection as Collection, Container as Container, @@ -59,8 +58,12 @@ __all__ = [ "ValuesView", "Sequence", "MutableSequence", - "ByteString", ] +if sys.version_info < (3, 14): + from typing import ByteString as ByteString # noqa: Y057 + + __all__ += ["ByteString"] + if sys.version_info >= (3, 12): __all__ += ["Buffer"] diff --git a/mypy/typeshed/stdlib/_ctypes.pyi b/mypy/typeshed/stdlib/_ctypes.pyi index f7da03a67ead..0fe7521d7749 100644 --- a/mypy/typeshed/stdlib/_ctypes.pyi +++ b/mypy/typeshed/stdlib/_ctypes.pyi @@ -51,8 +51,8 @@ class _CDataMeta(type): # By default mypy complains about the following two methods, because strictly speaking cls # might not be a Type[_CT]. However this can never actually happen, because the only class that # uses _CDataMeta as its metaclass is _CData. So it's safe to ignore the errors here. - def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] - def __rmul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] + def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] + def __rmul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ... # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] class _CData(metaclass=_CDataMeta): _b_base_: int diff --git a/mypy/typeshed/stdlib/_operator.pyi b/mypy/typeshed/stdlib/_operator.pyi index 69ee563b5cf4..1b0083f4e274 100644 --- a/mypy/typeshed/stdlib/_operator.pyi +++ b/mypy/typeshed/stdlib/_operator.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import SupportsGetItem from collections.abc import Callable, Container, Iterable, MutableMapping, MutableSequence, Sequence from typing import Any, AnyStr, Generic, Protocol, SupportsAbs, SupportsIndex, TypeVar, final, overload -from typing_extensions import ParamSpec, TypeAlias, TypeVarTuple, Unpack +from typing_extensions import ParamSpec, TypeAlias, TypeIs, TypeVarTuple, Unpack _R = TypeVar("_R") _T = TypeVar("_T") @@ -145,3 +145,7 @@ if sys.version_info >= (3, 11): def call(obj: Callable[_P, _R], /, *args: _P.args, **kwargs: _P.kwargs) -> _R: ... def _compare_digest(a: AnyStr, b: AnyStr, /) -> bool: ... + +if sys.version_info >= (3, 14): + def is_none(a: object, /) -> TypeIs[None]: ... + def is_not_none(a: _T | None, /) -> TypeIs[_T]: ... diff --git a/mypy/typeshed/stdlib/_stat.pyi b/mypy/typeshed/stdlib/_stat.pyi index 903571a64bca..7129a282b574 100644 --- a/mypy/typeshed/stdlib/_stat.pyi +++ b/mypy/typeshed/stdlib/_stat.pyi @@ -28,9 +28,9 @@ S_IFDIR: Final = 0o040000 # These are 0 on systems that don't support the specific kind of file. # Example: Linux doesn't support door files, so S_IFDOOR is 0 on linux. -S_IFDOOR: int -S_IFPORT: int -S_IFWHT: int +S_IFDOOR: Final[int] +S_IFPORT: Final[int] +S_IFWHT: Final[int] S_ISUID: Final = 0o4000 S_ISGID: Final = 0o2000 @@ -79,9 +79,9 @@ def S_ISWHT(mode: int, /) -> bool: ... def filemode(mode: int, /) -> str: ... if sys.platform == "win32": - IO_REPARSE_TAG_SYMLINK: int - IO_REPARSE_TAG_MOUNT_POINT: int - IO_REPARSE_TAG_APPEXECLINK: int + IO_REPARSE_TAG_SYMLINK: Final = 0xA000000C + IO_REPARSE_TAG_MOUNT_POINT: Final = 0xA0000003 + IO_REPARSE_TAG_APPEXECLINK: Final = 0x8000001B if sys.platform == "win32": FILE_ATTRIBUTE_ARCHIVE: Final = 32 diff --git a/mypy/typeshed/stdlib/_thread.pyi b/mypy/typeshed/stdlib/_thread.pyi index 304cb79ec96b..b75e7608fa77 100644 --- a/mypy/typeshed/stdlib/_thread.pyi +++ b/mypy/typeshed/stdlib/_thread.pyi @@ -1,3 +1,4 @@ +import signal import sys from _typeshed import structseq from collections.abc import Callable @@ -16,16 +17,39 @@ class LockType: def acquire(self, blocking: bool = True, timeout: float = -1) -> bool: ... def release(self) -> None: ... def locked(self) -> bool: ... + def acquire_lock(self, blocking: bool = True, timeout: float = -1) -> bool: ... + def release_lock(self) -> None: ... + def locked_lock(self) -> bool: ... def __enter__(self) -> bool: ... def __exit__( self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None ) -> None: ... +if sys.version_info >= (3, 13): + @final + class _ThreadHandle: + ident: int + + def join(self, timeout: float | None = None, /) -> None: ... + def is_done(self) -> bool: ... + def _set_done(self) -> None: ... + + def start_joinable_thread( + function: Callable[[], object], handle: _ThreadHandle | None = None, daemon: bool = True + ) -> _ThreadHandle: ... + lock = LockType + @overload def start_new_thread(function: Callable[[Unpack[_Ts]], object], args: tuple[Unpack[_Ts]], /) -> int: ... @overload def start_new_thread(function: Callable[..., object], args: tuple[Any, ...], kwargs: dict[str, Any], /) -> int: ... -def interrupt_main() -> None: ... + +if sys.version_info >= (3, 10): + def interrupt_main(signum: signal.Signals = ..., /) -> None: ... + +else: + def interrupt_main() -> None: ... + def exit() -> NoReturn: ... def allocate_lock() -> LockType: ... def get_ident() -> int: ... diff --git a/mypy/typeshed/stdlib/_tkinter.pyi b/mypy/typeshed/stdlib/_tkinter.pyi index a7293054d293..63b1e7ca7cb4 100644 --- a/mypy/typeshed/stdlib/_tkinter.pyi +++ b/mypy/typeshed/stdlib/_tkinter.pyi @@ -106,8 +106,8 @@ EXCEPTION: Final = 8 READABLE: Final = 2 WRITABLE: Final = 4 -TCL_VERSION: str -TK_VERSION: str +TCL_VERSION: Final[str] +TK_VERSION: Final[str] @final class TkttType: diff --git a/mypy/typeshed/stdlib/argparse.pyi b/mypy/typeshed/stdlib/argparse.pyi index 66fa4e15291f..2526322ac8f6 100644 --- a/mypy/typeshed/stdlib/argparse.pyi +++ b/mypy/typeshed/stdlib/argparse.pyi @@ -357,7 +357,17 @@ class Action(_AttributeHolder): if sys.version_info >= (3, 12): class BooleanOptionalAction(Action): - if sys.version_info >= (3, 13): + if sys.version_info >= (3, 14): + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: bool | None = None, + required: bool = False, + help: str | None = None, + deprecated: bool = False, + ) -> None: ... + elif sys.version_info >= (3, 13): @overload def __init__( self, diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index 90ede461fe3c..80049cff4ce0 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -10,27 +10,28 @@ class _ABC(type): if sys.version_info >= (3, 9): def __init__(cls, *args: Unused) -> None: ... -@deprecated("Replaced by ast.Constant; removal scheduled for Python 3.14") -class Num(Constant, metaclass=_ABC): - value: int | float | complex +if sys.version_info < (3, 14): + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Num(Constant, metaclass=_ABC): + value: int | float | complex -@deprecated("Replaced by ast.Constant; removal scheduled for Python 3.14") -class Str(Constant, metaclass=_ABC): - value: str - # Aliases for value, for backwards compatibility - s: str + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Str(Constant, metaclass=_ABC): + value: str + # Aliases for value, for backwards compatibility + s: str -@deprecated("Replaced by ast.Constant; removal scheduled for Python 3.14") -class Bytes(Constant, metaclass=_ABC): - value: bytes - # Aliases for value, for backwards compatibility - s: bytes + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Bytes(Constant, metaclass=_ABC): + value: bytes + # Aliases for value, for backwards compatibility + s: bytes -@deprecated("Replaced by ast.Constant; removal scheduled for Python 3.14") -class NameConstant(Constant, metaclass=_ABC): ... + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class NameConstant(Constant, metaclass=_ABC): ... -@deprecated("Replaced by ast.Constant; removal scheduled for Python 3.14") -class Ellipsis(Constant, metaclass=_ABC): ... + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Ellipsis(Constant, metaclass=_ABC): ... if sys.version_info >= (3, 9): class slice(AST): ... diff --git a/mypy/typeshed/stdlib/asyncio/tasks.pyi b/mypy/typeshed/stdlib/asyncio/tasks.pyi index f23ecef126d6..bb423e857399 100644 --- a/mypy/typeshed/stdlib/asyncio/tasks.pyi +++ b/mypy/typeshed/stdlib/asyncio/tasks.pyi @@ -151,13 +151,13 @@ if sys.version_info >= (3, 10): @overload def gather(*coros_or_futures: _FutureLike[_T], return_exceptions: Literal[False] = False) -> Future[list[_T]]: ... # type: ignore[overload-overlap] @overload - def gather(coro_or_future1: _FutureLike[_T1], /, *, return_exceptions: bool) -> Future[tuple[_T1 | BaseException]]: ... # type: ignore[overload-overlap] + def gather(coro_or_future1: _FutureLike[_T1], /, *, return_exceptions: bool) -> Future[tuple[_T1 | BaseException]]: ... @overload - def gather( # type: ignore[overload-overlap] + def gather( coro_or_future1: _FutureLike[_T1], coro_or_future2: _FutureLike[_T2], /, *, return_exceptions: bool ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException]]: ... @overload - def gather( # type: ignore[overload-overlap] + def gather( coro_or_future1: _FutureLike[_T1], coro_or_future2: _FutureLike[_T2], coro_or_future3: _FutureLike[_T3], @@ -166,7 +166,7 @@ if sys.version_info >= (3, 10): return_exceptions: bool, ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException]]: ... @overload - def gather( # type: ignore[overload-overlap] + def gather( coro_or_future1: _FutureLike[_T1], coro_or_future2: _FutureLike[_T2], coro_or_future3: _FutureLike[_T3], @@ -176,7 +176,7 @@ if sys.version_info >= (3, 10): return_exceptions: bool, ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException, _T4 | BaseException]]: ... @overload - def gather( # type: ignore[overload-overlap] + def gather( coro_or_future1: _FutureLike[_T1], coro_or_future2: _FutureLike[_T2], coro_or_future3: _FutureLike[_T3], @@ -189,7 +189,7 @@ if sys.version_info >= (3, 10): tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException, _T4 | BaseException, _T5 | BaseException] ]: ... @overload - def gather( # type: ignore[overload-overlap] + def gather( coro_or_future1: _FutureLike[_T1], coro_or_future2: _FutureLike[_T2], coro_or_future3: _FutureLike[_T3], diff --git a/mypy/typeshed/stdlib/asyncio/unix_events.pyi b/mypy/typeshed/stdlib/asyncio/unix_events.pyi index 5dd3831f9a0a..fb21c5b5fa05 100644 --- a/mypy/typeshed/stdlib/asyncio/unix_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/unix_events.pyi @@ -159,7 +159,7 @@ if sys.platform != "win32": class _UnixSelectorEventLoop(BaseSelectorEventLoop): if sys.version_info >= (3, 13): - async def create_unix_server( # type: ignore[override] + async def create_unix_server( self, protocol_factory: _ProtocolFactory, path: StrPath | None = None, diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index 07d2f1989558..a6065cc6777f 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -33,7 +33,8 @@ from collections.abc import Awaitable, Callable, Iterable, Iterator, MutableSet, from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper from types import CellType, CodeType, TracebackType -# mypy crashes if any of {ByteString, Sequence, MutableSequence, Mapping, MutableMapping} are imported from collections.abc in builtins.pyi +# mypy crashes if any of {ByteString, Sequence, MutableSequence, Mapping, MutableMapping} +# are imported from collections.abc in builtins.pyi from typing import ( # noqa: Y022 IO, Any, @@ -872,7 +873,9 @@ class tuple(Sequence[_T_co]): if sys.version_info >= (3, 9): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... -# Doesn't exist at runtime, but deleting this breaks mypy. See #2999 +# Doesn't exist at runtime, but deleting this breaks mypy and pyright. See: +# https://github.com/python/typeshed/issues/7580 +# https://github.com/python/mypy/issues/8240 @final @type_check_only class function: @@ -989,7 +992,8 @@ class dict(MutableMapping[_KT, _VT]): def keys(self) -> dict_keys[_KT, _VT]: ... def values(self) -> dict_values[_KT, _VT]: ... def items(self) -> dict_items[_KT, _VT]: ... - # Signature of `dict.fromkeys` should be kept identical to `fromkeys` methods of `OrderedDict`/`ChainMap`/`UserDict` in `collections` + # Signature of `dict.fromkeys` should be kept identical to + # `fromkeys` methods of `OrderedDict`/`ChainMap`/`UserDict` in `collections` # TODO: the true signature of `dict.fromkeys` is not expressible in the current type system. # See #3800 & https://github.com/python/typing/issues/548#issuecomment-683336963. @classmethod @@ -1649,7 +1653,7 @@ _SupportsSumNoDefaultT = TypeVar("_SupportsSumNoDefaultT", bound=_SupportsSumWit # without creating many false-positive errors (see #7578). # Instead, we special-case the most common examples of this: bool and literal integers. @overload -def sum(iterable: Iterable[bool], /, start: int = 0) -> int: ... # type: ignore[overload-overlap] +def sum(iterable: Iterable[bool], /, start: int = 0) -> int: ... @overload def sum(iterable: Iterable[_SupportsSumNoDefaultT], /) -> _SupportsSumNoDefaultT | Literal[0]: ... @overload @@ -1657,9 +1661,8 @@ def sum(iterable: Iterable[_AddableT1], /, start: _AddableT2) -> _AddableT1 | _A # The argument to `vars()` has to have a `__dict__` attribute, so the second overload can't be annotated with `object` # (A "SupportsDunderDict" protocol doesn't work) -# Use a type: ignore to make complaints about overlapping overloads go away @overload -def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... # type: ignore[overload-overlap] +def vars(object: type, /) -> types.MappingProxyType[str, Any]: ... @overload def vars(object: Any = ..., /) -> dict[str, Any]: ... diff --git a/mypy/typeshed/stdlib/cmd.pyi b/mypy/typeshed/stdlib/cmd.pyi index 0733857433be..6e84133572bf 100644 --- a/mypy/typeshed/stdlib/cmd.pyi +++ b/mypy/typeshed/stdlib/cmd.pyi @@ -1,10 +1,11 @@ from collections.abc import Callable from typing import IO, Any, Final +from typing_extensions import LiteralString __all__ = ["Cmd"] PROMPT: Final = "(Cmd) " -IDENTCHARS: str # Too big to be `Literal` +IDENTCHARS: Final[LiteralString] # Too big to be `Literal` class Cmd: prompt: str diff --git a/mypy/typeshed/stdlib/collections/__init__.pyi b/mypy/typeshed/stdlib/collections/__init__.pyi index 71e3c564dd57..b2ed53e4427e 100644 --- a/mypy/typeshed/stdlib/collections/__init__.pyi +++ b/mypy/typeshed/stdlib/collections/__init__.pyi @@ -345,15 +345,15 @@ class _OrderedDictValuesView(ValuesView[_VT_co], Reversible[_VT_co]): # but they are not exposed anywhere) # pyright doesn't have a specific error code for subclassing error! @final -class _odict_keys(dict_keys[_KT_co, _VT_co], Reversible[_KT_co]): # type: ignore[misc] # pyright: ignore +class _odict_keys(dict_keys[_KT_co, _VT_co], Reversible[_KT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] def __reversed__(self) -> Iterator[_KT_co]: ... @final -class _odict_items(dict_items[_KT_co, _VT_co], Reversible[tuple[_KT_co, _VT_co]]): # type: ignore[misc] # pyright: ignore +class _odict_items(dict_items[_KT_co, _VT_co], Reversible[tuple[_KT_co, _VT_co]]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... @final -class _odict_values(dict_values[_KT_co, _VT_co], Reversible[_VT_co], Generic[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore +class _odict_values(dict_values[_KT_co, _VT_co], Reversible[_VT_co], Generic[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] def __reversed__(self) -> Iterator[_VT_co]: ... class OrderedDict(dict[_KT, _VT], Reversible[_KT], Generic[_KT, _VT]): @@ -475,7 +475,8 @@ class ChainMap(MutableMapping[_KT, _VT]): def pop(self, key: _KT, default: _T) -> _VT | _T: ... def copy(self) -> Self: ... __copy__ = copy - # All arguments to `fromkeys` are passed to `dict.fromkeys` at runtime, so the signature should be kept in line with `dict.fromkeys`. + # All arguments to `fromkeys` are passed to `dict.fromkeys` at runtime, + # so the signature should be kept in line with `dict.fromkeys`. @classmethod @overload def fromkeys(cls, iterable: Iterable[_T]) -> ChainMap[_T, Any | None]: ... diff --git a/mypy/typeshed/stdlib/contextlib.pyi b/mypy/typeshed/stdlib/contextlib.pyi index 29ac7cde561a..daf218d5a138 100644 --- a/mypy/typeshed/stdlib/contextlib.pyi +++ b/mypy/typeshed/stdlib/contextlib.pyi @@ -55,6 +55,7 @@ class AbstractAsyncContextManager(Protocol[_T_co, _ExitT_co]): ) -> _ExitT_co: ... class ContextDecorator: + def _recreate_cm(self) -> Self: ... def __call__(self, func: _F) -> _F: ... class _GeneratorContextManager(AbstractContextManager[_T_co, bool | None], ContextDecorator): @@ -80,6 +81,7 @@ if sys.version_info >= (3, 10): _AF = TypeVar("_AF", bound=Callable[..., Awaitable[Any]]) class AsyncContextDecorator: + def _recreate_cm(self) -> Self: ... def __call__(self, func: _AF) -> _AF: ... class _AsyncGeneratorContextManager(AbstractAsyncContextManager[_T_co, bool | None], AsyncContextDecorator): diff --git a/mypy/typeshed/stdlib/crypt.pyi b/mypy/typeshed/stdlib/crypt.pyi index 1ad0a384eae7..294003859286 100644 --- a/mypy/typeshed/stdlib/crypt.pyi +++ b/mypy/typeshed/stdlib/crypt.pyi @@ -1,12 +1,13 @@ import sys +from typing import Final if sys.platform != "win32": class _Method: ... - METHOD_CRYPT: _Method - METHOD_MD5: _Method - METHOD_SHA256: _Method - METHOD_SHA512: _Method - METHOD_BLOWFISH: _Method + METHOD_CRYPT: Final[_Method] + METHOD_MD5: Final[_Method] + METHOD_SHA256: Final[_Method] + METHOD_SHA512: Final[_Method] + METHOD_BLOWFISH: Final[_Method] methods: list[_Method] def mksalt(method: _Method | None = None, *, rounds: int | None = None) -> str: ... def crypt(word: str, salt: str | _Method | None = None) -> str: ... diff --git a/mypy/typeshed/stdlib/ctypes/__init__.pyi b/mypy/typeshed/stdlib/ctypes/__init__.pyi index dfd61c8f8ffc..40a073d107c7 100644 --- a/mypy/typeshed/stdlib/ctypes/__init__.pyi +++ b/mypy/typeshed/stdlib/ctypes/__init__.pyi @@ -185,3 +185,8 @@ if sys.version_info >= (3, 12): c_time_t: type[c_int32 | c_int64] # alias for one or the other at runtime class py_object(_CanCastTo, _SimpleCData[_T]): ... + +if sys.version_info >= (3, 14): + class c_float_complex(_SimpleCData[complex]): ... + class c_double_complex(_SimpleCData[complex]): ... + class c_longdouble_complex(_SimpleCData[complex]): ... diff --git a/mypy/typeshed/stdlib/ctypes/_endian.pyi b/mypy/typeshed/stdlib/ctypes/_endian.pyi index add6365e615f..144f5ba5dd40 100644 --- a/mypy/typeshed/stdlib/ctypes/_endian.pyi +++ b/mypy/typeshed/stdlib/ctypes/_endian.pyi @@ -1,12 +1,5 @@ import sys -from _ctypes import RTLD_GLOBAL as RTLD_GLOBAL, RTLD_LOCAL as RTLD_LOCAL, Structure, Union -from ctypes import DEFAULT_MODE as DEFAULT_MODE, cdll as cdll, pydll as pydll, pythonapi as pythonapi - -if sys.version_info >= (3, 12): - from _ctypes import SIZEOF_TIME_T as SIZEOF_TIME_T - -if sys.platform == "win32": - from ctypes import oledll as oledll, windll as windll +from ctypes import Structure, Union # At runtime, the native endianness is an alias for Structure, # while the other is a subclass with a metaclass added in. diff --git a/mypy/typeshed/stdlib/dataclasses.pyi b/mypy/typeshed/stdlib/dataclasses.pyi index 626608e8a59d..3295b1c1f835 100644 --- a/mypy/typeshed/stdlib/dataclasses.pyi +++ b/mypy/typeshed/stdlib/dataclasses.pyi @@ -5,7 +5,7 @@ from _typeshed import DataclassInstance from builtins import type as Type # alias to avoid name clashes with fields named "type" from collections.abc import Callable, Iterable, Mapping from typing import Any, Generic, Literal, Protocol, TypeVar, overload -from typing_extensions import TypeAlias, TypeIs +from typing_extensions import Never, TypeAlias, TypeIs if sys.version_info >= (3, 9): from types import GenericAlias @@ -213,6 +213,10 @@ else: ) -> Any: ... def fields(class_or_instance: DataclassInstance | type[DataclassInstance]) -> tuple[Field[Any], ...]: ... + +# HACK: `obj: Never` typing matches if object argument is using `Any` type. +@overload +def is_dataclass(obj: Never) -> TypeIs[DataclassInstance | type[DataclassInstance]]: ... # type: ignore[narrowed-type-not-subtype] # pyright: ignore[reportGeneralTypeIssues] @overload def is_dataclass(obj: type) -> TypeIs[type[DataclassInstance]]: ... @overload @@ -225,18 +229,17 @@ if sys.version_info >= (3, 9): else: class _InitVarMeta(type): # Not used, instead `InitVar.__class_getitem__` is called. - # pyright ignore is needed because pyright (not unreasonably) thinks this - # is an invalid use of InitVar. - def __getitem__(self, params: Any) -> InitVar[Any]: ... # pyright: ignore + # pyright (not unreasonably) thinks this is an invalid use of InitVar. + def __getitem__(self, params: Any) -> InitVar[Any]: ... # pyright: ignore[reportInvalidTypeForm] class InitVar(Generic[_T], metaclass=_InitVarMeta): type: Type[_T] def __init__(self, type: Type[_T]) -> None: ... if sys.version_info >= (3, 9): @overload - def __class_getitem__(cls, type: Type[_T]) -> InitVar[_T]: ... # pyright: ignore + def __class_getitem__(cls, type: Type[_T]) -> InitVar[_T]: ... # pyright: ignore[reportInvalidTypeForm] @overload - def __class_getitem__(cls, type: Any) -> InitVar[Any]: ... # pyright: ignore + def __class_getitem__(cls, type: Any) -> InitVar[Any]: ... # pyright: ignore[reportInvalidTypeForm] if sys.version_info >= (3, 12): def make_dataclass( diff --git a/mypy/typeshed/stdlib/datetime.pyi b/mypy/typeshed/stdlib/datetime.pyi index 38d5ac4c0819..e8a4efdc61f3 100644 --- a/mypy/typeshed/stdlib/datetime.pyi +++ b/mypy/typeshed/stdlib/datetime.pyi @@ -265,12 +265,12 @@ class datetime(date): def fromtimestamp(cls, timestamp: float, /, tz: _TzInfo | None = ...) -> Self: ... @classmethod - @deprecated("Use timezone-aware objects to represent datetimes in UTC; e.g. by calling .fromtimestamp(datetime.UTC)") + @deprecated("Use timezone-aware objects to represent datetimes in UTC; e.g. by calling .fromtimestamp(datetime.timezone.utc)") def utcfromtimestamp(cls, t: float, /) -> Self: ... @classmethod def now(cls, tz: _TzInfo | None = None) -> Self: ... @classmethod - @deprecated("Use timezone-aware objects to represent datetimes in UTC; e.g. by calling .now(datetime.UTC)") + @deprecated("Use timezone-aware objects to represent datetimes in UTC; e.g. by calling .now(datetime.timezone.utc)") def utcnow(cls) -> Self: ... @classmethod def combine(cls, date: _Date, time: _Time, tzinfo: _TzInfo | None = ...) -> Self: ... diff --git a/mypy/typeshed/stdlib/distutils/cmd.pyi b/mypy/typeshed/stdlib/distutils/cmd.pyi index ca4fb3265324..1f3f31c9c48a 100644 --- a/mypy/typeshed/stdlib/distutils/cmd.pyi +++ b/mypy/typeshed/stdlib/distutils/cmd.pyi @@ -1,6 +1,26 @@ from _typeshed import BytesPath, Incomplete, StrOrBytesPath, StrPath, Unused from abc import abstractmethod from collections.abc import Callable, Iterable +from distutils.command.bdist import bdist +from distutils.command.bdist_dumb import bdist_dumb +from distutils.command.bdist_rpm import bdist_rpm +from distutils.command.build import build +from distutils.command.build_clib import build_clib +from distutils.command.build_ext import build_ext +from distutils.command.build_py import build_py +from distutils.command.build_scripts import build_scripts +from distutils.command.check import check +from distutils.command.clean import clean +from distutils.command.config import config +from distutils.command.install import install +from distutils.command.install_data import install_data +from distutils.command.install_egg_info import install_egg_info +from distutils.command.install_headers import install_headers +from distutils.command.install_lib import install_lib +from distutils.command.install_scripts import install_scripts +from distutils.command.register import register +from distutils.command.sdist import sdist +from distutils.command.upload import upload from distutils.dist import Distribution from distutils.file_util import _BytesPathT, _StrPathT from typing import Any, ClassVar, Literal, TypeVar, overload @@ -28,8 +48,108 @@ class Command: def ensure_dirname(self, option: str) -> None: ... def get_command_name(self) -> str: ... def set_undefined_options(self, src_cmd: str, *option_pairs: tuple[str, str]) -> None: ... + # NOTE: This list comes directly from the distutils/command folder. Minus bdist_msi and bdist_wininst. + @overload + def get_finalized_command(self, command: Literal["bdist"], create: bool | Literal[0, 1] = 1) -> bdist: ... + @overload + def get_finalized_command(self, command: Literal["bdist_dumb"], create: bool | Literal[0, 1] = 1) -> bdist_dumb: ... + @overload + def get_finalized_command(self, command: Literal["bdist_rpm"], create: bool | Literal[0, 1] = 1) -> bdist_rpm: ... + @overload + def get_finalized_command(self, command: Literal["build"], create: bool | Literal[0, 1] = 1) -> build: ... + @overload + def get_finalized_command(self, command: Literal["build_clib"], create: bool | Literal[0, 1] = 1) -> build_clib: ... + @overload + def get_finalized_command(self, command: Literal["build_ext"], create: bool | Literal[0, 1] = 1) -> build_ext: ... + @overload + def get_finalized_command(self, command: Literal["build_py"], create: bool | Literal[0, 1] = 1) -> build_py: ... + @overload + def get_finalized_command(self, command: Literal["build_scripts"], create: bool | Literal[0, 1] = 1) -> build_scripts: ... + @overload + def get_finalized_command(self, command: Literal["check"], create: bool | Literal[0, 1] = 1) -> check: ... + @overload + def get_finalized_command(self, command: Literal["clean"], create: bool | Literal[0, 1] = 1) -> clean: ... + @overload + def get_finalized_command(self, command: Literal["config"], create: bool | Literal[0, 1] = 1) -> config: ... + @overload + def get_finalized_command(self, command: Literal["install"], create: bool | Literal[0, 1] = 1) -> install: ... + @overload + def get_finalized_command(self, command: Literal["install_data"], create: bool | Literal[0, 1] = 1) -> install_data: ... + @overload + def get_finalized_command( + self, command: Literal["install_egg_info"], create: bool | Literal[0, 1] = 1 + ) -> install_egg_info: ... + @overload + def get_finalized_command(self, command: Literal["install_headers"], create: bool | Literal[0, 1] = 1) -> install_headers: ... + @overload + def get_finalized_command(self, command: Literal["install_lib"], create: bool | Literal[0, 1] = 1) -> install_lib: ... + @overload + def get_finalized_command(self, command: Literal["install_scripts"], create: bool | Literal[0, 1] = 1) -> install_scripts: ... + @overload + def get_finalized_command(self, command: Literal["register"], create: bool | Literal[0, 1] = 1) -> register: ... + @overload + def get_finalized_command(self, command: Literal["sdist"], create: bool | Literal[0, 1] = 1) -> sdist: ... + @overload + def get_finalized_command(self, command: Literal["upload"], create: bool | Literal[0, 1] = 1) -> upload: ... + @overload def get_finalized_command(self, command: str, create: bool | Literal[0, 1] = 1) -> Command: ... @overload + def reinitialize_command(self, command: Literal["bdist"], reinit_subcommands: bool | Literal[0, 1] = 0) -> bdist: ... + @overload + def reinitialize_command( + self, command: Literal["bdist_dumb"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> bdist_dumb: ... + @overload + def reinitialize_command(self, command: Literal["bdist_rpm"], reinit_subcommands: bool | Literal[0, 1] = 0) -> bdist_rpm: ... + @overload + def reinitialize_command(self, command: Literal["build"], reinit_subcommands: bool | Literal[0, 1] = 0) -> build: ... + @overload + def reinitialize_command( + self, command: Literal["build_clib"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> build_clib: ... + @overload + def reinitialize_command(self, command: Literal["build_ext"], reinit_subcommands: bool | Literal[0, 1] = 0) -> build_ext: ... + @overload + def reinitialize_command(self, command: Literal["build_py"], reinit_subcommands: bool | Literal[0, 1] = 0) -> build_py: ... + @overload + def reinitialize_command( + self, command: Literal["build_scripts"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> build_scripts: ... + @overload + def reinitialize_command(self, command: Literal["check"], reinit_subcommands: bool | Literal[0, 1] = 0) -> check: ... + @overload + def reinitialize_command(self, command: Literal["clean"], reinit_subcommands: bool | Literal[0, 1] = 0) -> clean: ... + @overload + def reinitialize_command(self, command: Literal["config"], reinit_subcommands: bool | Literal[0, 1] = 0) -> config: ... + @overload + def reinitialize_command(self, command: Literal["install"], reinit_subcommands: bool | Literal[0, 1] = 0) -> install: ... + @overload + def reinitialize_command( + self, command: Literal["install_data"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> install_data: ... + @overload + def reinitialize_command( + self, command: Literal["install_egg_info"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> install_egg_info: ... + @overload + def reinitialize_command( + self, command: Literal["install_headers"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> install_headers: ... + @overload + def reinitialize_command( + self, command: Literal["install_lib"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> install_lib: ... + @overload + def reinitialize_command( + self, command: Literal["install_scripts"], reinit_subcommands: bool | Literal[0, 1] = 0 + ) -> install_scripts: ... + @overload + def reinitialize_command(self, command: Literal["register"], reinit_subcommands: bool | Literal[0, 1] = 0) -> register: ... + @overload + def reinitialize_command(self, command: Literal["sdist"], reinit_subcommands: bool | Literal[0, 1] = 0) -> sdist: ... + @overload + def reinitialize_command(self, command: Literal["upload"], reinit_subcommands: bool | Literal[0, 1] = 0) -> upload: ... + @overload def reinitialize_command(self, command: str, reinit_subcommands: bool | Literal[0, 1] = 0) -> Command: ... @overload def reinitialize_command(self, command: _CommandT, reinit_subcommands: bool | Literal[0, 1] = 0) -> _CommandT: ... diff --git a/mypy/typeshed/stdlib/distutils/command/__init__.pyi b/mypy/typeshed/stdlib/distutils/command/__init__.pyi index e69de29bb2d1..4d7372858af3 100644 --- a/mypy/typeshed/stdlib/distutils/command/__init__.pyi +++ b/mypy/typeshed/stdlib/distutils/command/__init__.pyi @@ -0,0 +1,48 @@ +import sys + +from . import ( + bdist, + bdist_dumb, + bdist_rpm, + build, + build_clib, + build_ext, + build_py, + build_scripts, + check, + clean, + install, + install_data, + install_headers, + install_lib, + install_scripts, + register, + sdist, + upload, +) + +__all__ = [ + "build", + "build_py", + "build_ext", + "build_clib", + "build_scripts", + "clean", + "install", + "install_lib", + "install_headers", + "install_scripts", + "install_data", + "sdist", + "register", + "bdist", + "bdist_dumb", + "bdist_rpm", + "check", + "upload", +] + +if sys.version_info < (3, 10): + from . import bdist_wininst + + __all__ += ["bdist_wininst"] diff --git a/mypy/typeshed/stdlib/distutils/command/check.pyi b/mypy/typeshed/stdlib/distutils/command/check.pyi index c67e4cbfdfe0..e69627d20c7a 100644 --- a/mypy/typeshed/stdlib/distutils/command/check.pyi +++ b/mypy/typeshed/stdlib/distutils/command/check.pyi @@ -1,4 +1,4 @@ -from typing import Any, ClassVar, Literal +from typing import Any, ClassVar, Final, Literal from typing_extensions import TypeAlias from ..cmd import Command @@ -22,7 +22,7 @@ class SilentReporter(_Reporter): ) -> None: ... def system_message(self, level, message, *children, **kwargs): ... -HAS_DOCUTILS: bool +HAS_DOCUTILS: Final[bool] class check(Command): description: str diff --git a/mypy/typeshed/stdlib/distutils/command/config.pyi b/mypy/typeshed/stdlib/distutils/command/config.pyi index 2f528c2c290b..b0910091d5b6 100644 --- a/mypy/typeshed/stdlib/distutils/command/config.pyi +++ b/mypy/typeshed/stdlib/distutils/command/config.pyi @@ -1,12 +1,12 @@ from _typeshed import StrOrBytesPath from collections.abc import Sequence from re import Pattern -from typing import Any, ClassVar, Literal +from typing import Any, ClassVar, Final, Literal from ..ccompiler import CCompiler from ..cmd import Command -LANG_EXT: dict[str, str] +LANG_EXT: Final[dict[str, str]] class config(Command): description: str diff --git a/mypy/typeshed/stdlib/distutils/command/install.pyi b/mypy/typeshed/stdlib/distutils/command/install.pyi index b0a5a82fc3f6..24a4eff2fb10 100644 --- a/mypy/typeshed/stdlib/distutils/command/install.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install.pyi @@ -1,11 +1,16 @@ +import sys from collections.abc import Callable -from typing import Any, ClassVar +from typing import Any, ClassVar, Final, Literal from ..cmd import Command -HAS_USER_SITE: bool -SCHEME_KEYS: tuple[str, ...] -INSTALL_SCHEMES: dict[str, dict[Any, Any]] +HAS_USER_SITE: Final[bool] + +SCHEME_KEYS: Final[tuple[Literal["purelib"], Literal["platlib"], Literal["headers"], Literal["scripts"], Literal["data"]]] +INSTALL_SCHEMES: Final[dict[str, dict[str, str]]] + +if sys.version_info < (3, 10): + WINDOWS_SCHEME: Final[dict[str, str]] class install(Command): description: str diff --git a/mypy/typeshed/stdlib/distutils/command/install_lib.pyi b/mypy/typeshed/stdlib/distutils/command/install_lib.pyi index 718d082b7b07..149ecae89781 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_lib.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_lib.pyi @@ -1,8 +1,8 @@ -from typing import Any, ClassVar +from typing import Any, ClassVar, Final from ..cmd import Command -PYTHON_SOURCE_EXTENSION: str +PYTHON_SOURCE_EXTENSION: Final = ".py" class install_lib(Command): description: str diff --git a/mypy/typeshed/stdlib/distutils/core.pyi b/mypy/typeshed/stdlib/distutils/core.pyi index f3c434df0b1a..a4d21f8ddd7b 100644 --- a/mypy/typeshed/stdlib/distutils/core.pyi +++ b/mypy/typeshed/stdlib/distutils/core.pyi @@ -3,9 +3,9 @@ from collections.abc import Mapping from distutils.cmd import Command as Command from distutils.dist import Distribution as Distribution from distutils.extension import Extension as Extension -from typing import Any, Literal +from typing import Any, Final, Literal -USAGE: str +USAGE: Final[str] def gen_usage(script_name: StrOrBytesPath) -> str: ... diff --git a/mypy/typeshed/stdlib/distutils/cygwinccompiler.pyi b/mypy/typeshed/stdlib/distutils/cygwinccompiler.pyi index 5f2e623eeff6..80924d63e471 100644 --- a/mypy/typeshed/stdlib/distutils/cygwinccompiler.pyi +++ b/mypy/typeshed/stdlib/distutils/cygwinccompiler.pyi @@ -1,20 +1,20 @@ from distutils.unixccompiler import UnixCCompiler from distutils.version import LooseVersion from re import Pattern -from typing import Literal +from typing import Final, Literal def get_msvcr() -> list[str] | None: ... class CygwinCCompiler(UnixCCompiler): ... class Mingw32CCompiler(CygwinCCompiler): ... -CONFIG_H_OK: str -CONFIG_H_NOTOK: str -CONFIG_H_UNCERTAIN: str +CONFIG_H_OK: Final = "ok" +CONFIG_H_NOTOK: Final = "not ok" +CONFIG_H_UNCERTAIN: Final = "uncertain" def check_config_h() -> tuple[Literal["ok", "not ok", "uncertain"], str]: ... -RE_VERSION: Pattern[bytes] +RE_VERSION: Final[Pattern[bytes]] def get_versions() -> tuple[LooseVersion | None, ...]: ... def is_cygwingcc() -> bool: ... diff --git a/mypy/typeshed/stdlib/distutils/debug.pyi b/mypy/typeshed/stdlib/distutils/debug.pyi index 11f28a8bc8ae..30095883b064 100644 --- a/mypy/typeshed/stdlib/distutils/debug.pyi +++ b/mypy/typeshed/stdlib/distutils/debug.pyi @@ -1 +1,3 @@ -DEBUG: bool | None +from typing import Final + +DEBUG: Final[str | None] diff --git a/mypy/typeshed/stdlib/distutils/dist.pyi b/mypy/typeshed/stdlib/distutils/dist.pyi index 21ddbc425918..e32fd70f7baa 100644 --- a/mypy/typeshed/stdlib/distutils/dist.pyi +++ b/mypy/typeshed/stdlib/distutils/dist.pyi @@ -1,6 +1,26 @@ from _typeshed import Incomplete, StrOrBytesPath, StrPath, SupportsWrite from collections.abc import Iterable, MutableMapping from distutils.cmd import Command +from distutils.command.bdist import bdist +from distutils.command.bdist_dumb import bdist_dumb +from distutils.command.bdist_rpm import bdist_rpm +from distutils.command.build import build +from distutils.command.build_clib import build_clib +from distutils.command.build_ext import build_ext +from distutils.command.build_py import build_py +from distutils.command.build_scripts import build_scripts +from distutils.command.check import check +from distutils.command.clean import clean +from distutils.command.config import config +from distutils.command.install import install +from distutils.command.install_data import install_data +from distutils.command.install_egg_info import install_egg_info +from distutils.command.install_headers import install_headers +from distutils.command.install_lib import install_lib +from distutils.command.install_scripts import install_scripts +from distutils.command.register import register +from distutils.command.sdist import sdist +from distutils.command.upload import upload from re import Pattern from typing import IO, ClassVar, Literal, TypeVar, overload from typing_extensions import TypeAlias @@ -63,10 +83,6 @@ class Distribution: def __init__(self, attrs: MutableMapping[str, Incomplete] | None = None) -> None: ... def get_option_dict(self, command: str) -> dict[str, tuple[str, str]]: ... def parse_config_files(self, filenames: Iterable[str] | None = None) -> None: ... - @overload - def get_command_obj(self, command: str, create: Literal[1, True] = 1) -> Command: ... - @overload - def get_command_obj(self, command: str, create: Literal[0, False]) -> Command | None: ... global_options: ClassVar[_OptionsList] common_usage: ClassVar[str] display_options: ClassVar[_OptionsList] @@ -108,8 +124,137 @@ class Distribution: def print_commands(self) -> None: ... def get_command_list(self): ... def get_command_packages(self): ... + # NOTE: This list comes directly from the distutils/command folder. Minus bdist_msi and bdist_wininst. + @overload + def get_command_obj(self, command: Literal["bdist"], create: Literal[1, True] = 1) -> bdist: ... + @overload + def get_command_obj(self, command: Literal["bdist_dumb"], create: Literal[1, True] = 1) -> bdist_dumb: ... + @overload + def get_command_obj(self, command: Literal["bdist_rpm"], create: Literal[1, True] = 1) -> bdist_rpm: ... + @overload + def get_command_obj(self, command: Literal["build"], create: Literal[1, True] = 1) -> build: ... + @overload + def get_command_obj(self, command: Literal["build_clib"], create: Literal[1, True] = 1) -> build_clib: ... + @overload + def get_command_obj(self, command: Literal["build_ext"], create: Literal[1, True] = 1) -> build_ext: ... + @overload + def get_command_obj(self, command: Literal["build_py"], create: Literal[1, True] = 1) -> build_py: ... + @overload + def get_command_obj(self, command: Literal["build_scripts"], create: Literal[1, True] = 1) -> build_scripts: ... + @overload + def get_command_obj(self, command: Literal["check"], create: Literal[1, True] = 1) -> check: ... + @overload + def get_command_obj(self, command: Literal["clean"], create: Literal[1, True] = 1) -> clean: ... + @overload + def get_command_obj(self, command: Literal["config"], create: Literal[1, True] = 1) -> config: ... + @overload + def get_command_obj(self, command: Literal["install"], create: Literal[1, True] = 1) -> install: ... + @overload + def get_command_obj(self, command: Literal["install_data"], create: Literal[1, True] = 1) -> install_data: ... + @overload + def get_command_obj(self, command: Literal["install_egg_info"], create: Literal[1, True] = 1) -> install_egg_info: ... + @overload + def get_command_obj(self, command: Literal["install_headers"], create: Literal[1, True] = 1) -> install_headers: ... + @overload + def get_command_obj(self, command: Literal["install_lib"], create: Literal[1, True] = 1) -> install_lib: ... + @overload + def get_command_obj(self, command: Literal["install_scripts"], create: Literal[1, True] = 1) -> install_scripts: ... + @overload + def get_command_obj(self, command: Literal["register"], create: Literal[1, True] = 1) -> register: ... + @overload + def get_command_obj(self, command: Literal["sdist"], create: Literal[1, True] = 1) -> sdist: ... + @overload + def get_command_obj(self, command: Literal["upload"], create: Literal[1, True] = 1) -> upload: ... + @overload + def get_command_obj(self, command: str, create: Literal[1, True] = 1) -> Command: ... + # Not replicating the overloads for "Command | None", user may use "isinstance" + @overload + def get_command_obj(self, command: str, create: Literal[0, False]) -> Command | None: ... + @overload + def get_command_class(self, command: Literal["bdist"]) -> type[bdist]: ... + @overload + def get_command_class(self, command: Literal["bdist_dumb"]) -> type[bdist_dumb]: ... + @overload + def get_command_class(self, command: Literal["bdist_rpm"]) -> type[bdist_rpm]: ... + @overload + def get_command_class(self, command: Literal["build"]) -> type[build]: ... + @overload + def get_command_class(self, command: Literal["build_clib"]) -> type[build_clib]: ... + @overload + def get_command_class(self, command: Literal["build_ext"]) -> type[build_ext]: ... + @overload + def get_command_class(self, command: Literal["build_py"]) -> type[build_py]: ... + @overload + def get_command_class(self, command: Literal["build_scripts"]) -> type[build_scripts]: ... + @overload + def get_command_class(self, command: Literal["check"]) -> type[check]: ... + @overload + def get_command_class(self, command: Literal["clean"]) -> type[clean]: ... + @overload + def get_command_class(self, command: Literal["config"]) -> type[config]: ... + @overload + def get_command_class(self, command: Literal["install"]) -> type[install]: ... + @overload + def get_command_class(self, command: Literal["install_data"]) -> type[install_data]: ... + @overload + def get_command_class(self, command: Literal["install_egg_info"]) -> type[install_egg_info]: ... + @overload + def get_command_class(self, command: Literal["install_headers"]) -> type[install_headers]: ... + @overload + def get_command_class(self, command: Literal["install_lib"]) -> type[install_lib]: ... + @overload + def get_command_class(self, command: Literal["install_scripts"]) -> type[install_scripts]: ... + @overload + def get_command_class(self, command: Literal["register"]) -> type[register]: ... + @overload + def get_command_class(self, command: Literal["sdist"]) -> type[sdist]: ... + @overload + def get_command_class(self, command: Literal["upload"]) -> type[upload]: ... + @overload def get_command_class(self, command: str) -> type[Command]: ... @overload + def reinitialize_command(self, command: Literal["bdist"], reinit_subcommands: bool = False) -> bdist: ... + @overload + def reinitialize_command(self, command: Literal["bdist_dumb"], reinit_subcommands: bool = False) -> bdist_dumb: ... + @overload + def reinitialize_command(self, command: Literal["bdist_rpm"], reinit_subcommands: bool = False) -> bdist_rpm: ... + @overload + def reinitialize_command(self, command: Literal["build"], reinit_subcommands: bool = False) -> build: ... + @overload + def reinitialize_command(self, command: Literal["build_clib"], reinit_subcommands: bool = False) -> build_clib: ... + @overload + def reinitialize_command(self, command: Literal["build_ext"], reinit_subcommands: bool = False) -> build_ext: ... + @overload + def reinitialize_command(self, command: Literal["build_py"], reinit_subcommands: bool = False) -> build_py: ... + @overload + def reinitialize_command(self, command: Literal["build_scripts"], reinit_subcommands: bool = False) -> build_scripts: ... + @overload + def reinitialize_command(self, command: Literal["check"], reinit_subcommands: bool = False) -> check: ... + @overload + def reinitialize_command(self, command: Literal["clean"], reinit_subcommands: bool = False) -> clean: ... + @overload + def reinitialize_command(self, command: Literal["config"], reinit_subcommands: bool = False) -> config: ... + @overload + def reinitialize_command(self, command: Literal["install"], reinit_subcommands: bool = False) -> install: ... + @overload + def reinitialize_command(self, command: Literal["install_data"], reinit_subcommands: bool = False) -> install_data: ... + @overload + def reinitialize_command( + self, command: Literal["install_egg_info"], reinit_subcommands: bool = False + ) -> install_egg_info: ... + @overload + def reinitialize_command(self, command: Literal["install_headers"], reinit_subcommands: bool = False) -> install_headers: ... + @overload + def reinitialize_command(self, command: Literal["install_lib"], reinit_subcommands: bool = False) -> install_lib: ... + @overload + def reinitialize_command(self, command: Literal["install_scripts"], reinit_subcommands: bool = False) -> install_scripts: ... + @overload + def reinitialize_command(self, command: Literal["register"], reinit_subcommands: bool = False) -> register: ... + @overload + def reinitialize_command(self, command: Literal["sdist"], reinit_subcommands: bool = False) -> sdist: ... + @overload + def reinitialize_command(self, command: Literal["upload"], reinit_subcommands: bool = False) -> upload: ... + @overload def reinitialize_command(self, command: str, reinit_subcommands: bool = False) -> Command: ... @overload def reinitialize_command(self, command: _CommandT, reinit_subcommands: bool = False) -> _CommandT: ... diff --git a/mypy/typeshed/stdlib/distutils/fancy_getopt.pyi b/mypy/typeshed/stdlib/distutils/fancy_getopt.pyi index f9916d4511b2..c4d37419ed06 100644 --- a/mypy/typeshed/stdlib/distutils/fancy_getopt.pyi +++ b/mypy/typeshed/stdlib/distutils/fancy_getopt.pyi @@ -1,15 +1,15 @@ from collections.abc import Iterable, Mapping from re import Pattern -from typing import Any, overload +from typing import Any, Final, overload from typing_extensions import TypeAlias _Option: TypeAlias = tuple[str, str | None, str] _GR: TypeAlias = tuple[list[str], OptionDummy] -longopt_pat: str -longopt_re: Pattern[str] -neg_alias_re: Pattern[str] -longopt_xlate: dict[int, int] +longopt_pat: Final = r"[a-zA-Z](?:[a-zA-Z0-9-]*)" +longopt_re: Final[Pattern[str]] +neg_alias_re: Final[Pattern[str]] +longopt_xlate: Final[dict[int, int]] class FancyGetopt: def __init__(self, option_table: list[_Option] | None = None) -> None: ... @@ -25,7 +25,7 @@ def fancy_getopt( options: list[_Option], negative_opt: Mapping[_Option, _Option], object: Any, args: list[str] | None ) -> list[str] | _GR: ... -WS_TRANS: dict[int, str] +WS_TRANS: Final[dict[int, str]] def wrap_text(text: str, width: int) -> list[str]: ... def translate_longopt(opt: str) -> str: ... diff --git a/mypy/typeshed/stdlib/distutils/log.pyi b/mypy/typeshed/stdlib/distutils/log.pyi index 14ed8d8aefa8..0ea135c28371 100644 --- a/mypy/typeshed/stdlib/distutils/log.pyi +++ b/mypy/typeshed/stdlib/distutils/log.pyi @@ -1,10 +1,10 @@ -from typing import Any +from typing import Any, Final -DEBUG: int -INFO: int -WARN: int -ERROR: int -FATAL: int +DEBUG: Final = 1 +INFO: Final = 2 +WARN: Final = 3 +ERROR: Final = 4 +FATAL: Final = 5 class Log: def __init__(self, threshold: int = 3) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/sysconfig.pyi b/mypy/typeshed/stdlib/distutils/sysconfig.pyi index da72e3275fe3..4a9c45eb562a 100644 --- a/mypy/typeshed/stdlib/distutils/sysconfig.pyi +++ b/mypy/typeshed/stdlib/distutils/sysconfig.pyi @@ -1,15 +1,15 @@ import sys from collections.abc import Mapping from distutils.ccompiler import CCompiler -from typing import Literal, overload +from typing import Final, Literal, overload from typing_extensions import deprecated -PREFIX: str -EXEC_PREFIX: str -BASE_PREFIX: str -BASE_EXEC_PREFIX: str -project_base: str -python_build: bool +PREFIX: Final[str] +EXEC_PREFIX: Final[str] +BASE_PREFIX: Final[str] +BASE_EXEC_PREFIX: Final[str] +project_base: Final[str] +python_build: Final[bool] def expand_makefile_vars(s: str, vars: Mapping[str, str]) -> str: ... @overload diff --git a/mypy/typeshed/stdlib/email/message.pyi b/mypy/typeshed/stdlib/email/message.pyi index 4032bc6136d4..7e80f13adb8f 100644 --- a/mypy/typeshed/stdlib/email/message.pyi +++ b/mypy/typeshed/stdlib/email/message.pyi @@ -50,7 +50,8 @@ class Message(Generic[_HeaderT, _HeaderParamT]): def get_payload(self, i: None = None, *, decode: Literal[True]) -> _EncodedPayloadType | Any: ... @overload # not multipart, IDEM but w/o kwarg def get_payload(self, i: None, decode: Literal[True]) -> _EncodedPayloadType | Any: ... - # If `charset=None` and payload supports both `encode` AND `decode`, then an invalid payload could be passed, but this is unlikely + # If `charset=None` and payload supports both `encode` AND `decode`, + # then an invalid payload could be passed, but this is unlikely # Not[_SupportsEncodeToPayload] @overload def set_payload( @@ -146,7 +147,11 @@ class Message(Generic[_HeaderT, _HeaderParamT]): class MIMEPart(Message[_HeaderRegistryT, _HeaderRegistryParamT]): def __init__(self, policy: Policy | None = None) -> None: ... def get_body(self, preferencelist: Sequence[str] = ("related", "html", "plain")) -> MIMEPart[_HeaderRegistryT] | None: ... - def iter_attachments(self) -> Iterator[MIMEPart[_HeaderRegistryT]]: ... + def attach(self, payload: Self) -> None: ... # type: ignore[override] + # The attachments are created via type(self) in the attach method. It's theoretically + # possible to sneak other attachment types into a MIMEPart instance, but could cause + # cause unforseen consequences. + def iter_attachments(self) -> Iterator[Self]: ... def iter_parts(self) -> Iterator[MIMEPart[_HeaderRegistryT]]: ... def get_content(self, *args: Any, content_manager: ContentManager | None = None, **kw: Any) -> Any: ... def set_content(self, *args: Any, content_manager: ContentManager | None = None, **kw: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/email/utils.pyi b/mypy/typeshed/stdlib/email/utils.pyi index 2724dbf6ec2f..9dab22c18f6c 100644 --- a/mypy/typeshed/stdlib/email/utils.pyi +++ b/mypy/typeshed/stdlib/email/utils.pyi @@ -66,7 +66,10 @@ def mktime_tz(data: _PDTZ) -> int: ... def formatdate(timeval: float | None = None, localtime: bool = False, usegmt: bool = False) -> str: ... def format_datetime(dt: datetime.datetime, usegmt: bool = False) -> str: ... -if sys.version_info >= (3, 12): +if sys.version_info >= (3, 14): + def localtime(dt: datetime.datetime | None = None) -> datetime.datetime: ... + +elif sys.version_info >= (3, 12): @overload def localtime(dt: datetime.datetime | None = None) -> datetime.datetime: ... @overload diff --git a/mypy/typeshed/stdlib/filecmp.pyi b/mypy/typeshed/stdlib/filecmp.pyi index dfec2da72344..cb7b94596077 100644 --- a/mypy/typeshed/stdlib/filecmp.pyi +++ b/mypy/typeshed/stdlib/filecmp.pyi @@ -17,13 +17,24 @@ def cmpfiles( ) -> tuple[list[AnyStr], list[AnyStr], list[AnyStr]]: ... class dircmp(Generic[AnyStr]): - def __init__( - self, - a: GenericPath[AnyStr], - b: GenericPath[AnyStr], - ignore: Sequence[AnyStr] | None = None, - hide: Sequence[AnyStr] | None = None, - ) -> None: ... + if sys.version_info >= (3, 13): + def __init__( + self, + a: GenericPath[AnyStr], + b: GenericPath[AnyStr], + ignore: Sequence[AnyStr] | None = None, + hide: Sequence[AnyStr] | None = None, + *, + shallow: bool = True, + ) -> None: ... + else: + def __init__( + self, + a: GenericPath[AnyStr], + b: GenericPath[AnyStr], + ignore: Sequence[AnyStr] | None = None, + hide: Sequence[AnyStr] | None = None, + ) -> None: ... left: AnyStr right: AnyStr hide: Sequence[AnyStr] diff --git a/mypy/typeshed/stdlib/ftplib.pyi b/mypy/typeshed/stdlib/ftplib.pyi index 1b96e0d504b7..3693d7c52a26 100644 --- a/mypy/typeshed/stdlib/ftplib.pyi +++ b/mypy/typeshed/stdlib/ftplib.pyi @@ -86,7 +86,7 @@ class FTP: def makeport(self) -> socket: ... def makepasv(self) -> tuple[str, int]: ... def login(self, user: str = "", passwd: str = "", acct: str = "") -> str: ... - # In practice, `rest` rest can actually be anything whose str() is an integer sequence, so to make it simple we allow integers. + # In practice, `rest` can actually be anything whose str() is an integer sequence, so to make it simple we allow integers def ntransfercmd(self, cmd: str, rest: int | str | None = None) -> tuple[socket, int | None]: ... def transfercmd(self, cmd: str, rest: int | str | None = None) -> socket: ... def retrbinary( diff --git a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi index 37b9a3882179..5e26f8987277 100644 --- a/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi +++ b/mypy/typeshed/stdlib/importlib/metadata/__init__.pyi @@ -155,7 +155,7 @@ if sys.version_info >= (3, 10) and sys.version_info < (3, 12): @property def names(self) -> set[str]: ... @overload - def select(self) -> Self: ... # type: ignore[misc] + def select(self) -> Self: ... @overload def select( self, @@ -277,7 +277,7 @@ if sys.version_info >= (3, 12): elif sys.version_info >= (3, 10): @overload - def entry_points() -> SelectableGroups: ... # type: ignore[overload-overlap] + def entry_points() -> SelectableGroups: ... @overload def entry_points( *, name: str = ..., value: str = ..., group: str = ..., module: str = ..., attr: str = ..., extras: list[str] = ... diff --git a/mypy/typeshed/stdlib/json/encoder.pyi b/mypy/typeshed/stdlib/json/encoder.pyi index 473398a60b2a..aa4a3bdf61d4 100644 --- a/mypy/typeshed/stdlib/json/encoder.pyi +++ b/mypy/typeshed/stdlib/json/encoder.pyi @@ -1,12 +1,12 @@ from collections.abc import Callable, Iterator from re import Pattern -from typing import Any +from typing import Any, Final -ESCAPE: Pattern[str] -ESCAPE_ASCII: Pattern[str] -HAS_UTF8: Pattern[bytes] -ESCAPE_DCT: dict[str, str] -INFINITY: float +ESCAPE: Final[Pattern[str]] +ESCAPE_ASCII: Final[Pattern[str]] +HAS_UTF8: Final[Pattern[bytes]] +ESCAPE_DCT: Final[dict[str, str]] +INFINITY: Final[float] def py_encode_basestring(s: str) -> str: ... # undocumented def py_encode_basestring_ascii(s: str) -> str: ... # undocumented diff --git a/mypy/typeshed/stdlib/lib2to3/fixes/fix_unicode.pyi b/mypy/typeshed/stdlib/lib2to3/fixes/fix_unicode.pyi index 80d9d8b6e656..85d1315213b9 100644 --- a/mypy/typeshed/stdlib/lib2to3/fixes/fix_unicode.pyi +++ b/mypy/typeshed/stdlib/lib2to3/fixes/fix_unicode.pyi @@ -6,7 +6,7 @@ from ..pytree import Node class FixUnicode(fixer_base.BaseFix): BM_compatible: ClassVar[Literal[True]] - PATTERN: ClassVar[Literal["STRING | 'unicode' | 'unichr'"]] # type: ignore[name-defined] # Name "STRING" is not defined + PATTERN: ClassVar[str] unicode_literals: bool def start_tree(self, tree: Node, filename: StrPath) -> None: ... def transform(self, node, results): ... diff --git a/mypy/typeshed/stdlib/logging/__init__.pyi b/mypy/typeshed/stdlib/logging/__init__.pyi index e6e6e8f645a0..9a4827a8f626 100644 --- a/mypy/typeshed/stdlib/logging/__init__.pyi +++ b/mypy/typeshed/stdlib/logging/__init__.pyi @@ -55,10 +55,9 @@ __all__ = [ "setLogRecordFactory", "lastResort", "raiseExceptions", + "warn", ] -if sys.version_info < (3, 13): - __all__ += ["warn"] if sys.version_info >= (3, 11): __all__ += ["getLevelNamesMapping"] if sys.version_info >= (3, 12): @@ -157,17 +156,16 @@ class Logger(Filterer): stacklevel: int = 1, extra: Mapping[str, object] | None = None, ) -> None: ... - if sys.version_info < (3, 13): - def warn( - self, - msg: object, - *args: object, - exc_info: _ExcInfoType = None, - stack_info: bool = False, - stacklevel: int = 1, - extra: Mapping[str, object] | None = None, - ) -> None: ... - + @deprecated("Deprecated; use warning() instead.") + def warn( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + ) -> None: ... def error( self, msg: object, @@ -412,18 +410,17 @@ class LoggerAdapter(Generic[_L]): extra: Mapping[str, object] | None = None, **kwargs: object, ) -> None: ... - if sys.version_info < (3, 13): - def warn( - self, - msg: object, - *args: object, - exc_info: _ExcInfoType = None, - stack_info: bool = False, - stacklevel: int = 1, - extra: Mapping[str, object] | None = None, - **kwargs: object, - ) -> None: ... - + @deprecated("Deprecated; use warning() instead.") + def warn( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, + **kwargs: object, + ) -> None: ... def error( self, msg: object, @@ -523,17 +520,15 @@ def warning( stacklevel: int = 1, extra: Mapping[str, object] | None = None, ) -> None: ... - -if sys.version_info < (3, 13): - def warn( - msg: object, - *args: object, - exc_info: _ExcInfoType = None, - stack_info: bool = False, - stacklevel: int = 1, - extra: Mapping[str, object] | None = None, - ) -> None: ... - +@deprecated("Deprecated; use warning() instead.") +def warn( + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: Mapping[str, object] | None = None, +) -> None: ... def error( msg: object, *args: object, diff --git a/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi b/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi index 2b96ff047470..2b0498abc2c6 100644 --- a/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/sharedctypes.pyi @@ -73,7 +73,7 @@ def copy(obj: _CT) -> _CT: ... @overload def synchronized(obj: _SimpleCData[_T], lock: _LockLike | None = None, ctx: Any | None = None) -> Synchronized[_T]: ... @overload -def synchronized(obj: ctypes.Array[c_char], lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedString: ... # type: ignore +def synchronized(obj: ctypes.Array[c_char], lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedString: ... @overload def synchronized( obj: ctypes.Array[_SimpleCData[_T]], lock: _LockLike | None = None, ctx: Any | None = None @@ -115,12 +115,12 @@ class SynchronizedArray(SynchronizedBase[ctypes.Array[_SimpleCData[_T]]], Generi class SynchronizedString(SynchronizedArray[bytes]): @overload # type: ignore[override] def __getitem__(self, i: slice) -> bytes: ... - @overload # type: ignore[override] + @overload def __getitem__(self, i: int) -> bytes: ... @overload # type: ignore[override] def __setitem__(self, i: slice, value: bytes) -> None: ... - @overload # type: ignore[override] - def __setitem__(self, i: int, value: bytes) -> None: ... # type: ignore[override] + @overload + def __setitem__(self, i: int, value: bytes) -> None: ... def __getslice__(self, start: int, stop: int) -> bytes: ... # type: ignore[override] def __setslice__(self, start: int, stop: int, values: bytes) -> None: ... # type: ignore[override] diff --git a/mypy/typeshed/stdlib/operator.pyi b/mypy/typeshed/stdlib/operator.pyi index a0e5df7977da..1a817f00f3c1 100644 --- a/mypy/typeshed/stdlib/operator.pyi +++ b/mypy/typeshed/stdlib/operator.pyi @@ -61,6 +61,9 @@ __all__ = [ if sys.version_info >= (3, 11): __all__ += ["call"] +if sys.version_info >= (3, 14): + __all__ += ["is_none", "is_not_none"] + __lt__ = lt __le__ = le __eq__ = eq diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index e2d272cb4112..700e0e9df310 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -365,7 +365,9 @@ class stat_result(structseq[float], tuple[int, int, int, int, int, int, int, flo if sys.version_info >= (3, 12) and sys.platform == "win32": @property @deprecated( - "Use st_birthtime instead to retrieve the file creation time. In the future, this property will contain the last metadata change time." + """\ +Use st_birthtime instead to retrieve the file creation time. \ +In the future, this property will contain the last metadata change time.""" ) def st_ctime(self) -> float: ... else: diff --git a/mypy/typeshed/stdlib/pathlib.pyi b/mypy/typeshed/stdlib/pathlib.pyi index 116bf6431831..bdca375f626d 100644 --- a/mypy/typeshed/stdlib/pathlib.pyi +++ b/mypy/typeshed/stdlib/pathlib.pyi @@ -159,6 +159,20 @@ class Path(PurePath): def lchmod(self, mode: int) -> None: ... def lstat(self) -> stat_result: ... def mkdir(self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False) -> None: ... + + if sys.version_info >= (3, 14): + def copy(self, target: StrPath, *, follow_symlinks: bool = True, preserve_metadata: bool = False) -> None: ... + def copytree( + self, + target: StrPath, + *, + follow_symlinks: bool = True, + preserve_metadata: bool = False, + dirs_exist_ok: bool = False, + ignore: Callable[[Self], bool] | None = None, + on_error: Callable[[OSError], object] | None = None, + ) -> None: ... + # Adapted from builtins.open # Text mode: always returns a TextIOWrapper # The Traversable .open in stdlib/importlib/abc.pyi should be kept in sync with this. @@ -232,10 +246,18 @@ class Path(PurePath): if sys.version_info >= (3, 9): def readlink(self) -> Self: ... - def rename(self, target: str | PurePath) -> Self: ... - def replace(self, target: str | PurePath) -> Self: ... + if sys.version_info >= (3, 10): + def rename(self, target: StrPath) -> Self: ... + def replace(self, target: StrPath) -> Self: ... + else: + def rename(self, target: str | PurePath) -> Self: ... + def replace(self, target: str | PurePath) -> Self: ... + def resolve(self, strict: bool = False) -> Self: ... def rmdir(self) -> None: ... + if sys.version_info >= (3, 14): + def delete(self, ignore_errors: bool = False, on_error: Callable[[OSError], object] | None = None) -> None: ... + def symlink_to(self, target: StrOrBytesPath, target_is_directory: bool = False) -> None: ... if sys.version_info >= (3, 10): def hardlink_to(self, target: StrOrBytesPath) -> None: ... @@ -266,6 +288,9 @@ class Path(PurePath): self, top_down: bool = ..., on_error: Callable[[OSError], object] | None = ..., follow_symlinks: bool = ... ) -> Iterator[tuple[Self, list[str], list[str]]]: ... + if sys.version_info >= (3, 14): + def rmtree(self, ignore_errors: bool = False, on_error: Callable[[OSError], object] | None = None) -> None: ... + class PosixPath(Path, PurePosixPath): ... class WindowsPath(Path, PureWindowsPath): ... diff --git a/mypy/typeshed/stdlib/pdb.pyi b/mypy/typeshed/stdlib/pdb.pyi index d49315427813..61e8b7176e84 100644 --- a/mypy/typeshed/stdlib/pdb.pyi +++ b/mypy/typeshed/stdlib/pdb.pyi @@ -84,7 +84,7 @@ class Pdb(Bdb, Cmd): def _runscript(self, filename: str) -> None: ... if sys.version_info >= (3, 13): - def completedefault(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... # type: ignore[override] + def completedefault(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ... def do_commands(self, arg: str) -> bool | None: ... def do_break(self, arg: str, temporary: bool = ...) -> bool | None: ... diff --git a/mypy/typeshed/stdlib/poplib.pyi b/mypy/typeshed/stdlib/poplib.pyi index 7476f2991978..a1e41be86a7f 100644 --- a/mypy/typeshed/stdlib/poplib.pyi +++ b/mypy/typeshed/stdlib/poplib.pyi @@ -67,5 +67,6 @@ class POP3_SSL(POP3): timeout: float = ..., context: ssl.SSLContext | None = None, ) -> None: ... - # "context" is actually the last argument, but that breaks LSP and it doesn't really matter because all the arguments are ignored + # "context" is actually the last argument, + # but that breaks LSP and it doesn't really matter because all the arguments are ignored def stls(self, context: Any = None, keyfile: Any = None, certfile: Any = None) -> NoReturn: ... diff --git a/mypy/typeshed/stdlib/pty.pyi b/mypy/typeshed/stdlib/pty.pyi index 4c9e42b4ec5e..941915179c4a 100644 --- a/mypy/typeshed/stdlib/pty.pyi +++ b/mypy/typeshed/stdlib/pty.pyi @@ -1,7 +1,7 @@ import sys from collections.abc import Callable, Iterable from typing import Final -from typing_extensions import TypeAlias +from typing_extensions import TypeAlias, deprecated if sys.platform != "win32": __all__ = ["openpty", "fork", "spawn"] @@ -13,7 +13,12 @@ if sys.platform != "win32": CHILD: Final = 0 def openpty() -> tuple[int, int]: ... - def master_open() -> tuple[int, str]: ... # deprecated, use openpty() - def slave_open(tty_name: str) -> int: ... # deprecated, use openpty() + + if sys.version_info < (3, 14): + @deprecated("Deprecated in 3.12, to be removed in 3.14; use openpty() instead") + def master_open() -> tuple[int, str]: ... + @deprecated("Deprecated in 3.12, to be removed in 3.14; use openpty() instead") + def slave_open(tty_name: str) -> int: ... + def fork() -> tuple[int, int]: ... def spawn(argv: str | Iterable[str], master_read: _Reader = ..., stdin_read: _Reader = ...) -> int: ... diff --git a/mypy/typeshed/stdlib/pyexpat/__init__.pyi b/mypy/typeshed/stdlib/pyexpat/__init__.pyi index 64decd56bee6..dc0156ef13bd 100644 --- a/mypy/typeshed/stdlib/pyexpat/__init__.pyi +++ b/mypy/typeshed/stdlib/pyexpat/__init__.pyi @@ -15,9 +15,9 @@ class ExpatError(Exception): offset: int error = ExpatError -XML_PARAM_ENTITY_PARSING_NEVER: int -XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE: int -XML_PARAM_ENTITY_PARSING_ALWAYS: int +XML_PARAM_ENTITY_PARSING_NEVER: Final = 0 +XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE: Final = 1 +XML_PARAM_ENTITY_PARSING_ALWAYS: Final = 2 _Model: TypeAlias = tuple[int, int, str | None, tuple[Any, ...]] diff --git a/mypy/typeshed/stdlib/pyexpat/errors.pyi b/mypy/typeshed/stdlib/pyexpat/errors.pyi index 2e512eb12989..cae4da089161 100644 --- a/mypy/typeshed/stdlib/pyexpat/errors.pyi +++ b/mypy/typeshed/stdlib/pyexpat/errors.pyi @@ -1,49 +1,51 @@ import sys +from typing import Final +from typing_extensions import LiteralString codes: dict[str, int] messages: dict[int, str] -XML_ERROR_ABORTED: str -XML_ERROR_ASYNC_ENTITY: str -XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF: str -XML_ERROR_BAD_CHAR_REF: str -XML_ERROR_BINARY_ENTITY_REF: str -XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING: str -XML_ERROR_DUPLICATE_ATTRIBUTE: str -XML_ERROR_ENTITY_DECLARED_IN_PE: str -XML_ERROR_EXTERNAL_ENTITY_HANDLING: str -XML_ERROR_FEATURE_REQUIRES_XML_DTD: str -XML_ERROR_FINISHED: str -XML_ERROR_INCOMPLETE_PE: str -XML_ERROR_INCORRECT_ENCODING: str -XML_ERROR_INVALID_TOKEN: str -XML_ERROR_JUNK_AFTER_DOC_ELEMENT: str -XML_ERROR_MISPLACED_XML_PI: str -XML_ERROR_NOT_STANDALONE: str -XML_ERROR_NOT_SUSPENDED: str -XML_ERROR_NO_ELEMENTS: str -XML_ERROR_NO_MEMORY: str -XML_ERROR_PARAM_ENTITY_REF: str -XML_ERROR_PARTIAL_CHAR: str -XML_ERROR_PUBLICID: str -XML_ERROR_RECURSIVE_ENTITY_REF: str -XML_ERROR_SUSPENDED: str -XML_ERROR_SUSPEND_PE: str -XML_ERROR_SYNTAX: str -XML_ERROR_TAG_MISMATCH: str -XML_ERROR_TEXT_DECL: str -XML_ERROR_UNBOUND_PREFIX: str -XML_ERROR_UNCLOSED_CDATA_SECTION: str -XML_ERROR_UNCLOSED_TOKEN: str -XML_ERROR_UNDECLARING_PREFIX: str -XML_ERROR_UNDEFINED_ENTITY: str -XML_ERROR_UNEXPECTED_STATE: str -XML_ERROR_UNKNOWN_ENCODING: str -XML_ERROR_XML_DECL: str +XML_ERROR_ABORTED: Final[LiteralString] +XML_ERROR_ASYNC_ENTITY: Final[LiteralString] +XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF: Final[LiteralString] +XML_ERROR_BAD_CHAR_REF: Final[LiteralString] +XML_ERROR_BINARY_ENTITY_REF: Final[LiteralString] +XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING: Final[LiteralString] +XML_ERROR_DUPLICATE_ATTRIBUTE: Final[LiteralString] +XML_ERROR_ENTITY_DECLARED_IN_PE: Final[LiteralString] +XML_ERROR_EXTERNAL_ENTITY_HANDLING: Final[LiteralString] +XML_ERROR_FEATURE_REQUIRES_XML_DTD: Final[LiteralString] +XML_ERROR_FINISHED: Final[LiteralString] +XML_ERROR_INCOMPLETE_PE: Final[LiteralString] +XML_ERROR_INCORRECT_ENCODING: Final[LiteralString] +XML_ERROR_INVALID_TOKEN: Final[LiteralString] +XML_ERROR_JUNK_AFTER_DOC_ELEMENT: Final[LiteralString] +XML_ERROR_MISPLACED_XML_PI: Final[LiteralString] +XML_ERROR_NOT_STANDALONE: Final[LiteralString] +XML_ERROR_NOT_SUSPENDED: Final[LiteralString] +XML_ERROR_NO_ELEMENTS: Final[LiteralString] +XML_ERROR_NO_MEMORY: Final[LiteralString] +XML_ERROR_PARAM_ENTITY_REF: Final[LiteralString] +XML_ERROR_PARTIAL_CHAR: Final[LiteralString] +XML_ERROR_PUBLICID: Final[LiteralString] +XML_ERROR_RECURSIVE_ENTITY_REF: Final[LiteralString] +XML_ERROR_SUSPENDED: Final[LiteralString] +XML_ERROR_SUSPEND_PE: Final[LiteralString] +XML_ERROR_SYNTAX: Final[LiteralString] +XML_ERROR_TAG_MISMATCH: Final[LiteralString] +XML_ERROR_TEXT_DECL: Final[LiteralString] +XML_ERROR_UNBOUND_PREFIX: Final[LiteralString] +XML_ERROR_UNCLOSED_CDATA_SECTION: Final[LiteralString] +XML_ERROR_UNCLOSED_TOKEN: Final[LiteralString] +XML_ERROR_UNDECLARING_PREFIX: Final[LiteralString] +XML_ERROR_UNDEFINED_ENTITY: Final[LiteralString] +XML_ERROR_UNEXPECTED_STATE: Final[LiteralString] +XML_ERROR_UNKNOWN_ENCODING: Final[LiteralString] +XML_ERROR_XML_DECL: Final[LiteralString] if sys.version_info >= (3, 11): - XML_ERROR_RESERVED_PREFIX_XML: str - XML_ERROR_RESERVED_PREFIX_XMLNS: str - XML_ERROR_RESERVED_NAMESPACE_URI: str - XML_ERROR_INVALID_ARGUMENT: str - XML_ERROR_NO_BUFFER: str - XML_ERROR_AMPLIFICATION_LIMIT_BREACH: str + XML_ERROR_RESERVED_PREFIX_XML: Final[LiteralString] + XML_ERROR_RESERVED_PREFIX_XMLNS: Final[LiteralString] + XML_ERROR_RESERVED_NAMESPACE_URI: Final[LiteralString] + XML_ERROR_INVALID_ARGUMENT: Final[LiteralString] + XML_ERROR_NO_BUFFER: Final[LiteralString] + XML_ERROR_AMPLIFICATION_LIMIT_BREACH: Final[LiteralString] diff --git a/mypy/typeshed/stdlib/pyexpat/model.pyi b/mypy/typeshed/stdlib/pyexpat/model.pyi index f357cf6511a2..bac8f3692ce5 100644 --- a/mypy/typeshed/stdlib/pyexpat/model.pyi +++ b/mypy/typeshed/stdlib/pyexpat/model.pyi @@ -1,11 +1,13 @@ -XML_CTYPE_ANY: int -XML_CTYPE_CHOICE: int -XML_CTYPE_EMPTY: int -XML_CTYPE_MIXED: int -XML_CTYPE_NAME: int -XML_CTYPE_SEQ: int +from typing import Final -XML_CQUANT_NONE: int -XML_CQUANT_OPT: int -XML_CQUANT_PLUS: int -XML_CQUANT_REP: int +XML_CTYPE_ANY: Final = 2 +XML_CTYPE_EMPTY: Final = 1 +XML_CTYPE_MIXED: Final = 3 +XML_CTYPE_NAME: Final = 4 +XML_CTYPE_CHOICE: Final = 5 +XML_CTYPE_SEQ: Final = 6 + +XML_CQUANT_NONE: Final = 0 +XML_CQUANT_OPT: Final = 1 +XML_CQUANT_REP: Final = 2 +XML_CQUANT_PLUS: Final = 3 diff --git a/mypy/typeshed/stdlib/re.pyi b/mypy/typeshed/stdlib/re.pyi index b06f494c0b7d..76f98dd9f2a2 100644 --- a/mypy/typeshed/stdlib/re.pyi +++ b/mypy/typeshed/stdlib/re.pyi @@ -74,7 +74,7 @@ class Match(Generic[AnyStr]): @overload def expand(self: Match[str], template: str) -> str: ... @overload - def expand(self: Match[bytes], template: ReadableBuffer) -> bytes: ... # type: ignore[overload-overlap] + def expand(self: Match[bytes], template: ReadableBuffer) -> bytes: ... @overload def expand(self, template: AnyStr) -> AnyStr: ... # group() returns "AnyStr" or "AnyStr | None", depending on the pattern. @@ -124,19 +124,21 @@ class Pattern(Generic[AnyStr]): @overload def search(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Match[str] | None: ... @overload - def search(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ... # type: ignore[overload-overlap] + def search(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ... @overload def search(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Match[AnyStr] | None: ... @overload def match(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Match[str] | None: ... @overload - def match(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ... # type: ignore[overload-overlap] + def match(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ... @overload def match(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Match[AnyStr] | None: ... @overload def fullmatch(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Match[str] | None: ... @overload - def fullmatch(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ... # type: ignore[overload-overlap] + def fullmatch( + self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize + ) -> Match[bytes] | None: ... @overload def fullmatch(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Match[AnyStr] | None: ... @overload @@ -155,13 +157,15 @@ class Pattern(Generic[AnyStr]): @overload def finditer(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Iterator[Match[str]]: ... @overload - def finditer(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Iterator[Match[bytes]]: ... # type: ignore[overload-overlap] + def finditer( + self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize + ) -> Iterator[Match[bytes]]: ... @overload def finditer(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Iterator[Match[AnyStr]]: ... @overload def sub(self: Pattern[str], repl: str | Callable[[Match[str]], str], string: str, count: int = 0) -> str: ... @overload - def sub( # type: ignore[overload-overlap] + def sub( self: Pattern[bytes], repl: ReadableBuffer | Callable[[Match[bytes]], ReadableBuffer], string: ReadableBuffer, @@ -172,7 +176,7 @@ class Pattern(Generic[AnyStr]): @overload def subn(self: Pattern[str], repl: str | Callable[[Match[str]], str], string: str, count: int = 0) -> tuple[str, int]: ... @overload - def subn( # type: ignore[overload-overlap] + def subn( self: Pattern[bytes], repl: ReadableBuffer | Callable[[Match[bytes]], ReadableBuffer], string: ReadableBuffer, diff --git a/mypy/typeshed/stdlib/socketserver.pyi b/mypy/typeshed/stdlib/socketserver.pyi index 5753d1d661b9..ae6575d85082 100644 --- a/mypy/typeshed/stdlib/socketserver.pyi +++ b/mypy/typeshed/stdlib/socketserver.pyi @@ -3,8 +3,9 @@ import types from _socket import _Address, _RetAddress from _typeshed import ReadableBuffer from collections.abc import Callable +from io import BufferedIOBase from socket import socket as _socket -from typing import Any, BinaryIO, ClassVar +from typing import Any, ClassVar from typing_extensions import Self, TypeAlias __all__ = [ @@ -158,11 +159,11 @@ class StreamRequestHandler(BaseRequestHandler): timeout: ClassVar[float | None] # undocumented disable_nagle_algorithm: ClassVar[bool] # undocumented connection: Any # undocumented - rfile: BinaryIO - wfile: BinaryIO + rfile: BufferedIOBase + wfile: BufferedIOBase class DatagramRequestHandler(BaseRequestHandler): - packet: _socket # undocumented + packet: bytes # undocumented socket: _socket # undocumented - rfile: BinaryIO - wfile: BinaryIO + rfile: BufferedIOBase + wfile: BufferedIOBase diff --git a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi index 9e46012ee777..0ee511df4e37 100644 --- a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi +++ b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi @@ -29,7 +29,10 @@ def DateFromTicks(ticks: float) -> Date: ... def TimeFromTicks(ticks: float) -> Time: ... def TimestampFromTicks(ticks: float) -> Timestamp: ... -version_info: tuple[int, int, int] +if sys.version_info < (3, 14): + # Deprecated in 3.12, removed in 3.14. + version_info: tuple[int, int, int] + sqlite_version_info: tuple[int, int, int] Binary = memoryview @@ -90,7 +93,10 @@ SQLITE_UPDATE: Final[int] adapters: dict[tuple[type[Any], type[Any]], _Adapter[Any]] converters: dict[str, _Converter] sqlite_version: str -version: str + +if sys.version_info < (3, 14): + # Deprecated in 3.12, removed in 3.14. + version: str if sys.version_info >= (3, 11): SQLITE_ABORT: Final[int] diff --git a/mypy/typeshed/stdlib/symtable.pyi b/mypy/typeshed/stdlib/symtable.pyi index 5481d4d1dd4a..ee0a1eb2f1cb 100644 --- a/mypy/typeshed/stdlib/symtable.pyi +++ b/mypy/typeshed/stdlib/symtable.pyi @@ -2,6 +2,7 @@ import sys from _collections_abc import dict_keys from collections.abc import Sequence from typing import Any +from typing_extensions import deprecated __all__ = ["symtable", "SymbolTable", "Class", "Function", "Symbol"] @@ -51,7 +52,9 @@ class Function(SymbolTable): def get_nonlocals(self) -> tuple[str, ...]: ... class Class(SymbolTable): - def get_methods(self) -> tuple[str, ...]: ... + if sys.version_info < (3, 16): + @deprecated("deprecated in Python 3.14, will be removed in Python 3.16") + def get_methods(self) -> tuple[str, ...]: ... class Symbol: def __init__( diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi index d6adf21c1900..e46903bf610f 100644 --- a/mypy/typeshed/stdlib/tarfile.pyi +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -423,7 +423,7 @@ class TarInfo: name: str path: str size: int - mtime: int + mtime: int | float chksum: int devmajor: int devminor: int diff --git a/mypy/typeshed/stdlib/tempfile.pyi b/mypy/typeshed/stdlib/tempfile.pyi index d31fd74d3482..62422b84eb37 100644 --- a/mypy/typeshed/stdlib/tempfile.pyi +++ b/mypy/typeshed/stdlib/tempfile.pyi @@ -463,7 +463,7 @@ class TemporaryDirectory(Generic[AnyStr]): # The overloads overlap, but they should still work fine. @overload -def mkstemp( # type: ignore[overload-overlap] +def mkstemp( suffix: str | None = None, prefix: str | None = None, dir: StrPath | None = None, text: bool = False ) -> tuple[int, str]: ... @overload @@ -473,7 +473,7 @@ def mkstemp( # The overloads overlap, but they should still work fine. @overload -def mkdtemp(suffix: str | None = None, prefix: str | None = None, dir: StrPath | None = None) -> str: ... # type: ignore[overload-overlap] +def mkdtemp(suffix: str | None = None, prefix: str | None = None, dir: StrPath | None = None) -> str: ... @overload def mkdtemp(suffix: bytes | None = None, prefix: bytes | None = None, dir: BytesPath | None = None) -> bytes: ... def mktemp(suffix: str = "", prefix: str = "tmp", dir: StrPath | None = None) -> str: ... diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index 77953525bebe..2a42eb789731 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -2148,11 +2148,12 @@ class Listbox(Widget, XView, YView): selectborderwidth: _ScreenUnits = 0, selectforeground: str = ..., # from listbox man page: "The value of the [selectmode] option may be - # arbitrary, but the default bindings expect it to be ..." + # arbitrary, but the default bindings expect it to be either single, + # browse, multiple, or extended" # # I have never seen anyone setting this to something else than what # "the default bindings expect", but let's support it anyway. - selectmode: str = "browse", + selectmode: str | Literal["single", "browse", "multiple", "extended"] = "browse", # noqa: Y051 setgrid: bool = False, state: Literal["normal", "disabled"] = "normal", takefocus: _TakeFocusValue = "", @@ -2187,7 +2188,7 @@ class Listbox(Widget, XView, YView): selectbackground: str = ..., selectborderwidth: _ScreenUnits = ..., selectforeground: str = ..., - selectmode: str = ..., + selectmode: str | Literal["single", "browse", "multiple", "extended"] = ..., # noqa: Y051 setgrid: bool = ..., state: Literal["normal", "disabled"] = ..., takefocus: _TakeFocusValue = ..., @@ -2907,6 +2908,9 @@ class Scrollbar(Widget): def set(self, first: float | str, last: float | str) -> None: ... _TextIndex: TypeAlias = _tkinter.Tcl_Obj | str | float | Misc +_WhatToCount: TypeAlias = Literal[ + "chars", "displaychars", "displayindices", "displaylines", "indices", "lines", "xpixels", "ypixels" +] class Text(Widget, XView, YView): def __init__( @@ -3021,7 +3025,27 @@ class Text(Widget, XView, YView): config = configure def bbox(self, index: _TextIndex) -> tuple[int, int, int, int] | None: ... # type: ignore[override] def compare(self, index1: _TextIndex, op: Literal["<", "<=", "==", ">=", ">", "!="], index2: _TextIndex) -> bool: ... - def count(self, index1, index2, *args): ... # TODO + @overload + def count(self, index1: _TextIndex, index2: _TextIndex) -> tuple[int] | None: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], /) -> tuple[int] | None: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, arg1: Literal["update"], arg2: _WhatToCount, /) -> int | None: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: Literal["update"], /) -> int | None: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /) -> tuple[int, int]: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount | Literal["update"], + arg2: _WhatToCount | Literal["update"], + arg3: _WhatToCount | Literal["update"], + /, + *args: _WhatToCount | Literal["update"], + ) -> tuple[int, ...]: ... @overload def debug(self, boolean: None = None) -> bool: ... @overload @@ -3564,7 +3588,7 @@ class Spinbox(Widget, XView): def scan_dragto(self, x): ... def selection(self, *args) -> tuple[int, ...]: ... def selection_adjust(self, index): ... - def selection_clear(self): ... + def selection_clear(self): ... # type: ignore[override] def selection_element(self, element: Incomplete | None = None): ... def selection_from(self, index: int) -> None: ... def selection_present(self) -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/constants.pyi b/mypy/typeshed/stdlib/tkinter/constants.pyi index 0b497f3a42e4..fbfe8b49b997 100644 --- a/mypy/typeshed/stdlib/tkinter/constants.pyi +++ b/mypy/typeshed/stdlib/tkinter/constants.pyi @@ -1,12 +1,12 @@ from typing import Final # These are not actually bools. See #4669 -NO: bool -YES: bool -TRUE: bool -FALSE: bool -ON: bool -OFF: bool +NO: Final[bool] +YES: Final[bool] +TRUE: Final[bool] +FALSE: Final[bool] +ON: Final[bool] +OFF: Final[bool] N: Final = "n" S: Final = "s" W: Final = "w" diff --git a/mypy/typeshed/stdlib/tkinter/dialog.pyi b/mypy/typeshed/stdlib/tkinter/dialog.pyi index f76732a25460..b7d74c0fa71e 100644 --- a/mypy/typeshed/stdlib/tkinter/dialog.pyi +++ b/mypy/typeshed/stdlib/tkinter/dialog.pyi @@ -2,12 +2,12 @@ import sys from _typeshed import Incomplete from collections.abc import Mapping from tkinter import Widget -from typing import Any +from typing import Any, Final if sys.version_info >= (3, 9): __all__ = ["Dialog"] -DIALOG_ICON: str +DIALOG_ICON: Final = "questhead" class Dialog(Widget): widgetName: str diff --git a/mypy/typeshed/stdlib/tkinter/messagebox.pyi b/mypy/typeshed/stdlib/tkinter/messagebox.pyi index 5a04b66d7866..5cdfe512f9b7 100644 --- a/mypy/typeshed/stdlib/tkinter/messagebox.pyi +++ b/mypy/typeshed/stdlib/tkinter/messagebox.pyi @@ -1,6 +1,6 @@ import sys from tkinter.commondialog import Dialog -from typing import ClassVar +from typing import ClassVar, Final if sys.version_info >= (3, 9): __all__ = [ @@ -14,22 +14,22 @@ if sys.version_info >= (3, 9): "askretrycancel", ] -ERROR: str -INFO: str -QUESTION: str -WARNING: str -ABORTRETRYIGNORE: str -OK: str -OKCANCEL: str -RETRYCANCEL: str -YESNO: str -YESNOCANCEL: str -ABORT: str -RETRY: str -IGNORE: str -CANCEL: str -YES: str -NO: str +ERROR: Final = "error" +INFO: Final = "info" +QUESTION: Final = "question" +WARNING: Final = "warning" +ABORTRETRYIGNORE: Final = "abortretryignore" +OK: Final = "ok" +OKCANCEL: Final = "okcancel" +RETRYCANCEL: Final = "retrycancel" +YESNO: Final = "yesno" +YESNOCANCEL: Final = "yesnocancel" +ABORT: Final = "abort" +RETRY: Final = "retry" +IGNORE: Final = "ignore" +CANCEL: Final = "cancel" +YES: Final = "yes" +NO: Final = "no" class Message(Dialog): command: ClassVar[str] diff --git a/mypy/typeshed/stdlib/tkinter/ttk.pyi b/mypy/typeshed/stdlib/tkinter/ttk.pyi index 86a23ce82211..b851f478140a 100644 --- a/mypy/typeshed/stdlib/tkinter/ttk.pyi +++ b/mypy/typeshed/stdlib/tkinter/ttk.pyi @@ -556,7 +556,9 @@ class Notebook(Widget): sticky: str = ..., # consists of letters 'n', 's', 'w', 'e', no repeats, may be empty padding: _Padding = ..., text: str = ..., - image=..., # Sequence of an image name, followed by zero or more (sequences of one or more state names followed by an image name) + # `image` is a sequence of an image name, followed by zero or more + # (sequences of one or more state names followed by an image name) + image=..., compound: tkinter._Compound = ..., underline: int = ..., ) -> None: ... @@ -1040,7 +1042,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): @overload def heading(self, column: str | int, option: str) -> Any: ... @overload - def heading(self, column: str | int, option: None = None) -> _TreeviewHeaderDict: ... # type: ignore[overload-overlap] + def heading(self, column: str | int, option: None = None) -> _TreeviewHeaderDict: ... @overload def heading( self, @@ -1052,7 +1054,8 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): anchor: tkinter._Anchor = ..., command: str | Callable[[], object] = ..., ) -> None: ... - def identify(self, component, x, y): ... # Internal Method. Leave untyped + # Internal Method. Leave untyped: + def identify(self, component, x, y): ... # type: ignore[override] def identify_row(self, y: int) -> str: ... def identify_column(self, x: int) -> str: ... def identify_region(self, x: int, y: int) -> Literal["heading", "separator", "tree", "cell", "nothing"]: ... @@ -1084,7 +1087,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView): @overload def item(self, item: str | int, option: str) -> Any: ... @overload - def item(self, item: str | int, option: None = None) -> _TreeviewItemDict: ... # type: ignore[overload-overlap] + def item(self, item: str | int, option: None = None) -> _TreeviewItemDict: ... @overload def item( self, diff --git a/mypy/typeshed/stdlib/turtle.pyi b/mypy/typeshed/stdlib/turtle.pyi index 199feee746cb..29d289303927 100644 --- a/mypy/typeshed/stdlib/turtle.pyi +++ b/mypy/typeshed/stdlib/turtle.pyi @@ -338,7 +338,7 @@ class TPen: def isvisible(self) -> bool: ... # Note: signatures 1 and 2 overlap unsafely when no arguments are provided @overload - def pen(self) -> _PenState: ... # type: ignore[overload-overlap] + def pen(self) -> _PenState: ... @overload def pen( self, @@ -384,7 +384,7 @@ class RawTurtle(TPen, TNavigator): def shape(self, name: str) -> None: ... # Unsafely overlaps when no arguments are provided @overload - def shapesize(self) -> tuple[float, float, float]: ... # type: ignore[overload-overlap] + def shapesize(self) -> tuple[float, float, float]: ... @overload def shapesize( self, stretch_wid: float | None = None, stretch_len: float | None = None, outline: float | None = None @@ -395,7 +395,7 @@ class RawTurtle(TPen, TNavigator): def shearfactor(self, shear: float) -> None: ... # Unsafely overlaps when no arguments are provided @overload - def shapetransform(self) -> tuple[float, float, float, float]: ... # type: ignore[overload-overlap] + def shapetransform(self) -> tuple[float, float, float, float]: ... @overload def shapetransform( self, t11: float | None = None, t12: float | None = None, t21: float | None = None, t22: float | None = None @@ -622,7 +622,7 @@ def isvisible() -> bool: ... # Note: signatures 1 and 2 overlap unsafely when no arguments are provided @overload -def pen() -> _PenState: ... # type: ignore[overload-overlap] +def pen() -> _PenState: ... @overload def pen( pen: _PenState | None = None, @@ -661,7 +661,7 @@ if sys.version_info >= (3, 12): # Unsafely overlaps when no arguments are provided @overload -def shapesize() -> tuple[float, float, float]: ... # type: ignore[overload-overlap] +def shapesize() -> tuple[float, float, float]: ... @overload def shapesize(stretch_wid: float | None = None, stretch_len: float | None = None, outline: float | None = None) -> None: ... @overload @@ -671,7 +671,7 @@ def shearfactor(shear: float) -> None: ... # Unsafely overlaps when no arguments are provided @overload -def shapetransform() -> tuple[float, float, float, float]: ... # type: ignore[overload-overlap] +def shapetransform() -> tuple[float, float, float, float]: ... @overload def shapetransform( t11: float | None = None, t12: float | None = None, t21: float | None = None, t22: float | None = None diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 1e3eacd9f1fa..0f6592a9883e 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -305,9 +305,9 @@ class MappingProxyType(Mapping[_KT, _VT_co]): def values(self) -> ValuesView[_VT_co]: ... def items(self) -> ItemsView[_KT, _VT_co]: ... @overload - def get(self, key: _KT, /) -> _VT_co | None: ... # type: ignore[override] + def get(self, key: _KT, /) -> _VT_co | None: ... @overload - def get(self, key: _KT, default: _VT_co | _T2, /) -> _VT_co | _T2: ... # type: ignore[override] + def get(self, key: _KT, default: _VT_co | _T2, /) -> _VT_co | _T2: ... if sys.version_info >= (3, 9): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... def __reversed__(self) -> Iterator[_KT]: ... @@ -583,7 +583,7 @@ _P = ParamSpec("_P") # it's not really an Awaitable, but can be used in an await expression. Real type: Generator & Awaitable @overload -def coroutine(func: Callable[_P, Generator[Any, Any, _R]]) -> Callable[_P, Awaitable[_R]]: ... # type: ignore[overload-overlap] +def coroutine(func: Callable[_P, Generator[Any, Any, _R]]) -> Callable[_P, Awaitable[_R]]: ... @overload def coroutine(func: _Fn) -> _Fn: ... diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index f4de1fa86de5..cadd06358d4a 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -2,7 +2,7 @@ # ruff: noqa: F811 # TODO: The collections import is required, otherwise mypy crashes. # https://github.com/python/mypy/issues/16744 -import collections # noqa: F401 # pyright: ignore +import collections # noqa: F401 # pyright: ignore[reportUnusedImport] import sys import typing_extensions from _collections_abc import dict_items, dict_keys, dict_values @@ -800,18 +800,12 @@ class IO(Iterator[AnyStr]): def writable(self) -> bool: ... @abstractmethod @overload - def write(self: IO[str], s: str, /) -> int: ... - @abstractmethod - @overload def write(self: IO[bytes], s: ReadableBuffer, /) -> int: ... @abstractmethod @overload def write(self, s: AnyStr, /) -> int: ... @abstractmethod @overload - def writelines(self: IO[str], lines: Iterable[str], /) -> None: ... - @abstractmethod - @overload def writelines(self: IO[bytes], lines: Iterable[ReadableBuffer], /) -> None: ... @abstractmethod @overload @@ -846,7 +840,8 @@ class TextIO(IO[str]): @abstractmethod def __enter__(self) -> TextIO: ... -ByteString: typing_extensions.TypeAlias = bytes | bytearray | memoryview +if sys.version_info < (3, 14): + ByteString: typing_extensions.TypeAlias = bytes | bytearray | memoryview # Functions diff --git a/mypy/typeshed/stdlib/unittest/mock.pyi b/mypy/typeshed/stdlib/unittest/mock.pyi index 84620b7f3889..1cfd38f540a4 100644 --- a/mypy/typeshed/stdlib/unittest/mock.pyi +++ b/mypy/typeshed/stdlib/unittest/mock.pyi @@ -299,7 +299,7 @@ class _patcher: # Ideally we'd be able to add an overload for it so that the return type is _patch[MagicMock], # but that's impossible with the current type system. @overload - def __call__( # type: ignore[overload-overlap] + def __call__( self, target: str, new: _T, diff --git a/mypy/typeshed/stdlib/unittest/runner.pyi b/mypy/typeshed/stdlib/unittest/runner.pyi index 0033083ac406..46da85619d30 100644 --- a/mypy/typeshed/stdlib/unittest/runner.pyi +++ b/mypy/typeshed/stdlib/unittest/runner.pyi @@ -2,36 +2,52 @@ import sys import unittest.case import unittest.result import unittest.suite -from _typeshed import Incomplete +from _typeshed import SupportsFlush, SupportsWrite from collections.abc import Callable, Iterable -from typing import TextIO -from typing_extensions import TypeAlias +from typing import Any, Generic, Protocol, TypeVar +from typing_extensions import Never, TypeAlias -_ResultClassType: TypeAlias = Callable[[TextIO, bool, int], unittest.result.TestResult] +_ResultClassType: TypeAlias = Callable[[_TextTestStream, bool, int], TextTestResult] -class TextTestResult(unittest.result.TestResult): +class _SupportsWriteAndFlush(SupportsWrite[str], SupportsFlush, Protocol): ... + +# All methods used by unittest.runner.TextTestResult's stream +class _TextTestStream(_SupportsWriteAndFlush, Protocol): + def writeln(self, arg: str | None = None) -> str: ... + +# _WritelnDecorator should have all the same attrs as its stream param. +# But that's not feasible to do Generically +# We can expand the attributes if requested +class _WritelnDecorator(_TextTestStream): + def __init__(self, stream: _TextTestStream) -> None: ... + def __getattr__(self, attr: str) -> Any: ... # Any attribute from the stream type passed to __init__ + # These attributes are prevented by __getattr__ + stream: Never + __getstate__: Never + +_StreamT = TypeVar("_StreamT", bound=_TextTestStream, default=_WritelnDecorator) + +class TextTestResult(unittest.result.TestResult, Generic[_StreamT]): descriptions: bool # undocumented dots: bool # undocumented separator1: str separator2: str showAll: bool # undocumented - stream: TextIO # undocumented + stream: _StreamT # undocumented if sys.version_info >= (3, 12): durations: unittest.result._DurationsType | None def __init__( - self, stream: TextIO, descriptions: bool, verbosity: int, *, durations: unittest.result._DurationsType | None = None + self, stream: _StreamT, descriptions: bool, verbosity: int, *, durations: unittest.result._DurationsType | None = None ) -> None: ... else: - def __init__(self, stream: TextIO, descriptions: bool, verbosity: int) -> None: ... + def __init__(self, stream: _StreamT, descriptions: bool, verbosity: int) -> None: ... def getDescription(self, test: unittest.case.TestCase) -> str: ... def printErrorList(self, flavour: str, errors: Iterable[tuple[unittest.case.TestCase, str]]) -> None: ... class TextTestRunner: resultclass: _ResultClassType - # TODO: add `_WritelnDecorator` type - # stream: _WritelnDecorator - stream: Incomplete + stream: _WritelnDecorator descriptions: bool verbosity: int failfast: bool @@ -43,7 +59,7 @@ class TextTestRunner: durations: unittest.result._DurationsType | None def __init__( self, - stream: TextIO | None = None, + stream: _SupportsWriteAndFlush | None = None, descriptions: bool = True, verbosity: int = 1, failfast: bool = False, @@ -57,7 +73,7 @@ class TextTestRunner: else: def __init__( self, - stream: TextIO | None = None, + stream: _SupportsWriteAndFlush | None = None, descriptions: bool = True, verbosity: int = 1, failfast: bool = False, @@ -68,5 +84,5 @@ class TextTestRunner: tb_locals: bool = False, ) -> None: ... - def _makeResult(self) -> unittest.result.TestResult: ... - def run(self, test: unittest.suite.TestSuite | unittest.case.TestCase) -> unittest.result.TestResult: ... + def _makeResult(self) -> TextTestResult: ... + def run(self, test: unittest.suite.TestSuite | unittest.case.TestCase) -> TextTestResult: ... diff --git a/mypy/typeshed/stdlib/urllib/parse.pyi b/mypy/typeshed/stdlib/urllib/parse.pyi index 89a50995d553..785bb9678ec7 100644 --- a/mypy/typeshed/stdlib/urllib/parse.pyi +++ b/mypy/typeshed/stdlib/urllib/parse.pyi @@ -198,13 +198,13 @@ else: # Requires an iterable of length 6 @overload -def urlunparse(components: Iterable[None]) -> Literal[b""]: ... +def urlunparse(components: Iterable[None]) -> Literal[b""]: ... # type: ignore[overload-overlap] @overload def urlunparse(components: Iterable[AnyStr | None]) -> AnyStr: ... # Requires an iterable of length 5 @overload -def urlunsplit(components: Iterable[None]) -> Literal[b""]: ... +def urlunsplit(components: Iterable[None]) -> Literal[b""]: ... # type: ignore[overload-overlap] @overload def urlunsplit(components: Iterable[AnyStr | None]) -> AnyStr: ... def unwrap(url: str) -> str: ... diff --git a/mypy/typeshed/stdlib/urllib/request.pyi b/mypy/typeshed/stdlib/urllib/request.pyi index 2a6476f9e6d8..ad4f91fc31ae 100644 --- a/mypy/typeshed/stdlib/urllib/request.pyi +++ b/mypy/typeshed/stdlib/urllib/request.pyi @@ -79,6 +79,7 @@ else: def pathname2url(pathname: str) -> str: ... def getproxies() -> dict[str, str]: ... +def getproxies_environment() -> dict[str, str]: ... def parse_http_list(s: str) -> list[str]: ... def parse_keqv_list(l: list[str]) -> dict[str, str]: ... diff --git a/mypy/typeshed/stdlib/xml/dom/__init__.pyi b/mypy/typeshed/stdlib/xml/dom/__init__.pyi index e5b91bf2a795..8738015638a9 100644 --- a/mypy/typeshed/stdlib/xml/dom/__init__.pyi +++ b/mypy/typeshed/stdlib/xml/dom/__init__.pyi @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Final from .domreg import getDOMImplementation as getDOMImplementation, registerDOMImplementation as registerDOMImplementation @@ -17,22 +17,22 @@ class Node: NOTATION_NODE: int # ExceptionCode -INDEX_SIZE_ERR: int -DOMSTRING_SIZE_ERR: int -HIERARCHY_REQUEST_ERR: int -WRONG_DOCUMENT_ERR: int -INVALID_CHARACTER_ERR: int -NO_DATA_ALLOWED_ERR: int -NO_MODIFICATION_ALLOWED_ERR: int -NOT_FOUND_ERR: int -NOT_SUPPORTED_ERR: int -INUSE_ATTRIBUTE_ERR: int -INVALID_STATE_ERR: int -SYNTAX_ERR: int -INVALID_MODIFICATION_ERR: int -NAMESPACE_ERR: int -INVALID_ACCESS_ERR: int -VALIDATION_ERR: int +INDEX_SIZE_ERR: Final[int] +DOMSTRING_SIZE_ERR: Final[int] +HIERARCHY_REQUEST_ERR: Final[int] +WRONG_DOCUMENT_ERR: Final[int] +INVALID_CHARACTER_ERR: Final[int] +NO_DATA_ALLOWED_ERR: Final[int] +NO_MODIFICATION_ALLOWED_ERR: Final[int] +NOT_FOUND_ERR: Final[int] +NOT_SUPPORTED_ERR: Final[int] +INUSE_ATTRIBUTE_ERR: Final[int] +INVALID_STATE_ERR: Final[int] +SYNTAX_ERR: Final[int] +INVALID_MODIFICATION_ERR: Final[int] +NAMESPACE_ERR: Final[int] +INVALID_ACCESS_ERR: Final[int] +VALIDATION_ERR: Final[int] class DOMException(Exception): code: int @@ -62,8 +62,8 @@ class UserDataHandler: NODE_DELETED: int NODE_RENAMED: int -XML_NAMESPACE: str -XMLNS_NAMESPACE: str -XHTML_NAMESPACE: str -EMPTY_NAMESPACE: None -EMPTY_PREFIX: None +XML_NAMESPACE: Final[str] +XMLNS_NAMESPACE: Final[str] +XHTML_NAMESPACE: Final[str] +EMPTY_NAMESPACE: Final[None] +EMPTY_PREFIX: Final[None] diff --git a/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi b/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi index cbba15dd3ebe..5a15772ec2a9 100644 --- a/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi +++ b/mypy/typeshed/stdlib/xml/etree/ElementInclude.pyi @@ -1,14 +1,15 @@ import sys from _typeshed import FileDescriptorOrPath from collections.abc import Callable +from typing import Final from xml.etree.ElementTree import Element -XINCLUDE: str -XINCLUDE_INCLUDE: str -XINCLUDE_FALLBACK: str +XINCLUDE: Final[str] +XINCLUDE_INCLUDE: Final[str] +XINCLUDE_FALLBACK: Final[str] if sys.version_info >= (3, 9): - DEFAULT_MAX_INCLUSION_DEPTH: int + DEFAULT_MAX_INCLUSION_DEPTH: Final = 6 class FatalIncludeError(SyntaxError): ... diff --git a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi index 4849b0ea1c35..64ebbd3ee63f 100644 --- a/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi +++ b/mypy/typeshed/stdlib/xml/etree/ElementTree.pyi @@ -2,7 +2,7 @@ import sys from _collections_abc import dict_keys from _typeshed import FileDescriptorOrPath, ReadableBuffer, SupportsRead, SupportsWrite from collections.abc import Callable, Generator, ItemsView, Iterable, Iterator, Mapping, Sequence -from typing import Any, Literal, SupportsIndex, TypeVar, overload +from typing import Any, Final, Literal, SupportsIndex, TypeVar, overload from typing_extensions import TypeAlias, TypeGuard, deprecated __all__ = [ @@ -41,7 +41,7 @@ _FileRead: TypeAlias = FileDescriptorOrPath | SupportsRead[bytes] | SupportsRead _FileWriteC14N: TypeAlias = FileDescriptorOrPath | SupportsWrite[bytes] _FileWrite: TypeAlias = _FileWriteC14N | SupportsWrite[str] -VERSION: str +VERSION: Final[str] class ParseError(SyntaxError): code: int diff --git a/mypy/typeshed/stdlib/zipfile/__init__.pyi b/mypy/typeshed/stdlib/zipfile/__init__.pyi index 57a8a6aaa40a..5b8f02f61bce 100644 --- a/mypy/typeshed/stdlib/zipfile/__init__.pyi +++ b/mypy/typeshed/stdlib/zipfile/__init__.pyi @@ -94,6 +94,20 @@ class ZipExtFile(io.BufferedIOBase): class _Writer(Protocol): def write(self, s: str, /) -> object: ... +class _ZipReadable(Protocol): + def seek(self, offset: int, whence: int = 0, /) -> int: ... + def read(self, n: int = -1, /) -> bytes: ... + +class _ZipTellable(Protocol): + def tell(self) -> int: ... + +class _ZipReadableTellable(_ZipReadable, _ZipTellable, Protocol): ... + +class _ZipWritable(Protocol): + def flush(self) -> None: ... + def close(self) -> None: ... + def write(self, b: bytes, /) -> int: ... + class ZipFile: filename: str | None debug: int @@ -106,24 +120,50 @@ class ZipFile: compresslevel: int | None # undocumented mode: _ZipFileMode # undocumented pwd: bytes | None # undocumented + # metadata_encoding is new in 3.11 if sys.version_info >= (3, 11): @overload def __init__( self, file: StrPath | IO[bytes], + mode: _ZipFileMode = "r", + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + metadata_encoding: str | None = None, + ) -> None: ... + # metadata_encoding is only allowed for read mode + @overload + def __init__( + self, + file: StrPath | _ZipReadable, mode: Literal["r"] = "r", compression: int = 0, allowZip64: bool = True, compresslevel: int | None = None, *, strict_timestamps: bool = True, - metadata_encoding: str | None, + metadata_encoding: str | None = None, ) -> None: ... @overload def __init__( self, - file: StrPath | IO[bytes], - mode: _ZipFileMode = "r", + file: StrPath | _ZipWritable, + mode: Literal["w", "x"] = ..., + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + metadata_encoding: None = None, + ) -> None: ... + @overload + def __init__( + self, + file: StrPath | _ZipReadableTellable, + mode: Literal["a"] = ..., compression: int = 0, allowZip64: bool = True, compresslevel: int | None = None, @@ -132,6 +172,7 @@ class ZipFile: metadata_encoding: None = None, ) -> None: ... else: + @overload def __init__( self, file: StrPath | IO[bytes], @@ -142,6 +183,39 @@ class ZipFile: *, strict_timestamps: bool = True, ) -> None: ... + @overload + def __init__( + self, + file: StrPath | _ZipReadable, + mode: Literal["r"] = "r", + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + ) -> None: ... + @overload + def __init__( + self, + file: StrPath | _ZipWritable, + mode: Literal["w", "x"] = ..., + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + ) -> None: ... + @overload + def __init__( + self, + file: StrPath | _ZipReadableTellable, + mode: Literal["a"] = ..., + compression: int = 0, + allowZip64: bool = True, + compresslevel: int | None = None, + *, + strict_timestamps: bool = True, + ) -> None: ... def __enter__(self) -> Self: ... def __exit__( @@ -230,6 +304,7 @@ else: class Path: root: CompleteDirs + at: str def __init__(self, root: ZipFile | StrPath | IO[bytes], at: str = "") -> None: ... @property def name(self) -> str: ... diff --git a/mypy/typeshed/stdlib/zipfile/_path.pyi b/mypy/typeshed/stdlib/zipfile/_path.pyi index bafbbeeb0d0b..933acf2c4803 100644 --- a/mypy/typeshed/stdlib/zipfile/_path.pyi +++ b/mypy/typeshed/stdlib/zipfile/_path.pyi @@ -31,6 +31,7 @@ if sys.version_info >= (3, 12): class Path: root: CompleteDirs + at: str def __init__(self, root: ZipFile | StrPath | IO[bytes], at: str = "") -> None: ... @property def name(self) -> str: ... diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index b65a4cd59f79..dbf228623d7c 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -274,9 +274,7 @@ def bin(f: IO[bytes]) -> None: txt(sys.stdout) bin(sys.stdout) [out] -_program.py:5: error: No overload variant of "write" of "IO" matches argument type "bytes" -_program.py:5: note: Possible overload variants: -_program.py:5: note: def write(self, str, /) -> int +_program.py:5: error: Argument 1 to "write" of "IO" has incompatible type "bytes"; expected "str" _program.py:10: error: Argument 1 to "bin" has incompatible type "Union[TextIO, Any]"; expected "IO[bytes]" [case testBuiltinOpen] From 0ca47e8db3240d56b93eb75337b2393a09306dd6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 1 Sep 2024 00:27:57 -0700 Subject: [PATCH 128/247] Sync typeshed (#17729) Source commit: https://github.com/python/typeshed/commit/23d867efb2df6de5600f64656f1aa8a83e06109e Note that you will need to close and re-open the PR in order to trigger CI. --------- Co-authored-by: mypybot <> Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: AlexWaygood --- mypy/typeshed/stdlib/distutils/cmd.pyi | 1 + mypy/typeshed/stdlib/distutils/dist.pyi | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mypy/typeshed/stdlib/distutils/cmd.pyi b/mypy/typeshed/stdlib/distutils/cmd.pyi index 1f3f31c9c48a..dcb423a49b09 100644 --- a/mypy/typeshed/stdlib/distutils/cmd.pyi +++ b/mypy/typeshed/stdlib/distutils/cmd.pyi @@ -30,6 +30,7 @@ _CommandT = TypeVar("_CommandT", bound=Command) _Ts = TypeVarTuple("_Ts") class Command: + dry_run: Literal[0, 1] # Exposed from __getattr_. Same as Distribution.dry_run distribution: Distribution # Any to work around variance issues sub_commands: ClassVar[list[tuple[str, Callable[[Any], bool] | None]]] diff --git a/mypy/typeshed/stdlib/distutils/dist.pyi b/mypy/typeshed/stdlib/distutils/dist.pyi index e32fd70f7baa..7013167dddbf 100644 --- a/mypy/typeshed/stdlib/distutils/dist.pyi +++ b/mypy/typeshed/stdlib/distutils/dist.pyi @@ -88,9 +88,9 @@ class Distribution: display_options: ClassVar[_OptionsList] display_option_names: ClassVar[list[str]] negative_opt: ClassVar[dict[str, str]] - verbose: int - dry_run: int - help: int + verbose: Literal[0, 1] + dry_run: Literal[0, 1] + help: Literal[0, 1] command_packages: list[str] | None script_name: str | None script_args: list[str] | None From 0412590ba13395a04f122a5dd83764db9d001fe6 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 2 Sep 2024 06:26:53 -0700 Subject: [PATCH 129/247] [nit] conf.py: annotate the type, instead of ignoring the error (#17727) There was a type-ignore here. But we can simply do what mypy is asking us to do; in fact, the comment even already tells us these have to be strings, (like the other arrays). --- mypyc/doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/doc/conf.py b/mypyc/doc/conf.py index da887e0d8267..fdd98c12a221 100644 --- a/mypyc/doc/conf.py +++ b/mypyc/doc/conf.py @@ -36,7 +36,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] # type: ignore[var-annotated] +extensions: list[str] = [] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] From 0c1036717578b00e35625cc353a538e4eb63bc37 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 7 Sep 2024 13:42:54 -0700 Subject: [PATCH 130/247] Use newer Python in docs build (#17747) Helps with #17742 --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8bded1d380aa..c6ed3cf1a08d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.12' - name: Install tox run: pip install tox==4.11.0 - name: Setup tox environment From 72c413d2352da5ce1433ef241faca8f40fa1fe27 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Thu, 12 Sep 2024 16:26:36 +0200 Subject: [PATCH 131/247] stubgen: Use `Generator` type var defaults (#17670) Fixes #17669 --- mypy/stubgen.py | 13 ++++++++++--- test-data/unit/stubgen.test | 34 +++++++++++++++++----------------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 14417f55545e..02c0c1e58ab5 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -588,8 +588,8 @@ def _get_func_return(self, o: FuncDef, ctx: FunctionContext) -> str | None: if has_yield_expression(o) or has_yield_from_expression(o): generator_name = self.add_name("collections.abc.Generator") yield_name = "None" - send_name = "None" - return_name = "None" + send_name: str | None = None + return_name: str | None = None if has_yield_from_expression(o): yield_name = send_name = self.add_name("_typeshed.Incomplete") else: @@ -600,7 +600,14 @@ def _get_func_return(self, o: FuncDef, ctx: FunctionContext) -> str | None: send_name = self.add_name("_typeshed.Incomplete") if has_return_statement(o): return_name = self.add_name("_typeshed.Incomplete") - return f"{generator_name}[{yield_name}, {send_name}, {return_name}]" + if return_name is not None: + if send_name is None: + send_name = "None" + return f"{generator_name}[{yield_name}, {send_name}, {return_name}]" + elif send_name is not None: + return f"{generator_name}[{yield_name}, {send_name}]" + else: + return f"{generator_name}[{yield_name}]" if not has_return_statement(o) and o.abstract_status == NOT_ABSTRACT: return "None" return None diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index fe0538159aa3..69781e9d2143 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -1640,11 +1640,11 @@ def all(): from _typeshed import Incomplete from collections.abc import Generator -def f() -> Generator[Incomplete, None, None]: ... -def g() -> Generator[None, Incomplete, None]: ... -def h1() -> Generator[None, None, None]: ... +def f() -> Generator[Incomplete]: ... +def g() -> Generator[None, Incomplete]: ... +def h1() -> Generator[None]: ... def h2() -> Generator[None, None, Incomplete]: ... -def h3() -> Generator[None, None, None]: ... +def h3() -> Generator[None]: ... def all() -> Generator[Incomplete, Incomplete, Incomplete]: ... [case testFunctionYieldsNone] @@ -1656,8 +1656,8 @@ def g(): [out] from collections.abc import Generator -def f() -> Generator[None, None, None]: ... -def g() -> Generator[None, None, None]: ... +def f() -> Generator[None]: ... +def g() -> Generator[None]: ... [case testGeneratorAlreadyDefined] class Generator: @@ -1671,7 +1671,7 @@ from collections.abc import Generator as _Generator class Generator: ... -def f() -> _Generator[Incomplete, None, None]: ... +def f() -> _Generator[Incomplete]: ... [case testGeneratorYieldFrom] def g1(): @@ -1692,10 +1692,10 @@ def g5(): from _typeshed import Incomplete from collections.abc import Generator -def g1() -> Generator[Incomplete, Incomplete, None]: ... -def g2() -> Generator[Incomplete, Incomplete, None]: ... -def g3() -> Generator[Incomplete, Incomplete, None]: ... -def g4() -> Generator[Incomplete, Incomplete, None]: ... +def g1() -> Generator[Incomplete, Incomplete]: ... +def g2() -> Generator[Incomplete, Incomplete]: ... +def g3() -> Generator[Incomplete, Incomplete]: ... +def g4() -> Generator[Incomplete, Incomplete]: ... def g5() -> Generator[Incomplete, Incomplete, Incomplete]: ... [case testGeneratorYieldAndYieldFrom] @@ -1728,13 +1728,13 @@ def g7(): from _typeshed import Incomplete from collections.abc import Generator -def g1() -> Generator[Incomplete, Incomplete, None]: ... -def g2() -> Generator[Incomplete, Incomplete, None]: ... -def g3() -> Generator[Incomplete, Incomplete, None]: ... -def g4() -> Generator[Incomplete, Incomplete, None]: ... -def g5() -> Generator[Incomplete, Incomplete, None]: ... +def g1() -> Generator[Incomplete, Incomplete]: ... +def g2() -> Generator[Incomplete, Incomplete]: ... +def g3() -> Generator[Incomplete, Incomplete]: ... +def g4() -> Generator[Incomplete, Incomplete]: ... +def g5() -> Generator[Incomplete, Incomplete]: ... def g6() -> Generator[Incomplete, Incomplete, Incomplete]: ... -def g7() -> Generator[Incomplete, Incomplete, None]: ... +def g7() -> Generator[Incomplete, Incomplete]: ... [case testCallable] from typing import Callable From f8195bc735e4a4a424a863fd9e40b2448f79d616 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 15 Sep 2024 15:39:24 -0700 Subject: [PATCH 132/247] Sync typeshed (#17772) Source commit: https://github.com/python/typeshed/commit/9e506eb5e8fc2823db8c60ad561b1145ff114947 --- mypy/typeshed/stdlib/VERSIONS | 2 +- mypy/typeshed/stdlib/_curses.pyi | 2 +- mypy/typeshed/stdlib/_locale.pyi | 149 ++++++++++-------- mypy/typeshed/stdlib/_winapi.pyi | 28 ++++ mypy/typeshed/stdlib/builtins.pyi | 1 + mypy/typeshed/stdlib/codecs.pyi | 2 +- mypy/typeshed/stdlib/copy.pyi | 9 +- mypy/typeshed/stdlib/distutils/dist.pyi | 25 ++- mypy/typeshed/stdlib/doctest.pyi | 26 ++- .../stdlib/email/_header_value_parser.pyi | 4 + mypy/typeshed/stdlib/email/_policybase.pyi | 39 +++-- mypy/typeshed/stdlib/email/errors.pyi | 3 + mypy/typeshed/stdlib/email/utils.pyi | 16 +- mypy/typeshed/stdlib/io.pyi | 3 +- mypy/typeshed/stdlib/subprocess.pyi | 5 + mypy/typeshed/stdlib/tempfile.pyi | 6 +- 16 files changed, 214 insertions(+), 106 deletions(-) diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index 641f951ce3c0..66bf2bec7cb0 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -41,7 +41,7 @@ _json: 3.0- _locale: 3.0- _lsprof: 3.0- _markupbase: 3.0- -_msi: 3.0- +_msi: 3.0-3.12 _operator: 3.4- _osx_support: 3.0- _posixsubprocess: 3.2- diff --git a/mypy/typeshed/stdlib/_curses.pyi b/mypy/typeshed/stdlib/_curses.pyi index 505637574af1..b68c8925a041 100644 --- a/mypy/typeshed/stdlib/_curses.pyi +++ b/mypy/typeshed/stdlib/_curses.pyi @@ -493,7 +493,7 @@ class _CursesWindow: def instr(self, y: int, x: int, n: int = ...) -> bytes: ... def is_linetouched(self, line: int, /) -> bool: ... def is_wintouched(self) -> bool: ... - def keypad(self, yes: bool) -> None: ... + def keypad(self, yes: bool, /) -> None: ... def leaveok(self, yes: bool) -> None: ... def move(self, new_y: int, new_x: int) -> None: ... def mvderwin(self, y: int, x: int) -> None: ... diff --git a/mypy/typeshed/stdlib/_locale.pyi b/mypy/typeshed/stdlib/_locale.pyi index 0825e12034f4..ccce7a0d9d70 100644 --- a/mypy/typeshed/stdlib/_locale.pyi +++ b/mypy/typeshed/stdlib/_locale.pyi @@ -1,17 +1,38 @@ import sys from _typeshed import StrPath -from collections.abc import Mapping +from typing import Final, Literal, TypedDict, type_check_only -LC_CTYPE: int -LC_COLLATE: int -LC_TIME: int -LC_MONETARY: int -LC_NUMERIC: int -LC_ALL: int -CHAR_MAX: int +@type_check_only +class _LocaleConv(TypedDict): + decimal_point: str + grouping: list[int] + thousands_sep: str + int_curr_symbol: str + currency_symbol: str + p_cs_precedes: Literal[0, 1, 127] + n_cs_precedes: Literal[0, 1, 127] + p_sep_by_space: Literal[0, 1, 127] + n_sep_by_space: Literal[0, 1, 127] + mon_decimal_point: str + frac_digits: int + int_frac_digits: int + mon_thousands_sep: str + mon_grouping: list[int] + positive_sign: str + negative_sign: str + p_sign_posn: Literal[0, 1, 2, 3, 4, 127] + n_sign_posn: Literal[0, 1, 2, 3, 4, 127] + +LC_CTYPE: Final[int] +LC_COLLATE: Final[int] +LC_TIME: Final[int] +LC_MONETARY: Final[int] +LC_NUMERIC: Final[int] +LC_ALL: Final[int] +CHAR_MAX: Final = 127 def setlocale(category: int, locale: str | None = None, /) -> str: ... -def localeconv() -> Mapping[str, int | str | list[int]]: ... +def localeconv() -> _LocaleConv: ... if sys.version_info >= (3, 11): def getencoding() -> str: ... @@ -25,67 +46,67 @@ def strxfrm(string: str, /) -> str: ... if sys.platform != "win32": LC_MESSAGES: int - ABDAY_1: int - ABDAY_2: int - ABDAY_3: int - ABDAY_4: int - ABDAY_5: int - ABDAY_6: int - ABDAY_7: int + ABDAY_1: Final[int] + ABDAY_2: Final[int] + ABDAY_3: Final[int] + ABDAY_4: Final[int] + ABDAY_5: Final[int] + ABDAY_6: Final[int] + ABDAY_7: Final[int] - ABMON_1: int - ABMON_2: int - ABMON_3: int - ABMON_4: int - ABMON_5: int - ABMON_6: int - ABMON_7: int - ABMON_8: int - ABMON_9: int - ABMON_10: int - ABMON_11: int - ABMON_12: int + ABMON_1: Final[int] + ABMON_2: Final[int] + ABMON_3: Final[int] + ABMON_4: Final[int] + ABMON_5: Final[int] + ABMON_6: Final[int] + ABMON_7: Final[int] + ABMON_8: Final[int] + ABMON_9: Final[int] + ABMON_10: Final[int] + ABMON_11: Final[int] + ABMON_12: Final[int] - DAY_1: int - DAY_2: int - DAY_3: int - DAY_4: int - DAY_5: int - DAY_6: int - DAY_7: int + DAY_1: Final[int] + DAY_2: Final[int] + DAY_3: Final[int] + DAY_4: Final[int] + DAY_5: Final[int] + DAY_6: Final[int] + DAY_7: Final[int] - ERA: int - ERA_D_T_FMT: int - ERA_D_FMT: int - ERA_T_FMT: int + ERA: Final[int] + ERA_D_T_FMT: Final[int] + ERA_D_FMT: Final[int] + ERA_T_FMT: Final[int] - MON_1: int - MON_2: int - MON_3: int - MON_4: int - MON_5: int - MON_6: int - MON_7: int - MON_8: int - MON_9: int - MON_10: int - MON_11: int - MON_12: int + MON_1: Final[int] + MON_2: Final[int] + MON_3: Final[int] + MON_4: Final[int] + MON_5: Final[int] + MON_6: Final[int] + MON_7: Final[int] + MON_8: Final[int] + MON_9: Final[int] + MON_10: Final[int] + MON_11: Final[int] + MON_12: Final[int] - CODESET: int - D_T_FMT: int - D_FMT: int - T_FMT: int - T_FMT_AMPM: int - AM_STR: int - PM_STR: int + CODESET: Final[int] + D_T_FMT: Final[int] + D_FMT: Final[int] + T_FMT: Final[int] + T_FMT_AMPM: Final[int] + AM_STR: Final[int] + PM_STR: Final[int] - RADIXCHAR: int - THOUSEP: int - YESEXPR: int - NOEXPR: int - CRNCYSTR: int - ALT_DIGITS: int + RADIXCHAR: Final[int] + THOUSEP: Final[int] + YESEXPR: Final[int] + NOEXPR: Final[int] + CRNCYSTR: Final[int] + ALT_DIGITS: Final[int] def nl_langinfo(key: int, /) -> str: ... diff --git a/mypy/typeshed/stdlib/_winapi.pyi b/mypy/typeshed/stdlib/_winapi.pyi index 62ea124045cc..0f71a0687748 100644 --- a/mypy/typeshed/stdlib/_winapi.pyi +++ b/mypy/typeshed/stdlib/_winapi.pyi @@ -99,6 +99,20 @@ if sys.platform == "win32": SEC_RESERVE: Final = 0x4000000 SEC_WRITECOMBINE: Final = 0x40000000 + if sys.version_info >= (3, 13): + STARTF_FORCEOFFFEEDBACK: Final = 0x80 + STARTF_FORCEONFEEDBACK: Final = 0x40 + STARTF_PREVENTPINNING: Final = 0x2000 + STARTF_RUNFULLSCREEN: Final = 0x20 + STARTF_TITLEISAPPID: Final = 0x1000 + STARTF_TITLEISLINKNAME: Final = 0x800 + STARTF_UNTRUSTEDSOURCE: Final = 0x8000 + STARTF_USECOUNTCHARS: Final = 0x8 + STARTF_USEFILLATTRIBUTE: Final = 0x10 + STARTF_USEHOTKEY: Final = 0x200 + STARTF_USEPOSITION: Final = 0x4 + STARTF_USESIZE: Final = 0x2 + STARTF_USESHOWWINDOW: Final = 0x1 STARTF_USESTDHANDLES: Final = 0x100 @@ -250,6 +264,20 @@ if sys.platform == "win32": def cancel(self) -> None: ... def getbuffer(self) -> bytes | None: ... + if sys.version_info >= (3, 13): + def BatchedWaitForMultipleObjects( + handle_seq: Sequence[int], wait_all: bool, milliseconds: int = 0xFFFFFFFF + ) -> list[int]: ... + def CreateEventW(security_attributes: int, manual_reset: bool, initial_state: bool, name: str | None) -> int: ... + def CreateMutexW(security_attributes: int, initial_owner: bool, name: str) -> int: ... + def GetLongPathName(path: str) -> str: ... + def GetShortPathName(path: str) -> str: ... + def OpenEventW(desired_access: int, inherit_handle: bool, name: str) -> int: ... + def OpenMutexW(desired_access: int, inherit_handle: bool, name: str) -> int: ... + def ReleaseMutex(mutex: int) -> None: ... + def ResetEvent(event: int) -> None: ... + def SetEvent(event: int) -> None: ... + if sys.version_info >= (3, 12): def CopyFile2(existing_file_name: str, new_file_name: str, flags: int, progress_routine: int | None = None) -> int: ... def NeedCurrentDirectoryForExePath(exe_name: str, /) -> bool: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index a6065cc6777f..0999fb1d6c36 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -1,3 +1,4 @@ +# ruff: noqa: PYI036 # This is the module declaring BaseException import _ast import _typeshed import sys diff --git a/mypy/typeshed/stdlib/codecs.pyi b/mypy/typeshed/stdlib/codecs.pyi index 9bc098dbc6d7..a41df9752d33 100644 --- a/mypy/typeshed/stdlib/codecs.pyi +++ b/mypy/typeshed/stdlib/codecs.pyi @@ -80,7 +80,7 @@ class _Encoder(Protocol): def __call__(self, input: str, errors: str = ..., /) -> tuple[bytes, int]: ... # signature of Codec().encode class _Decoder(Protocol): - def __call__(self, input: bytes, errors: str = ..., /) -> tuple[str, int]: ... # signature of Codec().decode + def __call__(self, input: ReadableBuffer, errors: str = ..., /) -> tuple[str, int]: ... # signature of Codec().decode class _StreamReader(Protocol): def __call__(self, stream: _ReadableStream, errors: str = ..., /) -> StreamReader: ... diff --git a/mypy/typeshed/stdlib/copy.pyi b/mypy/typeshed/stdlib/copy.pyi index 020ce6c31b58..2cceec6a2250 100644 --- a/mypy/typeshed/stdlib/copy.pyi +++ b/mypy/typeshed/stdlib/copy.pyi @@ -1,16 +1,15 @@ import sys from typing import Any, Protocol, TypeVar -from typing_extensions import ParamSpec, Self +from typing_extensions import Self __all__ = ["Error", "copy", "deepcopy"] _T = TypeVar("_T") -_SR = TypeVar("_SR", bound=_SupportsReplace[Any]) -_P = ParamSpec("_P") +_SR = TypeVar("_SR", bound=_SupportsReplace) -class _SupportsReplace(Protocol[_P]): +class _SupportsReplace(Protocol): # In reality doesn't support args, but there's no other great way to express this. - def __replace__(self, *args: _P.args, **kwargs: _P.kwargs) -> Self: ... + def __replace__(self, *args: Any, **kwargs: Any) -> Self: ... # None in CPython but non-None in Jython PyStringMap: Any diff --git a/mypy/typeshed/stdlib/distutils/dist.pyi b/mypy/typeshed/stdlib/distutils/dist.pyi index 7013167dddbf..75fc7dbb388d 100644 --- a/mypy/typeshed/stdlib/distutils/dist.pyi +++ b/mypy/typeshed/stdlib/distutils/dist.pyi @@ -270,7 +270,7 @@ class Distribution: def has_data_files(self) -> bool: ... def is_pure(self) -> bool: ... - # Getter methods generated in __init__ + # Default getter methods generated in __init__ from self.metadata._METHOD_BASENAMES def get_name(self) -> str: ... def get_version(self) -> str: ... def get_fullname(self) -> str: ... @@ -292,3 +292,26 @@ class Distribution: def get_requires(self) -> list[str]: ... def get_provides(self) -> list[str]: ... def get_obsoletes(self) -> list[str]: ... + + # Default attributes generated in __init__ from self.display_option_names + help_commands: bool | Literal[0] + name: str | Literal[0] + version: str | Literal[0] + fullname: str | Literal[0] + author: str | Literal[0] + author_email: str | Literal[0] + maintainer: str | Literal[0] + maintainer_email: str | Literal[0] + contact: str | Literal[0] + contact_email: str | Literal[0] + url: str | Literal[0] + license: str | Literal[0] + licence: str | Literal[0] + description: str | Literal[0] + long_description: str | Literal[0] + platforms: str | list[str] | Literal[0] + classifiers: str | list[str] | Literal[0] + keywords: str | list[str] | Literal[0] + provides: list[str] | Literal[0] + requires: list[str] | Literal[0] + obsoletes: list[str] | Literal[0] diff --git a/mypy/typeshed/stdlib/doctest.pyi b/mypy/typeshed/stdlib/doctest.pyi index 7e334ef0c504..4380083027a6 100644 --- a/mypy/typeshed/stdlib/doctest.pyi +++ b/mypy/typeshed/stdlib/doctest.pyi @@ -1,9 +1,10 @@ +import sys import types import unittest from _typeshed import ExcInfo from collections.abc import Callable -from typing import Any, NamedTuple -from typing_extensions import TypeAlias +from typing import Any, ClassVar, NamedTuple +from typing_extensions import Self, TypeAlias __all__ = [ "register_optionflag", @@ -41,9 +42,22 @@ __all__ = [ "debug", ] -class TestResults(NamedTuple): - failed: int - attempted: int +# MyPy errors on conditionals within named tuples. + +if sys.version_info >= (3, 13): + class TestResults(NamedTuple): + def __new__(cls, failed: int, attempted: int, *, skipped: int = 0) -> Self: ... # type: ignore[misc] + skipped: int + failed: int + attempted: int + _fields: ClassVar = ("failed", "attempted") # type: ignore[misc] + __match_args__ = ("failed", "attempted") # type: ignore[misc] + __doc__: None # type: ignore[misc] + +else: + class TestResults(NamedTuple): + failed: int + attempted: int OPTIONFLAGS_BY_NAME: dict[str, int] @@ -134,6 +148,8 @@ class DocTestRunner: original_optionflags: int tries: int failures: int + if sys.version_info >= (3, 13): + skips: int test: DocTest def __init__(self, checker: OutputChecker | None = None, verbose: bool | None = None, optionflags: int = 0) -> None: ... def report_start(self, out: _Out, test: DocTest, example: Example) -> None: ... diff --git a/mypy/typeshed/stdlib/email/_header_value_parser.pyi b/mypy/typeshed/stdlib/email/_header_value_parser.pyi index 806fc84cf784..ff405a8b61d2 100644 --- a/mypy/typeshed/stdlib/email/_header_value_parser.pyi +++ b/mypy/typeshed/stdlib/email/_header_value_parser.pyi @@ -16,6 +16,10 @@ TOKEN_ENDS: Final[set[str]] ASPECIALS: Final[set[str]] ATTRIBUTE_ENDS: Final[set[str]] EXTENDED_ATTRIBUTE_ENDS: Final[set[str]] +# Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 +NLSET: Final[set[str]] +# Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 +SPECIALSNL: Final[set[str]] def quote_string(value: Any) -> str: ... diff --git a/mypy/typeshed/stdlib/email/_policybase.pyi b/mypy/typeshed/stdlib/email/_policybase.pyi index a3dd61a282ce..9e1f653c9d78 100644 --- a/mypy/typeshed/stdlib/email/_policybase.pyi +++ b/mypy/typeshed/stdlib/email/_policybase.pyi @@ -3,20 +3,9 @@ from collections.abc import Callable from email.errors import MessageDefect from email.header import Header from email.message import Message -from typing import Any from typing_extensions import Self class _PolicyBase: - def __add__(self, other: Any) -> Self: ... - def clone(self, **kw: Any) -> Self: ... - -class Policy(_PolicyBase, metaclass=ABCMeta): - max_line_length: int | None - linesep: str - cte_type: str - raise_on_defect: bool - mangle_from_: bool - message_factory: Callable[[Policy], Message] | None def __init__( self, *, @@ -24,9 +13,35 @@ class Policy(_PolicyBase, metaclass=ABCMeta): linesep: str = "\n", cte_type: str = "8bit", raise_on_defect: bool = False, - mangle_from_: bool = False, + mangle_from_: bool = ..., # default depends on sub-class message_factory: Callable[[Policy], Message] | None = None, + # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 + verify_generated_headers: bool = True, ) -> None: ... + def clone( + self, + *, + max_line_length: int | None = ..., + linesep: str = ..., + cte_type: str = ..., + raise_on_defect: bool = ..., + mangle_from_: bool = ..., + message_factory: Callable[[Policy], Message] | None = ..., + # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 + verify_generated_headers: bool = ..., + ) -> Self: ... + def __add__(self, other: Policy) -> Self: ... + +class Policy(_PolicyBase, metaclass=ABCMeta): + max_line_length: int | None + linesep: str + cte_type: str + raise_on_defect: bool + mangle_from_: bool + message_factory: Callable[[Policy], Message] | None + # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 + verify_generated_headers: bool + def handle_defect(self, obj: Message, defect: MessageDefect) -> None: ... def register_defect(self, obj: Message, defect: MessageDefect) -> None: ... def header_max_count(self, name: str) -> int | None: ... diff --git a/mypy/typeshed/stdlib/email/errors.pyi b/mypy/typeshed/stdlib/email/errors.pyi index c54f1560c9ae..f105576c5ee4 100644 --- a/mypy/typeshed/stdlib/email/errors.pyi +++ b/mypy/typeshed/stdlib/email/errors.pyi @@ -7,6 +7,9 @@ class BoundaryError(MessageParseError): ... class MultipartConversionError(MessageError, TypeError): ... class CharsetError(MessageError): ... +# Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 +class HeaderWriteError(MessageError): ... + class MessageDefect(ValueError): def __init__(self, line: str | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/email/utils.pyi b/mypy/typeshed/stdlib/email/utils.pyi index 9dab22c18f6c..dc3eecb5ef7f 100644 --- a/mypy/typeshed/stdlib/email/utils.pyi +++ b/mypy/typeshed/stdlib/email/utils.pyi @@ -30,20 +30,12 @@ _PDTZ: TypeAlias = tuple[int, int, int, int, int, int, int, int, int, int | None def quote(str: str) -> str: ... def unquote(str: str) -> str: ... -if sys.version_info >= (3, 13): - def parseaddr(addr: str | list[str], *, strict: bool = True) -> tuple[str, str]: ... - -else: - def parseaddr(addr: str) -> tuple[str, str]: ... - +# `strict` parameter added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 +def parseaddr(addr: str | list[str], *, strict: bool = True) -> tuple[str, str]: ... def formataddr(pair: tuple[str | None, str], charset: str | Charset = "utf-8") -> str: ... -if sys.version_info >= (3, 13): - def getaddresses(fieldvalues: Iterable[str], *, strict: bool = True) -> list[tuple[str, str]]: ... - -else: - def getaddresses(fieldvalues: Iterable[str]) -> list[tuple[str, str]]: ... - +# `strict` parameter added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 +def getaddresses(fieldvalues: Iterable[str], *, strict: bool = True) -> list[tuple[str, str]]: ... @overload def parsedate(data: None) -> None: ... @overload diff --git a/mypy/typeshed/stdlib/io.pyi b/mypy/typeshed/stdlib/io.pyi index 2d64d261951d..7607608696dd 100644 --- a/mypy/typeshed/stdlib/io.pyi +++ b/mypy/typeshed/stdlib/io.pyi @@ -84,7 +84,6 @@ class RawIOBase(IOBase): def read(self, size: int = -1, /) -> bytes | None: ... class BufferedIOBase(IOBase): - raw: RawIOBase # This is not part of the BufferedIOBase API and may not exist on some implementations. def detach(self) -> RawIOBase: ... def readinto(self, buffer: WriteableBuffer, /) -> int: ... def write(self, buffer: ReadableBuffer, /) -> int: ... @@ -119,11 +118,13 @@ class BytesIO(BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible d def read1(self, size: int | None = -1, /) -> bytes: ... class BufferedReader(BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of methods in the base classes + raw: RawIOBase def __enter__(self) -> Self: ... def __init__(self, raw: RawIOBase, buffer_size: int = ...) -> None: ... def peek(self, size: int = 0, /) -> bytes: ... class BufferedWriter(BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of writelines in the base classes + raw: RawIOBase def __enter__(self) -> Self: ... def __init__(self, raw: RawIOBase, buffer_size: int = ...) -> None: ... def write(self, buffer: ReadableBuffer, /) -> int: ... diff --git a/mypy/typeshed/stdlib/subprocess.pyi b/mypy/typeshed/stdlib/subprocess.pyi index 2a5859807b51..703a5012012c 100644 --- a/mypy/typeshed/stdlib/subprocess.pyi +++ b/mypy/typeshed/stdlib/subprocess.pyi @@ -2582,6 +2582,11 @@ else: def list2cmdline(seq: Iterable[StrOrBytesPath]) -> str: ... # undocumented if sys.platform == "win32": + if sys.version_info >= (3, 13): + from _winapi import STARTF_FORCEOFFFEEDBACK, STARTF_FORCEONFEEDBACK + + __all__ += ["STARTF_FORCEOFFFEEDBACK", "STARTF_FORCEONFEEDBACK"] + class STARTUPINFO: def __init__( self, diff --git a/mypy/typeshed/stdlib/tempfile.pyi b/mypy/typeshed/stdlib/tempfile.pyi index 62422b84eb37..0c19d56fc7a6 100644 --- a/mypy/typeshed/stdlib/tempfile.pyi +++ b/mypy/typeshed/stdlib/tempfile.pyi @@ -253,11 +253,11 @@ class _TemporaryFileWrapper(IO[AnyStr]): def truncate(self, size: int | None = ...) -> int: ... def writable(self) -> bool: ... @overload - def write(self: _TemporaryFileWrapper[str], s: str) -> int: ... + def write(self: _TemporaryFileWrapper[str], s: str, /) -> int: ... @overload - def write(self: _TemporaryFileWrapper[bytes], s: ReadableBuffer) -> int: ... + def write(self: _TemporaryFileWrapper[bytes], s: ReadableBuffer, /) -> int: ... @overload - def write(self, s: AnyStr) -> int: ... + def write(self, s: AnyStr, /) -> int: ... @overload def writelines(self: _TemporaryFileWrapper[str], lines: Iterable[str]) -> None: ... @overload From 8b1e6064928bdb173822df639a73c8cbac0845ec Mon Sep 17 00:00:00 2001 From: quinn-sasha Date: Mon, 16 Sep 2024 07:44:03 +0900 Subject: [PATCH 133/247] Make changelog visible in mypy documentation (#17742) Resolves #17717 --- CHANGELOG.md | 310 ++++++++++++++++++------------------- docs/requirements-docs.txt | 1 + docs/source/changelog.md | 3 + docs/source/conf.py | 2 +- docs/source/index.rst | 1 + 5 files changed, 161 insertions(+), 156 deletions(-) create mode 100644 docs/source/changelog.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b544e05ee573..9632cb39a8b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ We’ve just uploaded mypy 1.11 to the Python Package Index ([PyPI](https://pypi You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Support Python 3.12 Syntax for Generics (PEP 695) +### Support Python 3.12 Syntax for Generics (PEP 695) Mypy now supports the new type parameter syntax introduced in Python 3.12 ([PEP 695](https://peps.python.org/pep-0695/)). This feature is still experimental and must be enabled with the `--enable-incomplete-feature=NewGenericSyntax` flag, or with `enable_incomplete_feature = NewGenericSyntax` in the mypy configuration file. @@ -40,7 +40,7 @@ type A[T] = C[list[T]] This feature was contributed by Jukka Lehtosalo. -#### Support for `functools.partial` +### Support for `functools.partial` Mypy now type checks uses of `functools.partial`. Previously mypy would accept arbitrary arguments. @@ -60,7 +60,7 @@ g(11) This feature was contributed by Shantanu (PR [16939](https://github.com/python/mypy/pull/16939)). -#### Stricter Checks for Untyped Overrides +### Stricter Checks for Untyped Overrides Past mypy versions didn't check if untyped methods were compatible with overridden methods. This would result in false negatives. Now mypy performs these checks when using `--check-untyped-defs`. @@ -78,7 +78,7 @@ class Derived(Base): This feature was contributed by Steven Troxler (PR [17276](https://github.com/python/mypy/pull/17276)). -#### Type Inference Improvements +### Type Inference Improvements The new polymorphic inference algorithm introduced in mypy 1.5 is now used in more situations. This improves type inference involving generic higher-order functions, in particular. @@ -95,14 +95,14 @@ for x in (1, 'x'): This was also contributed by Ivan Levkivskyi (PR [17408](https://github.com/python/mypy/pull/17408)). -#### Improvements to Detection of Overlapping Overloads +### Improvements to Detection of Overlapping Overloads The details of how mypy checks if two `@overload` signatures are unsafely overlapping were overhauled. This both fixes some false positives, and allows mypy to detect additional unsafe signatures. This feature was contributed by Ivan Levkivskyi (PR [17392](https://github.com/python/mypy/pull/17392)). -#### Better Support for Type Hints in Expressions +### Better Support for Type Hints in Expressions Mypy now allows more expressions that evaluate to valid type annotations in all expression contexts. The inferred types of these expressions are also sometimes more precise. Previously they were often `object`. @@ -117,7 +117,7 @@ print(Callable[[], int] | None) # No error This feature was contributed by Jukka Lehtosalo (PR [17404](https://github.com/python/mypy/pull/17404)). -#### Mypyc Improvements +### Mypyc Improvements Mypyc now supports the new syntax for generics introduced in Python 3.12 (see above). Another notable improvement is signficantly faster basic operations on `int` values. @@ -129,18 +129,18 @@ Mypyc now supports the new syntax for generics introduced in Python 3.12 (see ab * Allow specifying primitives as pure (Jukka Lehtosalo, PR [17263](https://github.com/python/mypy/pull/17263)) -#### Changes to Stubtest +### Changes to Stubtest * Ignore `_ios_support` (Alex Waygood, PR [17270](https://github.com/python/mypy/pull/17270)) * Improve support for Python 3.13 (Shantanu, PR [17261](https://github.com/python/mypy/pull/17261)) -#### Changes to Stubgen +### Changes to Stubgen * Gracefully handle invalid `Optional` and recognize aliases to PEP 604 unions (Ali Hamdan, PR [17386](https://github.com/python/mypy/pull/17386)) * Fix for Python 3.13 (Jelle Zijlstra, PR [17290](https://github.com/python/mypy/pull/17290)) * Preserve enum value initialisers (Shantanu, PR [17125](https://github.com/python/mypy/pull/17125)) -#### Miscellaneous New Features +### Miscellaneous New Features * Add error format support and JSON output option via `--output json` (Tushar Sadhwani, PR [11396](https://github.com/python/mypy/pull/11396)) * Support `enum.member` in Python 3.11+ (Nikita Sobolev, PR [17382](https://github.com/python/mypy/pull/17382)) * Support `enum.nonmember` in Python 3.11+ (Nikita Sobolev, PR [17376](https://github.com/python/mypy/pull/17376)) @@ -149,7 +149,7 @@ Mypyc now supports the new syntax for generics introduced in Python 3.12 (see ab * Add support for `__spec__` (Shantanu, PR [14739](https://github.com/python/mypy/pull/14739)) -#### Changes to Error Reporting +### Changes to Error Reporting * Mention `--enable-incomplete-feature=NewGenericSyntax` in messages (Shantanu, PR [17462](https://github.com/python/mypy/pull/17462)) * Do not report plugin-generated methods with `explicit-override` (sobolevn, PR [17433](https://github.com/python/mypy/pull/17433)) * Use and display namespaces for function type variables (Ivan Levkivskyi, PR [17311](https://github.com/python/mypy/pull/17311)) @@ -162,7 +162,7 @@ Mypyc now supports the new syntax for generics introduced in Python 3.12 (see ab * Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) -#### Fixes for Crashes +### Fixes for Crashes * Fix daemon crash on invalid type in TypedDict (Ivan Levkivskyi, PR [17495](https://github.com/python/mypy/pull/17495)) * Fix crash and bugs related to `partial()` (Ivan Levkivskyi, PR [17423](https://github.com/python/mypy/pull/17423)) * Fix crash when overriding with unpacked TypedDict (Ivan Levkivskyi, PR [17359](https://github.com/python/mypy/pull/17359)) @@ -174,14 +174,14 @@ Mypyc now supports the new syntax for generics introduced in Python 3.12 (see ab * Fix crash on type comment inside generic definitions (Bénédikt Tran, PR [16849](https://github.com/python/mypy/pull/16849)) -#### Changes to Documentation +### Changes to Documentation * Use inline config in documentation for optional error codes (Shantanu, PR [17374](https://github.com/python/mypy/pull/17374)) * Use lower-case generics in documentation (Seo Sanghyeon, PR [17176](https://github.com/python/mypy/pull/17176)) * Add documentation for show-error-code-links (GiorgosPapoutsakis, PR [17144](https://github.com/python/mypy/pull/17144)) * Update CONTRIBUTING.md to include commands for Windows (GiorgosPapoutsakis, PR [17142](https://github.com/python/mypy/pull/17142)) -#### Other Notable Improvements and Fixes +### Other Notable Improvements and Fixes * Fix ParamSpec inference against TypeVarTuple (Ivan Levkivskyi, PR [17431](https://github.com/python/mypy/pull/17431)) * Fix explicit type for `partial` (Ivan Levkivskyi, PR [17424](https://github.com/python/mypy/pull/17424)) * Always allow lambda calls (Ivan Levkivskyi, PR [17430](https://github.com/python/mypy/pull/17430)) @@ -206,12 +206,12 @@ Mypyc now supports the new syntax for generics introduced in Python 3.12 (see ab * Fix Literal strings containing pipe characters (Jelle Zijlstra, PR [17148](https://github.com/python/mypy/pull/17148)) -#### Typeshed Updates +### Typeshed Updates Please see [git log](https://github.com/python/typeshed/commits/main?after=6dda799d8ad1d89e0f8aad7ac41d2d34bd838ace+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. -#### Acknowledgements +### Acknowledgements Thanks to all mypy contributors who contributed to this release: - Alex Waygood @@ -254,7 +254,7 @@ We’ve just uploaded mypy 1.10 to the Python Package Index ([PyPI](https://pypi You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Support TypeIs (PEP 742) +### Support TypeIs (PEP 742) Mypy now supports `TypeIs` ([PEP 742](https://peps.python.org/pep-0742/)), which allows functions to narrow the type of a value, similar to `isinstance()`. Unlike `TypeGuard`, @@ -281,7 +281,7 @@ can be used on earlier Python versions by importing it from This feature was contributed by Jelle Zijlstra (PR [16898](https://github.com/python/mypy/pull/16898)). -#### Support TypeVar Defaults (PEP 696) +### Support TypeVar Defaults (PEP 696) [PEP 696](https://peps.python.org/pep-0696/) adds support for type parameter defaults. Example: @@ -308,7 +308,7 @@ can be used with earlier Python releases by importing `TypeVar` from This feature was contributed by Marc Mueller (PR [16878](https://github.com/python/mypy/pull/16878) and PR [16925](https://github.com/python/mypy/pull/16925)). -#### Support TypeAliasType (PEP 695) +### Support TypeAliasType (PEP 695) As part of the initial steps towards implementing [PEP 695](https://peps.python.org/pep-0695/), mypy now supports `TypeAliasType`. `TypeAliasType` provides a backport of the new `type` statement in Python 3.12. @@ -344,7 +344,7 @@ c: ListOrSet[str] = 'test' # error: Incompatible types in assignment (expressio This feature was contributed by Ali Hamdan (PR [16926](https://github.com/python/mypy/pull/16926), PR [17038](https://github.com/python/mypy/pull/17038) and PR [17053](https://github.com/python/mypy/pull/17053)) -#### Detect Additional Unsafe Uses of super() +### Detect Additional Unsafe Uses of super() Mypy will reject unsafe uses of `super()` more consistently, when the target has a trivial (empty) body. Example: @@ -360,13 +360,13 @@ class Sub(Proto): This feature was contributed by Shantanu (PR [16756](https://github.com/python/mypy/pull/16756)). -#### Stubgen Improvements +### Stubgen Improvements - Preserve empty tuple annotation (Ali Hamdan, PR [16907](https://github.com/python/mypy/pull/16907)) - Add support for PEP 570 positional-only parameters (Ali Hamdan, PR [16904](https://github.com/python/mypy/pull/16904)) - Replace obsolete typing aliases with builtin containers (Ali Hamdan, PR [16780](https://github.com/python/mypy/pull/16780)) - Fix generated dataclass `__init__` signature (Ali Hamdan, PR [16906](https://github.com/python/mypy/pull/16906)) -#### Mypyc Improvements +### Mypyc Improvements - Provide an easier way to define IR-to-IR transforms (Jukka Lehtosalo, PR [16998](https://github.com/python/mypy/pull/16998)) - Implement lowering pass and add primitives for int (in)equality (Jukka Lehtosalo, PR [17027](https://github.com/python/mypy/pull/17027)) @@ -377,15 +377,15 @@ This feature was contributed by Shantanu (PR [16756](https://github.com/python/m - Fix compilation of unreachable comprehensions (Richard Si, PR [15721](https://github.com/python/mypy/pull/15721)) - Don't crash on non-inlinable final local reads (Richard Si, PR [15719](https://github.com/python/mypy/pull/15719)) -#### Documentation Improvements +### Documentation Improvements - Import `TypedDict` from `typing` instead of `typing_extensions` (Riccardo Di Maio, PR [16958](https://github.com/python/mypy/pull/16958)) - Add missing `mutable-override` to section title (James Braza, PR [16886](https://github.com/python/mypy/pull/16886)) -#### Error Reporting Improvements +### Error Reporting Improvements - Use lower-case generics more consistently in error messages (Jukka Lehtosalo, PR [17035](https://github.com/python/mypy/pull/17035)) -#### Other Notable Changes and Fixes +### Other Notable Changes and Fixes - Fix incorrect inferred type when accessing descriptor on union type (Matthieu Devlin, PR [16604](https://github.com/python/mypy/pull/16604)) - Fix crash when expanding invalid `Unpack` in a `Callable` alias (Ali Hamdan, PR [17028](https://github.com/python/mypy/pull/17028)) - Fix false positive when string formatting with string enum (roberfi, PR [16555](https://github.com/python/mypy/pull/16555)) @@ -403,15 +403,15 @@ This feature was contributed by Shantanu (PR [16756](https://github.com/python/m - Experimental: Support TypedDict within `type[...]` (Marc Mueller, PR [16963](https://github.com/python/mypy/pull/16963)) - Experimtental: Fix issue with TypedDict with optional keys in `type[...]` (Marc Mueller, PR [17068](https://github.com/python/mypy/pull/17068)) -#### Typeshed Updates +### Typeshed Updates Please see [git log](https://github.com/python/typeshed/commits/main?after=6dda799d8ad1d89e0f8aad7ac41d2d34bd838ace+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. -#### Mypy 1.10.1 +### Mypy 1.10.1 - Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) -#### Acknowledgements +### Acknowledgements Thanks to all mypy contributors who contributed to this release: - Alex Waygood @@ -454,7 +454,7 @@ We’ve just uploaded mypy 1.9 to the Python Package Index ([PyPI](https://pypi. You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Breaking Changes +### Breaking Changes Because the version of typeshed we use in mypy 1.9 doesn't support 3.7, neither does mypy 1.9. (Jared Hance, PR [16883](https://github.com/python/mypy/pull/16883)) @@ -476,7 +476,7 @@ projects to use `--local-partial-types`, but it's not yet clear whether this is practical. The migration usually involves adding some explicit type annotations to module-level and class-level variables. -#### Basic Support for Type Parameter Defaults (PEP 696) +### Basic Support for Type Parameter Defaults (PEP 696) This release contains new experimental support for type parameter defaults ([PEP 696](https://peps.python.org/pep-0696)). Please try it @@ -506,7 +506,7 @@ reveal_type(Context().bot) reveal_type(Context[MyBot]().bot) ``` -#### Type-checking Improvements +### Type-checking Improvements * Fix missing type store for overloads (Marc Mueller, PR [16803](https://github.com/python/mypy/pull/16803)) * Fix `'WriteToConn' object has no attribute 'flush'` (Charlie Denton, PR [16801](https://github.com/python/mypy/pull/16801)) * Improve TypeAlias error messages (Marc Mueller, PR [16831](https://github.com/python/mypy/pull/16831)) @@ -519,11 +519,11 @@ reveal_type(Context[MyBot]().bot) * Add `alias` support to `field()` in `attrs` plugin (Nikita Sobolev, PR [16610](https://github.com/python/mypy/pull/16610)) * Improve attrs hashability detection (Tin Tvrtković, PR [16556](https://github.com/python/mypy/pull/16556)) -#### Performance Improvements +### Performance Improvements * Speed up finding function type variables (Jukka Lehtosalo, PR [16562](https://github.com/python/mypy/pull/16562)) -#### Documentation Updates +### Documentation Updates * Document supported values for `--enable-incomplete-feature` in "mypy --help" (Froger David, PR [16661](https://github.com/python/mypy/pull/16661)) * Update new type system discussion links (thomaswhaley, PR [16841](https://github.com/python/mypy/pull/16841)) @@ -533,7 +533,7 @@ reveal_type(Context[MyBot]().bot) * Fix numbering error (Stefanie Molin, PR [16838](https://github.com/python/mypy/pull/16838)) * Various documentation improvements (Shantanu, PR [16836](https://github.com/python/mypy/pull/16836)) -#### Stubtest Improvements +### Stubtest Improvements * Ignore private function/method parameters when they are missing from the stub (private parameter names start with a single underscore and have a default) (PR [16507](https://github.com/python/mypy/pull/16507)) * Ignore a new protocol dunder (Alex Waygood, PR [16895](https://github.com/python/mypy/pull/16895)) * Private parameters can be omitted (Sebastian Rittau, PR [16507](https://github.com/python/mypy/pull/16507)) @@ -541,13 +541,13 @@ reveal_type(Context[MyBot]().bot) * Adjust symbol table logic (Shantanu, PR [16823](https://github.com/python/mypy/pull/16823)) * Fix posisitional-only handling in overload resolution (Shantanu, PR [16750](https://github.com/python/mypy/pull/16750)) -#### Stubgen Improvements +### Stubgen Improvements * Fix crash on star unpack of TypeVarTuple (Ali Hamdan, PR [16869](https://github.com/python/mypy/pull/16869)) * Use PEP 604 unions everywhere (Ali Hamdan, PR [16519](https://github.com/python/mypy/pull/16519)) * Do not ignore property deleter (Ali Hamdan, PR [16781](https://github.com/python/mypy/pull/16781)) * Support type stub generation for `staticmethod` (WeilerMarcel, PR [14934](https://github.com/python/mypy/pull/14934)) -#### Acknowledgements +### Acknowledgements ​Thanks to all mypy contributors who contributed to this release: @@ -591,7 +591,7 @@ We’ve just uploaded mypy 1.8 to the Python Package Index ([PyPI](https://pypi. You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Type-checking Improvements +### Type-checking Improvements * Do not intersect types in isinstance checks if at least one is final (Christoph Tyralla, PR [16330](https://github.com/python/mypy/pull/16330)) * Detect that `@final` class without `__bool__` cannot have falsey instances (Ilya Priven, PR [16566](https://github.com/python/mypy/pull/16566)) * Do not allow `TypedDict` classes with extra keywords (Nikita Sobolev, PR [16438](https://github.com/python/mypy/pull/16438)) @@ -601,44 +601,44 @@ You can read the full documentation for this release on [Read the Docs](http://m * Allow type ignores of PEP 695 constructs (Shantanu, PR [16608](https://github.com/python/mypy/pull/16608)) * Enable `type_check_only` support for `TypedDict` and `NamedTuple` (Nikita Sobolev, PR [16469](https://github.com/python/mypy/pull/16469)) -#### Performance Improvements +### Performance Improvements * Add fast path to analyzing special form assignments (Jukka Lehtosalo, PR [16561](https://github.com/python/mypy/pull/16561)) -#### Improvements to Error Reporting +### Improvements to Error Reporting * Don't show documentation links for plugin error codes (Ivan Levkivskyi, PR [16383](https://github.com/python/mypy/pull/16383)) * Improve error messages for `super` checks and add more tests (Nikita Sobolev, PR [16393](https://github.com/python/mypy/pull/16393)) * Add error code for mutable covariant override (Ivan Levkivskyi, PR [16399](https://github.com/python/mypy/pull/16399)) -#### Stubgen Improvements +### Stubgen Improvements * Preserve simple defaults in function signatures (Ali Hamdan, PR [15355](https://github.com/python/mypy/pull/15355)) * Include `__all__` in output (Jelle Zijlstra, PR [16356](https://github.com/python/mypy/pull/16356)) * Fix stubgen regressions with pybind11 and mypy 1.7 (Chad Dombrova, PR [16504](https://github.com/python/mypy/pull/16504)) -#### Stubtest Improvements +### Stubtest Improvements * Improve handling of unrepresentable defaults (Jelle Zijlstra, PR [16433](https://github.com/python/mypy/pull/16433)) * Print more helpful errors if a function is missing from stub (Alex Waygood, PR [16517](https://github.com/python/mypy/pull/16517)) * Support `@type_check_only` decorator (Nikita Sobolev, PR [16422](https://github.com/python/mypy/pull/16422)) * Warn about missing `__del__` (Shantanu, PR [16456](https://github.com/python/mypy/pull/16456)) * Fix crashes with some uses of `final` and `deprecated` (Shantanu, PR [16457](https://github.com/python/mypy/pull/16457)) -#### Fixes to Crashes +### Fixes to Crashes * Fix crash with type alias to `Callable[[Unpack[Tuple[Any, ...]]], Any]` (Alex Waygood, PR [16541](https://github.com/python/mypy/pull/16541)) * Fix crash on TypeGuard in `__call__` (Ivan Levkivskyi, PR [16516](https://github.com/python/mypy/pull/16516)) * Fix crash on invalid enum in method (Ivan Levkivskyi, PR [16511](https://github.com/python/mypy/pull/16511)) * Fix crash on unimported Any in TypedDict (Ivan Levkivskyi, PR [16510](https://github.com/python/mypy/pull/16510)) -#### Documentation Updates +### Documentation Updates * Update soft-error-limit default value to -1 (Sveinung Gundersen, PR [16542](https://github.com/python/mypy/pull/16542)) * Support Sphinx 7.x (Michael R. Crusoe, PR [16460](https://github.com/python/mypy/pull/16460)) -#### Other Notable Changes and Fixes +### Other Notable Changes and Fixes * Allow mypy to output a junit file with per-file results (Matthew Wright, PR [16388](https://github.com/python/mypy/pull/16388)) -#### Typeshed Updates +### Typeshed Updates Please see [git log](https://github.com/python/typeshed/commits/main?after=4a854366e03dee700109f8e758a08b2457ea2f51+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. -#### Acknowledgements +### Acknowledgements ​Thanks to all mypy contributors who contributed to this release: @@ -672,7 +672,7 @@ We’ve just uploaded mypy 1.7 to the Python Package Index ([PyPI](https://pypi. You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Using TypedDict for `**kwargs` Typing +### Using TypedDict for `**kwargs` Typing Mypy now has support for using `Unpack[...]` with a TypedDict type to annotate `**kwargs` arguments enabled by default. Example: @@ -702,7 +702,7 @@ Refer to [PEP 692](https://peps.python.org/pep-0692/) for more information. Note This was contributed by Ivan Levkivskyi back in 2022 (PR [13471](https://github.com/python/mypy/pull/13471)). -#### TypeVarTuple Support Enabled (Experimental) +### TypeVarTuple Support Enabled (Experimental) Mypy now has support for variadic generics (TypeVarTuple) enabled by default, as an experimental feature. Refer to [PEP 646](https://peps.python.org/pep-0646/) for the details. @@ -724,7 +724,7 @@ Changes included in this release: * Subtyping and inference of user-defined variadic types (Ivan Levkivskyi, PR [16076](https://github.com/python/mypy/pull/16076)) * Complete type analysis of variadic types (Ivan Levkivskyi, PR [15991](https://github.com/python/mypy/pull/15991)) -#### New Way of Installing Mypyc Dependencies +### New Way of Installing Mypyc Dependencies If you want to install package dependencies needed by mypyc (not just mypy), you should now install `mypy[mypyc]` instead of just `mypy`: @@ -736,13 +736,13 @@ Mypy has many more users than mypyc, so always installing mypyc dependencies wou This change was contributed by Shantanu (PR [16229](https://github.com/python/mypy/pull/16229)). -#### New Rules for Re-exports +### New Rules for Re-exports Mypy no longer considers an import such as `import a.b as b` as an explicit re-export. The old behavior was arguably inconsistent and surprising. This may impact some stub packages, such as older versions of `types-six`. You can change the import to `from a import b as b`, if treating the import as a re-export was intentional. This change was contributed by Anders Kaseorg (PR [14086](https://github.com/python/mypy/pull/14086)). -#### Improved Type Inference +### Improved Type Inference The new type inference algorithm that was recently introduced to mypy (but was not enabled by default) is now enabled by default. It improves type inference of calls to generic callables where an argument is also a generic callable, in particular. You can use `--old-type-inference` to disable the new behavior. @@ -750,7 +750,7 @@ The new algorithm can (rarely) produce different error messages, different error The new type inference algorithm was contributed by Ivan Levkivskyi. PR [16345](https://github.com/python/mypy/pull/16345) enabled it by default. -#### Narrowing Tuple Types Using len() +### Narrowing Tuple Types Using len() Mypy now can narrow tuple types using `len()` checks. Example: @@ -763,7 +763,7 @@ def f(t: tuple[int, int] | tuple[int, int, int]) -> None: This feature was contributed by Ivan Levkivskyi (PR [16237](https://github.com/python/mypy/pull/16237)). -#### More Precise Tuple Lengths (Experimental) +### More Precise Tuple Lengths (Experimental) Mypy supports experimental, more precise checking of tuple type lengths through `--enable-incomplete-feature=PreciseTupleTypes`. Refer to the [documentation](https://mypy.readthedocs.io/en/latest/command_line.html#enabling-incomplete-experimental-features) for more information. @@ -771,13 +771,13 @@ More generally, we are planning to use `--enable-incomplete-feature` to introduc This feature was contributed by Ivan Levkivskyi (PR [16237](https://github.com/python/mypy/pull/16237)). -#### Mypy Changelog +### Mypy Changelog We now maintain a [changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) in the mypy Git repository. It mirrors the contents of [mypy release blog posts](https://mypy-lang.blogspot.com/). We will continue to also publish release blog posts. In the future, release blog posts will be created based on the changelog near a release date. This was contributed by Shantanu (PR [16280](https://github.com/python/mypy/pull/16280)). -#### Mypy Daemon Improvements +### Mypy Daemon Improvements * Fix daemon crash caused by deleted submodule (Jukka Lehtosalo, PR [16370](https://github.com/python/mypy/pull/16370)) * Fix file reloading in dmypy with --export-types (Ivan Levkivskyi, PR [16359](https://github.com/python/mypy/pull/16359)) @@ -789,7 +789,7 @@ This was contributed by Shantanu (PR [16280](https://github.com/python/mypy/pull * Stream dmypy output instead of dumping everything at the end (Valentin Stanciu, PR [16252](https://github.com/python/mypy/pull/16252)) * Make sure all dmypy errors are shown (Valentin Stanciu, PR [16250](https://github.com/python/mypy/pull/16250)) -#### Mypyc Improvements +### Mypyc Improvements * Generate error on duplicate function definitions (Jukka Lehtosalo, PR [16309](https://github.com/python/mypy/pull/16309)) * Don't crash on unreachable statements (Jukka Lehtosalo, PR [16311](https://github.com/python/mypy/pull/16311)) @@ -797,7 +797,7 @@ This was contributed by Shantanu (PR [16280](https://github.com/python/mypy/pull * Fix direct `__dict__` access on inner functions in new Python (Shantanu, PR [16084](https://github.com/python/mypy/pull/16084)) * Make tuple packing and unpacking more efficient (Jukka Lehtosalo, PR [16022](https://github.com/python/mypy/pull/16022)) -#### Improvements to Error Reporting +### Improvements to Error Reporting * Update starred expression error message to match CPython (Cibin Mathew, PR [16304](https://github.com/python/mypy/pull/16304)) * Fix error code of "Maybe you forgot to use await" note (Jelle Zijlstra, PR [16203](https://github.com/python/mypy/pull/16203)) @@ -807,14 +807,14 @@ This was contributed by Shantanu (PR [16280](https://github.com/python/mypy/pull * Add hint for AsyncIterator incompatible return type (Ilya Priven, PR [15883](https://github.com/python/mypy/pull/15883)) * Don't suggest stubs packages where the runtime package now ships with types (Alex Waygood, PR [16226](https://github.com/python/mypy/pull/16226)) -#### Performance Improvements +### Performance Improvements * Speed up type argument checking (Jukka Lehtosalo, PR [16353](https://github.com/python/mypy/pull/16353)) * Add fast path for checking self types (Jukka Lehtosalo, PR [16352](https://github.com/python/mypy/pull/16352)) * Cache information about whether file is typeshed file (Jukka Lehtosalo, PR [16351](https://github.com/python/mypy/pull/16351)) * Skip expensive `repr()` in logging call when not needed (Jukka Lehtosalo, PR [16350](https://github.com/python/mypy/pull/16350)) -#### Attrs and Dataclass Improvements +### Attrs and Dataclass Improvements * `dataclass.replace`: Allow transformed classes (Ilya Priven, PR [15915](https://github.com/python/mypy/pull/15915)) * `dataclass.replace`: Fall through to typeshed signature (Ilya Priven, PR [15962](https://github.com/python/mypy/pull/15962)) @@ -823,7 +823,7 @@ This was contributed by Shantanu (PR [16280](https://github.com/python/mypy/pull * `attrs`, `dataclasses`: Don't enforce slots when base class doesn't (Ilya Priven, PR [15976](https://github.com/python/mypy/pull/15976)) * Fix crash on dataclass field / property collision (Nikita Sobolev, PR [16147](https://github.com/python/mypy/pull/16147)) -#### Stubgen Improvements +### Stubgen Improvements * Write stubs with utf-8 encoding (Jørgen Lind, PR [16329](https://github.com/python/mypy/pull/16329)) * Fix missing property setter in semantic analysis mode (Ali Hamdan, PR [16303](https://github.com/python/mypy/pull/16303)) @@ -831,7 +831,7 @@ This was contributed by Shantanu (PR [16280](https://github.com/python/mypy/pull * Multiple fixes to the generated imports (Ali Hamdan, PR [15624](https://github.com/python/mypy/pull/15624)) * Generate valid dataclass stubs (Ali Hamdan, PR [15625](https://github.com/python/mypy/pull/15625)) -#### Fixes to Crashes +### Fixes to Crashes * Fix incremental mode crash on TypedDict in method (Ivan Levkivskyi, PR [16364](https://github.com/python/mypy/pull/16364)) * Fix crash on star unpack in TypedDict (Ivan Levkivskyi, PR [16116](https://github.com/python/mypy/pull/16116)) @@ -840,7 +840,7 @@ This was contributed by Shantanu (PR [16280](https://github.com/python/mypy/pull * Fix crash when parsing error code config with typo (Shantanu, PR [16005](https://github.com/python/mypy/pull/16005)) * Fix `__post_init__()` internal error (Ilya Priven, PR [16080](https://github.com/python/mypy/pull/16080)) -#### Documentation Updates +### Documentation Updates * Make it easier to copy commands from README (Hamir Mahal, PR [16133](https://github.com/python/mypy/pull/16133)) * Document and rename `[overload-overlap]` error code (Shantanu, PR [16074](https://github.com/python/mypy/pull/16074)) @@ -848,7 +848,7 @@ This was contributed by Shantanu (PR [16280](https://github.com/python/mypy/pull * Document `force_union_syntax` and `force_uppercase_builtins` (Nikita Sobolev, PR [16048](https://github.com/python/mypy/pull/16048)) * Document we're not tracking relationships between symbols (Ilya Priven, PR [16018](https://github.com/python/mypy/pull/16018)) -#### Other Notable Changes and Fixes +### Other Notable Changes and Fixes * Propagate narrowed types to lambda expressions (Ivan Levkivskyi, PR [16407](https://github.com/python/mypy/pull/16407)) * Avoid importing from `setuptools._distutils` (Shantanu, PR [16348](https://github.com/python/mypy/pull/16348)) @@ -882,11 +882,11 @@ This was contributed by Shantanu (PR [16280](https://github.com/python/mypy/pull * Make iterable logic more consistent (Shantanu, PR [16006](https://github.com/python/mypy/pull/16006)) * Fix inference for properties with `__call__` (Shantanu, PR [15926](https://github.com/python/mypy/pull/15926)) -#### Typeshed Updates +### Typeshed Updates Please see [git log](https://github.com/python/typeshed/commits/main?after=4a854366e03dee700109f8e758a08b2457ea2f51+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. -#### Acknowledgements +### Acknowledgements Thanks to all mypy contributors who contributed to this release: @@ -932,7 +932,7 @@ We’ve just uploaded mypy 1.6 to the Python Package Index ([PyPI](https://pypi. You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Introduce Error Subcodes for Import Errors +### Introduce Error Subcodes for Import Errors Mypy now uses the error code import-untyped if an import targets an installed library that doesn’t support static type checking, and no stub files are available. Other invalid imports produce the import-not-found error code. They both are subcodes of the import error code, which was previously used for both kinds of import-related errors. @@ -942,17 +942,17 @@ If you use \--warn-unused-ignore or \--strict, mypy will complain if you use \# This feature was contributed by Shantanu (PR [15840](https://github.com/python/mypy/pull/15840), PR [14740](https://github.com/python/mypy/pull/14740)). -#### Remove Support for Targeting Python 3.6 and Earlier +### Remove Support for Targeting Python 3.6 and Earlier Running mypy with \--python-version 3.6, for example, is no longer supported. Python 3.6 hasn’t been properly supported by mypy for some time now, and this makes it explicit. This was contributed by Nikita Sobolev (PR [15668](https://github.com/python/mypy/pull/15668)). -#### Selective Filtering of \--disallow-untyped-calls Targets +### Selective Filtering of \--disallow-untyped-calls Targets Using \--disallow-untyped-calls could be annoying when using libraries with missing type information, as mypy would generate many errors about code that uses the library. Now you can use \--untyped-calls-exclude=acme, for example, to disable these errors about calls targeting functions defined in the acme package. Refer to the [documentation](https://mypy.readthedocs.io/en/latest/command_line.html#cmdoption-mypy-untyped-calls-exclude) for more information. This feature was contributed by Ivan Levkivskyi (PR [15845](https://github.com/python/mypy/pull/15845)). -#### Improved Type Inference between Callable Types +### Improved Type Inference between Callable Types Mypy now does a better job inferring type variables inside arguments of callable types. For example, this code fragment now type checks correctly: @@ -965,7 +965,7 @@ reveal_type(f(g)) # Callable[[str, int, int], None] This was contributed by Ivan Levkivskyi (PR [15910](https://github.com/python/mypy/pull/15910)). -#### Don’t Consider None and TypeVar to Overlap in Overloads +### Don’t Consider None and TypeVar to Overlap in Overloads Mypy now doesn’t consider an overload item with an argument type None to overlap with a type variable: @@ -981,7 +981,7 @@ Previously mypy would generate an error about the definition of f above. This is This feature was contributed by Ivan Levkivskyi (PR [15846](https://github.com/python/mypy/pull/15846)). -#### Improvements to \--new-type-inference +### Improvements to \--new-type-inference The experimental new type inference algorithm (polymorphic inference) introduced as an opt-in feature in mypy 1.5 has several improvements: @@ -993,7 +993,7 @@ The experimental new type inference algorithm (polymorphic inference) introduced **Note:** We are planning to enable \--new-type-inference by default in mypy 1.7. Please try this out and let us know if you encounter any issues. -#### ParamSpec Improvements +### ParamSpec Improvements * Support self-types containing ParamSpec (Ivan Levkivskyi, PR [15903](https://github.com/python/mypy/pull/15903)) * Allow “…” in Concatenate, and clean up ParamSpec literals (Ivan Levkivskyi, PR [15905](https://github.com/python/mypy/pull/15905)) @@ -1002,12 +1002,12 @@ The experimental new type inference algorithm (polymorphic inference) introduced * Fix crash on invalid type variable with ParamSpec (Ivan Levkivskyi, PR [15953](https://github.com/python/mypy/pull/15953)) * Fix subtyping between ParamSpecs (Ivan Levkivskyi, PR [15892](https://github.com/python/mypy/pull/15892)) -#### Stubgen Improvements +### Stubgen Improvements * Add option to include docstrings with stubgen (chylek, PR [13284](https://github.com/python/mypy/pull/13284)) * Add required ... initializer to NamedTuple fields with default values (Nikita Sobolev, PR [15680](https://github.com/python/mypy/pull/15680)) -#### Stubtest Improvements +### Stubtest Improvements * Fix \_\_mypy-replace false positives (Alex Waygood, PR [15689](https://github.com/python/mypy/pull/15689)) * Fix edge case for bytes enum subclasses (Alex Waygood, PR [15943](https://github.com/python/mypy/pull/15943)) @@ -1015,14 +1015,14 @@ The experimental new type inference algorithm (polymorphic inference) introduced * Fixes to new check for missing stdlib modules (Alex Waygood, PR [15960](https://github.com/python/mypy/pull/15960)) * Fix stubtest enum.Flag edge case (Alex Waygood, PR [15933](https://github.com/python/mypy/pull/15933)) -#### Documentation Improvements +### Documentation Improvements * Do not advertise to create your own assert\_never helper (Nikita Sobolev, PR [15947](https://github.com/python/mypy/pull/15947)) * Fix all the missing references found within the docs (Albert Tugushev, PR [15875](https://github.com/python/mypy/pull/15875)) * Document await-not-async error code (Shantanu, PR [15858](https://github.com/python/mypy/pull/15858)) * Improve documentation of disabling error codes (Shantanu, PR [15841](https://github.com/python/mypy/pull/15841)) -#### Other Notable Changes and Fixes +### Other Notable Changes and Fixes * Make unsupported PEP 695 features (introduced in Python 3.12) give a reasonable error message (Shantanu, PR [16013](https://github.com/python/mypy/pull/16013)) * Remove the \--py2 command-line argument (Marc Mueller, PR [15670](https://github.com/python/mypy/pull/15670)) @@ -1045,11 +1045,11 @@ The experimental new type inference algorithm (polymorphic inference) introduced * Add tox.ini to mypy sdist (Marcel Telka, PR [15853](https://github.com/python/mypy/pull/15853)) * Fix mypyc regression with pretty (Shantanu, PR [16124](https://github.com/python/mypy/pull/16124)) -#### Typeshed Updates +### Typeshed Updates Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=6a8d653a671925b0a3af61729ff8cf3f90c9c662+0&branch=main&path=stdlib) for full list of typeshed changes. -#### Acknowledgements +### Acknowledgements Thanks to Max Murin, who did most of the release manager work for this release (I just did the final steps). @@ -1088,11 +1088,11 @@ We’ve just uploaded mypy 1.5 to the Python Package Index ([PyPI](https://pypi. You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Drop Support for Python 3.7 +### Drop Support for Python 3.7 Mypy no longer supports running with Python 3.7, which has reached end-of-life. This was contributed by Shantanu (PR [15566](https://github.com/python/mypy/pull/15566)). -#### Optional Check to Require Explicit @override +### Optional Check to Require Explicit @override If you enable the explicit-override error code, mypy will generate an error if a method override doesn’t use the @typing.override decorator (as discussed in [PEP 698](https://peps.python.org/pep-0698/#strict-enforcement-per-project)). This way mypy will detect accidentally introduced overrides. Example: @@ -1122,7 +1122,7 @@ The override decorator will be available in typing in Python 3.12, but you can a This feature was contributed by Marc Mueller(PR [15512](https://github.com/python/mypy/pull/15512)). -#### More Flexible TypedDict Creation and Update +### More Flexible TypedDict Creation and Update Mypy was previously overly strict when type checking TypedDict creation and update operations. Though these checks were often technically correct, they sometimes triggered for apparently valid code. These checks have now been relaxed by default. You can enable stricter checking by using the new \--extra-checks flag. @@ -1150,11 +1150,11 @@ a.update(b) # OK (previously an error) This feature was contributed by Ivan Levkivskyi (PR [15425](https://github.com/python/mypy/pull/15425)). -#### Deprecated Flag: \--strict-concatenate +### Deprecated Flag: \--strict-concatenate The behavior of \--strict-concatenate is now included in the new \--extra-checks flag, and the old flag is deprecated. -#### Optionally Show Links to Error Code Documentation +### Optionally Show Links to Error Code Documentation If you use \--show-error-code-links, mypy will add documentation links to (many) reported errors. The links are not shown for error messages that are sufficiently obvious, and they are shown once per error code only. @@ -1165,19 +1165,19 @@ a.py:1: note: See https://mypy.rtfd.io/en/stable/_refs.html#code-var-annotated f ``` This was contributed by Ivan Levkivskyi (PR [15449](https://github.com/python/mypy/pull/15449)). -#### Consistently Avoid Type Checking Unreachable Code +### Consistently Avoid Type Checking Unreachable Code If a module top level has unreachable code, mypy won’t type check the unreachable statements. This is consistent with how functions behave. The behavior of \--warn-unreachable is also more consistent now. This was contributed by Ilya Priven (PR [15386](https://github.com/python/mypy/pull/15386)). -#### Experimental Improved Type Inference for Generic Functions +### Experimental Improved Type Inference for Generic Functions You can use \--new-type-inference to opt into an experimental new type inference algorithm. It fixes issues when calling a generic functions with an argument that is also a generic function, in particular. This current implementation is still incomplete, but we encourage trying it out and reporting bugs if you encounter regressions. We are planning to enable the new algorithm by default in a future mypy release. This feature was contributed by Ivan Levkivskyi (PR [15287](https://github.com/python/mypy/pull/15287)). -#### Partial Support for Python 3.12 +### Partial Support for Python 3.12 Mypy and mypyc now support running on recent Python 3.12 development versions. Not all new Python 3.12 features are supported, and we don’t ship compiled wheels for Python 3.12 yet. @@ -1193,7 +1193,7 @@ Mypy and mypyc now support running on recent Python 3.12 development versions. N * mypyc: Don't use \_PyErr\_ChainExceptions on 3.12, since it's deprecated (Jukka Lehtosalo, PR [15468](https://github.com/python/mypy/pull/15468)) * mypyc: Add Python 3.12 feature macro (Jukka Lehtosalo, PR [15465](https://github.com/python/mypy/pull/15465)) -#### Improvements to Dataclasses +### Improvements to Dataclasses * Improve signature of dataclasses.replace (Ilya Priven, PR [14849](https://github.com/python/mypy/pull/14849)) * Fix dataclass/protocol crash on joining types (Ilya Priven, PR [15629](https://github.com/python/mypy/pull/15629)) @@ -1202,7 +1202,7 @@ Mypy and mypyc now support running on recent Python 3.12 development versions. N * Add `__slots__` attribute to dataclasses (Nikita Sobolev, PR [15649](https://github.com/python/mypy/pull/15649)) * Support better \_\_post\_init\_\_ method signature for dataclasses (Nikita Sobolev, PR [15503](https://github.com/python/mypy/pull/15503)) -#### Mypyc Improvements +### Mypyc Improvements * Support unsigned 8-bit native integer type: mypy\_extensions.u8 (Jukka Lehtosalo, PR [15564](https://github.com/python/mypy/pull/15564)) * Support signed 16-bit native integer type: mypy\_extensions.i16 (Jukka Lehtosalo, PR [15464](https://github.com/python/mypy/pull/15464)) @@ -1212,20 +1212,20 @@ Mypy and mypyc now support running on recent Python 3.12 development versions. N * Use C99 compound literals for undefined tuple values (Jukka Lehtosalo, PR [15453](https://github.com/python/mypy/pull/15453)) * Don't explicitly assign NULL values in setup functions (Logan Hunt, PR [15379](https://github.com/python/mypy/pull/15379)) -#### Stubgen Improvements +### Stubgen Improvements * Teach stubgen to work with complex and unary expressions (Nikita Sobolev, PR [15661](https://github.com/python/mypy/pull/15661)) * Support ParamSpec and TypeVarTuple (Ali Hamdan, PR [15626](https://github.com/python/mypy/pull/15626)) * Fix crash on non-str docstring (Ali Hamdan, PR [15623](https://github.com/python/mypy/pull/15623)) -#### Documentation Updates +### Documentation Updates * Add documentation for additional error codes (Ivan Levkivskyi, PR [15539](https://github.com/python/mypy/pull/15539)) * Improve documentation of type narrowing (Ilya Priven, PR [15652](https://github.com/python/mypy/pull/15652)) * Small improvements to protocol documentation (Shantanu, PR [15460](https://github.com/python/mypy/pull/15460)) * Remove confusing instance variable example in cheat sheet (Adel Atallah, PR [15441](https://github.com/python/mypy/pull/15441)) -#### Other Notable Fixes and Improvements +### Other Notable Fixes and Improvements * Constant fold additional unary and binary expressions (Richard Si, PR [15202](https://github.com/python/mypy/pull/15202)) * Exclude the same special attributes from Protocol as CPython (Kyle Benesch, PR [15490](https://github.com/python/mypy/pull/15490)) @@ -1245,11 +1245,11 @@ Mypy and mypyc now support running on recent Python 3.12 development versions. N * Fix self types in subclass methods without Self annotation (Ivan Levkivskyi, PR [15541](https://github.com/python/mypy/pull/15541)) * Check for abstract class objects in tuples (Nikita Sobolev, PR [15366](https://github.com/python/mypy/pull/15366)) -#### Typeshed Updates +### Typeshed Updates Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=fc7d4722eaa54803926cee5730e1f784979c0531+0&branch=main&path=stdlib) for full list of typeshed changes. -#### Acknowledgements +### Acknowledgements Thanks to all mypy contributors who contributed to this release: @@ -1285,13 +1285,13 @@ We’ve just uploaded mypy 1.4 to the Python Package Index ([PyPI](https://pypi. You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### The Override Decorator +### The Override Decorator Mypy can now ensure that when renaming a method, overrides are also renamed. You can explicitly mark a method as overriding a base class method by using the @typing.override decorator ([PEP 698](https://peps.python.org/pep-0698/)). If the method is then renamed in the base class while the method override is not, mypy will generate an error. The decorator will be available in typing in Python 3.12, but you can also use the backport from a recent version of `typing_extensions` on all supported Python versions. This feature was contributed byThomas M Kehrenberg (PR [14609](https://github.com/python/mypy/pull/14609)). -#### Propagating Type Narrowing to Nested Functions +### Propagating Type Narrowing to Nested Functions Previously, type narrowing was not propagated to nested functions because it would not be sound if the narrowed variable changed between the definition of the nested function and the call site. Mypy will now propagate the narrowed type if the variable is not assigned to after the definition of the nested function: @@ -1311,7 +1311,7 @@ This may generate some new errors because asserts that were previously necessary This was contributed by Jukka Lehtosalo (PR [15133](https://github.com/python/mypy/pull/15133)). -#### Narrowing Enum Values Using “==” +### Narrowing Enum Values Using “==” Mypy now allows narrowing enum types using the \== operator. Previously this was only supported when using the is operator. This makes exhaustiveness checking with enum types more usable, as the requirement to use the is operator was not very intuitive. In this example mypy can detect that the developer forgot to handle the value MyEnum.C in example @@ -1380,18 +1380,18 @@ def test_something() -> None: This feature was contributed by Shantanu (PR [11521](https://github.com/python/mypy/pull/11521)). -#### Performance Improvements +### Performance Improvements * Speed up simplification of large union types and also fix a recursive tuple crash (Shantanu, PR [15128](https://github.com/python/mypy/pull/15128)) * Speed up union subtyping (Shantanu, PR [15104](https://github.com/python/mypy/pull/15104)) * Don't type check most function bodies when type checking third-party library code, or generally when ignoring errors (Jukka Lehtosalo, PR [14150](https://github.com/python/mypy/pull/14150)) -#### Improvements to Plugins +### Improvements to Plugins * attrs.evolve: Support generics and unions (Ilya Konstantinov, PR [15050](https://github.com/python/mypy/pull/15050)) * Fix ctypes plugin (Alex Waygood) -#### Fixes to Crashes +### Fixes to Crashes * Fix a crash when function-scope recursive alias appears as upper bound (Ivan Levkivskyi, PR [15159](https://github.com/python/mypy/pull/15159)) * Fix crash on follow\_imports\_for\_stubs (Ivan Levkivskyi, PR [15407](https://github.com/python/mypy/pull/15407)) @@ -1405,7 +1405,7 @@ This feature was contributed by Shantanu (PR [11521](https://github.com/python/m * Fix crash on lambda in generic context with generic method in body (Ivan Levkivskyi, PR [15155](https://github.com/python/mypy/pull/15155)) * Fix recursive type alias crash in make\_simplified\_union (Ivan Levkivskyi, PR [15216](https://github.com/python/mypy/pull/15216)) -#### Improvements to Error Messages +### Improvements to Error Messages * Use lower-case built-in collection types such as list\[…\] instead of List\[…\] in errors when targeting Python 3.9+ (Max Murin, PR [15070](https://github.com/python/mypy/pull/15070)) * Use X | Y union syntax in error messages when targeting Python 3.10+ (Omar Silva, PR [15102](https://github.com/python/mypy/pull/15102)) @@ -1418,7 +1418,7 @@ This feature was contributed by Shantanu (PR [11521](https://github.com/python/m * Add explanation if argument type is incompatible because of an unsupported numbers type (Jukka Lehtosalo, PR [15137](https://github.com/python/mypy/pull/15137)) * Add more detail to 'signature incompatible with supertype' messages for non-callables (Ilya Priven, PR [15263](https://github.com/python/mypy/pull/15263)) -#### Documentation Updates +### Documentation Updates * Add \--local-partial-types note to dmypy docs (Alan Du, PR [15259](https://github.com/python/mypy/pull/15259)) * Update getting started docs for mypyc for Windows (Valentin Stanciu, PR [15233](https://github.com/python/mypy/pull/15233)) @@ -1426,13 +1426,13 @@ This feature was contributed by Shantanu (PR [11521](https://github.com/python/m * Clarify difference between disallow\_untyped\_defs and disallow\_incomplete\_defs (Ilya Priven, PR [15247](https://github.com/python/mypy/pull/15247)) * Use attrs and @attrs.define in documentation and tests (Ilya Priven, PR [15152](https://github.com/python/mypy/pull/15152)) -#### Mypyc Improvements +### Mypyc Improvements * Fix unexpected TypeError for certain variables with an inferred optional type (Richard Si, PR [15206](https://github.com/python/mypy/pull/15206)) * Inline math literals (Logan Hunt, PR [15324](https://github.com/python/mypy/pull/15324)) * Support unpacking mappings in dict display (Richard Si, PR [15203](https://github.com/python/mypy/pull/15203)) -#### Changes to Stubgen +### Changes to Stubgen * Do not remove Generic from base classes (Ali Hamdan, PR [15316](https://github.com/python/mypy/pull/15316)) * Support yield from statements (Ali Hamdan, PR [15271](https://github.com/python/mypy/pull/15271)) @@ -1442,7 +1442,7 @@ This feature was contributed by Shantanu (PR [11521](https://github.com/python/m * Make stubgen respect MYPY\_CACHE\_DIR (Henrik Bäärnhielm, PR [14722](https://github.com/python/mypy/pull/14722)) * Fixes and simplifications (Ali Hamdan, PR [15232](https://github.com/python/mypy/pull/15232)) -#### Other Notable Fixes and Improvements +### Other Notable Fixes and Improvements * Fix nested async functions when using TypeVar value restriction (Jukka Lehtosalo, PR [14705](https://github.com/python/mypy/pull/14705)) * Always allow returning Any from lambda (Ivan Levkivskyi, PR [15413](https://github.com/python/mypy/pull/15413)) @@ -1455,11 +1455,11 @@ This feature was contributed by Shantanu (PR [11521](https://github.com/python/m * Fix match subject ignoring redefinitions (Vincent Vanlaer, PR [15306](https://github.com/python/mypy/pull/15306)) * Support `__all__`.remove (Shantanu, PR [15279](https://github.com/python/mypy/pull/15279)) -#### Typeshed Updates +### Typeshed Updates Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=877e06ad1cfd9fd9967c0b0340a86d0c23ea89ce+0&branch=main&path=stdlib) for full list of typeshed changes. -#### Acknowledgements +### Acknowledgements Thanks to all mypy contributors who contributed to this release: @@ -1515,12 +1515,12 @@ Posted by Jared Hance You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Performance Improvements +### Performance Improvements * Improve performance of union subtyping (Shantanu, PR [15104](https://github.com/python/mypy/pull/15104)) * Add negative subtype caches (Ivan Levkivskyi, PR [14884](https://github.com/python/mypy/pull/14884)) -#### Stub Tooling Improvements +### Stub Tooling Improvements * Stubtest: Check that the stub is abstract if the runtime is, even when the stub is an overloaded method (Alex Waygood, PR [14955](https://github.com/python/mypy/pull/14955)) * Stubtest: Verify stub methods or properties are decorated with @final if they are decorated with @final at runtime (Alex Waygood, PR [14951](https://github.com/python/mypy/pull/14951)) @@ -1528,12 +1528,12 @@ You can read the full documentation for this release on [Read the Docs](http://m * Stubgen: Support @functools.cached\_property (Nikita Sobolev, PR [14981](https://github.com/python/mypy/pull/14981)) * Improvements to stubgenc (Chad Dombrova, PR [14564](https://github.com/python/mypy/pull/14564)) -#### Improvements to attrs +### Improvements to attrs * Add support for converters with TypeVars on generic attrs classes (Chad Dombrova, PR [14908](https://github.com/python/mypy/pull/14908)) * Fix attrs.evolve on bound TypeVar (Ilya Konstantinov, PR [15022](https://github.com/python/mypy/pull/15022)) -#### Documentation Updates +### Documentation Updates * Improve async documentation (Shantanu, PR [14973](https://github.com/python/mypy/pull/14973)) * Improvements to cheat sheet (Shantanu, PR [14972](https://github.com/python/mypy/pull/14972)) @@ -1545,26 +1545,26 @@ You can read the full documentation for this release on [Read the Docs](http://m * Fix alignment of cheat sheet example (Ondřej Cvacho, PR [15039](https://github.com/python/mypy/pull/15039)) * Fix error for callback protocol matching against callable type object (Shantanu, PR [15042](https://github.com/python/mypy/pull/15042)) -#### Error Reporting Improvements +### Error Reporting Improvements * Improve bytes formatting error (Shantanu, PR [14959](https://github.com/python/mypy/pull/14959)) -#### Mypyc Improvements +### Mypyc Improvements * Fix unions of bools and ints (Tomer Chachamu, PR [15066](https://github.com/python/mypy/pull/15066)) -#### Other Fixes and Improvements +### Other Fixes and Improvements * Fix narrowing union types that include Self with isinstance (Christoph Tyralla, PR [14923](https://github.com/python/mypy/pull/14923)) * Allow objects matching SupportsKeysAndGetItem to be unpacked (Bryan Forbes, PR [14990](https://github.com/python/mypy/pull/14990)) * Check type guard validity for staticmethods (EXPLOSION, PR [14953](https://github.com/python/mypy/pull/14953)) * Fix sys.platform when cross-compiling with emscripten (Ethan Smith, PR [14888](https://github.com/python/mypy/pull/14888)) -#### Typeshed Updates +### Typeshed Updates Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=b0ed50e9392a23e52445b630a808153e0e256976+0&branch=main&path=stdlib) for full list of typeshed changes. -#### Acknowledgements +### Acknowledgements Thanks to all mypy contributors who contributed to this release: @@ -1603,14 +1603,14 @@ We’ve just uploaded mypy 1.2 to the Python Package Index ([PyPI](https://pypi. You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Improvements to Dataclass Transforms +### Improvements to Dataclass Transforms * Support implicit default for "init" parameter in field specifiers (Wesley Collin Wright and Jukka Lehtosalo, PR [15010](https://github.com/python/mypy/pull/15010)) * Support descriptors in dataclass transform (Jukka Lehtosalo, PR [15006](https://github.com/python/mypy/pull/15006)) * Fix frozen\_default in incremental mode (Wesley Collin Wright) * Fix frozen behavior for base classes with direct metaclasses (Wesley Collin Wright, PR [14878](https://github.com/python/mypy/pull/14878)) -#### Mypyc: Native Floats +### Mypyc: Native Floats Mypyc now uses a native, unboxed representation for values of type float. Previously these were heap-allocated Python objects. Native floats are faster and use less memory. Code that uses floating-point operations heavily can be several times faster when using native floats. @@ -1649,7 +1649,7 @@ Related changes: * Document native floats and integers (Jukka Lehtosalo, PR [14927](https://github.com/python/mypy/pull/14927)) * Fixes to float to int conversion (Jukka Lehtosalo, PR [14936](https://github.com/python/mypy/pull/14936)) -#### Mypyc: Native Integers +### Mypyc: Native Integers Mypyc now supports signed 32-bit and 64-bit integer types in addition to the arbitrary-precision int type. You can use the types mypy\_extensions.i32 and mypy\_extensions.i64 to speed up code that uses integer operations heavily. @@ -1663,33 +1663,33 @@ def inc(x: i64) -> i64: Refer to the [documentation](https://mypyc.readthedocs.io/en/latest/using_type_annotations.html#native-integer-types) for more information. This feature was contributed by Jukka Lehtosalo. -#### Other Mypyc Fixes and Improvements +### Other Mypyc Fixes and Improvements * Support iterating over a TypedDict (Richard Si, PR [14747](https://github.com/python/mypy/pull/14747)) * Faster coercions between different tuple types (Jukka Lehtosalo, PR [14899](https://github.com/python/mypy/pull/14899)) * Faster calls via type aliases (Jukka Lehtosalo, PR [14784](https://github.com/python/mypy/pull/14784)) * Faster classmethod calls via cls (Jukka Lehtosalo, PR [14789](https://github.com/python/mypy/pull/14789)) -#### Fixes to Crashes +### Fixes to Crashes * Fix crash on class-level import in protocol definition (Ivan Levkivskyi, PR [14926](https://github.com/python/mypy/pull/14926)) * Fix crash on single item union of alias (Ivan Levkivskyi, PR [14876](https://github.com/python/mypy/pull/14876)) * Fix crash on ParamSpec in incremental mode (Ivan Levkivskyi, PR [14885](https://github.com/python/mypy/pull/14885)) -#### Documentation Updates +### Documentation Updates * Update adopting \--strict documentation for 1.0 (Shantanu, PR [14865](https://github.com/python/mypy/pull/14865)) * Some minor documentation tweaks (Jukka Lehtosalo, PR [14847](https://github.com/python/mypy/pull/14847)) * Improve documentation of top level mypy: disable-error-code comment (Nikita Sobolev, PR [14810](https://github.com/python/mypy/pull/14810)) -#### Error Reporting Improvements +### Error Reporting Improvements * Add error code to `typing_extensions` suggestion (Shantanu, PR [14881](https://github.com/python/mypy/pull/14881)) * Add a separate error code for top-level await (Nikita Sobolev, PR [14801](https://github.com/python/mypy/pull/14801)) * Don’t suggest two obsolete stub packages (Jelle Zijlstra, PR [14842](https://github.com/python/mypy/pull/14842)) * Add suggestions for pandas-stubs and lxml-stubs (Shantanu, PR [14737](https://github.com/python/mypy/pull/14737)) -#### Other Fixes and Improvements +### Other Fixes and Improvements * Multiple inheritance considers callable objects as subtypes of functions (Christoph Tyralla, PR [14855](https://github.com/python/mypy/pull/14855)) * stubtest: Respect @final runtime decorator and enforce it in stubs (Nikita Sobolev, PR [14922](https://github.com/python/mypy/pull/14922)) @@ -1708,11 +1708,11 @@ Refer to the [documentation](https://mypyc.readthedocs.io/en/latest/using_type_a * Improve “used before definition” checks when a local definition has the same name as a global definition (Stas Ilinskiy, PR [14517](https://github.com/python/mypy/pull/14517)) * Honor NoReturn as \_\_setitem\_\_ return type to mark unreachable code (sterliakov, PR [12572](https://github.com/python/mypy/pull/12572)) -#### Typeshed Updates +### Typeshed Updates Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=a544b75320e97424d2d927605316383c755cdac0+0&branch=main&path=stdlib) for full list of typeshed changes. -#### Acknowledgements +### Acknowledgements Thanks to all mypy contributors who contributed to this release: @@ -1748,13 +1748,13 @@ Posted by Jukka Lehtosalo You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Support for `dataclass_transform`` +### Support for `dataclass_transform`` This release adds full support for the dataclass\_transform decorator defined in [PEP 681](https://peps.python.org/pep-0681/#decorator-function-example). This allows decorators, base classes, and metaclasses that generate a \_\_init\_\_ method or other methods based on the properties of that class (similar to dataclasses) to have those methods recognized by mypy. This was contributed by Wesley Collin Wright. -#### Dedicated Error Code for Method Assignments +### Dedicated Error Code for Method Assignments Mypy can’t safely check all assignments to methods (a form of monkey patching), so mypy generates an error by default. To make it easier to ignore this error, mypy now uses the new error code method-assign for this. By disabling this error code in a file or globally, mypy will no longer complain about assignments to methods if the signatures are compatible. @@ -1762,16 +1762,16 @@ Mypy also supports the old error code assignment for these assignments to preven This was contributed by Ivan Levkivskyi (PR [14570](https://github.com/python/mypy/pull/14570)). -#### Fixes to Crashes +### Fixes to Crashes * Fix a crash on walrus in comprehension at class scope (Ivan Levkivskyi, PR [14556](https://github.com/python/mypy/pull/14556)) * Fix crash related to value-constrained TypeVar (Shantanu, PR [14642](https://github.com/python/mypy/pull/14642)) -#### Fixes to Cache Corruption +### Fixes to Cache Corruption * Fix generic TypedDict/NamedTuple caching (Ivan Levkivskyi, PR [14675](https://github.com/python/mypy/pull/14675)) -#### Mypyc Fixes and Improvements +### Mypyc Fixes and Improvements * Raise "non-trait base must be first..." error less frequently (Richard Si, PR [14468](https://github.com/python/mypy/pull/14468)) * Generate faster code for bool comparisons and arithmetic (Jukka Lehtosalo, PR [14489](https://github.com/python/mypy/pull/14489)) @@ -1782,12 +1782,12 @@ This was contributed by Ivan Levkivskyi (PR [14570](https://github.com/python/my * Fix crash on star unpacking to underscore (Ivan Levkivskyi, PR [14624](https://github.com/python/mypy/pull/14624)) * Fix iterating over a union of dicts (Richard Si, PR [14713](https://github.com/python/mypy/pull/14713)) -#### Fixes to Detecting Undefined Names (used-before-def) +### Fixes to Detecting Undefined Names (used-before-def) * Correctly handle walrus operator (Stas Ilinskiy, PR [14646](https://github.com/python/mypy/pull/14646)) * Handle walrus declaration in match subject correctly (Stas Ilinskiy, PR [14665](https://github.com/python/mypy/pull/14665)) -#### Stubgen Improvements +### Stubgen Improvements Stubgen is a tool for automatically generating draft stubs for libraries. @@ -1795,14 +1795,14 @@ Stubgen is a tool for automatically generating draft stubs for libraries. * Fix crash with PEP 604 union in type variable bound (Shantanu, PR [14557](https://github.com/python/mypy/pull/14557)) * Preserve PEP 604 unions in generated .pyi files (hamdanal, PR [14601](https://github.com/python/mypy/pull/14601)) -#### Stubtest Improvements +### Stubtest Improvements Stubtest is a tool for testing that stubs conform to the implementations. * Update message format so that it’s easier to go to error location (Avasam, PR [14437](https://github.com/python/mypy/pull/14437)) * Handle name-mangling edge cases better (Alex Waygood, PR [14596](https://github.com/python/mypy/pull/14596)) -#### Changes to Error Reporting and Messages +### Changes to Error Reporting and Messages * Add new TypedDict error code typeddict-unknown-key (JoaquimEsteves, PR [14225](https://github.com/python/mypy/pull/14225)) * Give arguments a more reasonable location in error messages (Max Murin, PR [14562](https://github.com/python/mypy/pull/14562)) @@ -1814,7 +1814,7 @@ Stubtest is a tool for testing that stubs conform to the implementations. * Adjust inconsistent dataclasses plugin error messages (Wesley Collin Wright, PR [14637](https://github.com/python/mypy/pull/14637)) * Consolidate literal bool argument error messages (Wesley Collin Wright, PR [14693](https://github.com/python/mypy/pull/14693)) -#### Other Fixes and Improvements +### Other Fixes and Improvements * Check that type guards accept a positional argument (EXPLOSION, PR [14238](https://github.com/python/mypy/pull/14238)) * Fix bug with in operator used with a union of Container and Iterable (Max Murin, PR [14384](https://github.com/python/mypy/pull/14384)) @@ -1822,11 +1822,11 @@ Stubtest is a tool for testing that stubs conform to the implementations. * Allow overlapping comparisons between bytes-like types (Shantanu, PR [14658](https://github.com/python/mypy/pull/14658)) * Fix mypy daemon documentation link in README (Ivan Levkivskyi, PR [14644](https://github.com/python/mypy/pull/14644)) -#### Typeshed Updates +### Typeshed Updates Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=5ebf892d0710a6e87925b8d138dfa597e7bb11cc+0&branch=main&path=stdlib) for full list of typeshed changes. -#### Acknowledgements +### Acknowledgements Thanks to all mypy contributors who contributed to this release: @@ -1868,7 +1868,7 @@ We’ve just uploaded mypy 1.0 to the Python Package Index ([PyPI](https://pypi. You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### New Release Versioning Scheme +### New Release Versioning Scheme Now that mypy reached 1.0, we’ll switch to a new versioning scheme. Mypy version numbers will be of form x.y.z. @@ -1884,7 +1884,7 @@ Any significant backward incompatible change must be announced in the blog post See [”Release Process” in the mypy wiki](https://github.com/python/mypy/wiki/Release-Process) for more details and for the most up-to-date version of the versioning scheme. -#### Performance Improvements +### Performance Improvements Mypy 1.0 is up to 40% faster than mypy 0.991 when type checking the Dropbox internal codebase. We also set up a daily job to measure the performance of the most recent development version of mypy to make it easier to track changes in performance. @@ -1911,7 +1911,7 @@ Many optimizations contributed to this improvement: * Speed up freshening type variables (Jukka Lehtosalo, PR [14323](https://github.com/python/mypy/pull/14323)) * Optimize implementation of TypedDict types for \*\*kwds (Jukka Lehtosalo, PR [14316](https://github.com/python/mypy/pull/14316)) -#### Warn About Variables Used Before Definition +### Warn About Variables Used Before Definition Mypy will now generate an error if you use a variable before it’s defined. This feature is enabled by default. By default mypy reports an error when it infers that a variable is always undefined. ```python @@ -1920,7 +1920,7 @@ x = 0 ``` This feature was contributed by Stas Ilinskiy. -#### Detect Possibly Undefined Variables (Experimental) +### Detect Possibly Undefined Variables (Experimental) A new experimental possibly-undefined error code is now available that will detect variables that may be undefined: ```python @@ -1932,7 +1932,7 @@ The error code is disabled be default, since it can generate false positives. This feature was contributed by Stas Ilinskiy. -#### Support the “Self” Type +### Support the “Self” Type There is now a simpler syntax for declaring [generic self types](https://mypy.readthedocs.io/en/stable/generics.html#generic-methods-and-generic-self) introduced in [PEP 673](https://peps.python.org/pep-0673/): the Self type. You no longer have to define a type variable to use “self types”, and you can use them with attributes. Example from mypy documentation: ```python @@ -1958,7 +1958,7 @@ The feature was introduced in Python 3.11. In earlier Python versions a backport This was contributed by Ivan Levkivskyi (PR [14041](https://github.com/python/mypy/pull/14041)). -#### Support ParamSpec in Type Aliases +### Support ParamSpec in Type Aliases ParamSpec and Concatenate can now be used in type aliases. Example: ```python @@ -1972,11 +1972,11 @@ def f(c: A[int, str]) -> None: ``` This feature was contributed by Ivan Levkivskyi (PR [14159](https://github.com/python/mypy/pull/14159)). -#### ParamSpec and Generic Self Types No Longer Experimental +### ParamSpec and Generic Self Types No Longer Experimental Support for ParamSpec ([PEP 612](https://www.python.org/dev/peps/pep-0612/)) and generic self types are no longer considered experimental. -#### Miscellaneous New Features +### Miscellaneous New Features * Minimal, partial implementation of dataclass\_transform ([PEP 681](https://peps.python.org/pep-0681/)) (Wesley Collin Wright, PR [14523](https://github.com/python/mypy/pull/14523)) * Add basic support for `typing_extensions`.TypeVar (Marc Mueller, PR [14313](https://github.com/python/mypy/pull/14313)) @@ -1989,7 +1989,7 @@ Support for ParamSpec ([PEP 612](https://www.python.org/dev/peps/pep-0612/)) and * Generate error for class attribute access if attribute is defined with `__slots__` (Harrison McCarty, PR [14125](https://github.com/python/mypy/pull/14125)) * Support additional attributes in callback protocols (Ivan Levkivskyi, PR [14084](https://github.com/python/mypy/pull/14084)) -#### Fixes to Crashes +### Fixes to Crashes * Fix crash on prefixed ParamSpec with forward reference (Ivan Levkivskyi, PR [14569](https://github.com/python/mypy/pull/14569)) * Fix internal crash when resolving the same partial type twice (Shantanu, PR [14552](https://github.com/python/mypy/pull/14552)) @@ -2009,19 +2009,19 @@ Support for ParamSpec ([PEP 612](https://www.python.org/dev/peps/pep-0612/)) and * Fix crash with enums (Michael Lee, PR [14021](https://github.com/python/mypy/pull/14021)) * Fix crash with malformed TypedDicts and disllow-any-expr (Michael Lee, PR [13963](https://github.com/python/mypy/pull/13963)) -#### Error Reporting Improvements +### Error Reporting Improvements * More helpful error for missing self (Shantanu, PR [14386](https://github.com/python/mypy/pull/14386)) * Add error-code truthy-iterable (Marc Mueller, PR [13762](https://github.com/python/mypy/pull/13762)) * Fix pluralization in error messages (KotlinIsland, PR [14411](https://github.com/python/mypy/pull/14411)) -#### Mypyc: Support Match Statement +### Mypyc: Support Match Statement Mypyc can now compile Python 3.10 match statements. This was contributed by dosisod (PR [13953](https://github.com/python/mypy/pull/13953)). -#### Other Mypyc Fixes and Improvements +### Other Mypyc Fixes and Improvements * Optimize int(x)/float(x)/complex(x) on instances of native classes (Richard Si, PR [14450](https://github.com/python/mypy/pull/14450)) * Always emit warnings (Richard Si, PR [14451](https://github.com/python/mypy/pull/14451)) @@ -2037,7 +2037,7 @@ This was contributed by dosisod (PR [13953](https://github.com/python/mypy/pull/ * Allow use of enum.Enum (Shantanu, PR [13995](https://github.com/python/mypy/pull/13995)) * Fix compiling on Arch Linux (dosisod, PR [13978](https://github.com/python/mypy/pull/13978)) -#### Documentation Improvements +### Documentation Improvements * Various documentation and error message tweaks (Jukka Lehtosalo, PR [14574](https://github.com/python/mypy/pull/14574)) * Improve Generics documentation (Shantanu, PR [14587](https://github.com/python/mypy/pull/14587)) @@ -2057,7 +2057,7 @@ This was contributed by dosisod (PR [13953](https://github.com/python/mypy/pull/ * Flycheck-mypy is deprecated, since its functionality was merged to Flycheck (Ivan Levkivskyi, PR [14247](https://github.com/python/mypy/pull/14247)) * Update code example in "Declaring decorators" (ChristianWitzler, PR [14131](https://github.com/python/mypy/pull/14131)) -#### Stubtest Improvements +### Stubtest Improvements Stubtest is a tool for testing that stubs conform to the implementations. @@ -2068,13 +2068,13 @@ Stubtest is a tool for testing that stubs conform to the implementations. * Add \_\_warningregistry\_\_ to the list of ignored module dunders (Nikita Sobolev, PR [14218](https://github.com/python/mypy/pull/14218)) * If a default is present in the stub, check that it is correct (Jelle Zijlstra, PR [14085](https://github.com/python/mypy/pull/14085)) -#### Stubgen Improvements +### Stubgen Improvements Stubgen is a tool for automatically generating draft stubs for libraries. * Treat dlls as C modules (Shantanu, PR [14503](https://github.com/python/mypy/pull/14503)) -#### Other Notable Fixes and Improvements +### Other Notable Fixes and Improvements * Update stub suggestions based on recent typeshed changes (Alex Waygood, PR [14265](https://github.com/python/mypy/pull/14265)) * Fix attrs protocol check with cache (Marc Mueller, PR [14558](https://github.com/python/mypy/pull/14558)) @@ -2113,11 +2113,11 @@ Stubgen is a tool for automatically generating draft stubs for libraries. * Improve handling of redefinitions through imports (Shantanu, PR [13969](https://github.com/python/mypy/pull/13969)) * Preserve (some) implicitly exported types (Shantanu, PR [13967](https://github.com/python/mypy/pull/13967)) -#### Typeshed Updates +### Typeshed Updates Typeshed is now modular and distributed as separate PyPI packages for everything except the standard library stubs. Please see [git log](https://github.com/python/typeshed/commits/main?after=ea0ae2155e8a04c9837903c3aff8dd5ad5f36ebc+0&branch=main&path=stdlib) for full list of typeshed changes. -#### Acknowledgements +### Acknowledgements Thanks to all mypy contributors who contributed to this release: diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index a3504b07824d..dc502d121ffc 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,2 +1,3 @@ sphinx>=5.1.0 furo>=2022.3.4 +myst-parser>=4.0.0 diff --git a/docs/source/changelog.md b/docs/source/changelog.md new file mode 100644 index 000000000000..a490ada727a6 --- /dev/null +++ b/docs/source/changelog.md @@ -0,0 +1,3 @@ + +```{include} ../../CHANGELOG.md +``` diff --git a/docs/source/conf.py b/docs/source/conf.py index 5934c7474536..f8faa03a09b2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -35,7 +35,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.intersphinx", "docs.source.html_builder"] +extensions = ["sphinx.ext.intersphinx", "docs.source.html_builder", "myst_parser"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/source/index.rst b/docs/source/index.rst index c9dc6bc1f8c9..de3286d58ace 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -103,6 +103,7 @@ Contents error_code_list2 additional_features faq + changelog .. toctree:: :hidden: From 77919cfdd11340a7965fb3c850b5cd19bb0e289a Mon Sep 17 00:00:00 2001 From: Soubhik Kumar Mitra <59209034+x612skm@users.noreply.github.com> Date: Mon, 16 Sep 2024 04:15:22 +0530 Subject: [PATCH 134/247] Add missing lines-covered and lines-valid attributes (#17738) This PR resolves an issue where the Cobertura XML report generated by MyPy was missing the lines-covered and lines-valid attributes in the element. Changes made: - Added the lines-covered and lines-valid attributes to the element in the Cobertura XML report. - Updated the CoberturaReportSuite test suite to validate that these attributes are correctly included in the generated XML. Fixes #17689 --- mypy/report.py | 2 ++ test-data/unit/reports.test | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/report.py b/mypy/report.py index 764cfec7799a..73942b6c5ae3 100644 --- a/mypy/report.py +++ b/mypy/report.py @@ -689,6 +689,8 @@ def on_finish(self) -> None: self.root_package.covered_lines, self.root_package.total_lines ) self.root.attrib["branch-rate"] = "0" + self.root.attrib["lines-covered"] = str(self.root_package.covered_lines) + self.root.attrib["lines-valid"] = str(self.root_package.total_lines) sources = etree.SubElement(self.root, "sources") source_element = etree.SubElement(sources, "source") source_element.text = os.getcwd() diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 81e24240af2d..6e0fdba8aaa3 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -27,7 +27,7 @@ def bar() -> str: def untyped_function(): return 42 [outfile build/cobertura.xml] - + $PWD @@ -81,7 +81,7 @@ def foo(a: int) -> MyDict: return {"a": a} md: MyDict = MyDict(**foo(42)) [outfile build/cobertura.xml] - + $PWD From 5c38427a53f409bae068c314e119fecc77730d76 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 16 Sep 2024 01:46:08 +0300 Subject: [PATCH 135/247] Check for `truthy-bool` in `not ...` unary expressions (#17773) Closes https://github.com/python/mypy/issues/17769 --- mypy/checker.py | 13 +++++++++++-- mypy/checkexpr.py | 1 + test-data/unit/check-errorcodes.test | 26 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index db65660bbfbd..9c4f4ce88690 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5651,7 +5651,16 @@ def _is_truthy_type(self, t: ProperType) -> bool: ) ) - def _check_for_truthy_type(self, t: Type, expr: Expression) -> None: + def check_for_truthy_type(self, t: Type, expr: Expression) -> None: + """ + Check if a type can have a truthy value. + + Used in checks like:: + + if x: # <--- + + not x # <--- + """ if not state.strict_optional: return # if everything can be None, all bets are off @@ -6145,7 +6154,7 @@ def has_no_custom_eq_checks(t: Type) -> bool: if in_boolean_context: # We don't check `:=` values in expressions like `(a := A())`, # because they produce two error messages. - self._check_for_truthy_type(original_vartype, node) + self.check_for_truthy_type(original_vartype, node) vartype = try_expanding_sum_type_to_union(original_vartype, "builtins.bool") if_type = true_only(vartype) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 9dee743ad406..22595c85e702 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4256,6 +4256,7 @@ def visit_unary_expr(self, e: UnaryExpr) -> Type: op = e.op if op == "not": result: Type = self.bool_type() + self.chk.check_for_truthy_type(operand_type, e.expr) else: method = operators.unary_op_methods[op] result, method_type = self.check_method_call_by_name(method, operand_type, [], [], e) diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index c4d72388fba9..cca13347dd86 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -846,34 +846,48 @@ foo = Foo() if foo: # E: "__main__.foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] pass +not foo # E: "__main__.foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + zero = 0 if zero: pass +not zero + false = False if false: pass +not false + null = None if null: pass +not null + s = '' if s: pass +not s + good_union: Union[str, int] = 5 if good_union: pass if not good_union: pass +not good_union + bad_union: Union[Foo, Bar] = Foo() if bad_union: # E: "__main__.bad_union" has type "Union[Foo, Bar]" of which no members implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] pass if not bad_union: # E: "__main__.bad_union" has type "Union[Foo, Bar]" of which no members implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] pass +not bad_union # E: "__main__.bad_union" has type "Union[Foo, Bar]" of which no members implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] + # 'object' is special and is treated as potentially falsy obj: object = Foo() if obj: @@ -881,18 +895,26 @@ if obj: if not obj: pass +not obj + lst: List[int] = [] if lst: pass +not lst + a: Any if a: pass +not a + any_or_object: Union[object, Any] if any_or_object: pass +not any_or_object + if (my_foo := Foo()): # E: "__main__.my_foo" has type "Foo" which does not implement __bool__ or __len__ so it could always be true in boolean context [truthy-bool] pass @@ -909,6 +931,8 @@ if not f: # E: Function "f" could always be true in boolean context [truthy-fu pass conditional_result = 'foo' if f else 'bar' # E: Function "f" could always be true in boolean context [truthy-function] +not f # E: Function "f" could always be true in boolean context [truthy-function] + [case testTruthyIterable] # flags: --enable-error-code truthy-iterable from typing import Iterable @@ -916,6 +940,8 @@ def func(var: Iterable[str]) -> None: if var: # E: "var" has type "Iterable[str]" which can always be true in boolean context. Consider using "Collection[str]" instead. [truthy-iterable] ... + not var # E: "var" has type "Iterable[str]" which can always be true in boolean context. Consider using "Collection[str]" instead. [truthy-iterable] + [case testNoOverloadImplementation] from typing import overload From f68f76d7ab7a5a6277a47fbadd6012ddd16ea731 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 18 Sep 2024 12:22:21 +0100 Subject: [PATCH 136/247] [PEP 695] Fix nested generic classes (#17776) There was confusion about the fullnames of type variables in nested generic classes. A type variable could be defined internally as `m.OuterClass.T`, but it was sometimes accessed as `m.T`. The root cause was that the semantic analyzer didn't initialize the attribute that refers to the enclosing class consistently. Fixes #17596. Fixes #17630. --- mypy/semanal.py | 5 ++ test-data/unit/check-python312.test | 66 ++++++++++++++++++++++ test-data/unit/fine-grained-python312.test | 20 +++++++ 3 files changed, 91 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index 782985e3fbab..522b4abdd5e9 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -824,6 +824,11 @@ def file_context( self.num_incomplete_refs = 0 if active_type: + enclosing_fullname = active_type.fullname.rsplit(".", 1)[0] + if "." in enclosing_fullname: + enclosing_node = self.lookup_fully_qualified_or_none(enclosing_fullname) + if enclosing_node and isinstance(enclosing_node.node, TypeInfo): + self._type = enclosing_node.node self.push_type_args(active_type.defn.type_args, active_type.defn) self.incomplete_type_stack.append(False) scope.enter_class(active_type) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index a3f4c87120cd..89ced8be2c6f 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1713,3 +1713,69 @@ type XNested = (1 + (yield 1)) # E: Yield expression cannot be used within a ty type YNested = (1 + (yield from [])) # E: Yield expression cannot be used within a type alias type ZNested = (1 + (a := 1)) # E: Named expression cannot be used within a type alias type KNested = (1 + (await 1)) # E: Await expression cannot be used within a type alias + +[case testPEP695NestedGenericClass1] +# flags: --enable-incomplete-feature=NewGenericSyntax +class C[T]: + def f(self) -> T: ... + +class A: + class B[Q]: + def __init__(self, a: Q) -> None: + self.a = a + + def f(self) -> Q: + return self.a + + def g(self, x: Q) -> None: ... + + b: B[str] + +x: A.B[int] +x.g("x") # E: Argument 1 to "g" of "B" has incompatible type "str"; expected "int" +reveal_type(x.a) # N: Revealed type is "builtins.int" +reveal_type(x) # N: Revealed type is "__main__.A.B[builtins.int]" +reveal_type(A.b) # N: Revealed type is "__main__.A.B[builtins.str]" + +[case testPEP695NestedGenericClass2] +# flags: --enable-incomplete-feature=NewGenericSyntax +class A: + def m(self) -> None: + class B[T]: + def f(self) -> T: ... + x: B[int] + reveal_type(x.f()) # N: Revealed type is "builtins.int" + self.a = B[str]() + +reveal_type(A().a) # N: Revealed type is "__main__.B@4[builtins.str]" +reveal_type(A().a.f()) # N: Revealed type is "builtins.str" + +[case testPEP695NestedGenericClass3] +# flags: --enable-incomplete-feature=NewGenericSyntax +class C[T]: + def f(self) -> T: ... + class D[S]: + x: T # E: Name "T" is not defined + def g(self) -> S: ... + +a: C[int] +reveal_type(a.f()) # N: Revealed type is "builtins.int" +b: C.D[str] +reveal_type(b.g()) # N: Revealed type is "builtins.str" + +class E[T]: + class F[T]: # E: "T" already defined as a type parameter + x: T + +c: E.F[int] + +[case testPEP695NestedGenericClass4] +# flags: --enable-incomplete-feature=NewGenericSyntax +class A: + class B[T]: + def __get__(self, instance: A, owner: type[A]) -> T: + return None # E: Incompatible return value type (got "None", expected "T") + f = B[int]() + +a = A() +v = a.f diff --git a/test-data/unit/fine-grained-python312.test b/test-data/unit/fine-grained-python312.test index 3970c8cacfbf..80e4e4ca3bd8 100644 --- a/test-data/unit/fine-grained-python312.test +++ b/test-data/unit/fine-grained-python312.test @@ -80,3 +80,23 @@ from builtins import tuple as B == main:4: error: Incompatible types in assignment (expression has type "int", variable has type "tuple[int, str]") main:5: error: Incompatible types in assignment (expression has type "str", variable has type "tuple[int, str]") + +[case testPEP695NestedGenericClassMethodUpdated] +# flags: --enable-incomplete-feature=NewGenericSyntax +from a import f + +class C: + class D[T]: + x: T + def m(self) -> T: + f() + return self.x + +[file a.py] +def f() -> None: pass + +[file a.py.2] +def f(x: int) -> None: pass +[out] +== +main:8: error: Missing positional argument "x" in call to "f" From a47f301fc8f3fb71b0a4b4998c74f9a5b326db08 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 18 Sep 2024 12:31:29 +0100 Subject: [PATCH 137/247] [PEP 695] Support Annotated[...] in new-style type aliases (#17777) The rvalue expression isn't semantically analyzed, so we can't rely on the `fullname` attribute to check if there is a reference to `Annotated`. Instead, use a lookup function provided by the caller to determine the fullname. Error reporting in the second argument to `Annotated` is still inconsistent, but this seems lower priority. I'll create a follow-up issue about (or update an existing issue if one exists). Fixes #17751. --- mypy/exprtotype.py | 39 ++++++++++++++++++++--------- mypy/semanal.py | 4 ++- test-data/unit/check-python312.test | 22 ++++++++++++++++ 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 92316d11926d..c7df851668be 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -2,12 +2,15 @@ from __future__ import annotations +from typing import Callable + from mypy.fastparse import parse_type_string from mypy.nodes import ( MISSING_FALLBACK, BytesExpr, CallExpr, ComplexExpr, + Context, DictExpr, EllipsisExpr, Expression, @@ -21,6 +24,7 @@ RefExpr, StarExpr, StrExpr, + SymbolTableNode, TupleExpr, UnaryExpr, get_member_expr_fullname, @@ -63,12 +67,16 @@ def expr_to_unanalyzed_type( allow_new_syntax: bool = False, _parent: Expression | None = None, allow_unpack: bool = False, + lookup_qualified: Callable[[str, Context], SymbolTableNode | None] | None = None, ) -> ProperType: """Translate an expression to the corresponding type. The result is not semantically analyzed. It can be UnboundType or TypeList. Raise TypeTranslationError if the expression cannot represent a type. + If lookup_qualified is not provided, the expression is expected to be semantically + analyzed. + If allow_new_syntax is True, allow all type syntax independent of the target Python version (used in stubs). @@ -101,19 +109,26 @@ def expr_to_unanalyzed_type( else: args = [expr.index] - if isinstance(expr.base, RefExpr) and expr.base.fullname in ANNOTATED_TYPE_NAMES: - # TODO: this is not the optimal solution as we are basically getting rid - # of the Annotation definition and only returning the type information, - # losing all the annotations. + if isinstance(expr.base, RefExpr): + # Check if the type is Annotated[...]. For this we need the fullname, + # which must be looked up if the expression hasn't been semantically analyzed. + base_fullname = None + if lookup_qualified is not None: + sym = lookup_qualified(base.name, expr) + if sym and sym.node: + base_fullname = sym.node.fullname + else: + base_fullname = expr.base.fullname - return expr_to_unanalyzed_type(args[0], options, allow_new_syntax, expr) - else: - base.args = tuple( - expr_to_unanalyzed_type( - arg, options, allow_new_syntax, expr, allow_unpack=True - ) - for arg in args - ) + if base_fullname is not None and base_fullname in ANNOTATED_TYPE_NAMES: + # TODO: this is not the optimal solution as we are basically getting rid + # of the Annotation definition and only returning the type information, + # losing all the annotations. + return expr_to_unanalyzed_type(args[0], options, allow_new_syntax, expr) + base.args = tuple( + expr_to_unanalyzed_type(arg, options, allow_new_syntax, expr, allow_unpack=True) + for arg in args + ) if not base.args: base.empty_tuple_index = True return base diff --git a/mypy/semanal.py b/mypy/semanal.py index 522b4abdd5e9..27d2a3abf93f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3749,7 +3749,9 @@ def analyze_alias( dynamic = bool(self.function_stack and self.function_stack[-1].is_dynamic()) global_scope = not self.type and not self.function_stack try: - typ = expr_to_unanalyzed_type(rvalue, self.options, self.is_stub_file) + typ = expr_to_unanalyzed_type( + rvalue, self.options, self.is_stub_file, lookup_qualified=self.lookup_qualified + ) except TypeTranslationError: self.fail( "Invalid type alias: expression is not a valid type", rvalue, code=codes.VALID_TYPE diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 89ced8be2c6f..0b3055212d20 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1714,6 +1714,28 @@ type YNested = (1 + (yield from [])) # E: Yield expression cannot be used within type ZNested = (1 + (a := 1)) # E: Named expression cannot be used within a type alias type KNested = (1 + (await 1)) # E: Await expression cannot be used within a type alias +[case testPEP695TypeAliasAndAnnotated] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing_extensions import Annotated, Annotated as _Annotated +import typing_extensions as t + +def ann(*args): ... + +type A = Annotated[int, ann()] +type B = Annotated[int | str, ann((1, 2))] +type C = _Annotated[int, ann()] +type D = t.Annotated[str, ann()] + +x: A +y: B +z: C +zz: D +reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" +reveal_type(z) # N: Revealed type is "builtins.int" +reveal_type(zz) # N: Revealed type is "builtins.str" +[builtins fixtures/tuple.pyi] + [case testPEP695NestedGenericClass1] # flags: --enable-incomplete-feature=NewGenericSyntax class C[T]: From fce14a0284b2c32d373cee57c9ed06f764769d93 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 18 Sep 2024 14:42:11 +0100 Subject: [PATCH 138/247] [PEP 695] Allow covariance with attribute that has "_" name prefix (#17782) Fix this conformance test: ``` class ShouldBeCovariant5[T]: def __init__(self, x: T) -> None: self._x = x @property def x(self) -> T: return self._x vo5_1: ShouldBeCovariant5[float] = ShouldBeCovariant5[int](1) # OK vo5_2: ShouldBeCovariant5[int] = ShouldBeCovariant5[float](1) # E ``` My fix is to treat such attributes as not settable when inferring variance. Link: https://github.com/python/typing/blob/main/conformance/tests/generics_variance_inference.py#L79 --- mypy/subtypes.py | 22 +++++++--- test-data/unit/check-python312.test | 63 +++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 649cbae4c831..6e2366c4e0df 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2006,16 +2006,22 @@ def infer_variance(info: TypeInfo, i: int) -> bool: for member in all_non_object_members(info): if member in ("__init__", "__new__"): continue - node = info[member].node - if isinstance(node, Var) and node.type is None: - tv.variance = VARIANCE_NOT_READY - return False + if isinstance(self_type, TupleType): self_type = mypy.typeops.tuple_fallback(self_type) - flags = get_member_flags(member, self_type) - typ = find_member(member, self_type, self_type) settable = IS_SETTABLE in flags + + node = info[member].node + if isinstance(node, Var): + if node.type is None: + tv.variance = VARIANCE_NOT_READY + return False + if has_underscore_prefix(member): + # Special case to avoid false positives (and to pass conformance tests) + settable = False + + typ = find_member(member, self_type, self_type) if typ: typ2 = expand_type(typ, {tvar.id: object_type}) if not is_subtype(typ, typ2): @@ -2036,6 +2042,10 @@ def infer_variance(info: TypeInfo, i: int) -> bool: return True +def has_underscore_prefix(name: str) -> bool: + return name.startswith("_") and not (name.startswith("__") and name.endswith("__")) + + def infer_class_variances(info: TypeInfo) -> bool: if not info.defn.type_args: return True diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 0b3055212d20..9dc52d2c07b0 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -342,6 +342,69 @@ class Invariant[T]: reveal_type(c(a1, a2)) # N: Revealed type is "Never" +[case testPEP695InferVarianceUnderscorePrefix] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class Covariant1[T]: + def __init__(self, x: T) -> None: + self._x = x + + @property + def x(self) -> T: + return self._x + +co1_1: Covariant1[float] = Covariant1[int](1) +co1_2: Covariant1[int] = Covariant1[float](1) # E: Incompatible types in assignment (expression has type "Covariant1[float]", variable has type "Covariant1[int]") + +class Covariant2[T]: + def __init__(self, x: T) -> None: + self.__foo_bar = x + + @property + def x(self) -> T: + return self.__foo_bar + +co2_1: Covariant2[float] = Covariant2[int](1) +co2_2: Covariant2[int] = Covariant2[float](1) # E: Incompatible types in assignment (expression has type "Covariant2[float]", variable has type "Covariant2[int]") + +class Invariant1[T]: + def __init__(self, x: T) -> None: + self._x = x + + # Methods behave differently from attributes + def _f(self, x: T) -> None: ... + + @property + def x(self) -> T: + return self._x + +inv1_1: Invariant1[float] = Invariant1[int](1) # E: Incompatible types in assignment (expression has type "Invariant1[int]", variable has type "Invariant1[float]") +inv1_2: Invariant1[int] = Invariant1[float](1) # E: Incompatible types in assignment (expression has type "Invariant1[float]", variable has type "Invariant1[int]") + +class Invariant2[T]: + def __init__(self, x: T) -> None: + # Dunders are special + self.__x__ = x + + @property + def x(self) -> T: + return self.__x__ + +inv2_1: Invariant2[float] = Invariant2[int](1) # E: Incompatible types in assignment (expression has type "Invariant2[int]", variable has type "Invariant2[float]") +inv2_2: Invariant2[int] = Invariant2[float](1) # E: Incompatible types in assignment (expression has type "Invariant2[float]", variable has type "Invariant2[int]") + +class Invariant3[T]: + def __init__(self, x: T) -> None: + self._x = Invariant1(x) + + @property + def x(self) -> T: + return self._x._x + +inv3_1: Invariant3[float] = Invariant3[int](1) # E: Incompatible types in assignment (expression has type "Invariant3[int]", variable has type "Invariant3[float]") +inv3_2: Invariant3[int] = Invariant3[float](1) # E: Incompatible types in assignment (expression has type "Invariant3[float]", variable has type "Invariant3[int]") +[builtins fixtures/property.pyi] + [case testPEP695InheritInvariant] # flags: --enable-incomplete-feature=NewGenericSyntax From 9d7a04277ebb5edcfe649977fb5bb45ac91573f6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 18 Sep 2024 14:42:25 +0100 Subject: [PATCH 139/247] [PEP 695] Fix covariance of frozen dataclasses (#17783) Fix this conformance test: ``` @dataclass(frozen=True) class ShouldBeCovariant4[T]: x: T vo4_1: ShouldBeCovariant4[float] = ShouldBeCovariant4[int](1) # OK vo4_2: ShouldBeCovariant4[int] = ShouldBeCovariant4[float](1) # E ``` Link: https://github.com/python/typing/blob/main/conformance/tests/generics_variance_inference.py#L66 --- mypy/subtypes.py | 3 ++- test-data/unit/check-python312.test | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 6e2366c4e0df..5c4471cc5b62 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2004,7 +2004,8 @@ def infer_variance(info: TypeInfo, i: int) -> bool: tvar = info.defn.type_vars[i] self_type = fill_typevars(info) for member in all_non_object_members(info): - if member in ("__init__", "__new__"): + # __mypy-replace is an implementation detail of the dataclass plugin + if member in ("__init__", "__new__", "__mypy-replace"): continue if isinstance(self_type, TupleType): diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 9dc52d2c07b0..d0a39f7e56a6 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -249,6 +249,25 @@ if int(): if int(): f = e +[case testPEP695InferVarianceInFrozenDataclass] +# flags: --enable-incomplete-feature=NewGenericSyntax +from dataclasses import dataclass + +@dataclass(frozen=True) +class Covariant[T]: + x: T + +cov1: Covariant[float] = Covariant[int](1) +cov2: Covariant[int] = Covariant[float](1) # E: Incompatible types in assignment (expression has type "Covariant[float]", variable has type "Covariant[int]") + +@dataclass(frozen=True) +class Invariant[T]: + x: list[T] + +inv1: Invariant[float] = Invariant[int]([1]) # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[float]") +inv2: Invariant[int] = Invariant[float]([1]) # E: Incompatible types in assignment (expression has type "Invariant[float]", variable has type "Invariant[int]") +[builtins fixtures/tuple.pyi] + [case testPEP695InferVarianceCalculateOnDemand] # flags: --enable-incomplete-feature=NewGenericSyntax From 4554bd0c6d49506a4a1c77dcf344dca4efee92e1 Mon Sep 17 00:00:00 2001 From: Katrina Connors <32425204+katconnors@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:22:06 -0700 Subject: [PATCH 140/247] Added error code for overlapping function signatures (#17597) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #17570. This is my first contribution to mypy! 🐍 Added an error code for overlapping function signatures. Test in check-errorcodes.test is a derivative of this post: https://stackoverflow.com/q/69341607 Co-authored-by: Jelle Zijlstra Co-authored-by: Alex Waygood --- docs/source/error_code_list.rst | 28 ++++++++++++++++++++++++++++ mypy/errorcodes.py | 8 ++++++++ mypy/messages.py | 1 + test-data/unit/check-errorcodes.test | 14 ++++++++++++++ 4 files changed, 51 insertions(+) diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index 85c8d437a856..ad73bc999f00 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -1149,6 +1149,34 @@ types you expect. See :ref:`overloading ` for more explanation. + +.. _code-overload-cannot-match: + +Check for overload signatures that cannot match [overload-cannot-match] +-------------------------------------------------------------------------- + +Warn if an ``@overload`` variant can never be matched, because an earlier +overload has a wider signature. For example, this can happen if the two +overloads accept the same parameters and each parameter on the first overload +has the same type or a wider type than the corresponding parameter on the second +overload. + +Example: + +.. code-block:: python + + from typing import overload, Union + + @overload + def process(response1: object, response2: object) -> object: + ... + @overload + def process(response1: int, response2: int) -> int: # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader [overload-cannot-match] + ... + + def process(response1: object, response2: object) -> object: + return response1 + response2 + .. _code-annotation-unchecked: Notify about an annotation in an unchecked function [annotation-unchecked] diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index 6e8763264ddd..ad061b161af1 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -273,6 +273,14 @@ def __hash__(self) -> int: # This is a catch-all for remaining uncategorized errors. MISC: Final[ErrorCode] = ErrorCode("misc", "Miscellaneous other checks", "General") +OVERLOAD_CANNOT_MATCH: Final[ErrorCode] = ErrorCode( + "overload-cannot-match", + "Warn if an @overload signature can never be matched", + "General", + sub_code_of=MISC, +) + + OVERLOAD_OVERLAP: Final[ErrorCode] = ErrorCode( "overload-overlap", "Warn if multiple @overload variants overlap in unsafe ways", diff --git a/mypy/messages.py b/mypy/messages.py index 62846c536f3d..dadce149680e 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1653,6 +1653,7 @@ def overloaded_signature_will_never_match( index1=index1, index2=index2 ), context, + code=codes.OVERLOAD_CANNOT_MATCH, ) def overloaded_signatures_typevar_specific(self, index: int, context: Context) -> None: diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index cca13347dd86..10cc145d0c70 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -1222,3 +1222,17 @@ def f(x: str) -> TypeIs[int]: # E: Narrowed type "int" is not a subtype of inpu pass [builtins fixtures/tuple.pyi] + + +[case testOverloadedFunctionSignature] +from typing import overload, Union + +@overload +def process(response1: float,response2: float) -> float: + ... +@overload +def process(response1: int,response2: int) -> int: # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader [overload-cannot-match] + ... + +def process(response1,response2)-> Union[float,int]: + return response1 + response2 From 2a8c91e6413f07f800833d6e3409070b91d725b4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 19 Sep 2024 17:44:29 +0100 Subject: [PATCH 141/247] [PEP 695] Fix crash on invalid type var reference (#17788) Test case from typing conformance test suite: https://github.com/python/typing/blob/main/conformance/tests/generics_syntax_declarations.py#L45 --- mypy/semanal.py | 1 + test-data/unit/check-python312.test | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index 27d2a3abf93f..780d0b614ae3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -6955,6 +6955,7 @@ def name_not_defined(self, name: str, ctx: Context, namespace: str | None = None namespace is None and self.type and not self.is_func_scope() + and self.incomplete_type_stack and self.incomplete_type_stack[-1] and not self.final_iteration ): diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index d0a39f7e56a6..bf1115dc51c5 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1320,6 +1320,14 @@ class P[T](Protocol[T]): # E: No arguments expected for "Protocol" base class class P2[T](Protocol[S]): # E: No arguments expected for "Protocol" base class pass +[case testPEP695CannotUseTypeVarFromOuterClass] +# mypy: enable-incomplete-feature=NewGenericSyntax +class ClassG[V]: + # This used to crash + class ClassD[T: dict[str, V]]: # E: Name "V" is not defined + ... +[builtins fixtures/dict.pyi] + [case testPEP695MixNewAndOldStyleGenerics] # mypy: enable-incomplete-feature=NewGenericSyntax from typing import TypeVar From a646f330641422d5bc9bf71d1f7751ab2d8a8d5d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 19 Sep 2024 17:44:57 +0100 Subject: [PATCH 142/247] [PEP 695] Inherit variance if base class has explicit variance (#17787) Previously we only inferred variance based on member types, but if a base class has explicit variance for some type variables, we need to consider it as well. --- mypy/subtypes.py | 9 ++++++ test-data/unit/check-python312.test | 40 ++++++++++++++++++++++++ test-data/unit/fixtures/tuple-simple.pyi | 2 +- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 5c4471cc5b62..df040dcb1311 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2031,6 +2031,15 @@ def infer_variance(info: TypeInfo, i: int) -> bool: contra = False if settable: co = False + + # Infer variance from base classes, in case they have explicit variances + for base in info.bases: + base2 = expand_type(base, {tvar.id: object_type}) + if not is_subtype(base, base2): + co = False + if not is_subtype(base2, base): + contra = False + if co: v = COVARIANT elif contra: diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index bf1115dc51c5..5200395047bc 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1891,3 +1891,43 @@ class A: a = A() v = a.f + +[case testPEP695VarianceInheritedFromBaseWithExplicitVariance] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import TypeVar, Generic + +T = TypeVar("T") + +class ParentInvariant(Generic[T]): + pass + +class Invariant1[T](ParentInvariant[T]): + pass + +a1: Invariant1[int] = Invariant1[float]() # E: Incompatible types in assignment (expression has type "Invariant1[float]", variable has type "Invariant1[int]") +a2: Invariant1[float] = Invariant1[int]() # E: Incompatible types in assignment (expression has type "Invariant1[int]", variable has type "Invariant1[float]") + +T_contra = TypeVar("T_contra", contravariant=True) + +class ParentContravariant(Generic[T_contra]): + pass + +class Contravariant[T](ParentContravariant[T]): + pass + +b1: Contravariant[int] = Contravariant[float]() +b2: Contravariant[float] = Contravariant[int]() # E: Incompatible types in assignment (expression has type "Contravariant[int]", variable has type "Contravariant[float]") + +class Invariant2[T](ParentContravariant[T]): + def f(self) -> T: ... + +c1: Invariant2[int] = Invariant2[float]() # E: Incompatible types in assignment (expression has type "Invariant2[float]", variable has type "Invariant2[int]") +c2: Invariant2[float] = Invariant2[int]() # E: Incompatible types in assignment (expression has type "Invariant2[int]", variable has type "Invariant2[float]") + +class Multi[T, S](ParentInvariant[T], ParentContravariant[S]): + pass + +d1: Multi[int, str] = Multi[float, str]() # E: Incompatible types in assignment (expression has type "Multi[float, str]", variable has type "Multi[int, str]") +d2: Multi[float, str] = Multi[int, str]() # E: Incompatible types in assignment (expression has type "Multi[int, str]", variable has type "Multi[float, str]") +d3: Multi[str, int] = Multi[str, float]() +d4: Multi[str, float] = Multi[str, int]() # E: Incompatible types in assignment (expression has type "Multi[str, int]", variable has type "Multi[str, float]") diff --git a/test-data/unit/fixtures/tuple-simple.pyi b/test-data/unit/fixtures/tuple-simple.pyi index 6c816c1c5b7a..07f9edf63cdd 100644 --- a/test-data/unit/fixtures/tuple-simple.pyi +++ b/test-data/unit/fixtures/tuple-simple.pyi @@ -5,7 +5,7 @@ from typing import Iterable, TypeVar, Generic -T = TypeVar('T') +T = TypeVar('T', covariant=True) class object: def __init__(self): pass From 18fee78645a32b4333a1b80720c7b7765775ad4b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 20 Sep 2024 12:49:14 +0100 Subject: [PATCH 143/247] [PEP 695] Generate error if new-style type alias used as base class (#17789) It doesn't work at runtime. --- mypy/semanal.py | 15 ++++++++++++ test-data/unit/check-python312.test | 38 ++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 780d0b614ae3..e239fbf1f141 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1824,6 +1824,8 @@ def analyze_class(self, defn: ClassDef) -> None: defn, bases, context=defn ) + self.check_type_alias_bases(bases) + for tvd in tvar_defs: if isinstance(tvd, TypeVarType) and any( has_placeholder(t) for t in [tvd.upper_bound] + tvd.values @@ -1895,6 +1897,19 @@ def analyze_class(self, defn: ClassDef) -> None: self.analyze_class_body_common(defn) + def check_type_alias_bases(self, bases: list[Expression]) -> None: + for base in bases: + if isinstance(base, IndexExpr): + base = base.base + if ( + isinstance(base, RefExpr) + and isinstance(base.node, TypeAlias) + and base.node.python_3_12_type_alias + ): + self.fail( + 'Type alias defined using "type" statement not valid as base class', base + ) + def setup_type_vars(self, defn: ClassDef, tvar_defs: list[TypeVarLikeType]) -> None: defn.type_vars = tvar_defs defn.info.type_vars = [] diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 5200395047bc..d9737694c262 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -591,6 +591,40 @@ a4: A4 reveal_type(a4) # N: Revealed type is "Union[builtins.int, builtins.str]" [builtins fixtures/type.pyi] +[case testPEP695TypeAliasNotValidAsBaseClass] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import TypeAlias + +import m + +type A1 = int +class Bad1(A1): # E: Type alias defined using "type" statement not valid as base class + pass + +type A2[T] = list[T] +class Bad2(A2[int]): # E: Type alias defined using "type" statement not valid as base class + pass + +class Bad3(m.A1): # E: Type alias defined using "type" statement not valid as base class + pass + +class Bad4(m.A2[int]): # E: Type alias defined using "type" statement not valid as base class + pass + +B1 = int +B2 = list +B3: TypeAlias = int +class Good1(B1): pass +class Good2(B2[int]): pass +class Good3(list[A1]): pass +class Good4(list[A2[int]]): pass +class Good5(B3): pass + +[file m.py] +type A1 = str +type A2[T] = list[T] +[typing fixtures/typing-medium.pyi] + [case testPEP695TypeAliasWithUnusedTypeParams] # flags: --enable-incomplete-feature=NewGenericSyntax type A[T] = int @@ -637,9 +671,7 @@ class D: pass type A = C -# Note that this doesn't actually work at runtime, but we currently don't -# keep track whether a type alias is valid in various runtime type contexts. -class D(A): +class D(A): # E: Type alias defined using "type" statement not valid as base class pass class C: pass From 5dfc7d941253553ab77836e9845cb8fdfb9d23a9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 20 Sep 2024 14:31:03 +0100 Subject: [PATCH 144/247] [PEP 695] Enable new type parameter syntax by default (#17798) I think the PEP 695 syntax is supported well enough now to enable it by default. --- docs/source/command_line.rst | 15 +- mypy/fastparse.py | 66 ++---- mypy/options.py | 4 +- mypyc/test-data/run-python312.test | 2 - mypyc/test/test_run.py | 1 - test-data/unit/check-python312.test | 240 ++++----------------- test-data/unit/deps.test | 2 +- test-data/unit/diff.test | 16 +- test-data/unit/fine-grained-python312.test | 17 +- test-data/unit/parse-python312.test | 8 +- test-data/unit/pythoneval.test | 4 +- 11 files changed, 85 insertions(+), 290 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index c085b63107b0..a89a3c85d4ee 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1008,7 +1008,7 @@ format into the specified directory. Enabling incomplete/experimental features ***************************************** -.. option:: --enable-incomplete-feature {PreciseTupleTypes, NewGenericSyntax, InlineTypedDict} +.. option:: --enable-incomplete-feature {PreciseTupleTypes, InlineTypedDict} Some features may require several mypy releases to implement, for example due to their complexity, potential for backwards incompatibility, or @@ -1055,19 +1055,6 @@ List of currently incomplete/experimental features: # Without PreciseTupleTypes: tuple[int, ...] # With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int] -* ``NewGenericSyntax``: this feature enables support for syntax defined - by :pep:`695`. For example: - - .. code-block:: python - - class Container[T]: # defines a generic class - content: T - - def first[T](items: list[T]) -> T: # defines a generic function - return items[0] - - type Items[T] = list[tuple[T, T]] # defines a generic type alias - * ``InlineTypedDict``: this feature enables non-standard syntax for inline :ref:`TypedDicts `, for example: diff --git a/mypy/fastparse.py b/mypy/fastparse.py index abcce74c6064..18858b0fa0b8 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -92,7 +92,7 @@ YieldFromExpr, check_arg_names, ) -from mypy.options import NEW_GENERIC_SYNTAX, Options +from mypy.options import Options from mypy.patterns import ( AsPattern, ClassPattern, @@ -965,19 +965,7 @@ def do_func_def( return_type = AnyType(TypeOfAny.from_error) else: if sys.version_info >= (3, 12) and n.type_params: - if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: - explicit_type_params = self.translate_type_params(n.type_params) - else: - self.fail( - ErrorMessage( - "PEP 695 generics are not yet supported. " - "Use --enable-incomplete-feature=NewGenericSyntax for experimental support", - code=codes.VALID_TYPE, - ), - n.type_params[0].lineno, - n.type_params[0].col_offset, - blocker=False, - ) + explicit_type_params = self.translate_type_params(n.type_params) arg_types = [a.type_annotation for a in args] return_type = TypeConverter( @@ -1157,19 +1145,7 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: explicit_type_params: list[TypeParam] | None = None if sys.version_info >= (3, 12) and n.type_params: - if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: - explicit_type_params = self.translate_type_params(n.type_params) - else: - self.fail( - ErrorMessage( - "PEP 695 generics are not yet supported. " - "Use --enable-incomplete-feature=NewGenericSyntax for experimental support", - code=codes.VALID_TYPE, - ), - n.type_params[0].lineno, - n.type_params[0].col_offset, - blocker=False, - ) + explicit_type_params = self.translate_type_params(n.type_params) cdef = ClassDef( n.name, @@ -1843,31 +1819,17 @@ def validate_type_alias(self, n: ast_TypeAlias) -> None: # TypeAlias(identifier name, type_param* type_params, expr value) def visit_TypeAlias(self, n: ast_TypeAlias) -> TypeAliasStmt | AssignmentStmt: node: TypeAliasStmt | AssignmentStmt - if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: - type_params = self.translate_type_params(n.type_params) - self.validate_type_alias(n) - value = self.visit(n.value) - # Since the value is evaluated lazily, wrap the value inside a lambda. - # This helps mypyc. - ret = ReturnStmt(value) - self.set_line(ret, n.value) - value_func = LambdaExpr(body=Block([ret])) - self.set_line(value_func, n.value) - node = TypeAliasStmt(self.visit_Name(n.name), type_params, value_func) - return self.set_line(node, n) - else: - self.fail( - ErrorMessage( - "PEP 695 type aliases are not yet supported. " - "Use --enable-incomplete-feature=NewGenericSyntax for experimental support", - code=codes.VALID_TYPE, - ), - n.lineno, - n.col_offset, - blocker=False, - ) - node = AssignmentStmt([NameExpr(n.name.id)], self.visit(n.value)) - return self.set_line(node, n) + type_params = self.translate_type_params(n.type_params) + self.validate_type_alias(n) + value = self.visit(n.value) + # Since the value is evaluated lazily, wrap the value inside a lambda. + # This helps mypyc. + ret = ReturnStmt(value) + self.set_line(ret, n.value) + value_func = LambdaExpr(body=Block([ret])) + self.set_line(value_func, n.value) + node = TypeAliasStmt(self.visit_Name(n.name), type_params, value_func) + return self.set_line(node, n) class TypeConverter: diff --git a/mypy/options.py b/mypy/options.py index 5e64d5e40035..56bd92957b41 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -75,8 +75,8 @@ class BuildType: PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax" INLINE_TYPEDDICT: Final = "InlineTypedDict" -INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX, INLINE_TYPEDDICT)) -COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK)) +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, INLINE_TYPEDDICT)) +COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, NEW_GENERIC_SYNTAX)) class Options: diff --git a/mypyc/test-data/run-python312.test b/mypyc/test-data/run-python312.test index 5e8a388fd8d3..a5a3f058d1e2 100644 --- a/mypyc/test-data/run-python312.test +++ b/mypyc/test-data/run-python312.test @@ -1,5 +1,4 @@ [case testPEP695Basics] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import Any, TypeAliasType, cast from testutil import assertRaises @@ -192,7 +191,6 @@ def test_recursive_type_alias() -> None: [typing fixtures/typing-full.pyi] [case testPEP695GenericTypeAlias] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import Callable from types import GenericAlias diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 37de192a9291..668e5b124841 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -196,7 +196,6 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> options.preserve_asts = True options.allow_empty_bodies = True options.incremental = self.separate - options.enable_incomplete_feature.append("NewGenericSyntax") # Avoid checking modules/packages named 'unchecked', to provide a way # to test interacting with code we don't have types for. diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index d9737694c262..2f0b912c439e 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1,70 +1,56 @@ -[case test695TypeAlias] -type MyInt = int # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support +[case testPEP695TypeAliasBasic] +type MyInt = int def f(x: MyInt) -> MyInt: return reveal_type(x) # N: Revealed type is "builtins.int" -type MyList[T] = list[T] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ - # E: Name "T" is not defined +type MyList[T] = list[T] -def g(x: MyList[int]) -> MyList[int]: # E: Variable "__main__.MyList" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases - return reveal_type(x) # N: Revealed type is "MyList?[builtins.int]" +def g(x: MyList[int]) -> MyList[int]: + return reveal_type(x) # N: Revealed type is "builtins.list[builtins.int]" -type MyInt2 = int # type: ignore[valid-type] +type MyInt2 = int def h(x: MyInt2) -> MyInt2: return reveal_type(x) # N: Revealed type is "builtins.int" -[case test695Class] -class MyGen[T]: # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support - def __init__(self, x: T) -> None: # E: Name "T" is not defined +[case testPEP695Class] +class MyGen[T]: + def __init__(self, x: T) -> None: self.x = x -def f(x: MyGen[int]): # E: "MyGen" expects no type arguments, but 1 given - reveal_type(x.x) # N: Revealed type is "Any" +def f(x: MyGen[int]): + reveal_type(x.x) # N: Revealed type is "builtins.int" -[case test695Function] -def f[T](x: T) -> T: # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ - # E: Name "T" is not defined - return reveal_type(x) # N: Revealed type is "Any" +[case testPEP695Function] +def f[T](x: T) -> T: + return reveal_type(x) # N: Revealed type is "T`-1" -reveal_type(f(1)) # N: Revealed type is "Any" +reveal_type(f(1)) # N: Revealed type is "builtins.int" -async def g[T](x: T) -> T: # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ - # E: Name "T" is not defined - return reveal_type(x) # N: Revealed type is "Any" +async def g[T](x: T) -> T: + return reveal_type(x) # N: Revealed type is "T`-1" -reveal_type(g(1)) # E: Value of type "Coroutine[Any, Any, Any]" must be used \ +reveal_type(g(1)) # E: Value of type "Coroutine[Any, Any, int]" must be used \ # N: Are you missing an await? \ - # N: Revealed type is "typing.Coroutine[Any, Any, Any]" + # N: Revealed type is "typing.Coroutine[Any, Any, builtins.int]" -[case test695TypeVar] +[case testPEP695TypeVarBasic] from typing import Callable -type Alias1[T: int] = list[T] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ - # E: Name "T" is not defined -type Alias2[**P] = Callable[P, int] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ - # E: Value of type "int" is not indexable \ - # E: Name "P" is not defined -type Alias3[*Ts] = tuple[*Ts] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ - # E: Name "Ts" is not defined - -class Cls1[T: int]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support -class Cls2[**P]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support -class Cls3[*Ts]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support - -def func1[T: int](x: T) -> T: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ - # E: Name "T" is not defined - -def func2[**P](x: Callable[P, int]) -> Callable[P, str]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ - # E: The first argument to Callable must be a list of types, parameter specification, or "..." \ - # N: See https://mypy.readthedocs.io/en/stable/kinds_of_types.html#callable-types-and-lambdas \ - # E: Name "P" is not defined -def func3[*Ts](x: tuple[*Ts]) -> tuple[int, *Ts]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ - # E: Name "Ts" is not defined +type Alias1[T: int] = list[T] +type Alias2[**P] = Callable[P, int] +type Alias3[*Ts] = tuple[*Ts] + +class Cls1[T: int]: ... +class Cls2[**P]: ... +class Cls3[*Ts]: ... + +def func1[T: int](x: T) -> T: ... +def func2[**P](x: Callable[P, int]) -> Callable[P, str]: ... +def func3[*Ts](x: tuple[*Ts]) -> tuple[int, *Ts]: ... [builtins fixtures/tuple.pyi] -[case test695TypeAliasType] +[case testPEP695TypeAliasType] from typing import Callable, TypeAliasType, TypeVar, TypeVarTuple T = TypeVar("T") @@ -86,9 +72,13 @@ reveal_type(ba2) # N: Revealed type is "def (*Any) -> builtins.str" [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] -[case testPEP695GenericFunctionSyntax] -# flags: --enable-incomplete-feature=NewGenericSyntax +[case testPEP695IncompleteFeatureIsAcceptedButHasNoEffect] +# mypy: enable-incomplete-feature=NewGenericSyntax +def f[T](x: T) -> T: + return x +reveal_type(f(1)) # N: Revealed type is "builtins.int" +[case testPEP695GenericFunctionSyntax] def ident[TV](x: TV) -> TV: y: TV = x y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "TV") @@ -107,8 +97,6 @@ reveal_type(tup(1, 'x')) # N: Revealed type is "Tuple[builtins.int, builtins.st [builtins fixtures/tuple.pyi] [case testPEP695GenericClassSyntax] -# flags: --enable-incomplete-feature=NewGenericSyntax - class C[T]: x: T @@ -128,8 +116,6 @@ reveal_type(c.x) # N: Revealed type is "builtins.int" reveal_type(c.ident(1)) # N: Revealed type is "builtins.int" [case testPEP695GenericMethodInGenericClass] -# flags: --enable-incomplete-feature=NewGenericSyntax - class C[T]: def m[S](self, x: S) -> T | S: ... @@ -139,8 +125,6 @@ b: C[object] = C[int]() reveal_type(C[str]().m(1)) # N: Revealed type is "Union[builtins.str, builtins.int]" [case testPEP695InferVarianceSimpleFromMethod] -# flags: --enable-incomplete-feature=NewGenericSyntax - class Invariant[T]: def f(self, x: T) -> None: pass @@ -178,8 +162,6 @@ if int(): f = e [case testPEP695InferVarianceSimpleFromAttribute] -# flags: --enable-incomplete-feature=NewGenericSyntax - class Invariant1[T]: def __init__(self, x: T) -> None: self.x = x @@ -214,8 +196,6 @@ if int(): b3 = a3 # E: Incompatible types in assignment (expression has type "Invariant3[object]", variable has type "Invariant3[int]") [case testPEP695InferVarianceRecursive] -# flags: --enable-incomplete-feature=NewGenericSyntax - class Invariant[T]: def f(self, x: Invariant[T]) -> Invariant[T]: return x @@ -250,7 +230,6 @@ if int(): f = e [case testPEP695InferVarianceInFrozenDataclass] -# flags: --enable-incomplete-feature=NewGenericSyntax from dataclasses import dataclass @dataclass(frozen=True) @@ -269,8 +248,6 @@ inv2: Invariant[int] = Invariant[float]([1]) # E: Incompatible types in assignm [builtins fixtures/tuple.pyi] [case testPEP695InferVarianceCalculateOnDemand] -# flags: --enable-incomplete-feature=NewGenericSyntax - class Covariant[T]: def __init__(self) -> None: self.x = [1] @@ -286,8 +263,6 @@ class Covariant[T]: def h(self, x: Covariant[int]) -> None: pass [case testPEP695InferVarianceNotReadyWhenNeeded] -# flags: --enable-incomplete-feature=NewGenericSyntax - class Covariant[T]: def f(self) -> None: c = Covariant[int]() @@ -328,8 +303,6 @@ if int(): b = a # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") [case testPEP695InferVarianceNotReadyForJoin] -# flags: --enable-incomplete-feature=NewGenericSyntax - class Invariant[T]: def f(self) -> None: # Assume covariance if variance us not ready @@ -342,8 +315,6 @@ class Invariant[T]: reveal_type([Invariant(1), Invariant(object())]) # N: Revealed type is "builtins.list[builtins.object]" [case testPEP695InferVarianceNotReadyForMeet] -# flags: --enable-incomplete-feature=NewGenericSyntax - from typing import TypeVar, Callable S = TypeVar("S") @@ -362,8 +333,6 @@ class Invariant[T]: reveal_type(c(a1, a2)) # N: Revealed type is "Never" [case testPEP695InferVarianceUnderscorePrefix] -# flags: --enable-incomplete-feature=NewGenericSyntax - class Covariant1[T]: def __init__(self, x: T) -> None: self._x = x @@ -425,8 +394,6 @@ inv3_2: Invariant3[int] = Invariant3[float](1) # E: Incompatible types in assig [builtins fixtures/property.pyi] [case testPEP695InheritInvariant] -# flags: --enable-incomplete-feature=NewGenericSyntax - class Invariant[T]: x: T @@ -448,7 +415,6 @@ if int(): b = a # E: Incompatible types in assignment (expression has type "Subclass[int]", variable has type "Subclass[object]") [case testPEP695InheritanceMakesInvariant] -# flags: --enable-incomplete-feature=NewGenericSyntax class Covariant[T]: def f(self) -> T: ... @@ -463,7 +429,6 @@ a: Subclass[int] = Subclass[object]() # E: Incompatible types in assignment (ex b: Subclass[object] = Subclass[int]() # E: Incompatible types in assignment (expression has type "Subclass[int]", variable has type "Subclass[object]") [case testPEP695InheritCoOrContravariant] -# flags: --enable-incomplete-feature=NewGenericSyntax class Contravariant[T]: def f(self, x: T) -> None: pass @@ -489,7 +454,6 @@ e: InvSubclass[int] = InvSubclass[object]() # E: Incompatible types in assignme f: InvSubclass[object] = InvSubclass[int]() # E: Incompatible types in assignment (expression has type "InvSubclass[int]", variable has type "InvSubclass[object]") [case testPEP695FinalAttribute] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import Final class C[T]: @@ -500,8 +464,6 @@ a: C[int] = C[object](1) # E: Incompatible types in assignment (expression has b: C[object] = C[int](1) [case testPEP695TwoTypeVariables] -# flags: --enable-incomplete-feature=NewGenericSyntax - class C[T, S]: def f(self, x: T) -> None: ... def g(self) -> S: ... @@ -512,8 +474,6 @@ c: C[int, int] = C[int, object]() # E: Incompatible types in assignment (expres d: C[int, object] = C[int, int]() [case testPEP695Properties] -# flags: --enable-incomplete-feature=NewGenericSyntax - class R[T]: @property def p(self) -> T: ... @@ -531,7 +491,6 @@ d: RW[object] = RW[int]() # E: Incompatible types in assignment (expression has [builtins fixtures/property.pyi] [case testPEP695Protocol] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import Protocol class PContra[T](Protocol): @@ -568,8 +527,6 @@ if int(): f = e # E: Incompatible types in assignment (expression has type "PInv[int]", variable has type "PInv[object]") [case testPEP695TypeAlias] -# flags: --enable-incomplete-feature=NewGenericSyntax - class C[T]: pass class D[T, S]: pass @@ -592,7 +549,6 @@ reveal_type(a4) # N: Revealed type is "Union[builtins.int, builtins.str]" [builtins fixtures/type.pyi] [case testPEP695TypeAliasNotValidAsBaseClass] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import TypeAlias import m @@ -626,14 +582,11 @@ type A2[T] = list[T] [typing fixtures/typing-medium.pyi] [case testPEP695TypeAliasWithUnusedTypeParams] -# flags: --enable-incomplete-feature=NewGenericSyntax type A[T] = int a: A[str] reveal_type(a) # N: Revealed type is "builtins.int" [case testPEP695TypeAliasForwardReference1] -# flags: --enable-incomplete-feature=NewGenericSyntax - type A[T] = C[T] a: A[int] @@ -642,8 +595,6 @@ reveal_type(a) # N: Revealed type is "__main__.C[builtins.int]" class C[T]: pass [case testPEP695TypeAliasForwardReference2] -# flags: --enable-incomplete-feature=NewGenericSyntax - type X = C type A = X @@ -655,8 +606,6 @@ class C: pass [typing fixtures/typing-full.pyi] [case testPEP695TypeAliasForwardReference3] -# flags: --enable-incomplete-feature=NewGenericSyntax - type X = D type A = C[X] @@ -667,8 +616,6 @@ class C[T]: pass class D: pass [case testPEP695TypeAliasForwardReference4] -# flags: --enable-incomplete-feature=NewGenericSyntax - type A = C class D(A): # E: Type alias defined using "type" statement not valid as base class @@ -680,7 +627,6 @@ x: C = D() y: D = C() # E: Incompatible types in assignment (expression has type "C", variable has type "D") [case testPEP695TypeAliasForwardReference5] -# flags: --enable-incomplete-feature=NewGenericSyntax type A = str type B[T] = C[T] class C[T]: pass @@ -692,13 +638,11 @@ reveal_type(b) # N: Revealed type is "__main__.C[builtins.int]" reveal_type(c) # N: Revealed type is "__main__.C[builtins.str]" [case testPEP695TypeAliasWithUndefineName] -# flags: --enable-incomplete-feature=NewGenericSyntax type A[T] = XXX # E: Name "XXX" is not defined a: A[int] reveal_type(a) # N: Revealed type is "Any" [case testPEP695TypeAliasInvalidType] -# flags: --enable-incomplete-feature=NewGenericSyntax type A = int | 1 # E: Invalid type: try using Literal[1] instead? a: A @@ -709,13 +653,10 @@ reveal_type(b) # N: Revealed type is "Any" [builtins fixtures/type.pyi] [case testPEP695TypeAliasBoundForwardReference] -# mypy: enable-incomplete-feature=NewGenericSyntax type B[T: Foo] = list[T] class Foo: pass [case testPEP695UpperBound] -# flags: --enable-incomplete-feature=NewGenericSyntax - class D: x: int class E(D): pass @@ -738,8 +679,6 @@ reveal_type(f(E())) # N: Revealed type is "__main__.E" f(1) # E: Value of type variable "T" of "f" cannot be "int" [case testPEP695UpperBoundForwardReference1] -# flags: --enable-incomplete-feature=NewGenericSyntax - class C[T: D]: pass a: C[D] @@ -753,8 +692,6 @@ class D: pass class E(D): pass [case testPEP695UpperBoundForwardReference2] -# flags: --enable-incomplete-feature=NewGenericSyntax - type A = D class C[T: A]: pass @@ -769,8 +706,6 @@ reveal_type(b) # N: Revealed type is "__main__.C[__main__.E]" c: C[int] # E: Type argument "int" of "C" must be a subtype of "D" [case testPEP695UpperBoundForwardReference3] -# flags: --enable-incomplete-feature=NewGenericSyntax - class D[T]: pass class E[T](D[T]): pass @@ -788,8 +723,6 @@ reveal_type(b) # N: Revealed type is "__main__.C[__main__.E[__main__.X]]" c: C[D[int]] # E: Type argument "D[int]" of "C" must be a subtype of "D[X]" [case testPEP695UpperBoundForwardReference4] -# flags: --enable-incomplete-feature=NewGenericSyntax - def f[T: D](a: T) -> T: reveal_type(a.x) # N: Revealed type is "builtins.int" return a @@ -803,8 +736,6 @@ reveal_type(f(E())) # N: Revealed type is "__main__.E" f(1) # E: Value of type variable "T" of "f" cannot be "int" [case testPEP695UpperBoundUndefinedName] -# flags: --enable-incomplete-feature=NewGenericSyntax - class C[T: XX]: # E: Name "XX" is not defined pass @@ -815,8 +746,6 @@ def f[T: YY](x: T) -> T: # E: Name "YY" is not defined reveal_type(f) # N: Revealed type is "def [T <: Any] (x: T`-1) -> T`-1" [case testPEP695UpperBoundWithMultipleParams] -# flags: --enable-incomplete-feature=NewGenericSyntax - class C[T, S: int]: pass class D[A: int, B]: pass @@ -834,8 +763,6 @@ f('x', None) # E: Value of type variable "T" of "f" cannot be "str" \ # E: Value of type variable "S" of "f" cannot be "None" [case testPEP695InferVarianceOfTupleType] -# flags: --enable-incomplete-feature=NewGenericSyntax - class Cov[T](tuple[int, str]): def f(self) -> T: pass @@ -855,9 +782,7 @@ e: Contra[int] = Contra[object]() f: Contra[object] = Contra[int]() # E: Incompatible types in assignment (expression has type "Contra[int]", variable has type "Contra[object]") [builtins fixtures/tuple-simple.pyi] -[case testPEP695ValueRestiction] -# flags: --enable-incomplete-feature=NewGenericSyntax - +[case testPEP695ValueRestriction] def f[T: (int, str)](x: T) -> T: reveal_type(x) # N: Revealed type is "builtins.int" \ # N: Revealed type is "builtins.str" @@ -873,9 +798,7 @@ a: C[object] b: C[None] c: C[int] # E: Value of type variable "T" of "C" cannot be "int" -[case testPEP695ValueRestictionForwardReference] -# flags: --enable-incomplete-feature=NewGenericSyntax - +[case testPEP695ValueRestrictionForwardReference] class C[T: (int, D)]: def __init__(self, x: T) -> None: a = x @@ -891,9 +814,7 @@ class D: pass C(D()) -[case testPEP695ValueRestictionUndefinedName] -# flags: --enable-incomplete-feature=NewGenericSyntax - +[case testPEP695ValueRestrictionUndefinedName] class C[T: (int, XX)]: # E: Name "XX" is not defined pass @@ -901,7 +822,6 @@ def f[S: (int, YY)](x: S) -> S: # E: Name "YY" is not defined return x [case testPEP695ParamSpec] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import Callable def g[**P](f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: @@ -923,7 +843,6 @@ reveal_type(a.m) # N: Revealed type is "def (builtins.int, builtins.str)" [builtins fixtures/tuple.pyi] [case testPEP695ParamSpecTypeAlias] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import Callable type C[**P] = Callable[P, int] @@ -934,8 +853,6 @@ reveal_type(f) # N: Revealed type is "def (builtins.str, Union[builtins.int, No [typing fixtures/typing-full.pyi] [case testPEP695TypeVarTuple] -# flags: --enable-incomplete-feature=NewGenericSyntax - def f[*Ts](t: tuple[*Ts]) -> tuple[*Ts]: reveal_type(t) # N: Revealed type is "Tuple[Unpack[Ts`-1]]" return t @@ -955,7 +872,6 @@ b = c # E: Incompatible types in assignment (expression has type "C[str]", vari [builtins fixtures/tuple.pyi] [case testPEP695TypeVarTupleAlias] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import Callable type C[*Ts] = tuple[*Ts, int] @@ -965,7 +881,6 @@ reveal_type(a) # N: Revealed type is "Tuple[builtins.str, None, builtins.int]" [builtins fixtures/tuple.pyi] [case testPEP695IncrementalFunction] -# flags: --enable-incomplete-feature=NewGenericSyntax import a [file a.py] @@ -992,7 +907,6 @@ tmp/a.py:4: error: Value of type variable "T" of "g" cannot be "str" tmp/a.py:5: error: Value of type variable "S" of "g" cannot be "int" [case testPEP695IncrementalClass] -# flags: --enable-incomplete-feature=NewGenericSyntax import a [file a.py] @@ -1025,7 +939,6 @@ tmp/a.py:12: error: Value of type variable "S" of "D" cannot be "SS" tmp/a.py:13: error: Type argument "object" of "D" must be a subtype of "int" [case testPEP695IncrementalParamSpecAndTypeVarTuple] -# flags: --enable-incomplete-feature=NewGenericSyntax import a [file a.py] @@ -1049,7 +962,6 @@ class D[**P]: tmp/a.py:6: note: Revealed type is "def (builtins.int, builtins.str)" [case testPEP695IncrementalTypeAlias] -# flags: --enable-incomplete-feature=NewGenericSyntax import a [file a.py] @@ -1073,8 +985,6 @@ tmp/a.py:3: note: Revealed type is "builtins.str" tmp/a.py:5: note: Revealed type is "b.Foo[builtins.int]" [case testPEP695UndefinedNameInGenericFunction] -# mypy: enable-incomplete-feature=NewGenericSyntax - def f[T](x: T) -> T: return unknown() # E: Name "unknown" is not defined @@ -1083,7 +993,6 @@ class C: return unknown() # E: Name "unknown" is not defined [case testPEP695FunctionTypeVarAccessInFunction] -# mypy: enable-incomplete-feature=NewGenericSyntax from typing import cast class C: @@ -1095,8 +1004,6 @@ class C: reveal_type(C().m(1)) # N: Revealed type is "builtins.int" [case testPEP695ScopingBasics] -# mypy: enable-incomplete-feature=NewGenericSyntax - T = 1 def f[T](x: T) -> T: @@ -1113,8 +1020,6 @@ class C[T]: reveal_type(T) # N: Revealed type is "builtins.int" [case testPEP695ClassScoping] -# mypy: enable-incomplete-feature=NewGenericSyntax - class C: class D: pass @@ -1125,7 +1030,6 @@ C().m(C.D(), C.D()) C().m(1, C.D()) # E: Value of type variable "T" of "m" of "C" cannot be "int" [case testPEP695NestedGenericFunction] -# mypy: enable-incomplete-feature=NewGenericSyntax def f[T](x: T) -> T: reveal_type(f(x)) # N: Revealed type is "T`-1" reveal_type(f(1)) # N: Revealed type is "builtins.int" @@ -1149,7 +1053,6 @@ def f[T](x: T) -> T: return x [case testPEP695NonLocalAndGlobal] -# mypy: enable-incomplete-feature=NewGenericSyntax def f() -> None: T = 1 def g[T](x: T) -> T: @@ -1179,7 +1082,6 @@ class C[T]: return a [case testPEP695ArgumentDefault] -# mypy: enable-incomplete-feature=NewGenericSyntax from typing import cast def f[T]( @@ -1199,7 +1101,6 @@ class C: [typing fixtures/typing-full.pyi] [case testPEP695ListComprehension] -# mypy: enable-incomplete-feature=NewGenericSyntax from typing import cast def f[T](x: T) -> T: @@ -1208,8 +1109,6 @@ def f[T](x: T) -> T: return x [case testPEP695ReuseNameInSameScope] -# mypy: enable-incomplete-feature=NewGenericSyntax - class C[T]: def m[S](self, x: S, y: T) -> S | T: return x @@ -1231,7 +1130,6 @@ def g[T](x: T) -> T: return x [case testPEP695NestedScopingSpecialCases] -# mypy: enable-incomplete-feature=NewGenericSyntax # This is adapted from PEP 695 S = 0 @@ -1248,7 +1146,6 @@ def outer1[S]() -> None: global S [case testPEP695ScopingWithBaseClasses] -# mypy: enable-incomplete-feature=NewGenericSyntax # This is adapted from PEP 695 class Outer: class Private: @@ -1264,7 +1161,6 @@ class Outer: return a [case testPEP695RedefineTypeParameterInScope] -# mypy: enable-incomplete-feature=NewGenericSyntax class C[T]: def m[T](self, x: T) -> T: # E: "T" already defined as a type parameter return x @@ -1276,7 +1172,6 @@ def f[S, S](x: S) -> S: # E: "S" already defined as a type parameter return x [case testPEP695ClassDecorator] -# mypy: enable-incomplete-feature=NewGenericSyntax from typing import Any T = 0 @@ -1288,8 +1183,6 @@ class C[T]: pass [case testPEP695RecursiceTypeAlias] -# mypy: enable-incomplete-feature=NewGenericSyntax - type A = str | list[A] a: A reveal_type(a) # N: Revealed type is "Union[builtins.str, builtins.list[...]]" @@ -1302,8 +1195,6 @@ reveal_type(b) # N: Revealed type is "Union[__main__.C[builtins.int], builtins. [builtins fixtures/type.pyi] [case testPEP695BadRecursiveTypeAlias] -# mypy: enable-incomplete-feature=NewGenericSyntax - type A = A # E: Cannot resolve name "A" (possible cyclic definition) type B = B | int # E: Invalid recursive alias: a union item of itself a: A @@ -1314,8 +1205,6 @@ reveal_type(b) # N: Revealed type is "Any" [typing fixtures/typing-full.pyi] [case testPEP695RecursiveTypeAliasForwardReference] -# mypy: enable-incomplete-feature=NewGenericSyntax - def f(a: A) -> None: if isinstance(a, str): reveal_type(a) # N: Revealed type is "builtins.str" @@ -1334,7 +1223,6 @@ f(C[int]()) # E: Argument 1 to "f" has incompatible type "C[int]"; expected "A" [builtins fixtures/isinstance.pyi] [case testPEP695InvalidGenericOrProtocolBaseClass] -# mypy: enable-incomplete-feature=NewGenericSyntax from typing import Generic, Protocol, TypeVar S = TypeVar("S") @@ -1353,7 +1241,6 @@ class P2[T](Protocol[S]): # E: No arguments expected for "Protocol" base class pass [case testPEP695CannotUseTypeVarFromOuterClass] -# mypy: enable-incomplete-feature=NewGenericSyntax class ClassG[V]: # This used to crash class ClassD[T: dict[str, V]]: # E: Name "V" is not defined @@ -1361,7 +1248,6 @@ class ClassG[V]: [builtins fixtures/dict.pyi] [case testPEP695MixNewAndOldStyleGenerics] -# mypy: enable-incomplete-feature=NewGenericSyntax from typing import TypeVar S = TypeVar("S") @@ -1382,7 +1268,6 @@ class D[T](C[S]): # E: All type parameters should be declared ("S" not declared pass [case testPEP695MixNewAndOldStyleTypeVarTupleAndParamSpec] -# mypy: enable-incomplete-feature=NewGenericSyntax from typing import TypeVarTuple, ParamSpec, Callable Ts = TypeVarTuple("Ts") P = ParamSpec("P") @@ -1394,7 +1279,6 @@ def g[T](x: T, f: tuple[*Ts] # E: All type parameters should be declared ("Ts" [builtins fixtures/tuple.pyi] [case testPEP695MixNewAndOldStyleGenericsInTypeAlias] -# mypy: enable-incomplete-feature=NewGenericSyntax from typing import TypeVar, ParamSpec, TypeVarTuple, Callable T = TypeVar("T") @@ -1411,7 +1295,6 @@ type C = Callable[P, None] # E: All type parameters should be declared ("P" not [typing fixtures/typing-full.pyi] [case testPEP695NonGenericAliasToGenericClass] -# mypy: enable-incomplete-feature=NewGenericSyntax class C[T]: pass type A = C x: C @@ -1421,7 +1304,6 @@ reveal_type(y) # N: Revealed type is "__main__.C[Any]" z: A[int] # E: Bad number of arguments for type alias, expected 0, given 1 [case testPEP695SelfType] -# mypy: enable-incomplete-feature=NewGenericSyntax from typing import Self class C: @@ -1452,8 +1334,6 @@ reveal_type(F[str]().mm(b'x')) # N: Revealed type is "Tuple[__main__.F[builtins [builtins fixtures/tuple.pyi] [case testPEP695CallAlias] -# mypy: enable-incomplete-feature=NewGenericSyntax - class C: def __init__(self, x: str) -> None: ... type A = C @@ -1476,7 +1356,6 @@ B2[int]() [typing fixtures/typing-full.pyi] [case testPEP695IncrementalTypeAliasKinds] -# flags: --enable-incomplete-feature=NewGenericSyntax import a [file a.py] @@ -1499,7 +1378,6 @@ C: TypeAlias = int tmp/a.py:2: error: "TypeAliasType" not callable [case testPEP695TypeAliasBoundAndValueChecking] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import Any, cast class C: pass @@ -1540,8 +1418,6 @@ c4: A3[int, str] # E: Type argument "int" of "A3" must be a subtype of "C" [typing fixtures/typing-full.pyi] [case testPEP695TypeAliasInClassBodyOrFunction] -# flags: --enable-incomplete-feature=NewGenericSyntax - class C: type A = int type B[T] = list[T] | None @@ -1594,16 +1470,14 @@ reveal_type(E[str]().a) # N: Revealed type is "builtins.list[Any]" [typing fixtures/typing-full.pyi] [case testPEP695RedefineAsTypeAlias1] -# flags: --enable-incomplete-feature=NewGenericSyntax class C: pass -type C = int # E: Name "C" already defined on line 2 +type C = int # E: Name "C" already defined on line 1 A = 0 -type A = str # E: Name "A" already defined on line 5 +type A = str # E: Name "A" already defined on line 4 reveal_type(A) # N: Revealed type is "builtins.int" [case testPEP695RedefineAsTypeAlias2] -# flags: --enable-incomplete-feature=NewGenericSyntax from m import D type D = int # E: Name "D" already defined (possibly by an import) a: D @@ -1612,30 +1486,26 @@ reveal_type(a) # N: Revealed type is "m.D" class D: pass [case testPEP695RedefineAsTypeAlias3] -# flags: --enable-incomplete-feature=NewGenericSyntax D = list["Forward"] -type D = int # E: Name "D" already defined on line 2 +type D = int # E: Name "D" already defined on line 1 Forward = str x: D reveal_type(x) # N: Revealed type is "builtins.list[builtins.str]" [case testPEP695MultiDefinitionsForTypeAlias] -# flags: --enable-incomplete-feature=NewGenericSyntax if int(): type A[T] = list[T] else: - type A[T] = str # E: Name "A" already defined on line 3 + type A[T] = str # E: Name "A" already defined on line 2 x: T # E: Name "T" is not defined a: A[int] reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" [case testPEP695UndefinedNameInAnnotation] -# flags: --enable-incomplete-feature=NewGenericSyntax def f[T](x: foobar, y: T) -> T: ... # E: Name "foobar" is not defined reveal_type(f) # N: Revealed type is "def [T] (x: Any, y: T`-1) -> T`-1" [case testPEP695WrongNumberOfConstrainedTypes] -# flags: --enable-incomplete-feature=NewGenericSyntax type A[T: ()] = list[T] # E: Type variable must have at least two constrained types a: A[int] reveal_type(a) # N: Revealed type is "builtins.list[builtins.int]" @@ -1645,7 +1515,6 @@ b: B[str] reveal_type(b) # N: Revealed type is "builtins.list[builtins.str]" [case testPEP695UsingTypeVariableInOwnBoundOrConstraint] -# flags: --enable-incomplete-feature=NewGenericSyntax type A[T: list[T]] = str # E: Name "T" is not defined type B[S: (list[S], str)] = str # E: Name "S" is not defined type C[T, S: list[T]] = str # E: Name "T" is not defined @@ -1655,7 +1524,6 @@ class D[T: T]: # E: Name "T" is not defined pass [case testPEP695InvalidType] -# flags: --enable-incomplete-feature=NewGenericSyntax def f[T: 1](x: T) -> T: ... # E: Invalid type: try using Literal[1] instead? class C[T: (int, (1 + 2))]: pass # E: Invalid type comment or annotation type A = list[1] # E: Invalid type: try using Literal[1] instead? @@ -1666,7 +1534,6 @@ b: B reveal_type(b) # N: Revealed type is "Any" [case testPEP695GenericNamedTuple] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import NamedTuple # Invariant because of the signature of the generated _replace method @@ -1693,7 +1560,6 @@ e: M[bool] # E: Value of type variable "T" of "M" cannot be "bool" [builtins fixtures/tuple.pyi] [case testPEP695GenericTypedDict] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import TypedDict class D[T](TypedDict): @@ -1714,7 +1580,6 @@ d: E[int] # E: Type argument "int" of "E" must be a subtype of "str" [typing fixtures/typing-full.pyi] [case testCurrentClassWorksAsBound] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import Protocol class Comparable[T: Comparable](Protocol): @@ -1727,7 +1592,6 @@ x: Comparable[Good] y: Comparable[int] # E: Type argument "int" of "Comparable" must be a subtype of "Comparable[Any]" [case testPEP695TypeAliasWithDifferentTargetTypes] -# flags: --enable-incomplete-feature=NewGenericSyntax import types # We need GenericAlias from here, and test stubs don't bring in 'types' from typing import Any, Callable, List, Literal, TypedDict @@ -1779,7 +1643,7 @@ type I3 = None | C[TD] [typing fixtures/typing-full.pyi] [case testTypedDictInlineYesNewStyleAlias] -# flags: --enable-incomplete-feature=NewGenericSyntax --enable-incomplete-feature=InlineTypedDict +# flags: --enable-incomplete-feature=InlineTypedDict type X[T] = {"item": T, "other": X[T] | None} x: X[str] reveal_type(x) # N: Revealed type is "TypedDict({'item': builtins.str, 'other': Union[..., None]})" @@ -1791,8 +1655,6 @@ type Y[T] = {"item": T, **Y[T]} # E: Overwriting TypedDict field "item" while m [typing fixtures/typing-full.pyi] [case testPEP695UsingIncorrectExpressionsInTypeVariableBound] -# flags: --enable-incomplete-feature=NewGenericSyntax - type X[T: (yield 1)] = Any # E: Yield expression cannot be used as a type variable bound type Y[T: (yield from [])] = Any # E: Yield expression cannot be used as a type variable bound type Z[T: (a := 1)] = Any # E: Named expression cannot be used as a type variable bound @@ -1824,8 +1686,6 @@ def fooz_nested[T: (1 + (a := 1))](): pass # E: Named expression cannot be used def fook_nested[T: (1 +(await 1))](): pass # E: Await expression cannot be used as a type variable bound [case testPEP695UsingIncorrectExpressionsInTypeAlias] -# flags: --enable-incomplete-feature=NewGenericSyntax - type X = (yield 1) # E: Yield expression cannot be used within a type alias type Y = (yield from []) # E: Yield expression cannot be used within a type alias type Z = (a := 1) # E: Named expression cannot be used within a type alias @@ -1837,7 +1697,6 @@ type ZNested = (1 + (a := 1)) # E: Named expression cannot be used within a typ type KNested = (1 + (await 1)) # E: Await expression cannot be used within a type alias [case testPEP695TypeAliasAndAnnotated] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing_extensions import Annotated, Annotated as _Annotated import typing_extensions as t @@ -1859,7 +1718,6 @@ reveal_type(zz) # N: Revealed type is "builtins.str" [builtins fixtures/tuple.pyi] [case testPEP695NestedGenericClass1] -# flags: --enable-incomplete-feature=NewGenericSyntax class C[T]: def f(self) -> T: ... @@ -1882,7 +1740,6 @@ reveal_type(x) # N: Revealed type is "__main__.A.B[builtins.int]" reveal_type(A.b) # N: Revealed type is "__main__.A.B[builtins.str]" [case testPEP695NestedGenericClass2] -# flags: --enable-incomplete-feature=NewGenericSyntax class A: def m(self) -> None: class B[T]: @@ -1891,11 +1748,10 @@ class A: reveal_type(x.f()) # N: Revealed type is "builtins.int" self.a = B[str]() -reveal_type(A().a) # N: Revealed type is "__main__.B@4[builtins.str]" +reveal_type(A().a) # N: Revealed type is "__main__.B@3[builtins.str]" reveal_type(A().a.f()) # N: Revealed type is "builtins.str" [case testPEP695NestedGenericClass3] -# flags: --enable-incomplete-feature=NewGenericSyntax class C[T]: def f(self) -> T: ... class D[S]: @@ -1914,7 +1770,6 @@ class E[T]: c: E.F[int] [case testPEP695NestedGenericClass4] -# flags: --enable-incomplete-feature=NewGenericSyntax class A: class B[T]: def __get__(self, instance: A, owner: type[A]) -> T: @@ -1925,7 +1780,6 @@ a = A() v = a.f [case testPEP695VarianceInheritedFromBaseWithExplicitVariance] -# flags: --enable-incomplete-feature=NewGenericSyntax from typing import TypeVar, Generic T = TypeVar("T") diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index 3364dee6c696..84cea99bf2f6 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -1433,7 +1433,7 @@ class B(A): -> m [case testPEP695TypeAliasDeps] -# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +# flags: --python-version=3.12 from a import C, E type A = C type A2 = A diff --git a/test-data/unit/diff.test b/test-data/unit/diff.test index 9212d902e8b2..10fb0a80cf38 100644 --- a/test-data/unit/diff.test +++ b/test-data/unit/diff.test @@ -1532,7 +1532,7 @@ __main__.C.get_by_team_and_id __main__.Optional [case testPEP695TypeAlias] -# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +# flags: --python-version=3.12 from typing_extensions import TypeAlias, TypeAliasType type A = int type B = str @@ -1544,7 +1544,7 @@ G = TypeAliasType("G", int) type H = int [file next.py] -# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +# flags: --python-version=3.12 from typing_extensions import TypeAlias, TypeAliasType type A = str type B = str @@ -1566,7 +1566,7 @@ __main__.G __main__.H [case testPEP695TypeAlias2] -# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +# flags: --python-version=3.12 type A[T: int] = list[T] type B[T: int] = list[T] type C[T: (int, str)] = list[T] @@ -1575,7 +1575,7 @@ type E[T: int] = list[T] type F[T: (int, str)] = list[T] [file next.py] -# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +# flags: --python-version=3.12 type A[T] = list[T] type B[T: str] = list[T] type C[T: (int, None)] = list[T] @@ -1590,13 +1590,13 @@ __main__.C __main__.D [case testPEP695GenericFunction] -# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +# flags: --python-version=3.12 def f[T](x: T) -> T: return x def g[T](x: T, y: T) -> T: return x [file next.py] -# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +# flags: --python-version=3.12 def f[T](x: T) -> T: return x def g[T, S](x: T, y: S) -> S: @@ -1605,7 +1605,7 @@ def g[T, S](x: T, y: S) -> S: __main__.g [case testPEP695GenericClass] -# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +# flags: --python-version=3.12 class C[T]: pass class D[T]: @@ -1615,7 +1615,7 @@ class E[T]: class F[T]: def f(self, x: object) -> T: ... [file next.py] -# flags: --enable-incomplete-feature=NewGenericSyntax --python-version=3.12 +# flags: --python-version=3.12 class C[T]: pass class D[T: int]: diff --git a/test-data/unit/fine-grained-python312.test b/test-data/unit/fine-grained-python312.test index 80e4e4ca3bd8..0e438ca06574 100644 --- a/test-data/unit/fine-grained-python312.test +++ b/test-data/unit/fine-grained-python312.test @@ -1,5 +1,4 @@ [case testPEP695TypeAliasDep] -# flags: --enable-incomplete-feature=NewGenericSyntax import m def g() -> m.C: return m.f() @@ -15,10 +14,9 @@ def f() -> int: pass [out] == -main:4: error: Incompatible return value type (got "int", expected "str") +main:3: error: Incompatible return value type (got "int", expected "str") [case testPEP695ChangeOldStyleToNewStyleTypeAlias] -# flags: --enable-incomplete-feature=NewGenericSyntax from m import A A() @@ -31,10 +29,9 @@ type A = int [builtins fixtures/tuple.pyi] [out] == -main:3: error: "TypeAliasType" not callable +main:2: error: "TypeAliasType" not callable [case testPEP695VarianceChangesDueToDependency] -# flags: --enable-incomplete-feature=NewGenericSyntax from a import C x: C[object] = C[int]() @@ -55,10 +52,9 @@ class A[T]: [out] == -main:4: error: Incompatible types in assignment (expression has type "C[int]", variable has type "C[object]") +main:3: error: Incompatible types in assignment (expression has type "C[int]", variable has type "C[object]") [case testPEP695TypeAliasChangesDueToDependency] -# flags: --enable-incomplete-feature=NewGenericSyntax from a import A x: A x = 0 @@ -78,11 +74,10 @@ from builtins import tuple as B [typing fixtures/typing-full.pyi] [out] == -main:4: error: Incompatible types in assignment (expression has type "int", variable has type "tuple[int, str]") -main:5: error: Incompatible types in assignment (expression has type "str", variable has type "tuple[int, str]") +main:3: error: Incompatible types in assignment (expression has type "int", variable has type "Tuple[int, str]") +main:4: error: Incompatible types in assignment (expression has type "str", variable has type "Tuple[int, str]") [case testPEP695NestedGenericClassMethodUpdated] -# flags: --enable-incomplete-feature=NewGenericSyntax from a import f class C: @@ -99,4 +94,4 @@ def f() -> None: pass def f(x: int) -> None: pass [out] == -main:8: error: Missing positional argument "x" in call to "f" +main:7: error: Missing positional argument "x" in call to "f" diff --git a/test-data/unit/parse-python312.test b/test-data/unit/parse-python312.test index 90ee96f38deb..2b1f9b42e0f7 100644 --- a/test-data/unit/parse-python312.test +++ b/test-data/unit/parse-python312.test @@ -1,5 +1,5 @@ [case testPEP695TypeAlias] -# mypy: enable-incomplete-feature=NewGenericSyntax +# comment type A[T] = C[T] [out] MypyFile:1( @@ -15,7 +15,7 @@ MypyFile:1( NameExpr(T))))))) [case testPEP695GenericFunction] -# mypy: enable-incomplete-feature=NewGenericSyntax +# comment def f[T](): pass def g[T: str](): pass @@ -46,7 +46,7 @@ MypyFile:1( PassStmt:5()))) [case testPEP695ParamSpec] -# mypy: enable-incomplete-feature=NewGenericSyntax +# comment def f[**P](): pass class C[T: int, **P]: pass @@ -68,7 +68,7 @@ MypyFile:1( PassStmt:4())) [case testPEP695TypeVarTuple] -# mypy: enable-incomplete-feature=NewGenericSyntax +# comment def f[*Ts](): pass class C[T: int, *Ts]: pass diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index dbf228623d7c..26fc419030bf 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -2099,7 +2099,7 @@ def f(d: Description) -> None: _testDataclassStrictOptionalAlwaysSet.py:9: note: Revealed type is "def (Union[builtins.int, None]) -> Union[builtins.str, None]" [case testPEP695VarianceInference] -# flags: --python-version=3.12 --enable-incomplete-feature=NewGenericSyntax +# flags: --python-version=3.12 from typing import Callable, Final class Job[_R_co]: @@ -2120,7 +2120,7 @@ def func( _testPEP695VarianceInference.py:17: error: Incompatible types in assignment (expression has type "Job[None]", variable has type "Job[int]") [case testPEP695TypeAliasWithDifferentTargetTypes] -# flags: --python-version=3.12 --enable-incomplete-feature=NewGenericSyntax +# flags: --python-version=3.12 from typing import Any, Callable, List, Literal, TypedDict, overload, TypeAlias, TypeVar, Never class C[T]: pass From 94109aaabf87be093200b19760a8e75318c62f9d Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sat, 21 Sep 2024 18:13:18 -0400 Subject: [PATCH 145/247] Fix TypeVar upper bounds sometimes not being displayed in pretty callables (#17802) Fixes #17792. Related to #17791. Currently, [`pretty_callable`](https://github.com/python/mypy/blob/5dfc7d941253553ab77836e9845cb8fdfb9d23a9/mypy/messages.py#L2862) only renders `TypeVar` upper bounds if they are of type `Instance`: https://github.com/python/mypy/blob/5dfc7d941253553ab77836e9845cb8fdfb9d23a9/mypy/messages.py#L2943-L2949 However, there are some types that can appear as `TypeVar` upper bounds which are not represented by `Instance`, such as `UnionType` and `CallableType`. This PR allows such non-`Instance` upper bounds to be rendered as well. ## Effect Consider the below code. Playground link: https://mypy-play.net/?mypy=1.11.2&python=3.12&enable-incomplete-feature=NewGenericSyntax&gist=ba30c820cc3668e0919dadf2f391ff4b ```python from collections.abc import Callable from typing import Any, overload ### No matching overloads @overload def f1[T: int](x: T) -> T: ... @overload def f1[T: Callable[..., None]](x: T) -> T: ... @overload def f1[T: tuple[int]](x: T) -> T: ... @overload def f1[T: None](x: T) -> T: ... @overload def f1[T: type[int]](x: T) -> T: ... @overload def f1[T: bytes | bytearray](x: T) -> T: ... def f1(x): return x f1(1.23) ### Mismatching conditional definitions if input(): def f2[T](x: T) -> T: return x else: def f2[T: Callable[..., None]](x: T) -> T: return x ``` ### Before * In the first error on line 20, all overloads aside from the first one are displayed as `def [T] f1(x: T) -> T` (upper bound missing). Duplicate entries are suppressed. * In the second error on line 28, the second definition is displayed as `def [T] f2(x: T) -> T` (upper bound missing), and is removed as an apparent duplicate of the first. ```none main.py:20: error: No overload variant of "f1" matches argument type "float" [call-overload] main.py:20: note: Possible overload variants: main.py:20: note: def [T: int] f1(x: T) -> T main.py:20: note: def [T] f1(x: T) -> T main.py:28: error: All conditional function variants must have identical signatures [misc] main.py:28: note: Original: main.py:28: note: def [T] f2(x: T) -> T main.py:28: note: Redefinition: Found 2 errors in 1 file (checked 1 source file) ``` ### After * All type var upper bounds are rendered. ```none main.py:20: error: No overload variant of "f1" matches argument type "float" [call-overload] main.py:20: note: Possible overload variants: main.py:20: note: def [T: int] f1(x: T) -> T main.py:20: note: def [T: Callable[..., None]] f1(x: T) -> T main.py:20: note: def [T: tuple[int]] f1(x: T) -> T main.py:20: note: def [T: None] f1(x: T) -> T main.py:20: note: def [T: type[int]] f1(x: T) -> T main.py:20: note: def [T: bytes | bytearray] f1(x: T) -> T main.py:28: error: All conditional function variants must have identical signatures [misc] main.py:28: note: Original: main.py:28: note: def [T] f2(x: T) -> T main.py:28: note: Redefinition: main.py:28: note: def [T: Callable[..., None]] f2(x: T) -> T Found 2 errors in 1 file (checked 1 source file) ``` --- mypy/messages.py | 4 ++-- test-data/unit/check-functions.test | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index dadce149680e..6567d9d96d0b 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2942,9 +2942,9 @@ def [T <: int] f(self, x: int, y: T) -> None for tvar in tp.variables: if isinstance(tvar, TypeVarType): upper_bound = get_proper_type(tvar.upper_bound) - if ( + if not ( isinstance(upper_bound, Instance) - and upper_bound.type.fullname != "builtins.object" + and upper_bound.type.fullname == "builtins.object" ): tvars.append(f"{tvar.name}: {format_type_bare(upper_bound, options)}") elif tvar.values: diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index b681c544a6bd..96f9815019e6 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1428,6 +1428,20 @@ else: # N: Redefinition: \ # N: def f(x: int = ...) -> None +[case testIncompatibleConditionalFunctionDefinition4] +from typing import Any, Union, TypeVar +T1 = TypeVar('T1') +T2 = TypeVar('T2', bound=Union[int, str]) +x = None # type: Any +if x: + def f(x: T1) -> T1: pass +else: + def f(x: T2) -> T2: pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def [T1] f(x: T1) -> T1 \ + # N: Redefinition: \ + # N: def [T2: Union[int, str]] f(x: T2) -> T2 + [case testConditionalFunctionDefinitionUsingDecorator1] from typing import Callable From 9ffb9ddf80fbf92956a55946c8f023fc09638959 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sat, 21 Sep 2024 19:45:25 -0400 Subject: [PATCH 146/247] Fix crash when passing too many type arguments to generic base class accepting single ParamSpec (#17770) Fixes #17765 The offender for this crash appears to be this snippet: https://github.com/python/mypy/blob/72c413d2352da5ce1433ef241faca8f40fa1fe27/mypy/semanal.py#L5905-L5910 This branch triggers when applying type args to a type that is generic with respect to a single `ParamSpec`. It allows double brackets to be omitted when providing a parameter specification by wrapping all of the provided type arguments into a single parameter specification argument (i.e. equating `Foo[int, int]` to `Foo[[int, int]]`). This wrapping occurs *unless*: * there is only a single type argument, and it resolves to `Any` (e.g. `Foo[Any]`) * **there is only a single type argument**, and it's a bracketed parameter specification or a `ParamSpec` (e.g. `Foo[[int, int]]`) The problem occurs when multiple type arguments provided and at least one of them is a bracketed parameter specification, as in `Foo[[int, int], str]`. Per the rules above, since there is more than 1 type argument, mypy attempts to wrap the arguments into a single parameter specification. This results in the attempted creation of a `Parameters` instance that contains another `Parameters` instance, which triggers this assert inside `Parameters.__init__`: https://github.com/python/mypy/blob/72c413d2352da5ce1433ef241faca8f40fa1fe27/mypy/types.py#L1634 I think a reasonable solution is to forgo wrapping the type arguments into a single `Parameters` if **any** of the provided type arguments are a `Parameters`/`ParamSpecType`. That is, don't transform `Foo[A1, A2, ...]` to `Foo[[A1, A2, ...]]` if any of `A1, A2, ...` are a parameter specification. This change brings the crash case inline with mypy's current behavior for a similar case: ```python # Current behavior P = ParamSpec("P") class C(Generic[P]): ... c: C[int, [int, str], str] # E: Nested parameter specifications are not allowed ``` Before this change: ```python P = ParamSpec("P") class C(Generic[P]): ... class D(C[int, [int, str], str]): ... # !!! CRASH !!! ``` After this change: ```python P = ParamSpec("P") class C(Generic[P]): ... class D(C[int, [int, str], str]): ... # E: Nested parameter specifications are not allowed ```` --- mypy/semanal.py | 5 ++--- test-data/unit/check-parameter-specification.test | 9 +++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index e239fbf1f141..0b654d6b145f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5926,9 +5926,8 @@ def analyze_type_application_args(self, expr: IndexExpr) -> list[Type] | None: if has_param_spec and num_args == 1 and types: first_arg = get_proper_type(types[0]) - if not ( - len(types) == 1 and isinstance(first_arg, (Parameters, ParamSpecType, AnyType)) - ): + single_any = len(types) == 1 and isinstance(first_arg, AnyType) + if not (single_any or any(isinstance(t, (Parameters, ParamSpecType)) for t in types)): types = [Parameters(types, [ARG_POS] * len(types), [None] * len(types))] return types diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index c2afb61586a8..654f36775172 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1836,6 +1836,15 @@ c: C[int, [int, str], str] # E: Nested parameter specifications are not allowed reveal_type(c) # N: Revealed type is "__main__.C[Any]" [builtins fixtures/paramspec.pyi] +[case testParamSpecInheritNoCrashOnNested] +from typing import Generic +from typing_extensions import ParamSpec + +P = ParamSpec("P") +class C(Generic[P]): ... +class D(C[int, [int, str], str]): ... # E: Nested parameter specifications are not allowed +[builtins fixtures/paramspec.pyi] + [case testParamSpecConcatenateSelfType] from typing import Callable from typing_extensions import ParamSpec, Concatenate From cf3db994c1288a4b3ba445a0d11b85cf119a50a3 Mon Sep 17 00:00:00 2001 From: Michael I Chen Date: Tue, 24 Sep 2024 05:24:41 +0100 Subject: [PATCH 147/247] Copyedit final_attrs.rst (#17813) --- docs/source/final_attrs.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/source/final_attrs.rst b/docs/source/final_attrs.rst index 297b97eca787..81bfba650430 100644 --- a/docs/source/final_attrs.rst +++ b/docs/source/final_attrs.rst @@ -25,8 +25,8 @@ Final names You can use the ``typing.Final`` qualifier to indicate that a name or attribute should not be reassigned, redefined, or -overridden. This is often useful for module and class level constants -as a way to prevent unintended modification. Mypy will prevent +overridden. This is often useful for module and class-level +constants to prevent unintended modification. Mypy will prevent further assignments to final names in type-checked code: .. code-block:: python @@ -70,7 +70,7 @@ You can use ``Final`` in one of these forms: ID: Final[int] = 1 - Here mypy will infer type ``int`` for ``ID``. + Here, mypy will infer type ``int`` for ``ID``. * You can omit the type: @@ -78,15 +78,15 @@ You can use ``Final`` in one of these forms: ID: Final = 1 - Here mypy will infer type ``Literal[1]`` for ``ID``. Note that unlike for - generic classes this is *not* the same as ``Final[Any]``. + Here, mypy will infer type ``Literal[1]`` for ``ID``. Note that unlike for + generic classes, this is *not* the same as ``Final[Any]``. -* In class bodies and stub files you can omit the right hand side and just write +* In class bodies and stub files, you can omit the right-hand side and just write ``ID: Final[int]``. * Finally, you can write ``self.id: Final = 1`` (also optionally with a type in square brackets). This is allowed *only* in - :py:meth:`__init__ ` methods, so that the final instance attribute is + :py:meth:`__init__ ` methods so the final instance attribute is assigned only once when an instance is created. Details of using ``Final`` @@ -129,7 +129,7 @@ the scope of a final declaration automatically depending on whether it was initialized in the class body or in :py:meth:`__init__ `. A final attribute can't be overridden by a subclass (even with another -explicit final declaration). Note however that a final attribute can +explicit final declaration). Note, however, that a final attribute can override a read-only property: .. code-block:: python @@ -176,12 +176,12 @@ overriding. You can use the ``typing.final`` decorator for this purpose: This ``@final`` decorator can be used with instance methods, class methods, static methods, and properties. -For overloaded methods you should add ``@final`` on the implementation +For overloaded methods, you should add ``@final`` on the implementation to make it final (or on the first overload in stubs): .. code-block:: python - from typing import Any, overload + from typing import final, overload class Base: @overload @@ -224,7 +224,7 @@ Here are some situations where using a final class may be useful: An abstract class that defines at least one abstract method or property and has ``@final`` decorator will generate an error from -mypy, since those attributes could never be implemented. +mypy since those attributes could never be implemented. .. code-block:: python From 9518b6a96778b72f2f90dbd0edf7ae94837757d5 Mon Sep 17 00:00:00 2001 From: Stanislav Terliakov <50529348+sterliakov@users.noreply.github.com> Date: Tue, 24 Sep 2024 08:26:32 +0200 Subject: [PATCH 148/247] Reject ParamSpec-typed callables calls with insufficient arguments (#17323) Fixes #14571. When type checking a call of a `ParamSpec`-typed callable, currently there is an incorrect "fast path" (if there are two arguments of shape `(*args: P.args, **kwargs: P.kwargs)`, accept), which breaks with `Concatenate` (such call was accepted even for `Concatenate[int, P]`). Also there was no checking that args and kwargs are actually present: since `*args` and `**kwargs` are not required, their absence was silently accepted. --- mypy/checkexpr.py | 23 +++- .../unit/check-parameter-specification.test | 111 ++++++++++++++++++ 2 files changed, 128 insertions(+), 6 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 22595c85e702..55c42335744d 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1756,7 +1756,11 @@ def check_callable_call( ) param_spec = callee.param_spec() - if param_spec is not None and arg_kinds == [ARG_STAR, ARG_STAR2]: + if ( + param_spec is not None + and arg_kinds == [ARG_STAR, ARG_STAR2] + and len(formal_to_actual) == 2 + ): arg1 = self.accept(args[0]) arg2 = self.accept(args[1]) if ( @@ -2362,6 +2366,9 @@ def check_argument_count( # Positional argument when expecting a keyword argument. self.msg.too_many_positional_arguments(callee, context) ok = False + elif callee.param_spec() is not None and not formal_to_actual[i]: + self.msg.too_few_arguments(callee, context, actual_names) + ok = False return ok def check_for_extra_actual_arguments( @@ -2763,9 +2770,9 @@ def plausible_overload_call_targets( ) -> list[CallableType]: """Returns all overload call targets that having matching argument counts. - If the given args contains a star-arg (*arg or **kwarg argument), this method - will ensure all star-arg overloads appear at the start of the list, instead - of their usual location. + If the given args contains a star-arg (*arg or **kwarg argument, including + ParamSpec), this method will ensure all star-arg overloads appear at the start + of the list, instead of their usual location. The only exception is if the starred argument is something like a Tuple or a NamedTuple, which has a definitive "shape". If so, we don't move the corresponding @@ -2793,9 +2800,13 @@ def has_shape(typ: Type) -> bool: formal_to_actual = map_actuals_to_formals( arg_kinds, arg_names, typ.arg_kinds, typ.arg_names, lambda i: arg_types[i] ) - with self.msg.filter_errors(): - if self.check_argument_count( + if typ.param_spec() is not None: + # ParamSpec can be expanded in a lot of different ways. We may try + # to expand it here instead, but picking an impossible overload + # is safe: it will be filtered out later. + star_matches.append(typ) + elif self.check_argument_count( typ, arg_types, arg_kinds, arg_names, formal_to_actual, None ): if args_have_var_arg and typ.is_var_arg: diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 654f36775172..38fb62fe78e0 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -2192,3 +2192,114 @@ parametrize(_test, Case(1, b=2), Case(3, b=4)) parametrize(_test, Case(1, 2), Case(3)) parametrize(_test, Case(1, 2), Case(3, b=4)) [builtins fixtures/paramspec.pyi] + +[case testRunParamSpecInsufficientArgs] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable + +_P = ParamSpec("_P") + +def run(predicate: Callable[_P, None], *args: _P.args, **kwargs: _P.kwargs) -> None: # N: "run" defined here + predicate() # E: Too few arguments + predicate(*args) # E: Too few arguments + predicate(**kwargs) # E: Too few arguments + predicate(*args, **kwargs) + +def fn() -> None: ... +def fn_args(x: int) -> None: ... +def fn_posonly(x: int, /) -> None: ... + +run(fn) +run(fn_args, 1) +run(fn_args, x=1) +run(fn_posonly, 1) +run(fn_posonly, x=1) # E: Unexpected keyword argument "x" for "run" + +[builtins fixtures/paramspec.pyi] + +[case testRunParamSpecConcatenateInsufficientArgs] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable + +_P = ParamSpec("_P") + +def run(predicate: Callable[Concatenate[int, _P], None], *args: _P.args, **kwargs: _P.kwargs) -> None: # N: "run" defined here + predicate() # E: Too few arguments + predicate(1) # E: Too few arguments + predicate(1, *args) # E: Too few arguments + predicate(1, *args) # E: Too few arguments + predicate(1, **kwargs) # E: Too few arguments + predicate(*args, **kwargs) # E: Argument 1 has incompatible type "*_P.args"; expected "int" + predicate(1, *args, **kwargs) + +def fn() -> None: ... +def fn_args(x: int, y: str) -> None: ... +def fn_posonly(x: int, /) -> None: ... +def fn_posonly_args(x: int, /, y: str) -> None: ... + +run(fn) # E: Argument 1 to "run" has incompatible type "Callable[[], None]"; expected "Callable[[int], None]" +run(fn_args, 1, 'a') # E: Too many arguments for "run" \ + # E: Argument 2 to "run" has incompatible type "int"; expected "str" +run(fn_args, y='a') +run(fn_args, 'a') +run(fn_posonly) +run(fn_posonly, x=1) # E: Unexpected keyword argument "x" for "run" +run(fn_posonly_args) # E: Missing positional argument "y" in call to "run" +run(fn_posonly_args, 'a') +run(fn_posonly_args, y='a') + +[builtins fixtures/paramspec.pyi] + +[case testRunParamSpecConcatenateInsufficientArgsInDecorator] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable + +P = ParamSpec("P") + +def decorator(fn: Callable[Concatenate[str, P], None]) -> Callable[P, None]: + def inner(*args: P.args, **kwargs: P.kwargs) -> None: + fn("value") # E: Too few arguments + fn("value", *args) # E: Too few arguments + fn("value", **kwargs) # E: Too few arguments + fn(*args, **kwargs) # E: Argument 1 has incompatible type "*P.args"; expected "str" + fn("value", *args, **kwargs) + return inner + +@decorator +def foo(s: str, s2: str) -> None: ... + +[builtins fixtures/paramspec.pyi] + +[case testRunParamSpecOverload] +from typing_extensions import ParamSpec +from typing import Callable, NoReturn, TypeVar, Union, overload + +P = ParamSpec("P") +T = TypeVar("T") + +@overload +def capture( + sync_fn: Callable[P, NoReturn], + *args: P.args, + **kwargs: P.kwargs, +) -> int: ... +@overload +def capture( + sync_fn: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, +) -> Union[T, int]: ... +def capture( + sync_fn: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, +) -> Union[T, int]: + return sync_fn(*args, **kwargs) + +def fn() -> str: return '' +def err() -> NoReturn: ... + +reveal_type(capture(fn)) # N: Revealed type is "Union[builtins.str, builtins.int]" +reveal_type(capture(err)) # N: Revealed type is "builtins.int" + +[builtins fixtures/paramspec.pyi] From ecfab6a02415c46eda5717ec6ee9bfac8115c1e9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 24 Sep 2024 15:30:23 +0100 Subject: [PATCH 149/247] [PEP 695] Allow Self return types with contravariance (#17786) Fix variance inference in this fragment from a typing conformance test: ``` class ClassA[T1, T2, T3](list[T1]): def method1(self, a: T2) -> None: ... def method2(self) -> T3: ... ``` Previously T2 was incorrectly inferred as invariant due to `list` having methods that return `Self`. Be more flexible with return types to allow inferring contravariance for type variables even if there are `Self` return types, in particular. We could probably make this even more lenient, but after thinking about this for a while, I wasn't sure what the most general rule would be, so I decided to just make a tweak to support the likely most common use case (which is probably actually not that common either). Link to conformance test: https://github.com/python/typing/blob/main/conformance/tests/generics_variance_inference.py#L15C1-L20C12 --- mypy/subtypes.py | 27 ++++++++++++++ test-data/unit/check-python312.test | 58 ++++++++++++++++++++++++++++- test-data/unit/pythoneval.test | 20 ++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index df040dcb1311..c76b3569fdd4 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2024,6 +2024,16 @@ def infer_variance(info: TypeInfo, i: int) -> bool: typ = find_member(member, self_type, self_type) if typ: + # It's okay for a method in a generic class with a contravariant type + # variable to return a generic instance of the class, if it doesn't involve + # variance (i.e. values of type variables are propagated). Our normal rules + # would disallow this. Replace such return types with 'Any' to allow this. + # + # This could probably be more lenient (e.g. allow self type be nested, don't + # require all type arguments to be identical to self_type), but this will + # hopefully cover the vast majority of such cases, including Self. + typ = erase_return_self_types(typ, self_type) + typ2 = expand_type(typ, {tvar.id: object_type}) if not is_subtype(typ, typ2): co = False @@ -2066,3 +2076,20 @@ def infer_class_variances(info: TypeInfo) -> bool: if not infer_variance(info, i): success = False return success + + +def erase_return_self_types(typ: Type, self_type: Instance) -> Type: + """If a typ is function-like and returns self_type, replace return type with Any.""" + proper_type = get_proper_type(typ) + if isinstance(proper_type, CallableType): + ret = get_proper_type(proper_type.ret_type) + if isinstance(ret, Instance) and ret == self_type: + return proper_type.copy_modified(ret_type=AnyType(TypeOfAny.implementation_artifact)) + elif isinstance(proper_type, Overloaded): + return Overloaded( + [ + cast(CallableType, erase_return_self_types(it, self_type)) + for it in proper_type.items + ] + ) + return typ diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 2f0b912c439e..085cc052705d 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -213,7 +213,7 @@ b: Invariant[int] if int(): a = b # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") if int(): - b = a # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") + b = a c: Covariant[object] d: Covariant[int] @@ -393,6 +393,62 @@ inv3_1: Invariant3[float] = Invariant3[int](1) # E: Incompatible types in assig inv3_2: Invariant3[int] = Invariant3[float](1) # E: Incompatible types in assignment (expression has type "Invariant3[float]", variable has type "Invariant3[int]") [builtins fixtures/property.pyi] +[case testPEP695InferVarianceWithInheritedSelf] +from typing import overload, Self, TypeVar, Generic + +T = TypeVar("T") +S = TypeVar("S") + +class C(Generic[T]): + def f(self, x: T) -> Self: ... + def g(self) -> T: ... + +class D[T1, T2](C[T1]): + def m(self, x: T2) -> None: ... + +a1: D[int, int] = D[int, object]() +a2: D[int, object] = D[int, int]() # E: Incompatible types in assignment (expression has type "D[int, int]", variable has type "D[int, object]") +a3: D[int, int] = D[object, object]() # E: Incompatible types in assignment (expression has type "D[object, object]", variable has type "D[int, int]") +a4: D[object, int] = D[int, object]() # E: Incompatible types in assignment (expression has type "D[int, object]", variable has type "D[object, int]") + +[case testPEP695InferVarianceWithReturnSelf] +from typing import Self, overload + +class Cov[T]: + def f(self) -> Self: ... + +a1: Cov[int] = Cov[float]() # E: Incompatible types in assignment (expression has type "Cov[float]", variable has type "Cov[int]") +a2: Cov[float] = Cov[int]() + +class Contra[T]: + def f(self) -> Self: ... + def g(self, x: T) -> None: ... + +b1: Contra[int] = Contra[float]() +b2: Contra[float] = Contra[int]() # E: Incompatible types in assignment (expression has type "Contra[int]", variable has type "Contra[float]") + +class Cov2[T]: + @overload + def f(self, x): ... + @overload + def f(self) -> Self: ... + def f(self, x=None): ... + +c1: Cov2[int] = Cov2[float]() # E: Incompatible types in assignment (expression has type "Cov2[float]", variable has type "Cov2[int]") +c2: Cov2[float] = Cov2[int]() + +class Contra2[T]: + @overload + def f(self, x): ... + @overload + def f(self) -> Self: ... + def f(self, x=None): ... + + def g(self, x: T) -> None: ... + +d1: Contra2[int] = Contra2[float]() +d2: Contra2[float] = Contra2[int]() # E: Incompatible types in assignment (expression has type "Contra2[int]", variable has type "Contra2[float]") + [case testPEP695InheritInvariant] class Invariant[T]: x: T diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 26fc419030bf..89f01bff963e 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -2196,3 +2196,23 @@ type K4 = None | B[int] type L1 = Never type L2 = list[Never] + +[case testPEP695VarianceInferenceSpecialCaseWithTypeshed] +# flags: --python-version=3.12 +class C1[T1, T2](list[T1]): + def m(self, a: T2) -> None: ... + +def func1(p: C1[int, object]): + x: C1[int, int] = p + +class C2[T1, T2, T3](dict[T2, T3]): + def m(self, a: T1) -> None: ... + +def func2(p: C2[object, int, int]): + x: C2[int, int, int] = p + +class C3[T1, T2](tuple[T1, ...]): + def m(self, a: T2) -> None: ... + +def func3(p: C3[int, object]): + x: C3[int, int] = p From 5c5e0511116868a893b13052d4e2a8b6837bb9f0 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:07:37 -0700 Subject: [PATCH 150/247] Test against latest Python 3.13, make testing 3.14 easy (#17812) --- .github/workflows/test.yml | 82 +++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 24 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01d5876635b0..6e29feef7b00 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,6 +71,22 @@ jobs: tox_extra_args: "-n 4" test_mypyc: true + - name: Test suit with py313-dev-ubuntu, mypyc-compiled + python: '3.13-dev' + arch: x64 + os: ubuntu-latest + toxenv: py + tox_extra_args: "-n 4" + test_mypyc: true + # - name: Test suit with py314-dev-ubuntu + # python: '3.14-dev' + # arch: x64 + # os: ubuntu-latest + # toxenv: py + # tox_extra_args: "-n 4" + # allow_failure: true + # test_mypyc: true + - name: mypyc runtime tests with py39-macos python: '3.9.18' arch: x64 @@ -119,12 +135,10 @@ jobs: MYPY_FORCE_TERMINAL_WIDTH: 200 # Pytest PYTEST_ADDOPTS: --color=yes + steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - architecture: ${{ matrix.arch }} + - name: Debug build if: ${{ matrix.debug_build }} run: | @@ -132,38 +146,58 @@ jobs: PYTHONDIR=~/python-debug/python-$PYTHONVERSION VENV=$PYTHONDIR/env ./misc/build-debug-python.sh $PYTHONVERSION $PYTHONDIR $VENV + # TODO: does this do anything? env vars aren't passed to the next step right source $VENV/bin/activate + - name: Latest Dev build + if: ${{ endsWith(matrix.python, '-dev') }} + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential gdb lcov libbz2-dev libffi-dev libgdbm-dev liblzma-dev libncurses5-dev \ + libreadline6-dev libsqlite3-dev libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g-dev + git clone --depth 1 https://github.com/python/cpython.git /tmp/cpython --branch $( echo ${{ matrix.python }} | sed 's/-dev//' ) + cd /tmp/cpython + echo git rev-parse HEAD; git rev-parse HEAD + git show --no-patch + ./configure --prefix=/opt/pythondev + make -j$(nproc) + sudo make install + sudo ln -s /opt/pythondev/bin/python3 /opt/pythondev/bin/python + sudo ln -s /opt/pythondev/bin/pip3 /opt/pythondev/bin/pip + echo "/opt/pythondev/bin" >> $GITHUB_PATH + - uses: actions/setup-python@v5 + if: ${{ !(matrix.debug_build || endsWith(matrix.python, '-dev')) }} + with: + python-version: ${{ matrix.python }} + architecture: ${{ matrix.arch }} + - name: Install tox - run: pip install setuptools==68.2.2 tox==4.11.0 + run: | + echo PATH; echo $PATH + echo which python; which python + echo which pip; which pip + echo python version; python -c 'import sys; print(sys.version)' + echo debug build; python -c 'import sysconfig; print(bool(sysconfig.get_config_var("Py_DEBUG")))' + echo os.cpu_count; python -c 'import os; print(os.cpu_count())' + echo os.sched_getaffinity; python -c 'import os; print(len(getattr(os, "sched_getaffinity", lambda *args: [])(0)))' + pip install setuptools==68.2.2 tox==4.11.0 + - name: Compiled with mypyc if: ${{ matrix.test_mypyc }} run: | pip install -r test-requirements.txt CC=clang MYPYC_OPT_LEVEL=0 MYPY_USE_MYPYC=1 pip install -e . + - name: Setup tox environment run: | - tox run -e ${{ matrix.toxenv }} --notest - python -c 'import os; print("os.cpu_count", os.cpu_count(), "os.sched_getaffinity", len(getattr(os, "sched_getaffinity", lambda *args: [])(0)))' + tox run -e ${{ matrix.toxenv }} --notes - name: Test run: tox run -e ${{ matrix.toxenv }} --skip-pkg-install -- ${{ matrix.tox_extra_args }} + continue-on-error: ${{ matrix.allow_failure == 'true' }} - python-nightly: - runs-on: ubuntu-latest - name: Test suite with Python nightly - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.13-dev' - - name: Install tox - run: pip install setuptools==68.2.2 tox==4.11.0 - - name: Setup tox environment - run: tox run -e py --notest - - name: Test - run: tox run -e py --skip-pkg-install -- "-n 4" - continue-on-error: true - - name: Mark as a success - run: exit 0 + - name: Mark as success (check failures manually) + if: ${{ matrix.allow_failure == 'true' }} + run: exit 0 python_32bits: runs-on: ubuntu-latest From 58825f748096797cd4e844a4d6e3b161556cf749 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 25 Sep 2024 12:45:57 +0100 Subject: [PATCH 151/247] [PEP 695] Document Python 3.12 type parameter syntax (#17816) Provide examples using both syntax variants, and give both of the syntax variants similar prominence. It's likely that both syntax variants will continue to be widely used for a long time. Also adjust terminology (e.g. use 'type parameter' / 'type argument' more consistently), since otherwise some descriptions would be unclear. I didn't update examples outside the generics chapter. I'll do this in a follow-up PR. Work on #17810. --------- Co-authored-by: Jelle Zijlstra Co-authored-by: Brian Schubert --- docs/source/generics.rst | 785 ++++++++++++++++++++++++++++++--------- 1 file changed, 619 insertions(+), 166 deletions(-) diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 01ae7534ba93..4eb6463e4bd4 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -2,7 +2,7 @@ Generics ======== This section explains how you can define your own generic classes that take -one or more type parameters, similar to built-in types such as ``list[X]``. +one or more type arguments, similar to built-in types such as ``list[T]``. User-defined generics are a moderately advanced feature and you can get far without ever using them -- feel free to skip this section and come back later. @@ -12,18 +12,48 @@ Defining generic classes ************************ The built-in collection classes are generic classes. Generic types -have one or more type parameters, which can be arbitrary types. For -example, ``dict[int, str]`` has the type parameters ``int`` and -``str``, and ``list[int]`` has a type parameter ``int``. +accept one or more type arguments within ``[...]``, which can be +arbitrary types. For example, the type ``dict[int, str]`` has the +type arguments ``int`` and ``str``, and ``list[int]`` has the type +argument ``int``. Programs can also define new generic classes. Here is a very simple -generic class that represents a stack: +generic class that represents a stack (using the syntax introduced in +Python 3.12): + +.. code-block:: python + + class Stack[T]: + def __init__(self) -> None: + # Create an empty list with items of type T + self.items: list[T] = [] + + def push(self, item: T) -> None: + self.items.append(item) + + def pop(self) -> T: + return self.items.pop() + + def empty(self) -> bool: + return not self.items + +There are two syntax variants for defining generic classes in Python. +Python 3.12 introduced a +`new dedicated syntax `_ +for defining generic classes (and also functions and type aliases, which +we will discuss later). The above example used the new syntax. Most examples are +given using both the new and the old (or legacy) syntax variants. +Unless mentioned otherwise, they work the same -- but the new syntax +is more readable and more convenient. + +Here is the same example using the old syntax (required for Python 3.11 +and earlier, but also supported on newer Python versions): .. code-block:: python from typing import TypeVar, Generic - T = TypeVar('T') + T = TypeVar('T') # Define type variable "T" class Stack(Generic[T]): def __init__(self) -> None: @@ -39,8 +69,16 @@ generic class that represents a stack: def empty(self) -> bool: return not self.items +.. note:: + + There are currently no plans to deprecate the legacy syntax. + You can freely mix code using the new and old syntax variants, + even within a single file (but *not* within a single class). + The ``Stack`` class can be used to represent a stack of any type: -``Stack[int]``, ``Stack[tuple[int, str]]``, etc. +``Stack[int]``, ``Stack[tuple[int, str]]``, etc. You can think of +``Stack[int]`` as referring to the definition of ``Stack`` above, +but with all instances of ``T`` replaced with ``int``. Using ``Stack`` is similar to built-in container types: @@ -50,19 +88,49 @@ Using ``Stack`` is similar to built-in container types: stack = Stack[int]() stack.push(2) stack.pop() - stack.push('x') # error: Argument 1 to "push" of "Stack" has incompatible type "str"; expected "int" -Construction of instances of generic types is type checked: + # error: Argument 1 to "push" of "Stack" has incompatible type "str"; expected "int" + stack.push('x') + + stack2: Stack[str] = Stack() + stack2.append('x') + +Construction of instances of generic types is type checked (Python 3.12 syntax): .. code-block:: python - class Box(Generic[T]): + class Box[T]: def __init__(self, content: T) -> None: self.content = content Box(1) # OK, inferred type is Box[int] Box[int](1) # Also OK - Box[int]('some string') # error: Argument 1 to "Box" has incompatible type "str"; expected "int" + + # error: Argument 1 to "Box" has incompatible type "str"; expected "int" + Box[int]('some string') + +Here is the definition of ``Box`` using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from typing import TypeVar, Generic + + T = TypeVar('T') + + class Box(Generic[T]): + def __init__(self, content: T) -> None: + self.content = content + +.. note:: + + Before moving on, let's clarify some terminology. + The name ``T`` in ``class Stack[T]`` or ``class Stack(Generic[T])`` + declares a *type parameter* ``T`` (of class ``Stack``). + ``T`` is also called a *type variable*, especially in a type annotation, + such as in the signature of ``push`` above. + When the type ``Stack[...]`` is used in a type annotation, the type + within square brackets is called a *type argument*. + This is similar to the distinction between function parameters and arguments. .. _generic-subclasses: @@ -70,7 +138,37 @@ Defining subclasses of generic classes ************************************** User-defined generic classes and generic classes defined in :py:mod:`typing` -can be used as a base class for another class (generic or non-generic). For example: +can be used as a base class for another class (generic or non-generic). For +example (Python 3.12 syntax): + +.. code-block:: python + + from typing import Mapping, Iterator + + # This is a generic subclass of Mapping + class MyMapp[KT, VT](Mapping[KT, VT]): + def __getitem__(self, k: KT) -> VT: ... + def __iter__(self) -> Iterator[KT]: ... + def __len__(self) -> int: ... + + items: MyMap[str, int] # OK + + # This is a non-generic subclass of dict + class StrDict(dict[str, str]): + def __str__(self) -> str: + return f'StrDict({super().__str__()})' + + data: StrDict[int, int] # Error! StrDict is not generic + data2: StrDict # OK + + # This is a user-defined generic class + class Receiver[T]: + def accept(self, value: T) -> None: ... + + # This is a generic subclass of Receiver + class AdvancedReceiver[T](Receiver[T]): ... + +Here is the above example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python @@ -92,7 +190,6 @@ can be used as a base class for another class (generic or non-generic). For exam def __str__(self) -> str: return f'StrDict({super().__str__()})' - data: StrDict[int, int] # Error! StrDict is not generic data2: StrDict # OK @@ -105,25 +202,27 @@ can be used as a base class for another class (generic or non-generic). For exam .. note:: - You have to add an explicit :py:class:`~typing.Mapping` base class + You have to add an explicit :py:class:`~collections.abc.Mapping` base class if you want mypy to consider a user-defined class as a mapping (and - :py:class:`~typing.Sequence` for sequences, etc.). This is because mypy doesn't use - *structural subtyping* for these ABCs, unlike simpler protocols - like :py:class:`~typing.Iterable`, which use :ref:`structural subtyping `. + :py:class:`~collections.abc.Sequence` for sequences, etc.). This is because + mypy doesn't use *structural subtyping* for these ABCs, unlike simpler protocols + like :py:class:`~collections.abc.Iterable`, which use + :ref:`structural subtyping `. -:py:class:`Generic ` can be omitted from bases if there are +When using the legacy syntax, :py:class:`Generic ` can be omitted +from bases if there are other base classes that include type variables, such as ``Mapping[KT, VT]`` in the above example. If you include ``Generic[...]`` in bases, then it should list all type variables present in other bases (or more, -if needed). The order of type variables is defined by the following +if needed). The order of type parameters is defined by the following rules: -* If ``Generic[...]`` is present, then the order of variables is +* If ``Generic[...]`` is present, then the order of parameters is always determined by their order in ``Generic[...]``. -* If there are no ``Generic[...]`` in bases, then all type variables +* If there are no ``Generic[...]`` in bases, then all type parameters are collected in the lexicographic order (i.e. by first appearance). -For example: +Example: .. code-block:: python @@ -142,12 +241,26 @@ For example: x: First[int, str] # Here T is bound to int, S is bound to str y: Second[int, str, Any] # Here T is Any, S is int, and U is str +When using the Python 3.12 syntax, all type parameters must always be +explicitly defined immediately after the class name within ``[...]``, and the +``Generic[...]`` base class is never used. + .. _generic-functions: Generic functions ***************** -Type variables can be used to define generic functions: +Functions can also be generic, i.e. they can have type parameters (Python 3.12 syntax): + +.. code-block:: python + + from collections.abc import Sequence + + # A generic function! + def first[T](seq: Sequence[T]) -> T: + return seq[0] + +Here is the same example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python @@ -159,24 +272,25 @@ Type variables can be used to define generic functions: def first(seq: Sequence[T]) -> T: return seq[0] -As with generic classes, the type variable can be replaced with any -type. That means ``first`` can be used with any sequence type, and the -return type is derived from the sequence item type. For example: +As with generic classes, the type parameter ``T`` can be replaced with any +type. That means ``first`` can be passed an argument with any sequence type, +and the return type is derived from the sequence item type. Example: .. code-block:: python reveal_type(first([1, 2, 3])) # Revealed type is "builtins.int" - reveal_type(first(['a', 'b'])) # Revealed type is "builtins.str" + reveal_type(first(('a', 'b'))) # Revealed type is "builtins.str" -Note also that a single definition of a type variable (such as ``T`` -above) can be used in multiple generic functions or classes. In this -example we use the same type variable in two generic functions: +When using the legacy syntax, a single definition of a type variable +(such as ``T`` above) can be used in multiple generic functions or +classes. In this example we use the same type variable in two generic +functions to declarare type parameters: .. code-block:: python from typing import TypeVar, Sequence - T = TypeVar('T') # Declare type variable + T = TypeVar('T') # Define type variable def first(seq: Sequence[T]) -> T: return seq[0] @@ -184,20 +298,62 @@ example we use the same type variable in two generic functions: def last(seq: Sequence[T]) -> T: return seq[-1] +Since the Python 3.12 syntax is more concise, it doesn't need (or have) +an equivalent way of sharing type parameter definitions. + A variable cannot have a type variable in its type unless the type variable is bound in a containing generic class or function. +When calling a generic function, you can't explicitly pass the values of +type parameters as type arguments. The values of type parameters are always +inferred by mypy. This is not valid: + +.. code-block:: python + + first[int]([1, 2]) # Error: can't use [...] with generic function + +If you really need this, you can define a generic class with a ``__call__`` +method. + .. _generic-methods-and-generic-self: Generic methods and generic self ******************************** -You can also define generic methods — just use a type variable in the -method signature that is different from class type variables. In -particular, the ``self`` argument may also be generic, allowing a +You can also define generic methods. In +particular, the ``self`` parameter may also be generic, allowing a method to return the most precise type known at the point of access. In this way, for example, you can type check a chain of setter -methods: +methods (Python 3.12 syntax): + +.. code-block:: python + + class Shape: + def set_scale[T: Shape](self: T, scale: float) -> T: + self.scale = scale + return self + + class Circle(Shape): + def set_radius(self, r: float) -> 'Circle': + self.radius = r + return self + + class Square(Shape): + def set_width(self, w: float) -> 'Square': + self.width = w + return self + + circle: Circle = Circle().set_scale(0.5).set_radius(2.7) + square: Square = Square().set_scale(0.5).set_width(3.2) + +Without using generic ``self``, the last two lines could not be type +checked properly, since the return type of ``set_scale`` would be +``Shape``, which doesn't define ``set_radius`` or ``set_width``. + +When using the legacy syntax, just use a type variable in the +method signature that is different from class type parameters (if any +are defined). Here is the above example using the legacy +syntax (3.11 and earlier): .. code-block:: python @@ -223,24 +379,41 @@ methods: circle: Circle = Circle().set_scale(0.5).set_radius(2.7) square: Square = Square().set_scale(0.5).set_width(3.2) -Without using generic ``self``, the last two lines could not be type -checked properly, since the return type of ``set_scale`` would be -``Shape``, which doesn't define ``set_radius`` or ``set_width``. +Other uses include factory methods, such as copy and deserialization methods. +For class methods, you can also define generic ``cls``, using ``type[T]`` +or :py:class:`Type[T] ` (Python 3.12 syntax): + +.. code-block:: python + + class Friend: + other: "Friend | None" = None + + @classmethod + def make_pair[T: Friend](cls: type[T]) -> tuple[T, T]: + a, b = cls(), cls() + a.other = b + b.other = a + return a, b -Other uses are factory methods, such as copy and deserialization. -For class methods, you can also define generic ``cls``, using :py:class:`Type[T] `: + class SuperFriend(Friend): + pass + + a, b = SuperFriend.make_pair() + +Here is the same example using the legacy syntax (3.11 and earlier, though +3.9 and later can use lower-case ``type[T]``): .. code-block:: python - from typing import TypeVar, Type + from typing import TypeVar T = TypeVar('T', bound='Friend') class Friend: - other: "Friend" = None + other: "Friend | None" = None @classmethod - def make_pair(cls: Type[T]) -> tuple[T, T]: + def make_pair(cls: type[T]) -> tuple[T, T]: a, b = cls(), cls() a.other = b b.other = a @@ -262,16 +435,13 @@ possibly by making use of the ``Any`` type or a ``# type: ignore`` comment. Note that mypy lets you use generic self types in certain unsafe ways in order to support common idioms. For example, using a generic -self type in an argument type is accepted even though it's unsafe: +self type in an argument type is accepted even though it's unsafe (Python 3.12 +syntax): .. code-block:: python - from typing import TypeVar - - T = TypeVar("T") - class Base: - def compare(self: T, other: T) -> bool: + def compare[T: Base](self: T, other: T) -> bool: return False class Sub(Base): @@ -280,7 +450,7 @@ self type in an argument type is accepted even though it's unsafe: # This is unsafe (see below) but allowed because it's # a common pattern and rarely causes issues in practice. - def compare(self, other: Sub) -> bool: + def compare(self, other: 'Sub') -> bool: return self.x > other.x b: Base = Sub(42) @@ -293,12 +463,12 @@ Automatic self types using typing.Self Since the patterns described above are quite common, mypy supports a simpler syntax, introduced in :pep:`673`, to make them easier to use. -Instead of defining a type variable and using an explicit annotation +Instead of introducing a type parameter and using an explicit annotation for ``self``, you can import the special type ``typing.Self`` that is -automatically transformed into a type variable with the current class -as the upper bound, and you don't need an annotation for ``self`` (or -``cls`` in class methods). The example from the previous section can -be made simpler by using ``Self``: +automatically transformed into a method-level type parameter with the +current class as the upper bound, and you don't need an annotation for +``self`` (or ``cls`` in class methods). The example from the previous +section can be made simpler by using ``Self``: .. code-block:: python @@ -319,7 +489,7 @@ be made simpler by using ``Self``: a, b = SuperFriend.make_pair() -This is more compact than using explicit type variables. Also, you can +This is more compact than using explicit type parameters. Also, you can use ``Self`` in attribute annotations in addition to methods. .. note:: @@ -354,8 +524,8 @@ Let us illustrate this by few simple examples: class Triangle(Shape): ... class Square(Shape): ... -* Most immutable containers, such as :py:class:`~typing.Sequence` and - :py:class:`~typing.FrozenSet` are covariant. :py:data:`~typing.Union` is +* Most immutable container types, such as :py:class:`~collections.abc.Sequence` + and :py:class:`~frozenset` are covariant. :py:data:`~typing.Union` is also covariant in all variables: ``Union[Triangle, int]`` is a subtype of ``Union[Shape, int]``. @@ -367,7 +537,7 @@ Let us illustrate this by few simple examples: triangles: Sequence[Triangle] count_lines(triangles) # OK - def foo(triangle: Triangle, num: int): + def foo(triangle: Triangle, num: int) -> None: shape_or_number: Union[Shape, int] # a Triangle is a Shape, and a Shape is a valid Union[Shape, int] shape_or_number = triangle @@ -400,8 +570,8 @@ Let us illustrate this by few simple examples: triangle. If we give it a callable that can calculate the area of an arbitrary shape (not just triangles), everything still works. -* :py:class:`~typing.List` is an invariant generic type. Naively, one would think - that it is covariant, like :py:class:`~typing.Sequence` above, but consider this code: +* ``list`` is an invariant generic type. Naively, one would think + that it is covariant, like :py:class:`~collections.abc.Sequence` above, but consider this code: .. code-block:: python @@ -416,13 +586,48 @@ Let us illustrate this by few simple examples: add_one(my_circles) # This may appear safe, but... my_circles[-1].rotate() # ...this will fail, since my_circles[0] is now a Shape, not a Circle - Another example of invariant type is :py:class:`~typing.Dict`. Most mutable containers + Another example of invariant type is ``dict``. Most mutable containers are invariant. -By default, mypy assumes that all user-defined generics are invariant. -To declare a given generic class as covariant or contravariant use -type variables defined with special keyword arguments ``covariant`` or -``contravariant``. For example: +When using the Python 3.12 syntax for generics, mypy will automatically +infer the most flexible variance for each class type variable. Here +``Box`` will be inferred as covariant: + +.. code-block:: python + + class Box[T]: # this type is implilicitly covariant + def __init__(self, content: T) -> None: + self._content = content + + def get_content(self) -> T: + return self._content + + def look_into(box: Box[Shape]): ... + + my_box = Box(Square()) + look_into(my_box) # OK, but mypy would complain here for an invariant type + +Here the underscore prefix for ``_content`` is significant. Without an +underscore prefix, the class would be invariant, as the attribute would +be understood as a public, mutable attribute (a single underscore prefix +has no special significance for mypy in most other contexts). By declaring +the attribute as ``Final``, the class could still be made covariant: + +.. code-block:: python + + from typing import Final + + class Box[T]: # this type is implilicitly covariant + def __init__(self, content: T) -> None: + self.content: Final = content + + def get_content(self) -> T: + return self._content + +When using the legacy syntax, mypy assumes that all user-defined generics +are invariant by default. To declare a given generic class as covariant or +contravariant, use type variables defined with special keyword arguments +``covariant`` or ``contravariant``. For example (Python 3.11 or earlier): .. code-block:: python @@ -437,9 +642,9 @@ type variables defined with special keyword arguments ``covariant`` or def get_content(self) -> T_co: return self._content - def look_into(box: Box[Animal]): ... + def look_into(box: Box[Shape]): ... - my_box = Box(Cat()) + my_box = Box(Square()) look_into(my_box) # OK, but mypy would complain here for an invariant type .. _type-variable-upper-bound: @@ -449,24 +654,32 @@ Type variables with upper bounds A type variable can also be restricted to having values that are subtypes of a specific type. This type is called the upper bound of -the type variable, and is specified with the ``bound=...`` keyword -argument to :py:class:`~typing.TypeVar`. +the type variable, and it is specified using ``T: `` when using the +Python 3.12 syntax. In the definition of a generic function that uses +such a type variable ``T``, the type represented by ``T`` is assumed +to be a subtype of its upper bound, so the function can use methods +of the upper bound on values of type ``T`` (Python 3.12 syntax): .. code-block:: python - from typing import TypeVar, SupportsAbs + from typing import SupportsAbs - T = TypeVar('T', bound=SupportsAbs[float]) + def max_by_abs[T: SupportsAbs[float]](*xs: T) -> T: + # Okay, because T is a subtype of SupportsAbs[float]. + return max(xs, key=abs) -In the definition of a generic function that uses such a type variable -``T``, the type represented by ``T`` is assumed to be a subtype of -its upper bound, so the function can use methods of the upper bound on -values of type ``T``. +An upper bound can also be specified with the ``bound=...`` keyword +argument to :py:class:`~typing.TypeVar`. +Here is the example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python - def largest_in_absolute_value(*xs: T) -> T: - return max(xs, key=abs) # Okay, because T is a subtype of SupportsAbs[float]. + from typing import TypeVar, SupportsAbs + + T = TypeVar('T', bound=SupportsAbs[float]) + + def max_by_abs(*xs: T) -> T: + return max(xs, key=abs) In a call to such a function, the type ``T`` must be replaced by a type that is a subtype of its upper bound. Continuing the example @@ -474,9 +687,9 @@ above: .. code-block:: python - largest_in_absolute_value(-3.5, 2) # Okay, has type float. - largest_in_absolute_value(5+6j, 7) # Okay, has type complex. - largest_in_absolute_value('a', 'b') # Error: 'str' is not a subtype of SupportsAbs[float]. + max_by_abs(-3.5, 2) # Okay, has type float. + max_by_abs(5+6j, 7) # Okay, has type complex. + max_by_abs('a', 'b') # Error: 'str' is not a subtype of SupportsAbs[float]. Type parameters of generic classes may also have upper bounds, which restrict the valid values for the type parameter in the same way. @@ -489,34 +702,34 @@ Type variables with value restriction By default, a type variable can be replaced with any type. However, sometimes it's useful to have a type variable that can only have some specific types as its value. A typical example is a type variable that can only have values -``str`` and ``bytes``: +``str`` and ``bytes``. This lets us define a function that can concatenate +two strings or bytes objects, but it can't be called with other argument +types (Python 3.12 syntax): .. code-block:: python - from typing import TypeVar + def concat[S: (str, bytes)](x: S, y: S) -> S: + return x + y - AnyStr = TypeVar('AnyStr', str, bytes) + concat('a', 'b') # Okay + concat(b'a', b'b') # Okay + concat(1, 2) # Error! -This is actually such a common type variable that :py:data:`~typing.AnyStr` is -defined in :py:mod:`typing` and we don't need to define it ourselves. -We can use :py:data:`~typing.AnyStr` to define a function that can concatenate -two strings or bytes objects, but it can't be called with other -argument types: +The same thing is also possibly using the legacy syntax (Python 3.11 or earlier): .. code-block:: python - from typing import AnyStr + from typing import TypeVar + + AnyStr = TypeVar('AnyStr', str, bytes) def concat(x: AnyStr, y: AnyStr) -> AnyStr: return x + y - concat('a', 'b') # Okay - concat(b'a', b'b') # Okay - concat(1, 2) # Error! - -Importantly, this is different from a union type, since combinations -of ``str`` and ``bytes`` are not accepted: +No matter which syntax you use, such a type variable is called a type variable +with a value restriction. Importantly, this is different from a union type, +since combinations of ``str`` and ``bytes`` are not accepted: .. code-block:: python @@ -524,11 +737,11 @@ of ``str`` and ``bytes`` are not accepted: In this case, this is exactly what we want, since it's not possible to concatenate a string and a bytes object! If we tried to use -``Union``, the type checker would complain about this possibility: +a union type, the type checker would complain about this possibility: .. code-block:: python - def union_concat(x: Union[str, bytes], y: Union[str, bytes]) -> Union[str, bytes]: + def union_concat(x: str | bytes, y: str | bytes) -> str | bytes: return x + y # Error: can't concatenate str and bytes Another interesting special case is calling ``concat()`` with a @@ -556,13 +769,19 @@ this is correct for ``concat``, since ``concat`` actually returns a >>> print(type(ss)) -You can also use a :py:class:`~typing.TypeVar` with a restricted set of possible -values when defining a generic class. For example, mypy uses the type -:py:class:`Pattern[AnyStr] ` for the return value of :py:func:`re.compile`, -since regular expressions can be based on a string or a bytes pattern. +You can also use type variables with a restricted set of possible +values when defining a generic class. For example, the type +:py:class:`Pattern[S] ` is used for the return +value of :py:func:`re.compile`, where ``S`` can be either ``str`` +or ``bytes``. Regular expressions can be based on a string or a +bytes pattern. + +A type variable may not have both a value restriction and an upper bound +(see :ref:`type-variable-upper-bound`). -A type variable may not have both a value restriction (see -:ref:`type-variable-upper-bound`) and an upper bound. +Note that you may come across :py:data:`~typing.AnyStr` imported from +:py:mod:`typing`. This feature is now deprecated, but it means the same +as our definition of ``AnyStr`` above. .. _declaring-decorators: @@ -571,11 +790,12 @@ Declaring decorators Decorators are typically functions that take a function as an argument and return another function. Describing this behaviour in terms of types can -be a little tricky; we'll show how you can use ``TypeVar`` and a special +be a little tricky; we'll show how you can use type variables and a special kind of type variable called a *parameter specification* to do so. Suppose we have the following decorator, not type annotated yet, -that preserves the original function's signature and merely prints the decorated function's name: +that preserves the original function's signature and merely prints the decorated +function's name: .. code-block:: python @@ -585,7 +805,7 @@ that preserves the original function's signature and merely prints the decorated return func(*args, **kwds) return wrapper -and we use it to decorate function ``add_forty_two``: +We can use it to decorate function ``add_forty_two``: .. code-block:: python @@ -611,7 +831,28 @@ Note that class decorators are handled differently than function decorators in mypy: decorating a class does not erase its type, even if the decorator has incomplete type annotations. -Here's how one could annotate the decorator: +Here's how one could annotate the decorator (Python 3.12 syntax): + +.. code-block:: python + + from typing import Any, Callable, cast + + # A decorator that preserves the signature. + def printing_decorator[F: Callable[..., Any]](func: F) -> F: + def wrapper(*args, **kwds): + print("Calling", func) + return func(*args, **kwds) + return cast(F, wrapper) + + @printing_decorator + def add_forty_two(value: int) -> int: + return value + 42 + + a = add_forty_two(3) + reveal_type(a) # Revealed type is "builtins.int" + add_forty_two('x') # Argument 1 to "add_forty_two" has incompatible type "str"; expected "int" + +Here is the example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python @@ -636,15 +877,28 @@ Here's how one could annotate the decorator: This still has some shortcomings. First, we need to use the unsafe :py:func:`~typing.cast` to convince mypy that ``wrapper()`` has the same -signature as ``func``. See :ref:`casts `. +signature as ``func`` (see :ref:`casts `). Second, the ``wrapper()`` function is not tightly type checked, although wrapper functions are typically small enough that this is not a big problem. This is also the reason for the :py:func:`~typing.cast` call in the ``return`` statement in ``printing_decorator()``. -However, we can use a parameter specification (:py:class:`~typing.ParamSpec`), -for a more faithful type annotation: +However, we can use a parameter specification, introduced using ``**P``, +for a more faithful type annotation (Python 3.12 syntax): + +.. code-block:: python + + from typing import Callable + + def printing_decorator[**P, T](func: Callable[P, T]) -> Callable[P, T]: + def wrapper(*args: P.args, **kwds: P.kwargs) -> T: + print("Calling", func) + return func(*args, **kwds) + return wrapper + +The same is possible using the legacy syntax with :py:class:`~typing.ParamSpec` +(Python 3.11 and earlier): .. code-block:: python @@ -661,7 +915,27 @@ for a more faithful type annotation: return wrapper Parameter specifications also allow you to describe decorators that -alter the signature of the input function: +alter the signature of the input function (Python 3.12 syntax): + +.. code-block:: python + + from typing import Callable + + # We reuse 'P' in the return type, but replace 'T' with 'str' + def stringify[**P, T](func: Callable[P, T]) -> Callable[P, str]: + def wrapper(*args: P.args, **kwds: P.kwargs) -> str: + return str(func(*args, **kwds)) + return wrapper + + @stringify + def add_forty_two(value: int) -> int: + return value + 42 + + a = add_forty_two(3) + reveal_type(a) # Revealed type is "builtins.str" + add_forty_two('x') # error: Argument 1 to "add_forty_two" has incompatible type "str"; expected "int" + +Here is the above example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python @@ -671,21 +945,31 @@ alter the signature of the input function: P = ParamSpec('P') T = TypeVar('T') - # We reuse 'P' in the return type, but replace 'T' with 'str' + # We reuse 'P' in the return type, but replace 'T' with 'str' def stringify(func: Callable[P, T]) -> Callable[P, str]: def wrapper(*args: P.args, **kwds: P.kwargs) -> str: return str(func(*args, **kwds)) return wrapper - @stringify +You can also insert an argument in a decorator (Python 3.12 syntax): + +.. code-block:: python + + from typing import Callable, Concatenate + + def printing_decorator[**P, T](func: Callable[P, T]) -> Callable[Concatenate[str, P], T]: + def wrapper(msg: str, /, *args: P.args, **kwds: P.kwargs) -> T: + print("Calling", func, "with", msg) + return func(*args, **kwds) + return wrapper + + @printing_decorator def add_forty_two(value: int) -> int: return value + 42 - a = add_forty_two(3) - reveal_type(a) # Revealed type is "builtins.str" - add_forty_two('x') # error: Argument 1 to "add_forty_two" has incompatible type "str"; expected "int" + a = add_forty_two('three', 3) -Or insert an argument: +Here is the same function using the legacy syntax (Python 3.11 and earlier): .. code-block:: python @@ -701,19 +985,31 @@ Or insert an argument: return func(*args, **kwds) return wrapper - @printing_decorator - def add_forty_two(value: int) -> int: - return value + 42 - - a = add_forty_two('three', 3) - .. _decorator-factories: Decorator factories ------------------- Functions that take arguments and return a decorator (also called second-order decorators), are -similarly supported via generics: +similarly supported via generics (Python 3.12 syntax): + +.. code-block:: python + + from typing import Any, Callable + + def route[F: Callable[..., Any]](url: str) -> Callable[[F], F]: + ... + + @route(url='/') + def index(request: Any) -> str: + return 'Hello world' + +Note that mypy infers that ``F`` is used to make the ``Callable`` return value +of ``route`` generic, instead of making ``route`` itself generic, since ``F`` is +only used in the return type. Python has no explicit syntax to mark that ``F`` +is only bound in the return value. + +Here is the example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python @@ -729,23 +1025,21 @@ similarly supported via generics: return 'Hello world' Sometimes the same decorator supports both bare calls and calls with arguments. This can be -achieved by combining with :py:func:`@overload `: +achieved by combining with :py:func:`@overload ` (Python 3.12 syntax): .. code-block:: python - from typing import Any, Callable, Optional, TypeVar, overload - - F = TypeVar('F', bound=Callable[..., Any]) + from typing import Any, Callable, overload # Bare decorator usage @overload - def atomic(__func: F) -> F: ... + def atomic[F: Callable[..., Any]](func: F, /) -> F: ... # Decorator with arguments @overload - def atomic(*, savepoint: bool = True) -> Callable[[F], F]: ... + def atomic[F: Callable[..., Any]](*, savepoint: bool = True) -> Callable[[F], F]: ... # Implementation - def atomic(__func: Optional[Callable[..., Any]] = None, *, savepoint: bool = True): + def atomic(func: Callable[..., Any] | None = None, /, *, savepoint: bool = True): def decorator(func: Callable[..., Any]): ... # Code goes here if __func is not None: @@ -760,21 +1054,40 @@ achieved by combining with :py:func:`@overload `: @atomic(savepoint=False) def func2() -> None: ... +Here is the decorator from the example using the legacy syntax +(Python 3.11 and earlier): + +.. code-block:: python + + from typing import Any, Callable, Optional, TypeVar, overload + + F = TypeVar('F', bound=Callable[..., Any]) + + # Bare decorator usage + @overload + def atomic(func: F, /) -> F: ... + # Decorator with arguments + @overload + def atomic(*, savepoint: bool = True) -> Callable[[F], F]: ... + + # Implementation + def atomic(func: Optional[Callable[..., Any]] = None, /, *, savepoint: bool = True): + ... # Same as above + Generic protocols ***************** Mypy supports generic protocols (see also :ref:`protocol-types`). Several :ref:`predefined protocols ` are generic, such as -:py:class:`Iterable[T] `, and you can define additional generic protocols. Generic -protocols mostly follow the normal rules for generic classes. Example: +:py:class:`Iterable[T] `, and you can define additional +generic protocols. Generic protocols mostly follow the normal rules for +generic classes. Example (Python 3.12 syntax): .. code-block:: python - from typing import Protocol, TypeVar - - T = TypeVar('T') + from typing import Protocol - class Box(Protocol[T]): + class Box[T](Protocol): content: T def do_stuff(one: Box[str], other: Box[bytes]) -> None: @@ -794,15 +1107,29 @@ protocols mostly follow the normal rules for generic classes. Example: y: Box[int] = ... x = y # Error -- Box is invariant +Here is the definition of ``Box`` from the above example using the legacy +syntax (Python 3.11 and earlier): + +.. code-block:: python + + from typing import Protocol, TypeVar + + T = TypeVar('T') + + class Box(Protocol[T]): + content: T + Note that ``class ClassName(Protocol[T])`` is allowed as a shorthand for -``class ClassName(Protocol, Generic[T])``, as per :pep:`PEP 544: Generic protocols <544#generic-protocols>`, +``class ClassName(Protocol, Generic[T])`` when using the legacy syntax, +as per :pep:`PEP 544: Generic protocols <544#generic-protocols>`. +This form is only valid when using the legacy syntax. -The main difference between generic protocols and ordinary generic -classes is that mypy checks that the declared variances of generic -type variables in a protocol match how they are used in the protocol -definition. The protocol in this example is rejected, since the type -variable ``T`` is used covariantly as a return type, but the type -variable is invariant: +When using the legacy syntax, there is an important difference between +generic protocols and ordinary generic classes: mypy checks that the +declared variances of generic type variables in a protocol match how +they are used in the protocol definition. The protocol in this example +is rejected, since the type variable ``T`` is used covariantly as +a return type, but the type variable is invariant: .. code-block:: python @@ -830,13 +1157,11 @@ This example correctly uses a covariant type variable: See :ref:`variance-of-generics` for more about variance. -Generic protocols can also be recursive. Example: +Generic protocols can also be recursive. Example (Python 3.12 synta): .. code-block:: python - T = TypeVar('T') - - class Linked(Protocol[T]): + class Linked[T](Protocol): val: T def next(self) -> 'Linked[T]': ... @@ -849,17 +1174,63 @@ Generic protocols can also be recursive. Example: result = last(L()) reveal_type(result) # Revealed type is "builtins.int" +Here is the definition of ``Linked`` using the legacy syntax +(Python 3.11 and earlier): + +.. code-block:: python + + from typing import TypeVar + + T = TypeVar('T') + + class Linked(Protocol[T]): + val: T + def next(self) -> 'Linked[T]': ... + .. _generic-type-aliases: Generic type aliases ******************** -Type aliases can be generic. In this case they can be used in two ways: -Subscripted aliases are equivalent to original types with substituted type -variables, so the number of type arguments must match the number of free type variables -in the generic type alias. Unsubscripted aliases are treated as original types with free -variables replaced with ``Any``. Examples (following :pep:`PEP 484: Type aliases -<484#type-aliases>`): +Type aliases can be generic. In this case they can be used in two ways. +First, subscripted aliases are equivalent to original types with substituted type +variables. Second, unsubscripted aliases are treated as original types with type +parameters replaced with ``Any``. + +The ``type`` statement introduced in Python 3.12 is used to define generic +type aliases (it also supports non-generic type aliases): + +.. code-block:: python + + from typing import Iterable, Callable + + type TInt[S] = tuple[int, S] + type UInt[S] = S | int + type CBack[S] = Callable[..., S] + + def response(query: str) -> UInt[str]: # Same as str | int + ... + def activate[S](cb: CBack[S]) -> S: # Same as Callable[..., S] + ... + table_entry: TInt # Same as tuple[int, Any] + + type Vec[T: (int, float, complex)] = Iterable[tuple[T, T]] + + def inproduct[T: (int, float, complex)](v: Vec[T]) -> T: + return sum(x*y for x, y in v) + + def dilate[T: (int, float, complex)](v: Vec[T], scale: T) -> Vec[T]: + return ((x * scale, y * scale) for x, y in v) + + v1: Vec[int] = [] # Same as Iterable[tuple[int, int]] + v2: Vec = [] # Same as Iterable[tuple[Any, Any]] + v3: Vec[int, int] = [] # Error: Invalid alias, too many type arguments! + +There is also a legacy syntax that relies on ``TypeVar``. +Here the number of type arguments must match the number of free type variables +in the generic type alias definition. A type variables is free if it's not +a type parameter of a surrounding class or function. Example (following +:pep:`PEP 484: Type aliases <484#type-aliases>`, Python 3.11 and earlier): .. code-block:: python @@ -867,7 +1238,7 @@ variables replaced with ``Any``. Examples (following :pep:`PEP 484: Type aliases S = TypeVar('S') - TInt = tuple[int, S] + TInt = tuple[int, S] # 1 type parameter, since only S is free UInt = Union[S, int] CBack = Callable[..., S] @@ -894,7 +1265,36 @@ variables replaced with ``Any``. Examples (following :pep:`PEP 484: Type aliases Type aliases can be imported from modules just like other names. An alias can also target another alias, although building complex chains of aliases is not recommended -- this impedes code readability, thus -defeating the purpose of using aliases. Example: +defeating the purpose of using aliases. Example (Python 3.12 syntax): + +.. code-block:: python + + from example1 import AliasType + from example2 import Vec + + # AliasType and Vec are type aliases (Vec as defined above) + + def fun() -> AliasType: + ... + + type OIntVec = Vec[int] | None + +Type aliases defined using the ``type`` statement are not valid as +base classes, and they can't be used to construct instances: + +.. code-block:: python + + from example1 import AliasType + from example2 import Vec + + # AliasType and Vec are type aliases (Vec as defined above) + + class NewVec[T](Vec[T]): # Error: not valid as base class + ... + + x = AliasType() # Error: can't be used to create instances + +Here are examples using the legacy syntax (Python 3.11 and earlier): .. code-block:: python @@ -907,18 +1307,49 @@ defeating the purpose of using aliases. Example: def fun() -> AliasType: ... + OIntVec = Optional[Vec[int]] + T = TypeVar('T') + # Old-style type aliases can be used as base classes and you can + # construct instances using them + class NewVec(Vec[T]): ... + x = AliasType() + for i, j in NewVec[int](): ... - OIntVec = Optional[Vec[int]] +Using type variable bounds or value restriction in generic aliases has +the same effect as in generic classes and functions. + + +Differences between the new and old syntax +****************************************** -Using type variable bounds or values in generic aliases has the same effect -as in generic classes/functions. +There are a few notable differences between the new (Python 3.12 and later) +and the old syntax for generic classes, functions and type aliases, beyond +the obvious syntactic differences: + + * Type variables defined using the old syntax create definitions at runtime + in the surrounding namespace, whereas the type variables defined using the + new syntax are only defined within the class, function or type variable + that uses them. + * Type variable definitions can be shared when using the old syntax, but + the new syntax doesn't support this. + * When using the new syntax, the variance of class type variables is always + inferred. + * Type aliases defined using the new syntax can contain forward references + and recursive references without using string literal escaping. The + same is true for the bounds and constraints of type variables. + * The new syntax lets you define a generic alias where the definition doesn't + contain a reference to a type parameter. This is occasionally useful, at + least when conditionally defining type aliases. + * Type aliases defined using the new syntax can't be used as base classes + and can't be used to construct instances, unlike aliases defined using the + old syntax. Generic class internals @@ -926,7 +1357,20 @@ Generic class internals You may wonder what happens at runtime when you index a generic class. Indexing returns a *generic alias* to the original class that returns instances -of the original class on instantiation: +of the original class on instantiation (Python 3.12 syntax): + +.. code-block:: python + + >>> class Stack[T]: ... + >>> Stack + __main__.Stack + >>> Stack[int] + __main__.Stack[int] + >>> instance = Stack[int]() + >>> instance.__class__ + __main__.Stack + +Here is the same example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python @@ -945,10 +1389,17 @@ Generic aliases can be instantiated or subclassed, similar to real classes, but the above examples illustrate that type variables are erased at runtime. Generic ``Stack`` instances are just ordinary Python objects, and they have no extra runtime overhead or magic due -to being generic, other than a metaclass that overloads the indexing -operator. +to being generic, other than the ``Generic`` base class that overloads +the indexing operator using ``__class_getitem__``. ``typing.Generic`` +is included as an implicit base class even when using the new syntax: + +.. code-block:: python + + >>> class Stack[T]: ... + >>> Stack.mro() + [, , ] -Note that in Python 3.8 and lower, the built-in types +Note that in Python 3.8 and earlier, the built-in types :py:class:`list`, :py:class:`dict` and others do not support indexing. This is why we have the aliases :py:class:`~typing.List`, :py:class:`~typing.Dict` and so on in the :py:mod:`typing` @@ -959,16 +1410,18 @@ class in more recent versions of Python: .. code-block:: python >>> # Only relevant for Python 3.8 and below - >>> # For Python 3.9 onwards, prefer `list[int]` syntax + >>> # If using Python 3.9 or newer, prefer the 'list[int]' syntax >>> from typing import List >>> List[int] typing.List[int] Note that the generic aliases in ``typing`` don't support constructing -instances: +instances, unlike the corresponding built-in classes: .. code-block:: python + >>> list[int]() + [] >>> from typing import List >>> List[int]() Traceback (most recent call last): From 7f3d7f8f10938618bcf8648d92f6a3da0a01117f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 25 Sep 2024 14:49:22 +0100 Subject: [PATCH 152/247] [PEP 695] Further documentation updates (#17826) Finish work started in #17816. Document `type` statement when discussing type aliases. Update some examples to have both old-style and new-style variants. In less common scenarios, examples only use a single syntax variant to reduce verbosity. Also update some examples to generally use more modern features. Closes #17810. --- docs/source/additional_features.rst | 18 +++----- docs/source/cheat_sheet_py3.rst | 15 ++++++- docs/source/error_code_list.rst | 20 +++------ docs/source/kinds_of_types.rst | 47 +++++++++++++++----- docs/source/literal_types.rst | 10 ++--- docs/source/metaclasses.rst | 8 ++-- docs/source/more_types.rst | 67 +++++++++++++++++++---------- docs/source/protocols.rst | 9 ++-- docs/source/runtime_troubles.rst | 14 +++--- docs/source/type_narrowing.rst | 14 ++---- 10 files changed, 135 insertions(+), 87 deletions(-) diff --git a/docs/source/additional_features.rst b/docs/source/additional_features.rst index ae625c157654..a9c3177d32a2 100644 --- a/docs/source/additional_features.rst +++ b/docs/source/additional_features.rst @@ -46,21 +46,18 @@ define dataclasses. For example: UnorderedPoint(1, 2) < UnorderedPoint(3, 4) # Error: Unsupported operand types Dataclasses can be generic and can be used in any other way a normal -class can be used: +class can be used (Python 3.12 syntax): .. code-block:: python from dataclasses import dataclass - from typing import Generic, TypeVar - - T = TypeVar('T') @dataclass - class BoxedData(Generic[T]): + class BoxedData[T]: data: T label: str - def unbox(bd: BoxedData[T]) -> T: + def unbox[T](bd: BoxedData[T]) -> T: ... val = unbox(BoxedData(42, "")) # OK, inferred type is int @@ -98,17 +95,16 @@ does **not** work: To have Mypy recognize a wrapper of :py:func:`dataclasses.dataclass ` -as a dataclass decorator, consider using the :py:func:`~typing.dataclass_transform` decorator: +as a dataclass decorator, consider using the :py:func:`~typing.dataclass_transform` +decorator (example uses Python 3.12 syntax): .. code-block:: python from dataclasses import dataclass, Field - from typing import TypeVar, dataclass_transform - - T = TypeVar('T') + from typing import dataclass_transform @dataclass_transform(field_specifiers=(Field,)) - def my_dataclass(cls: type[T]) -> type[T]: + def my_dataclass[T](cls: type[T]) -> type[T]: ... return dataclass(cls) diff --git a/docs/source/cheat_sheet_py3.rst b/docs/source/cheat_sheet_py3.rst index b8e43960fd09..ca6dfc8eb63a 100644 --- a/docs/source/cheat_sheet_py3.rst +++ b/docs/source/cheat_sheet_py3.rst @@ -349,7 +349,20 @@ Decorators ********** Decorator functions can be expressed via generics. See -:ref:`declaring-decorators` for more details. +:ref:`declaring-decorators` for more details. Example using Python 3.12 +syntax: + +.. code-block:: python + + from typing import Any, Callable + + def bare_decorator[F: Callable[..., Any]](func: F) -> F: + ... + + def decorator_args[F: Callable[..., Any]](url: str) -> Callable[[F], F]: + ... + +The same example using pre-3.12 syntax: .. code-block:: python diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index ad73bc999f00..ec069fdcba1d 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -434,15 +434,11 @@ Check type variable values [type-var] Mypy checks that value of a type variable is compatible with a value restriction or the upper bound type. -Example: +Example (Python 3.12 syntax): .. code-block:: python - from typing import TypeVar - - T1 = TypeVar('T1', int, float) - - def add(x: T1, y: T1) -> T1: + def add[T1: (int, float)](x: T1, y: T1) -> T1: return x + y add(4, 5.5) # OK @@ -783,27 +779,25 @@ Example: Safe handling of abstract type object types [type-abstract] ----------------------------------------------------------- -Mypy always allows instantiating (calling) type objects typed as ``Type[t]``, +Mypy always allows instantiating (calling) type objects typed as ``type[t]``, even if it is not known that ``t`` is non-abstract, since it is a common pattern to create functions that act as object factories (custom constructors). Therefore, to prevent issues described in the above section, when an abstract -type object is passed where ``Type[t]`` is expected, mypy will give an error. -Example: +type object is passed where ``type[t]`` is expected, mypy will give an error. +Example (Python 3.12 syntax): .. code-block:: python from abc import ABCMeta, abstractmethod - from typing import List, Type, TypeVar class Config(metaclass=ABCMeta): @abstractmethod def get_value(self, attr: str) -> str: ... - T = TypeVar("T") - def make_many(typ: Type[T], n: int) -> List[T]: + def make_many[T](typ: type[T], n: int) -> list[T]: return [typ() for _ in range(n)] # This will raise if typ is abstract - # Error: Only concrete class can be given where "Type[Config]" is expected [type-abstract] + # Error: Only concrete class can be given where "type[Config]" is expected [type-abstract] make_many(Config, 5) .. _code-safe-super: diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index d07eb40753f3..e645a27095d1 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -434,19 +434,20 @@ the runtime with some limitations (see :ref:`runtime_troubles`). Type aliases ************ -In certain situations, type names may end up being long and painful to type: +In certain situations, type names may end up being long and painful to type, +especially if they are used frequently: .. code-block:: python - def f() -> Union[list[dict[tuple[int, str], set[int]]], tuple[str, list[str]]]: + def f() -> list[dict[tuple[int, str], set[int]]] | tuple[str, list[str]]: ... When cases like this arise, you can define a type alias by simply -assigning the type to a variable: +assigning the type to a variable (this is an *implicit type alias*): .. code-block:: python - AliasType = Union[list[dict[tuple[int, str], set[int]]], tuple[str, list[str]]] + AliasType = list[dict[tuple[int, str], set[int]]] | tuple[str, list[str]] # Now we can use AliasType in place of the full name: @@ -459,8 +460,18 @@ assigning the type to a variable: another type -- it's equivalent to the target type except for :ref:`generic aliases `. -Since Mypy 0.930 you can also use *explicit type aliases*, which were -introduced in :pep:`613`. +Python 3.12 introduced the ``type`` statement for defining *explicit type aliases*. +Explicit type aliases are unambiguous and can also improve readability by +making the intent clear: + +.. code-block:: python + + type AliasType = list[dict[tuple[int, str], set[int]]] | tuple[str, list[str]] + + # Now we can use AliasType in place of the full name: + + def f() -> AliasType: + ... There can be confusion about exactly when an assignment defines an implicit type alias -- for example, when the alias contains forward references, invalid types, or violates some other @@ -469,8 +480,17 @@ distinction between an unannotated variable and a type alias is implicit, ambiguous or incorrect type alias declarations default to defining a normal variable instead of a type alias. -Explicit type aliases are unambiguous and can also improve readability by -making the intent clear: +Aliases defined using the ``type`` statement have these properties, which +distinguish them from implicit type aliases: + +* The definition may contain forward references without having to use string + literal escaping, since it is evaluated lazily. +* The alias can be used in type annotations, type arguments, and casts, but + it can't be used in contexts which require a class object. For example, it's + not valid as a base class and it can't be used to construct instances. + +There is also use an older syntax for defining explicit type aliases, which was +introduced in Python 3.10 (:pep:`613`): .. code-block:: python @@ -604,14 +624,21 @@ doesn't see that the ``buyer`` variable has type ``ProUser``: buyer.pay() # Rejected, not a method on User However, using the ``type[C]`` syntax and a type variable with an upper bound (see -:ref:`type-variable-upper-bound`) we can do better: +:ref:`type-variable-upper-bound`) we can do better (Python 3.12 syntax): + +.. code-block:: python + + def new_user[U: User](user_class: type[U]) -> U: + # Same implementation as before + +Here is the example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python U = TypeVar('U', bound=User) def new_user(user_class: type[U]) -> U: - # Same implementation as before + # Same implementation as before Now mypy will infer the correct type of the result when we call ``new_user()`` with a specific subclass of ``User``: diff --git a/docs/source/literal_types.rst b/docs/source/literal_types.rst index 283bf7f9dba1..972fce72740f 100644 --- a/docs/source/literal_types.rst +++ b/docs/source/literal_types.rst @@ -264,19 +264,15 @@ use the same technique with regular objects, tuples, or namedtuples. Similarly, tags do not need to be specifically str Literals: they can be any type you can normally narrow within ``if`` statements and the like. For example, you could have your tags be int or Enum Literals or even regular classes you narrow -using ``isinstance()``: +using ``isinstance()`` (Python 3.12 syntax): .. code-block:: python - from typing import Generic, TypeVar, Union - - T = TypeVar('T') - - class Wrapper(Generic[T]): + class Wrapper[T]: def __init__(self, inner: T) -> None: self.inner = inner - def process(w: Union[Wrapper[int], Wrapper[str]]) -> None: + def process(w: Wrapper[int] | Wrapper[str]) -> None: # Doing `if isinstance(w, Wrapper[int])` does not work: isinstance requires # that the second argument always be an *erased* type, with no generics. # This is because generics are a typing-only concept and do not exist at diff --git a/docs/source/metaclasses.rst b/docs/source/metaclasses.rst index 396d7dbb42cc..a3ee25f16054 100644 --- a/docs/source/metaclasses.rst +++ b/docs/source/metaclasses.rst @@ -34,13 +34,12 @@ Mypy supports the lookup of attributes in the metaclass: .. code-block:: python - from typing import Type, TypeVar, ClassVar - T = TypeVar('T') + from typing import ClassVar, Self class M(type): count: ClassVar[int] = 0 - def make(cls: Type[T]) -> T: + def make(cls) -> Self: M.count += 1 return cls() @@ -56,6 +55,9 @@ Mypy supports the lookup of attributes in the metaclass: b: B = B.make() # metaclasses are inherited print(B.count + " objects were created") # Error: Unsupported operand types for + ("int" and "str") +.. note:: + In Python 3.10 and earlier, ``Self`` is available in ``typing_extensions``. + .. _limitations: Gotchas and limitations of metaclass support diff --git a/docs/source/more_types.rst b/docs/source/more_types.rst index cb3ef64b39a7..2d3ce5925c02 100644 --- a/docs/source/more_types.rst +++ b/docs/source/more_types.rst @@ -256,11 +256,34 @@ method receives an integer we return a single item. If it receives a ``slice``, we return a :py:class:`~typing.Sequence` of items. We can precisely encode this relationship between the argument and the -return type by using overloads like so: +return type by using overloads like so (Python 3.12 syntax): .. code-block:: python - from typing import Sequence, TypeVar, Union, overload + from collections.abc import Sequence + from typing import overload + + class MyList[T](Sequence[T]): + @overload + def __getitem__(self, index: int) -> T: ... + + @overload + def __getitem__(self, index: slice) -> Sequence[T]: ... + + def __getitem__(self, index: int | slice) -> T | Sequence[T]: + if isinstance(index, int): + # Return a T here + elif isinstance(index, slice): + # Return a sequence of Ts here + else: + raise TypeError(...) + +Here is the same example using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from collections.abc import Sequence + from typing import TypeVar, Union, overload T = TypeVar('T') @@ -697,14 +720,13 @@ Restricted methods in generic classes ------------------------------------- In generic classes some methods may be allowed to be called only -for certain values of type arguments: +for certain values of type arguments (Python 3.12 syntax): .. code-block:: python - T = TypeVar('T') - - class Tag(Generic[T]): + class Tag[T]: item: T + def uppercase_item(self: Tag[str]) -> str: return self.item.upper() @@ -714,18 +736,18 @@ for certain values of type arguments: ts.uppercase_item() # This is OK This pattern also allows matching on nested types in situations where the type -argument is itself generic: +argument is itself generic (Python 3.12 syntax): .. code-block:: python - T = TypeVar('T', covariant=True) - S = TypeVar('S') + from collections.abc import Sequence - class Storage(Generic[T]): + class Storage[T]: def __init__(self, content: T) -> None: - self.content = content - def first_chunk(self: Storage[Sequence[S]]) -> S: - return self.content[0] + self._content = content + + def first_chunk[S](self: Storage[Sequence[S]]) -> S: + return self._content[0] page: Storage[list[str]] page.first_chunk() # OK, type is "str" @@ -734,13 +756,13 @@ argument is itself generic: # "first_chunk" with type "Callable[[Storage[Sequence[S]]], S]" Finally, one can use overloads on self-type to express precise types of -some tricky methods: +some tricky methods (Python 3.12 syntax): .. code-block:: python - T = TypeVar('T') + from typing import overload, Callable - class Tag(Generic[T]): + class Tag[T]: @overload def export(self: Tag[str]) -> str: ... @overload @@ -799,23 +821,22 @@ Precise typing of alternative constructors ------------------------------------------ Some classes may define alternative constructors. If these -classes are generic, self-type allows giving them precise signatures: +classes are generic, self-type allows giving them precise +signatures (Python 3.12 syntax): .. code-block:: python - T = TypeVar('T') - - class Base(Generic[T]): - Q = TypeVar('Q', bound='Base[T]') + from typing import Self + class Base[T]: def __init__(self, item: T) -> None: self.item = item @classmethod - def make_pair(cls: Type[Q], item: T) -> tuple[Q, Q]: + def make_pair(cls, item: T) -> tuple[Self, Self]: return cls(item), cls(item) - class Sub(Base[T]): + class Sub[T](Base[T]): ... pair = Sub.make_pair('yes') # Type is "tuple[Sub[str], Sub[str]]" diff --git a/docs/source/protocols.rst b/docs/source/protocols.rst index 731562867691..e143808e6c25 100644 --- a/docs/source/protocols.rst +++ b/docs/source/protocols.rst @@ -225,7 +225,7 @@ such as trees and linked lists: .. code-block:: python - from typing import TypeVar, Optional, Protocol + from typing import Optional, Protocol class TreeLike(Protocol): value: int @@ -315,8 +315,8 @@ member: # different name and kind in the callback Callback protocols and :py:data:`~typing.Callable` types can be used mostly interchangeably. -Argument names in :py:meth:`__call__ ` methods must be identical, unless -a double underscore prefix is used. For example: +Parameter names in :py:meth:`__call__ ` methods must be identical, unless +the parameters are positional-only. Example (using the legacy syntax for generic functions): .. code-block:: python @@ -325,7 +325,8 @@ a double underscore prefix is used. For example: T = TypeVar('T') class Copy(Protocol): - def __call__(self, __origin: T) -> T: ... + # '/' marks the end of positional-only parameters + def __call__(self, origin: T, /) -> T: ... copy_a: Callable[[T], T] copy_b: Copy diff --git a/docs/source/runtime_troubles.rst b/docs/source/runtime_troubles.rst index 66ab7b3a84c7..ee4cdf274ebe 100644 --- a/docs/source/runtime_troubles.rst +++ b/docs/source/runtime_troubles.rst @@ -69,7 +69,7 @@ Future annotations import (PEP 563) ----------------------------------- Many of the issues described here are caused by Python trying to evaluate -annotations. Future Python versions (potentially Python 3.12) will by default no +annotations. Future Python versions (potentially Python 3.14) will by default no longer attempt to evaluate function and variable annotations. This behaviour is made available in Python 3.7 and later through the use of ``from __future__ import annotations``. @@ -84,7 +84,7 @@ required to be valid Python syntax. For more details, see :pep:`563`. still require string literals or result in errors, typically involving use of forward references or generics in: - * :ref:`type aliases `; + * :ref:`type aliases ` not defined using the ``type`` statement; * :ref:`type narrowing `; * type definitions (see :py:class:`~typing.TypeVar`, :py:class:`~typing.NewType`, :py:class:`~typing.NamedTuple`); * base classes. @@ -93,6 +93,7 @@ required to be valid Python syntax. For more details, see :pep:`563`. # base class example from __future__ import annotations + class A(tuple['B', 'C']): ... # String literal types needed here class B: ... class C: ... @@ -244,7 +245,8 @@ complicated and you need to use :ref:`typing.TYPE_CHECKING task_queue: Tasks reveal_type(task_queue.get()) # Reveals str -If your subclass is also generic, you can use the following: +If your subclass is also generic, you can use the following (using the +legacy syntax for generic classes): .. code-block:: python @@ -262,9 +264,11 @@ If your subclass is also generic, you can use the following: task_queue: MyQueue[str] reveal_type(task_queue.get()) # Reveals str -In Python 3.9, we can just inherit directly from ``Queue[str]`` or ``Queue[T]`` +In Python 3.9 and later, we can just inherit directly from ``Queue[str]`` or ``Queue[T]`` since its :py:class:`queue.Queue` implements :py:meth:`~object.__class_getitem__`, so -the class object can be subscripted at runtime without issue. +the class object can be subscripted at runtime. You may still encounter issues (even if +you use a recent Python version) when subclassing generic classes defined in third-party +libraries if types are generic only in stubs. Using types defined in stubs but not at runtime ----------------------------------------------- diff --git a/docs/source/type_narrowing.rst b/docs/source/type_narrowing.rst index d698f35c44bc..131171844dfe 100644 --- a/docs/source/type_narrowing.rst +++ b/docs/source/type_narrowing.rst @@ -255,16 +255,13 @@ to the type specified as the first type parameter (``list[str]``). Generic TypeGuards ~~~~~~~~~~~~~~~~~~ -``TypeGuard`` can also work with generic types: +``TypeGuard`` can also work with generic types (Python 3.12 syntax): .. code-block:: python - from typing import TypeVar from typing import TypeGuard # use `typing_extensions` for `python<3.10` - _T = TypeVar("_T") - - def is_two_element_tuple(val: tuple[_T, ...]) -> TypeGuard[tuple[_T, _T]]: + def is_two_element_tuple[T](val: tuple[T, ...]) -> TypeGuard[tuple[T, T]]: return len(val) == 2 def func(names: tuple[str, ...]): @@ -276,16 +273,13 @@ Generic TypeGuards TypeGuards with parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Type guard functions can accept extra arguments: +Type guard functions can accept extra arguments (Python 3.12 syntax): .. code-block:: python - from typing import Type, TypeVar from typing import TypeGuard # use `typing_extensions` for `python<3.10` - _T = TypeVar("_T") - - def is_set_of(val: set[Any], type: Type[_T]) -> TypeGuard[set[_T]]: + def is_set_of[T](val: set[Any], type: type[T]) -> TypeGuard[set[T]]: return all(isinstance(x, type) for x in val) items: set[Any] From f6520c84746072e72e8a017963d3d2a3a9361771 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Wed, 25 Sep 2024 21:07:44 -0400 Subject: [PATCH 153/247] Narrow falsey str/bytes/int to literal type (#17818) Closes #16891 ### Before ```python from typing import Literal def f1(a: str) -> Literal[""]: return a and exit() # E: Incompatible return value type (got "str", expected "Literal['']") def f2(a: int) -> Literal[0]: return a and exit() # E: Incompatible return value type (got "int", expected "Literal[0]") def f3(a: bytes) -> Literal[b""]: return a and exit() # E: Incompatible return value type (got "bytes", expected "Literal[b'']") ``` ### After ```none Success: no issues found in 1 source file ``` --- mypy/typeops.py | 4 ++++ test-data/unit/check-expressions.test | 2 +- test-data/unit/check-narrowing.test | 18 +++++++++++++++++- test-data/unit/check-optional.test | 10 +++++----- test-data/unit/check-python38.test | 8 ++++---- 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index d22448a715e5..7f530d13d4e2 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -657,6 +657,10 @@ def false_only(t: Type) -> ProperType: new_items = [false_only(item) for item in t.items] can_be_false_items = [item for item in new_items if item.can_be_false] return make_simplified_union(can_be_false_items, line=t.line, column=t.column) + elif isinstance(t, Instance) and t.type.fullname in ("builtins.str", "builtins.bytes"): + return LiteralType("", fallback=t) + elif isinstance(t, Instance) and t.type.fullname == "builtins.int": + return LiteralType(0, fallback=t) else: ret_type = _get_type_method_ret_type(t, name="__bool__") or _get_type_method_ret_type( t, name="__len__" diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 61cee1d00c58..d5ddc910bcd6 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -342,7 +342,7 @@ b: bool i: str j = b or i if not j: - reveal_type(j) # N: Revealed type is "builtins.str" + reveal_type(j) # N: Revealed type is "Literal['']" [builtins fixtures/bool.pyi] [case testAndOr] diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index e142fdd5d060..0cb4bf8e4a19 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -1007,7 +1007,7 @@ str_or_false: Union[Literal[False], str] if str_or_false: reveal_type(str_or_false) # N: Revealed type is "builtins.str" else: - reveal_type(str_or_false) # N: Revealed type is "Union[Literal[False], builtins.str]" + reveal_type(str_or_false) # N: Revealed type is "Union[Literal[False], Literal['']]" true_or_false: Literal[True, False] @@ -1017,6 +1017,22 @@ else: reveal_type(true_or_false) # N: Revealed type is "Literal[False]" [builtins fixtures/primitives.pyi] +[case testNarrowingFalseyToLiteral] +from typing import Union + +a: str +b: bytes +c: int +d: Union[str, bytes, int] + +if not a: + reveal_type(a) # N: Revealed type is "Literal['']" +if not b: + reveal_type(b) # N: Revealed type is "Literal[b'']" +if not c: + reveal_type(c) # N: Revealed type is "Literal[0]" +if not d: + reveal_type(d) # N: Revealed type is "Union[Literal[''], Literal[b''], Literal[0]]" [case testNarrowingIsInstanceFinalSubclass] # flags: --warn-unreachable diff --git a/test-data/unit/check-optional.test b/test-data/unit/check-optional.test index f80aa5115bc3..683ce0446915 100644 --- a/test-data/unit/check-optional.test +++ b/test-data/unit/check-optional.test @@ -53,14 +53,14 @@ x = None # type: Optional[int] if x: reveal_type(x) # N: Revealed type is "builtins.int" else: - reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + reveal_type(x) # N: Revealed type is "Union[Literal[0], None]" [builtins fixtures/bool.pyi] [case testIfNotCases] from typing import Optional x = None # type: Optional[int] if not x: - reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" + reveal_type(x) # N: Revealed type is "Union[Literal[0], None]" else: reveal_type(x) # N: Revealed type is "builtins.int" [builtins fixtures/bool.pyi] @@ -109,13 +109,13 @@ reveal_type(z2) # N: Revealed type is "Union[builtins.int, builtins.str, None]" from typing import Optional x = None # type: Optional[str] y1 = x and 'b' -reveal_type(y1) # N: Revealed type is "Union[builtins.str, None]" +reveal_type(y1) # N: Revealed type is "Union[Literal[''], None, builtins.str]" y2 = x and 1 # x could be '', so... -reveal_type(y2) # N: Revealed type is "Union[builtins.str, None, builtins.int]" +reveal_type(y2) # N: Revealed type is "Union[Literal[''], None, builtins.int]" z1 = 'b' and x reveal_type(z1) # N: Revealed type is "Union[builtins.str, None]" z2 = int() and x -reveal_type(z2) # N: Revealed type is "Union[builtins.int, builtins.str, None]" +reveal_type(z2) # N: Revealed type is "Union[Literal[0], builtins.str, None]" [case testLambdaReturningNone] f = lambda: None diff --git a/test-data/unit/check-python38.test b/test-data/unit/check-python38.test index dfb918defb0a..199014a66fed 100644 --- a/test-data/unit/check-python38.test +++ b/test-data/unit/check-python38.test @@ -463,9 +463,9 @@ def check_partial_list() -> None: if (x := 0): reveal_type(x) # E: Statement is unreachable else: - reveal_type(x) # N: Revealed type is "builtins.int" + reveal_type(x) # N: Revealed type is "Literal[0]" -reveal_type(x) # N: Revealed type is "builtins.int" +reveal_type(x) # N: Revealed type is "Literal[0]" [case testWalrusAssignmentAndConditionScopeForProperty] # flags: --warn-unreachable @@ -483,7 +483,7 @@ wrapper = PropertyWrapper() if x := wrapper.f: reveal_type(x) # N: Revealed type is "builtins.str" else: - reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Literal['']" reveal_type(x) # N: Revealed type is "builtins.str" @@ -505,7 +505,7 @@ def f() -> str: ... if x := f(): reveal_type(x) # N: Revealed type is "builtins.str" else: - reveal_type(x) # N: Revealed type is "builtins.str" + reveal_type(x) # N: Revealed type is "Literal['']" reveal_type(x) # N: Revealed type is "builtins.str" From 6336ce117d857ac56c4cd140b8ab0ae8f70ecc83 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 26 Sep 2024 10:07:29 +0100 Subject: [PATCH 154/247] Discuss upper bounds before self types in documentation (#17827) The explanation of self types uses upper bounds, so it makes sense to discuss them earlier. I also made some other minor documentation tweaks to improve clarity. --- docs/source/generics.rst | 114 +++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 58 deletions(-) diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 4eb6463e4bd4..3d175cedbb93 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -315,6 +315,53 @@ inferred by mypy. This is not valid: If you really need this, you can define a generic class with a ``__call__`` method. +.. _type-variable-upper-bound: + +Type variables with upper bounds +******************************** + +A type variable can also be restricted to having values that are +subtypes of a specific type. This type is called the upper bound of +the type variable, and it is specified using ``T: `` when using the +Python 3.12 syntax. In the definition of a generic function or a generic +class that uses such a type variable ``T``, the type represented by ``T`` +is assumed to be a subtype of its upper bound, so you can use methods +of the upper bound on values of type ``T`` (Python 3.12 syntax): + +.. code-block:: python + + from typing import SupportsAbs + + def max_by_abs[T: SupportsAbs[float]](*xs: T) -> T: + # We can use abs(), because T is a subtype of SupportsAbs[float]. + return max(xs, key=abs) + +An upper bound can also be specified with the ``bound=...`` keyword +argument to :py:class:`~typing.TypeVar`. +Here is the example using the legacy syntax (Python 3.11 and earlier): + +.. code-block:: python + + from typing import TypeVar, SupportsAbs + + T = TypeVar('T', bound=SupportsAbs[float]) + + def max_by_abs(*xs: T) -> T: + return max(xs, key=abs) + +In a call to such a function, the type ``T`` must be replaced by a +type that is a subtype of its upper bound. Continuing the example +above: + +.. code-block:: python + + max_by_abs(-3.5, 2) # Okay, has type 'float' + max_by_abs(5+6j, 7) # Okay, has type 'complex' + max_by_abs('a', 'b') # Error: 'str' is not a subtype of SupportsAbs[float] + +Type parameters of generic classes may also have upper bounds, which +restrict the valid values for the type parameter in the same way. + .. _generic-methods-and-generic-self: Generic methods and generic self @@ -400,8 +447,7 @@ or :py:class:`Type[T] ` (Python 3.12 syntax): a, b = SuperFriend.make_pair() -Here is the same example using the legacy syntax (3.11 and earlier, though -3.9 and later can use lower-case ``type[T]``): +Here is the same example using the legacy syntax (3.11 and earlier): .. code-block:: python @@ -433,7 +479,7 @@ or a deserialization method returns the actual type of self. Therefore you may need to silence mypy inside these methods (but not at the call site), possibly by making use of the ``Any`` type or a ``# type: ignore`` comment. -Note that mypy lets you use generic self types in certain unsafe ways +Mypy lets you use generic self types in certain unsafe ways in order to support common idioms. For example, using a generic self type in an argument type is accepted even though it's unsafe (Python 3.12 syntax): @@ -647,59 +693,13 @@ contravariant, use type variables defined with special keyword arguments my_box = Box(Square()) look_into(my_box) # OK, but mypy would complain here for an invariant type -.. _type-variable-upper-bound: - -Type variables with upper bounds -******************************** - -A type variable can also be restricted to having values that are -subtypes of a specific type. This type is called the upper bound of -the type variable, and it is specified using ``T: `` when using the -Python 3.12 syntax. In the definition of a generic function that uses -such a type variable ``T``, the type represented by ``T`` is assumed -to be a subtype of its upper bound, so the function can use methods -of the upper bound on values of type ``T`` (Python 3.12 syntax): - -.. code-block:: python - - from typing import SupportsAbs - - def max_by_abs[T: SupportsAbs[float]](*xs: T) -> T: - # Okay, because T is a subtype of SupportsAbs[float]. - return max(xs, key=abs) - -An upper bound can also be specified with the ``bound=...`` keyword -argument to :py:class:`~typing.TypeVar`. -Here is the example using the legacy syntax (Python 3.11 and earlier): - -.. code-block:: python - - from typing import TypeVar, SupportsAbs - - T = TypeVar('T', bound=SupportsAbs[float]) - - def max_by_abs(*xs: T) -> T: - return max(xs, key=abs) - -In a call to such a function, the type ``T`` must be replaced by a -type that is a subtype of its upper bound. Continuing the example -above: - -.. code-block:: python - - max_by_abs(-3.5, 2) # Okay, has type float. - max_by_abs(5+6j, 7) # Okay, has type complex. - max_by_abs('a', 'b') # Error: 'str' is not a subtype of SupportsAbs[float]. - -Type parameters of generic classes may also have upper bounds, which -restrict the valid values for the type parameter in the same way. - .. _type-variable-value-restriction: Type variables with value restriction ************************************* -By default, a type variable can be replaced with any type. However, sometimes +By default, a type variable can be replaced with any type -- or any type that +is a subtype of the upper bound, which defaults to ``object``. However, sometimes it's useful to have a type variable that can only have some specific types as its value. A typical example is a type variable that can only have values ``str`` and ``bytes``. This lets us define a function that can concatenate @@ -758,11 +758,10 @@ You may expect that the type of ``ss`` is ``S``, but the type is actually ``str``: a subtype gets promoted to one of the valid values for the type variable, which in this case is ``str``. -This is thus -subtly different from *bounded quantification* in languages such as -Java, where the return type would be ``S``. The way mypy implements -this is correct for ``concat``, since ``concat`` actually returns a -``str`` instance in the above example: +This is thus subtly different from using ``str | bytes`` as an upper bound, +where the return type would be ``S`` (see :ref:`type-variable-upper-bound`). +Using a value restriction is correct for ``concat``, since ``concat`` +actually returns a ``str`` instance in the above example: .. code-block:: python @@ -776,8 +775,7 @@ value of :py:func:`re.compile`, where ``S`` can be either ``str`` or ``bytes``. Regular expressions can be based on a string or a bytes pattern. -A type variable may not have both a value restriction and an upper bound -(see :ref:`type-variable-upper-bound`). +A type variable may not have both a value restriction and an upper bound. Note that you may come across :py:data:`~typing.AnyStr` imported from :py:mod:`typing`. This feature is now deprecated, but it means the same From 0dfb7184d72d1aad951b1eed1541e3162d7ca040 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 26 Sep 2024 11:19:15 +0100 Subject: [PATCH 155/247] Update various references to deprecated type aliases in docs (#17829) Use `collections.abc.Iterable`, for example, instead of `typing.Iterable`, unless we are discussing support for older Python versions. The latter has been deprecated since Python 3.9. Also update a few other out-of-date things that I happened to notice. Part of the motivation is that Python 3.8 will reach end of life in October, so we can soon start mostly assuming that users are on 3.9 or newer (even if we'll continue supporting 3.8 for a while still). --------- Co-authored-by: Jelle Zijlstra --- docs/source/additional_features.rst | 12 +-- docs/source/cheat_sheet_py3.rst | 12 ++- docs/source/class_basics.rst | 3 +- docs/source/command_line.rst | 4 +- docs/source/common_issues.rst | 7 +- docs/source/dynamic_typing.rst | 4 +- docs/source/error_code_list.rst | 6 +- docs/source/error_code_list2.rst | 4 +- docs/source/faq.rst | 9 +- docs/source/generics.rst | 40 +++++---- docs/source/getting_started.rst | 2 +- docs/source/kinds_of_types.rst | 13 ++- docs/source/more_types.rst | 22 +++-- docs/source/protocols.rst | 133 ++++++++++++++++------------ docs/source/runtime_troubles.rst | 5 +- docs/source/type_narrowing.rst | 5 +- docs/source/typed_dict.rst | 4 +- 17 files changed, 161 insertions(+), 124 deletions(-) diff --git a/docs/source/additional_features.rst b/docs/source/additional_features.rst index a9c3177d32a2..e7c162a0b0df 100644 --- a/docs/source/additional_features.rst +++ b/docs/source/additional_features.rst @@ -363,20 +363,20 @@ Extended Callable types This feature is deprecated. You can use :ref:`callback protocols ` as a replacement. -As an experimental mypy extension, you can specify :py:data:`~typing.Callable` types +As an experimental mypy extension, you can specify :py:class:`~collections.abc.Callable` types that support keyword arguments, optional arguments, and more. When -you specify the arguments of a :py:data:`~typing.Callable`, you can choose to supply just +you specify the arguments of a :py:class:`~collections.abc.Callable`, you can choose to supply just the type of a nameless positional argument, or an "argument specifier" representing a more complicated form of argument. This allows one to more closely emulate the full range of possibilities given by the ``def`` statement in Python. As an example, here's a complicated function definition and the -corresponding :py:data:`~typing.Callable`: +corresponding :py:class:`~collections.abc.Callable`: .. code-block:: python - from typing import Callable + from collections.abc import Callable from mypy_extensions import (Arg, DefaultArg, NamedArg, DefaultNamedArg, VarArg, KwArg) @@ -449,7 +449,7 @@ purpose: In all cases, the ``type`` argument defaults to ``Any``, and if the ``name`` argument is omitted the argument has no name (the name is required for ``NamedArg`` and ``DefaultNamedArg``). A basic -:py:data:`~typing.Callable` such as +:py:class:`~collections.abc.Callable` such as .. code-block:: python @@ -461,7 +461,7 @@ is equivalent to the following: MyFunc = Callable[[Arg(int), Arg(str), Arg(int)], float] -A :py:data:`~typing.Callable` with unspecified argument types, such as +A :py:class:`~collections.abc.Callable` with unspecified argument types, such as .. code-block:: python diff --git a/docs/source/cheat_sheet_py3.rst b/docs/source/cheat_sheet_py3.rst index ca6dfc8eb63a..eb2f8228cf52 100644 --- a/docs/source/cheat_sheet_py3.rst +++ b/docs/source/cheat_sheet_py3.rst @@ -88,7 +88,8 @@ Functions .. code-block:: python - from typing import Callable, Iterator, Union, Optional + from collections.abc import Iterator, Callable + from typing import Union, Optional # This is how you annotate a function definition def stringify(num: int) -> str: @@ -274,7 +275,8 @@ that are common in idiomatic Python are standardized. .. code-block:: python - from typing import Mapping, MutableMapping, Sequence, Iterable + from collections.abc import Mapping, MutableMapping, Sequence, Iterable + # or 'from typing import ...' (required in Python 3.8) # Use Iterable for generic iterables (anything usable in "for"), # and Sequence where a sequence (supporting "len" and "__getitem__") is @@ -354,7 +356,8 @@ syntax: .. code-block:: python - from typing import Any, Callable + from collections.abc import Callable + from typing import Any def bare_decorator[F: Callable[..., Any]](func: F) -> F: ... @@ -366,7 +369,8 @@ The same example using pre-3.12 syntax: .. code-block:: python - from typing import Any, Callable, TypeVar + from collections.abc import Callable + from typing import Any, TypeVar F = TypeVar('F', bound=Callable[..., Any]) diff --git a/docs/source/class_basics.rst b/docs/source/class_basics.rst index 1d80da5830ec..241dbeae0f44 100644 --- a/docs/source/class_basics.rst +++ b/docs/source/class_basics.rst @@ -152,7 +152,8 @@ between class and instance variables with callable types. For example: .. code-block:: python - from typing import Callable, ClassVar + from collections.abc import Callable + from typing import ClassVar class A: foo: Callable[[int], None] diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index a89a3c85d4ee..3d4b841e0a7c 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -630,13 +630,11 @@ of the above sections. .. code-block:: python - from typing import Text - items: list[int] if 'some string' in items: # Error: non-overlapping container check! ... - text: Text + text: str if text != b'other bytes': # Error: non-overlapping equality check! ... diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index cfe82e19e77b..c34d20775cfd 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -363,7 +363,8 @@ explicit type cast: .. code-block:: python - from typing import Sequence, cast + from collections.abc import Sequence + from typing import cast def find_first_str(a: Sequence[object]) -> str: index = next((i for i, s in enumerate(a) if isinstance(s, str)), -1) @@ -700,7 +701,7 @@ This example demonstrates both safe and unsafe overrides: .. code-block:: python - from typing import Sequence, List, Iterable + from collections.abc import Sequence, Iterable class A: def test(self, t: Sequence[int]) -> Sequence[str]: @@ -713,7 +714,7 @@ This example demonstrates both safe and unsafe overrides: class NarrowerArgument(A): # A more specific argument type isn't accepted - def test(self, t: List[int]) -> Sequence[str]: # Error + def test(self, t: list[int]) -> Sequence[str]: # Error ... class NarrowerReturn(A): diff --git a/docs/source/dynamic_typing.rst b/docs/source/dynamic_typing.rst index d3476de2ca64..1c31a535bdc1 100644 --- a/docs/source/dynamic_typing.rst +++ b/docs/source/dynamic_typing.rst @@ -81,9 +81,7 @@ treated as ``Any``: .. code-block:: python - from typing import List - - def f(x: List) -> None: + def f(x: list) -> None: reveal_type(x) # Revealed type is "builtins.list[Any]" reveal_type(x[0]) # Revealed type is "Any" x[0].anything_goes() # OK diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index ec069fdcba1d..bb745d9d5c98 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -124,8 +124,6 @@ Example: .. code-block:: python - from typing import Sequence - def greet(name: str) -> None: print('hello', name) @@ -210,11 +208,11 @@ This example incorrectly uses the function ``log`` as a type: for x in objs: f(x) -You can use :py:data:`~typing.Callable` as the type for callable objects: +You can use :py:class:`~collections.abc.Callable` as the type for callable objects: .. code-block:: python - from typing import Callable + from collections.abc import Callable # OK def log_all(objs: list[object], f: Callable[[object], None]) -> None: diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst index 0655ef2d35d8..6d50e217a77d 100644 --- a/docs/source/error_code_list2.rst +++ b/docs/source/error_code_list2.rst @@ -270,7 +270,7 @@ example: # mypy: enable-error-code="possibly-undefined" - from typing import Iterable + from collections.abc import Iterable def test(values: Iterable[int], flag: bool) -> None: if flag: @@ -318,7 +318,7 @@ Example: .. code-block:: python - from typing import Iterable + from collections.abc import Iterable def transform(items: Iterable[int]) -> list[int]: # Error: "items" has type "Iterable[int]" which can always be true in boolean context. Consider using "Collection[int]" instead. [truthy-iterable] diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 195805382cd3..b7f5e3759a7e 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -102,8 +102,8 @@ Structural subtyping can be thought of as "static duck typing". Some argue that structural subtyping is better suited for languages with duck typing such as Python. Mypy however primarily uses nominal subtyping, leaving structural subtyping mostly opt-in (except for built-in protocols -such as :py:class:`~typing.Iterable` that always support structural subtyping). Here are some -reasons why: +such as :py:class:`~collections.abc.Iterable` that always support structural +subtyping). Here are some reasons why: 1. It is easy to generate short and informative error messages when using a nominal type system. This is especially important when @@ -140,13 +140,14 @@ How are mypy programs different from normal Python? Since you use a vanilla Python implementation to run mypy programs, mypy programs are also Python programs. The type checker may give warnings for some valid Python code, but the code is still always -runnable. Also, some Python features and syntax are still not +runnable. Also, a few Python features are still not supported by mypy, but this is gradually improving. The obvious difference is the availability of static type checking. The section :ref:`common_issues` mentions some modifications to Python code that may be required to make code type -check without errors. Also, your code must make attributes explicit. +check without errors. Also, your code must make defined +attributes explicit. Mypy supports modular, efficient type checking, and this seems to rule out type checking some language features, such as arbitrary diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 3d175cedbb93..982adf9324af 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -591,7 +591,7 @@ Let us illustrate this by few simple examples: Covariance should feel relatively intuitive, but contravariance and invariance can be harder to reason about. -* :py:data:`~typing.Callable` is an example of type that behaves contravariant +* :py:class:`~collections.abc.Callable` is an example of type that behaves contravariant in types of arguments. That is, ``Callable[[Shape], int]`` is a subtype of ``Callable[[Triangle], int]``, despite ``Shape`` being a supertype of ``Triangle``. To understand this, consider: @@ -833,7 +833,8 @@ Here's how one could annotate the decorator (Python 3.12 syntax): .. code-block:: python - from typing import Any, Callable, cast + from collections.abc import Callable + from typing import Any, cast # A decorator that preserves the signature. def printing_decorator[F: Callable[..., Any]](func: F) -> F: @@ -854,7 +855,8 @@ Here is the example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python - from typing import Any, Callable, TypeVar, cast + from collections.abc import Callable + from typing import Any, TypeVar, cast F = TypeVar('F', bound=Callable[..., Any]) @@ -887,7 +889,7 @@ for a more faithful type annotation (Python 3.12 syntax): .. code-block:: python - from typing import Callable + from collections.abc import Callable def printing_decorator[**P, T](func: Callable[P, T]) -> Callable[P, T]: def wrapper(*args: P.args, **kwds: P.kwargs) -> T: @@ -900,7 +902,8 @@ The same is possible using the legacy syntax with :py:class:`~typing.ParamSpec` .. code-block:: python - from typing import Callable, TypeVar + from collections.abc import Callable + from typing import TypeVar from typing_extensions import ParamSpec P = ParamSpec('P') @@ -917,7 +920,7 @@ alter the signature of the input function (Python 3.12 syntax): .. code-block:: python - from typing import Callable + from collections.abc import Callable # We reuse 'P' in the return type, but replace 'T' with 'str' def stringify[**P, T](func: Callable[P, T]) -> Callable[P, str]: @@ -937,7 +940,8 @@ Here is the above example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python - from typing import Callable, TypeVar + from collections.abc import Callable + from typing import TypeVar from typing_extensions import ParamSpec P = ParamSpec('P') @@ -953,7 +957,8 @@ You can also insert an argument in a decorator (Python 3.12 syntax): .. code-block:: python - from typing import Callable, Concatenate + from collections.abc import Callable + from typing import Concatenate def printing_decorator[**P, T](func: Callable[P, T]) -> Callable[Concatenate[str, P], T]: def wrapper(msg: str, /, *args: P.args, **kwds: P.kwargs) -> T: @@ -971,7 +976,8 @@ Here is the same function using the legacy syntax (Python 3.11 and earlier): .. code-block:: python - from typing import Callable, TypeVar + from collections.abc import Callable + from typing import TypeVar from typing_extensions import Concatenate, ParamSpec P = ParamSpec('P') @@ -993,7 +999,8 @@ similarly supported via generics (Python 3.12 syntax): .. code-block:: python - from typing import Any, Callable + from colletions.abc import Callable + from typing import Any def route[F: Callable[..., Any]](url: str) -> Callable[[F], F]: ... @@ -1011,7 +1018,8 @@ Here is the example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python - from typing import Any, Callable, TypeVar + from collections.abc import Callable + from typing import Any, TypeVar F = TypeVar('F', bound=Callable[..., Any]) @@ -1027,7 +1035,8 @@ achieved by combining with :py:func:`@overload ` (Python 3.12 s .. code-block:: python - from typing import Any, Callable, overload + from collections.abc import Callable + from typing import Any, overload # Bare decorator usage @overload @@ -1057,7 +1066,8 @@ Here is the decorator from the example using the legacy syntax .. code-block:: python - from typing import Any, Callable, Optional, TypeVar, overload + from collections.abc import Callable + from typing import Any, Optional, TypeVar, overload F = TypeVar('F', bound=Callable[..., Any]) @@ -1077,7 +1087,7 @@ Generic protocols Mypy supports generic protocols (see also :ref:`protocol-types`). Several :ref:`predefined protocols ` are generic, such as -:py:class:`Iterable[T] `, and you can define additional +:py:class:`Iterable[T] `, and you can define additional generic protocols. Generic protocols mostly follow the normal rules for generic classes. Example (Python 3.12 syntax): @@ -1200,7 +1210,7 @@ type aliases (it also supports non-generic type aliases): .. code-block:: python - from typing import Iterable, Callable + from collections.abc import Callable, Iterable type TInt[S] = tuple[int, S] type UInt[S] = S | int diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 049d7af003b5..12422ef85bde 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -215,7 +215,7 @@ generic types or your own type aliases), look through the For brevity, we often omit imports from :py:mod:`typing` or :py:mod:`collections.abc` in code examples, but mypy will give an error if you use types such as - :py:class:`~typing.Iterable` without first importing them. + :py:class:`~collections.abc.Iterable` without first importing them. .. note:: diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index e645a27095d1..afdf07a559da 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -136,7 +136,7 @@ purpose. Example: .. note:: Usually it's a better idea to use ``Sequence[T]`` instead of ``tuple[T, ...]``, as - :py:class:`~typing.Sequence` is also compatible with lists and other non-tuple sequences. + :py:class:`~collections.abc.Sequence` is also compatible with lists and other non-tuple sequences. .. note:: @@ -155,7 +155,7 @@ and returns ``Rt`` is ``Callable[[A1, ..., An], Rt]``. Example: .. code-block:: python - from typing import Callable + from collections.abc import Callable def twice(i: int, next: Callable[[int], int]) -> int: return next(next(i)) @@ -165,6 +165,11 @@ and returns ``Rt`` is ``Callable[[A1, ..., An], Rt]``. Example: print(twice(3, add)) # 5 +.. note:: + + Import :py:data:`Callable[...] ` from ``typing`` instead + of ``collections.abc`` if you use Python 3.8 or earlier. + You can only have positional arguments, and only ones without default values, in callable types. These cover the vast majority of uses of callable types, but sometimes this isn't quite enough. Mypy recognizes @@ -178,7 +183,7 @@ Any)`` function signature. Example: .. code-block:: python - from typing import Callable + from collections.abc import Callable def arbitrary_call(f: Callable[..., int]) -> int: return f('x') + f(y=2) # OK @@ -205,7 +210,7 @@ Callables can also be used against type objects, matching their .. code-block:: python - from typing import Callable + from collections.abc import Callable class C: def __init__(self, app: str) -> None: diff --git a/docs/source/more_types.rst b/docs/source/more_types.rst index 2d3ce5925c02..26edd9ef252c 100644 --- a/docs/source/more_types.rst +++ b/docs/source/more_types.rst @@ -253,7 +253,7 @@ calls like ``mouse_event(5, 25, 2)``. As another example, suppose we want to write a custom container class that implements the :py:meth:`__getitem__ ` method (``[]`` bracket indexing). If this method receives an integer we return a single item. If it receives a -``slice``, we return a :py:class:`~typing.Sequence` of items. +``slice``, we return a :py:class:`~collections.abc.Sequence` of items. We can precisely encode this relationship between the argument and the return type by using overloads like so (Python 3.12 syntax): @@ -760,7 +760,8 @@ some tricky methods (Python 3.12 syntax): .. code-block:: python - from typing import overload, Callable + from collections.abc import Callable + from typing import overload class Tag[T]: @overload @@ -875,8 +876,8 @@ expect to get back when ``await``-ing the coroutine. The result of calling an ``async def`` function *without awaiting* will automatically be inferred to be a value of type -:py:class:`Coroutine[Any, Any, T] `, which is a subtype of -:py:class:`Awaitable[T] `: +:py:class:`Coroutine[Any, Any, T] `, which is a subtype of +:py:class:`Awaitable[T] `: .. code-block:: python @@ -889,11 +890,12 @@ Asynchronous iterators ---------------------- If you have an asynchronous iterator, you can use the -:py:class:`~typing.AsyncIterator` type in your annotations: +:py:class:`~collections.abc.AsyncIterator` type in your annotations: .. code-block:: python - from typing import Optional, AsyncIterator + from collections.abc import AsyncIterator + from typing import Optional import asyncio class arange: @@ -926,7 +928,8 @@ async iterators: .. code-block:: python - from typing import AsyncGenerator, Optional + from collections.abc import AsyncGenerator + from typing import Optional import asyncio # Could also type this as returning AsyncIterator[int] @@ -943,7 +946,7 @@ One common confusion is that the presence of a ``yield`` statement in an .. code-block:: python - from typing import AsyncIterator + from collections.abc import AsyncIterator async def arange(stop: int) -> AsyncIterator[int]: # When called, arange gives you an async iterator @@ -969,7 +972,8 @@ This can sometimes come up when trying to define base classes, Protocols or over .. code-block:: python - from typing import AsyncIterator, Protocol, overload + from collections.abc import AsyncIterator + from typing import Protocol, overload class LauncherIncorrect(Protocol): # Because launch does not have yield, this has type diff --git a/docs/source/protocols.rst b/docs/source/protocols.rst index e143808e6c25..8cb53b1fd5ce 100644 --- a/docs/source/protocols.rst +++ b/docs/source/protocols.rst @@ -27,15 +27,17 @@ of protocols and structural subtyping in Python. Predefined protocols ******************** -The :py:mod:`typing` module defines various protocol classes that correspond -to common Python protocols, such as :py:class:`Iterable[T] `. If a class +The :py:mod:`collections.abc`, :py:mod:`typing` and other stdlib modules define +various protocol classes that correspond to common Python protocols, such as +:py:class:`Iterable[T] `. If a class defines a suitable :py:meth:`__iter__ ` method, mypy understands that it -implements the iterable protocol and is compatible with :py:class:`Iterable[T] `. +implements the iterable protocol and is compatible with :py:class:`Iterable[T] `. For example, ``IntList`` below is iterable, over ``int`` values: .. code-block:: python - from typing import Iterator, Iterable, Optional + from collections.abc import Iterator, Iterable + from typing import Optional class IntList: def __init__(self, value: int, next: Optional['IntList']) -> None: @@ -56,9 +58,18 @@ For example, ``IntList`` below is iterable, over ``int`` values: print_numbered(x) # OK print_numbered([4, 5]) # Also OK -:ref:`predefined_protocols_reference` lists all protocols defined in -:py:mod:`typing` and the signatures of the corresponding methods you need to define -to implement each protocol. +:ref:`predefined_protocols_reference` lists various protocols defined in +:py:mod:`collections.abc` and :py:mod:`typing` and the signatures of the corresponding methods +you need to define to implement each protocol. + +.. note:: + ``typing`` also contains deprecated aliases to protocols and ABCs defined in + :py:mod:`collections.abc`, such as :py:class:`Iterable[T] `. + These are only necessary in Python 3.8 and earlier, since the protocols in + ``collections.abc`` didn't yet support subscripting (``[]``) in Python 3.8, + but the aliases in ``typing`` have always supported + subscripting. In Python 3.9 and later, the aliases in ``typing`` don't provide + any extra functionality. Simple user-defined protocols ***************************** @@ -68,7 +79,8 @@ class: .. code-block:: python - from typing import Iterable, Protocol + from collections.abc import Iterable + from typing import Protocol class SupportsClose(Protocol): # Empty method body (explicit '...') @@ -290,13 +302,15 @@ Callback protocols ****************** Protocols can be used to define flexible callback types that are hard -(or even impossible) to express using the :py:data:`Callable[...] ` syntax, such as variadic, -overloaded, and complex generic callbacks. They are defined with a special :py:meth:`__call__ ` -member: +(or even impossible) to express using the +:py:class:`Callable[...] ` syntax, +such as variadic, overloaded, and complex generic callbacks. They are defined with a +special :py:meth:`__call__ ` member: .. code-block:: python - from typing import Optional, Iterable, Protocol + from collections.abc import Iterable + from typing import Optional, Protocol class Combiner(Protocol): def __call__(self, *vals: bytes, maxlen: Optional[int] = None) -> list[bytes]: ... @@ -314,13 +328,14 @@ member: batch_proc([], bad_cb) # Error! Argument 2 has incompatible type because of # different name and kind in the callback -Callback protocols and :py:data:`~typing.Callable` types can be used mostly interchangeably. +Callback protocols and :py:class:`~collections.abc.Callable` types can be used mostly interchangeably. Parameter names in :py:meth:`__call__ ` methods must be identical, unless the parameters are positional-only. Example (using the legacy syntax for generic functions): .. code-block:: python - from typing import Callable, Protocol, TypeVar + from collections.abc import Callable + from typing import Protocol, TypeVar T = TypeVar('T') @@ -345,8 +360,8 @@ Iteration protocols The iteration protocols are useful in many contexts. For example, they allow iteration of objects in for loops. -Iterable[T] ------------ +collections.abc.Iterable[T] +--------------------------- The :ref:`example above ` has a simple implementation of an :py:meth:`__iter__ ` method. @@ -355,17 +370,17 @@ The :ref:`example above ` has a simple implementation of a def __iter__(self) -> Iterator[T] -See also :py:class:`~typing.Iterable`. +See also :py:class:`~collections.abc.Iterable`. -Iterator[T] ------------ +collections.abc.Iterator[T] +--------------------------- .. code-block:: python def __next__(self) -> T def __iter__(self) -> Iterator[T] -See also :py:class:`~typing.Iterator`. +See also :py:class:`~collections.abc.Iterator`. Collection protocols .................... @@ -374,8 +389,8 @@ Many of these are implemented by built-in container types such as :py:class:`list` and :py:class:`dict`, and these are also useful for user-defined collection objects. -Sized ------ +collections.abc.Sized +--------------------- This is a type for objects that support :py:func:`len(x) `. @@ -383,10 +398,10 @@ This is a type for objects that support :py:func:`len(x) `. def __len__(self) -> int -See also :py:class:`~typing.Sized`. +See also :py:class:`~collections.abc.Sized`. -Container[T] ------------- +collections.abc.Container[T] +---------------------------- This is a type for objects that support the ``in`` operator. @@ -394,10 +409,10 @@ This is a type for objects that support the ``in`` operator. def __contains__(self, x: object) -> bool -See also :py:class:`~typing.Container`. +See also :py:class:`~collections.abc.Container`. -Collection[T] -------------- +collections.abc.Collection[T] +----------------------------- .. code-block:: python @@ -405,7 +420,7 @@ Collection[T] def __iter__(self) -> Iterator[T] def __contains__(self, x: object) -> bool -See also :py:class:`~typing.Collection`. +See also :py:class:`~collections.abc.Collection`. One-off protocols ................. @@ -413,8 +428,8 @@ One-off protocols These protocols are typically only useful with a single standard library function or class. -Reversible[T] -------------- +collections.abc.Reversible[T] +----------------------------- This is a type for objects that support :py:func:`reversed(x) `. @@ -422,10 +437,10 @@ This is a type for objects that support :py:func:`reversed(x) `. def __reversed__(self) -> Iterator[T] -See also :py:class:`~typing.Reversible`. +See also :py:class:`~collections.abc.Reversible`. -SupportsAbs[T] --------------- +typing.SupportsAbs[T] +--------------------- This is a type for objects that support :py:func:`abs(x) `. ``T`` is the type of value returned by :py:func:`abs(x) `. @@ -436,8 +451,8 @@ value returned by :py:func:`abs(x) `. See also :py:class:`~typing.SupportsAbs`. -SupportsBytes -------------- +typing.SupportsBytes +-------------------- This is a type for objects that support :py:class:`bytes(x) `. @@ -449,8 +464,8 @@ See also :py:class:`~typing.SupportsBytes`. .. _supports-int-etc: -SupportsComplex ---------------- +typing.SupportsComplex +---------------------- This is a type for objects that support :py:class:`complex(x) `. Note that no arithmetic operations are supported. @@ -461,8 +476,8 @@ are supported. See also :py:class:`~typing.SupportsComplex`. -SupportsFloat -------------- +typing.SupportsFloat +-------------------- This is a type for objects that support :py:class:`float(x) `. Note that no arithmetic operations are supported. @@ -473,8 +488,8 @@ are supported. See also :py:class:`~typing.SupportsFloat`. -SupportsInt ------------ +typing.SupportsInt +------------------ This is a type for objects that support :py:class:`int(x) `. Note that no arithmetic operations are supported. @@ -485,8 +500,8 @@ are supported. See also :py:class:`~typing.SupportsInt`. -SupportsRound[T] ----------------- +typing.SupportsRound[T] +----------------------- This is a type for objects that support :py:func:`round(x) `. @@ -502,33 +517,33 @@ Async protocols These protocols can be useful in async code. See :ref:`async-and-await` for more information. -Awaitable[T] ------------- +collections.abc.Awaitable[T] +---------------------------- .. code-block:: python def __await__(self) -> Generator[Any, None, T] -See also :py:class:`~typing.Awaitable`. +See also :py:class:`~collections.abc.Awaitable`. -AsyncIterable[T] ----------------- +collections.abc.AsyncIterable[T] +-------------------------------- .. code-block:: python def __aiter__(self) -> AsyncIterator[T] -See also :py:class:`~typing.AsyncIterable`. +See also :py:class:`~collections.abc.AsyncIterable`. -AsyncIterator[T] ----------------- +collections.abc.AsyncIterator[T] +-------------------------------- .. code-block:: python def __anext__(self) -> Awaitable[T] def __aiter__(self) -> AsyncIterator[T] -See also :py:class:`~typing.AsyncIterator`. +See also :py:class:`~collections.abc.AsyncIterator`. Context manager protocols ......................... @@ -537,8 +552,8 @@ There are two protocols for context managers -- one for regular context managers and one for async ones. These allow defining objects that can be used in ``with`` and ``async with`` statements. -ContextManager[T] ------------------ +contextlib.AbstractContextManager[T] +------------------------------------ .. code-block:: python @@ -548,10 +563,10 @@ ContextManager[T] exc_value: Optional[BaseException], traceback: Optional[TracebackType]) -> Optional[bool] -See also :py:class:`~typing.ContextManager`. +See also :py:class:`~contextlib.AbstractContextManager`. -AsyncContextManager[T] ----------------------- +contextlib.AbstractAsyncContextManager[T] +----------------------------------------- .. code-block:: python @@ -561,4 +576,4 @@ AsyncContextManager[T] exc_value: Optional[BaseException], traceback: Optional[TracebackType]) -> Awaitable[Optional[bool]] -See also :py:class:`~typing.AsyncContextManager`. +See also :py:class:`~contextlib.AbstractAsyncContextManager`. diff --git a/docs/source/runtime_troubles.rst b/docs/source/runtime_troubles.rst index ee4cdf274ebe..9e7df05e879e 100644 --- a/docs/source/runtime_troubles.rst +++ b/docs/source/runtime_troubles.rst @@ -21,8 +21,9 @@ problems you may encounter. String literal types and type comments -------------------------------------- -Mypy allows you to add type annotations using ``# type:`` type comments. -For example: +Mypy lets you add type annotations using the (now deprecated) ``# type:`` +type comment syntax. These were required with Python versions older than 3.6, +since they didn't support type annotations on variables. Example: .. code-block:: python diff --git a/docs/source/type_narrowing.rst b/docs/source/type_narrowing.rst index 131171844dfe..0f016497113e 100644 --- a/docs/source/type_narrowing.rst +++ b/docs/source/type_narrowing.rst @@ -114,7 +114,7 @@ So, we know what ``callable()`` will return. For example: .. code-block:: python - from typing import Callable + from collections.abc import Callable x: Callable[[], int] @@ -128,7 +128,8 @@ for callable and non-callable parts: .. code-block:: python - from typing import Callable, Union + from collections.abc import Callable + from typing import Union x: Union[int, Callable[[], int]] diff --git a/docs/source/typed_dict.rst b/docs/source/typed_dict.rst index c379b5449eae..e69b3895c668 100644 --- a/docs/source/typed_dict.rst +++ b/docs/source/typed_dict.rst @@ -89,7 +89,7 @@ A ``TypedDict`` object is not a subtype of the regular ``dict[...]`` type (and vice versa), since :py:class:`dict` allows arbitrary keys to be added and removed, unlike ``TypedDict``. However, any ``TypedDict`` object is a subtype of (that is, compatible with) ``Mapping[str, object]``, since -:py:class:`~typing.Mapping` only provides read-only access to the dictionary items: +:py:class:`~collections.abc.Mapping` only provides read-only access to the dictionary items: .. code-block:: python @@ -158,7 +158,7 @@ You must use string literals as keys when calling most of the methods, as otherwise mypy won't be able to check that the key is valid. List of supported operations: -* Anything included in :py:class:`~typing.Mapping`: +* Anything included in :py:class:`~collections.abc.Mapping`: * ``d[key]`` * ``key in d`` From db7b61b82a385bf0f9ef40e9f5a1e9c3185fdae1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 26 Sep 2024 15:01:35 +0100 Subject: [PATCH 156/247] Make "X | Y" union syntax more prominent in documentation (#17835) Soon 4 out of 5 supported Python versions will support the `X | Y` syntax, so we can make it more prominent (Python 3.13 will be out soon, and 3.8 will reach end of life). Use it in most examples. The syntax is available starting from Python 3.10, but it can be used in annotations in earlier Python versions as well if using `from __future__ import annotations`. --------- Co-authored-by: Brian Schubert Co-authored-by: Jelle Zijlstra --- docs/source/cheat_sheet_py3.rst | 23 ++-- docs/source/command_line.rst | 18 ++-- docs/source/common_issues.rst | 2 +- docs/source/config_file.rst | 6 +- docs/source/error_code_list.rst | 20 ++-- docs/source/generics.rst | 6 +- docs/source/getting_started.rst | 16 +-- docs/source/kinds_of_types.rst | 101 ++++++++++-------- docs/source/literal_types.rst | 2 +- docs/source/more_types.rst | 38 ++++--- docs/source/protocols.rst | 35 +++--- docs/source/runtime_troubles.rst | 4 +- .../source/type_inference_and_annotations.rst | 10 +- docs/source/type_narrowing.rst | 7 +- 14 files changed, 149 insertions(+), 139 deletions(-) diff --git a/docs/source/cheat_sheet_py3.rst b/docs/source/cheat_sheet_py3.rst index eb2f8228cf52..7385a66863bf 100644 --- a/docs/source/cheat_sheet_py3.rst +++ b/docs/source/cheat_sheet_py3.rst @@ -72,9 +72,9 @@ Useful built-in types # On earlier versions, use Union x: list[Union[int, str]] = [3, 5, "test", "fun"] - # Use Optional[X] for a value that could be None - # Optional[X] is the same as X | None or Union[X, None] - x: Optional[str] = "something" if some_condition() else None + # Use X | None for a value that could be None on Python 3.10+ + # Use Optional[X] on 3.9 and earlier; Optional[X] is the same as 'X | None' + x: str | None = "something" if some_condition() else None if x is not None: # Mypy understands x won't be None here because of the if-statement print(x.upper()) @@ -122,13 +122,14 @@ Functions i += 1 # You can of course split a function annotation over multiple lines - def send_email(address: Union[str, list[str]], - sender: str, - cc: Optional[list[str]], - bcc: Optional[list[str]], - subject: str = '', - body: Optional[list[str]] = None - ) -> bool: + def send_email( + address: str | list[str], + sender: str, + cc: list[str] | None, + bcc: list[str] | None, + subject: str = '', + body: list[str] | None = None, + ) -> bool: ... # Mypy understands positional-only and keyword-only arguments @@ -231,7 +232,7 @@ When you're puzzled or when things are complicated # If you initialize a variable with an empty container or "None" # you may have to help mypy a bit by providing an explicit type annotation x: list[str] = [] - x: Optional[str] = None + x: str | None = None # Use Any if you don't know the type of something or it's too # dynamic to write a type for diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 3d4b841e0a7c..062739540ade 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -420,11 +420,11 @@ The following flags adjust how mypy handles values of type ``None``. .. option:: --implicit-optional - This flag causes mypy to treat arguments with a ``None`` - default value as having an implicit :py:data:`~typing.Optional` type. + This flag causes mypy to treat parameters with a ``None`` + default value as having an implicit optional type (``T | None``). For example, if this flag is set, mypy would assume that the ``x`` - parameter is actually of type ``Optional[int]`` in the code snippet below + parameter is actually of type ``int | None`` in the code snippet below, since the default parameter is ``None``: .. code-block:: python @@ -438,7 +438,7 @@ The following flags adjust how mypy handles values of type ``None``. .. option:: --no-strict-optional - This flag effectively disables checking of :py:data:`~typing.Optional` + This flag effectively disables checking of optional types and ``None`` values. With this option, mypy doesn't generally check the use of ``None`` values -- it is treated as compatible with every type. @@ -575,26 +575,24 @@ of the above sections. .. option:: --local-partial-types In mypy, the most common cases for partial types are variables initialized using ``None``, - but without explicit ``Optional`` annotations. By default, mypy won't check partial types + but without explicit ``X | None`` annotations. By default, mypy won't check partial types spanning module top level or class top level. This flag changes the behavior to only allow partial types at local level, therefore it disallows inferring variable type for ``None`` from two assignments in different scopes. For example: .. code-block:: python - from typing import Optional - a = None # Need type annotation here if using --local-partial-types - b: Optional[int] = None + b: int | None = None class Foo: bar = None # Need type annotation here if using --local-partial-types - baz: Optional[int] = None + baz: int | None = None def __init__(self) -> None: self.bar = 1 - reveal_type(Foo().bar) # Union[int, None] without --local-partial-types + reveal_type(Foo().bar) # 'int | None' without --local-partial-types Note: this option is always implicitly enabled in mypy daemon and will become enabled by default for mypy in a future release. diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index c34d20775cfd..39954b8e332a 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -803,7 +803,7 @@ This is best understood via an example: .. code-block:: python - def foo(x: Optional[int]) -> Callable[[], int]: + def foo(x: int | None) -> Callable[[], int]: if x is None: x = 5 print(x + 1) # mypy correctly deduces x must be an int here diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index b0e82a33255a..ded8476b60e3 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -574,8 +574,8 @@ section of the command line docs. :type: boolean :default: False - Causes mypy to treat arguments with a ``None`` - default value as having an implicit :py:data:`~typing.Optional` type. + Causes mypy to treat parameters with a ``None`` + default value as having an implicit optional type (``T | None``). **Note:** This was True by default in mypy versions 0.980 and earlier. @@ -584,7 +584,7 @@ section of the command line docs. :type: boolean :default: True - Effectively disables checking of :py:data:`~typing.Optional` + Effectively disables checking of optional types and ``None`` values. With this option, mypy doesn't generally check the use of ``None`` values -- it is treated as compatible with every type. diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index bb745d9d5c98..2739894dd3ec 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -59,8 +59,6 @@ Example: .. code-block:: python - from typing import Union - class Cat: def sleep(self) -> None: ... def miaow(self) -> None: ... @@ -69,10 +67,10 @@ Example: def sleep(self) -> None: ... def follow_me(self) -> None: ... - def func(animal: Union[Cat, Dog]) -> None: + def func(animal: Cat | Dog) -> None: # OK: 'sleep' is defined for both Cat and Dog animal.sleep() - # Error: Item "Cat" of "Union[Cat, Dog]" has no attribute "follow_me" [union-attr] + # Error: Item "Cat" of "Cat | Dog" has no attribute "follow_me" [union-attr] animal.follow_me() You can often work around these errors by using ``assert isinstance(obj, ClassName)`` @@ -142,9 +140,7 @@ Example: .. code-block:: python - from typing import Optional - - def first(x: list[int]) -> Optional[int]: + def first(x: list[int]) -> int: return x[0] if x else 0 t = (5, 4) @@ -165,7 +161,7 @@ Example: .. code-block:: python - from typing import overload, Optional + from typing import overload @overload def inc_maybe(x: None) -> None: ... @@ -173,7 +169,7 @@ Example: @overload def inc_maybe(x: int) -> int: ... - def inc_maybe(x: Optional[int]) -> Optional[int]: + def inc_maybe(x: int | None) -> int | None: if x is None: return None else: @@ -273,16 +269,14 @@ Example: .. code-block:: python - from typing import Optional, Union - class Base: def method(self, - arg: int) -> Optional[int]: + arg: int) -> int | None: ... class Derived(Base): def method(self, - arg: Union[int, str]) -> int: # OK + arg: int | str) -> int: # OK ... class DerivedBad(Base): diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 982adf9324af..9c0a308ee39a 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -571,9 +571,9 @@ Let us illustrate this by few simple examples: class Square(Shape): ... * Most immutable container types, such as :py:class:`~collections.abc.Sequence` - and :py:class:`~frozenset` are covariant. :py:data:`~typing.Union` is - also covariant in all variables: ``Union[Triangle, int]`` is - a subtype of ``Union[Shape, int]``. + and :py:class:`~frozenset` are covariant. Union types are + also covariant in all union items: ``Triangle | int`` is + a subtype of ``Shape | int``. .. code-block:: python diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 12422ef85bde..28a4481e502e 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -186,19 +186,23 @@ For example, a ``RuntimeError`` instance can be passed to a function that is ann as taking an ``Exception``. As another example, suppose you want to write a function that can accept *either* -ints or strings, but no other types. You can express this using the -:py:data:`~typing.Union` type. For example, ``int`` is a subtype of ``Union[int, str]``: +ints or strings, but no other types. You can express this using a +union type. For example, ``int`` is a subtype of ``int | str``: .. code-block:: python - from typing import Union - - def normalize_id(user_id: Union[int, str]) -> str: + def normalize_id(user_id: int | str) -> str: if isinstance(user_id, int): return f'user-{100_000 + user_id}' else: return user_id +.. note:: + + If using Python 3.9 or earlier, use ``typing.Union[int, str]`` instead of + ``int | str``, or use ``from __future__ import annotations`` at the top of + the file (see :ref:`runtime_troubles`). + The :py:mod:`typing` module contains many other useful types. For a quick overview, look through the :ref:`mypy cheatsheet `. @@ -210,7 +214,7 @@ generic types or your own type aliases), look through the .. note:: When adding types, the convention is to import types - using the form ``from typing import Union`` (as opposed to doing + using the form ``from typing import `` (as opposed to doing just ``import typing`` or ``import typing as t`` or ``from typing import *``). For brevity, we often omit imports from :py:mod:`typing` or :py:mod:`collections.abc` diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index afdf07a559da..54693cddf953 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -228,6 +228,7 @@ instance of ``C`` or the type of ``C`` itself. This also works with .. _union-types: +.. _alternative_union_syntax: Union types *********** @@ -236,8 +237,8 @@ Python functions often accept values of two or more different types. You can use :ref:`overloading ` to represent this, but union types are often more convenient. -Use the ``Union[T1, ..., Tn]`` type constructor to construct a union -type. For example, if an argument has type ``Union[int, str]``, both +Use ``T1 | ... | Tn`` to construct a union +type. For example, if an argument has type ``int | str``, both integers and strings are valid argument values. You can use an :py:func:`isinstance` check to narrow down a union type to a @@ -245,9 +246,7 @@ more specific type: .. code-block:: python - from typing import Union - - def f(x: Union[int, str]) -> None: + def f(x: int | str) -> None: x + 1 # Error: str + int is not valid if isinstance(x, int): # Here type of x is int. @@ -269,20 +268,38 @@ more specific type: since the caller may have to use :py:func:`isinstance` before doing anything interesting with the value. +Python 3.9 and older only partially support this syntax. Instead, you can +use the legacy ``Union[T1, ..., Tn]`` type constructor. Example: + +.. code-block:: python + + from typing import Union + + def f(x: Union[int, str]) -> None: + ... + +It is also possible to use the new syntax with versions of Python where it +isn't supported by the runtime with some limitations, if you use +``from __future__ import annotations`` (see :ref:`runtime_troubles`): + +.. code-block:: python + + from __future__ import annotations + + def f(x: int | str) -> None: # OK on Python 3.7 and later + ... + .. _strict_optional: Optional types and the None type ******************************** -You can use the :py:data:`~typing.Optional` type modifier to define a type variant -that allows ``None``, such as ``Optional[int]`` (``Optional[X]`` is -the preferred shorthand for ``Union[X, None]``): +You can use ``T | None`` to define a type variant that allows ``None`` values, +such as ``int | None``. This is called an *optional type*: .. code-block:: python - from typing import Optional - - def strlen(s: str) -> Optional[int]: + def strlen(s: str) -> int | None: if not s: return None # OK return len(s) @@ -292,12 +309,23 @@ the preferred shorthand for ``Union[X, None]``): return None # Error: None not compatible with int return len(s) -Most operations will not be allowed on unguarded ``None`` or :py:data:`~typing.Optional` -values: +To support Python 3.9 and earlier, you can use the :py:data:`~typing.Optional` +type modifier instead, such as ``Optional[int]`` (``Optional[X]`` is +the preferred shorthand for ``Union[X, None]``): .. code-block:: python - def my_inc(x: Optional[int]) -> int: + from typing import Optional + + def strlen(s: str) -> Optional[int]: + ... + +Most operations will not be allowed on unguarded ``None`` or *optional* values +(values with an optional type): + +.. code-block:: python + + def my_inc(x: int | None) -> int: return x + 1 # Error: Cannot add None and int Instead, an explicit ``None`` check is required. Mypy has @@ -307,7 +335,7 @@ recognizes ``is None`` checks: .. code-block:: python - def my_inc(x: Optional[int]) -> int: + def my_inc(x: int | None) -> int: if x is None: return 0 else: @@ -323,7 +351,7 @@ Other supported checks for guarding against a ``None`` value include .. code-block:: python - def concat(x: Optional[str], y: Optional[str]) -> Optional[str]: + def concat(x: str | None, y: str | None) -> str | None: if x is not None and y is not None: # Both x and y are not None here return x + y @@ -340,7 +368,7 @@ will complain about the possible ``None`` value. You can use .. code-block:: python class Resource: - path: Optional[str] = None + path: str | None = None def initialize(self, path: str) -> None: self.path = path @@ -363,7 +391,7 @@ This is why you need to annotate an attribute in cases like the class .. code-block:: python class Resource: - path: Optional[str] = None + path: str | None = None ... This also works for attributes defined within methods: @@ -372,10 +400,11 @@ This also works for attributes defined within methods: class Counter: def __init__(self) -> None: - self.count: Optional[int] = None + self.count: int | None = None -This is not a problem when using variable annotations, since no initial -value is needed: +Often it's easier to not use any initial value for an attribute. +This way you don't need to use an optional type and can avoid ``assert ... is not None`` +checks. No initial value is needed if you annotate an attribute in the class body: .. code-block:: python @@ -390,13 +419,13 @@ the right thing without an annotation: .. code-block:: python def f(i: int) -> None: - n = None # Inferred type Optional[int] because of the assignment below + n = None # Inferred type 'int | None' because of the assignment below if i > 0: n = i ... Sometimes you may get the error "Cannot determine type of ". In this -case you should add an explicit ``Optional[...]`` annotation (or type comment). +case you should add an explicit ``... | None`` annotation. .. note:: @@ -414,25 +443,11 @@ case you should add an explicit ``Optional[...]`` annotation (or type comment). .. note:: - ``Optional[...]`` *does not* mean a function argument with a default value. - It simply means that ``None`` is a valid value for the argument. This is - a common confusion because ``None`` is a common default value for arguments. - -.. _alternative_union_syntax: - -X | Y syntax for Unions ------------------------ - -:pep:`604` introduced an alternative way for spelling union types. In Python -3.10 and later, you can write ``Union[int, str]`` as ``int | str``. It is -possible to use this syntax in versions of Python where it isn't supported by -the runtime with some limitations (see :ref:`runtime_troubles`). - -.. code-block:: python - - t1: int | str # equivalent to Union[int, str] - - t2: int | None # equivalent to Optional[int] + The type ``Optional[T]`` *does not* mean a function parameter with a default value. + It simply means that ``None`` is a valid argument value. This is + a common confusion because ``None`` is a common default value for parameters, + and parameters with default values are sometimes called *optional* parameters + (or arguments). .. _type-aliases: @@ -501,7 +516,7 @@ introduced in Python 3.10 (:pep:`613`): from typing import TypeAlias # "from typing_extensions" in Python 3.9 and earlier - AliasType: TypeAlias = Union[list[dict[tuple[int, str], set[int]]], tuple[str, list[str]]] + AliasType: TypeAlias = list[dict[tuple[int, str], set[int]]] | tuple[str, list[str]] .. _named-tuples: diff --git a/docs/source/literal_types.rst b/docs/source/literal_types.rst index 972fce72740f..877ab5de9087 100644 --- a/docs/source/literal_types.rst +++ b/docs/source/literal_types.rst @@ -70,7 +70,7 @@ complex types involving literals a little more convenient. Literal types may also contain ``None``. Mypy will treat ``Literal[None]`` as being equivalent to just ``None``. This means that ``Literal[4, None]``, -``Union[Literal[4], None]``, and ``Optional[Literal[4]]`` are all equivalent. +``Literal[4] | None``, and ``Optional[Literal[4]]`` are all equivalent. Literals may also contain aliases to other literal types. For example, the following program is legal: diff --git a/docs/source/more_types.rst b/docs/source/more_types.rst index 26edd9ef252c..cbf40d5dcaa5 100644 --- a/docs/source/more_types.rst +++ b/docs/source/more_types.rst @@ -116,7 +116,7 @@ implicitly casting from ``UserId`` where ``int`` is expected. Examples: :py:class:`~typing.NewType` accepts exactly two arguments. The first argument must be a string literal containing the name of the new type and must equal the name of the variable to which the new type is assigned. The second argument must be a properly subclassable class, i.e., -not a type construct like :py:data:`~typing.Union`, etc. +not a type construct like a :ref:`union type `, etc. The callable returned by :py:class:`~typing.NewType` accepts only one argument; this is equivalent to supporting only one constructor accepting an instance of the base class (see above). @@ -179,7 +179,7 @@ Function overloading ******************** Sometimes the arguments and types in a function depend on each other -in ways that can't be captured with a :py:data:`~typing.Union`. For example, suppose +in ways that can't be captured with a :ref:`union types `. For example, suppose we want to write a function that can accept x-y coordinates. If we pass in just a single x-y coordinate, we return a ``ClickEvent`` object. However, if we pass in two x-y coordinates, we return a ``DragEvent`` object. @@ -188,12 +188,10 @@ Our first attempt at writing this function might look like this: .. code-block:: python - from typing import Union, Optional - def mouse_event(x1: int, y1: int, - x2: Optional[int] = None, - y2: Optional[int] = None) -> Union[ClickEvent, DragEvent]: + x2: int | None = None, + y2: int | None = None) -> ClickEvent | DragEvent: if x2 is None and y2 is None: return ClickEvent(x1, y1) elif x2 is not None and y2 is not None: @@ -213,7 +211,7 @@ to more accurately describe the function's behavior: .. code-block:: python - from typing import Union, overload + from typing import overload # Overload *variants* for 'mouse_event'. # These variants give extra information to the type checker. @@ -236,8 +234,8 @@ to more accurately describe the function's behavior: def mouse_event(x1: int, y1: int, - x2: Optional[int] = None, - y2: Optional[int] = None) -> Union[ClickEvent, DragEvent]: + x2: int | None = None, + y2: int | None = None) -> ClickEvent | DragEvent: if x2 is None and y2 is None: return ClickEvent(x1, y1) elif x2 is not None and y2 is not None: @@ -283,7 +281,7 @@ Here is the same example using the legacy syntax (Python 3.11 and earlier): .. code-block:: python from collections.abc import Sequence - from typing import TypeVar, Union, overload + from typing import TypeVar, overload T = TypeVar('T') @@ -294,7 +292,7 @@ Here is the same example using the legacy syntax (Python 3.11 and earlier): @overload def __getitem__(self, index: slice) -> Sequence[T]: ... - def __getitem__(self, index: Union[int, slice]) -> Union[T, Sequence[T]]: + def __getitem__(self, index: int | slice) -> T | Sequence[T]: if isinstance(index, int): # Return a T here elif isinstance(index, slice): @@ -412,9 +410,9 @@ matching variant returns: .. code-block:: python - some_list: Union[list[int], list[str]] + some_list: list[int] | list[str] - # output3 is of type 'Union[float, str]' + # output3 is of type 'float | str' output3 = summarize(some_list) .. note:: @@ -441,7 +439,7 @@ types: .. code-block:: python - from typing import overload, Union + from typing import overload class Expression: # ...snip... @@ -492,7 +490,7 @@ the following unsafe overload definition: .. code-block:: python - from typing import overload, Union + from typing import overload @overload def unsafe_func(x: int) -> int: ... @@ -500,7 +498,7 @@ the following unsafe overload definition: @overload def unsafe_func(x: object) -> str: ... - def unsafe_func(x: object) -> Union[int, str]: + def unsafe_func(x: object) -> int | str: if isinstance(x, int): return 42 else: @@ -569,8 +567,8 @@ Type checking the implementation The body of an implementation is type-checked against the type hints provided on the implementation. For example, in the ``MyList`` example up above, the code in the body is checked with -argument list ``index: Union[int, slice]`` and a return type of -``Union[T, Sequence[T]]``. If there are no annotations on the +argument list ``index: int | slice`` and a return type of +``T | Sequence[T]``. If there are no annotations on the implementation, then the body is not type checked. If you want to force mypy to check the body anyways, use the :option:`--check-untyped-defs ` flag (:ref:`more details here `). @@ -578,10 +576,10 @@ flag (:ref:`more details here `). The variants must also also be compatible with the implementation type hints. In the ``MyList`` example, mypy will check that the parameter type ``int`` and the return type ``T`` are compatible with -``Union[int, slice]`` and ``Union[T, Sequence]`` for the +``int | slice`` and ``T | Sequence`` for the first variant. For the second variant it verifies the parameter type ``slice`` and the return type ``Sequence[T]`` are compatible -with ``Union[int, slice]`` and ``Union[T, Sequence]``. +with ``int | slice`` and ``T | Sequence``. .. note:: diff --git a/docs/source/protocols.rst b/docs/source/protocols.rst index 8cb53b1fd5ce..ed8d94f62ef1 100644 --- a/docs/source/protocols.rst +++ b/docs/source/protocols.rst @@ -36,11 +36,12 @@ For example, ``IntList`` below is iterable, over ``int`` values: .. code-block:: python + from __future__ import annotations + from collections.abc import Iterator, Iterable - from typing import Optional class IntList: - def __init__(self, value: int, next: Optional['IntList']) -> None: + def __init__(self, value: int, next: IntList | None) -> None: self.value = value self.next = next @@ -237,22 +238,24 @@ such as trees and linked lists: .. code-block:: python - from typing import Optional, Protocol + from __future__ import annotations + + from typing import Protocol class TreeLike(Protocol): value: int @property - def left(self) -> Optional['TreeLike']: ... + def left(self) -> TreeLike | None: ... @property - def right(self) -> Optional['TreeLike']: ... + def right(self) -> TreeLike | None: ... class SimpleTree: def __init__(self, value: int) -> None: self.value = value - self.left: Optional['SimpleTree'] = None - self.right: Optional['SimpleTree'] = None + self.left: SimpleTree | None = None + self.right: SimpleTree | None = None root: TreeLike = SimpleTree(0) # OK @@ -313,15 +316,15 @@ special :py:meth:`__call__ ` member: from typing import Optional, Protocol class Combiner(Protocol): - def __call__(self, *vals: bytes, maxlen: Optional[int] = None) -> list[bytes]: ... + def __call__(self, *vals: bytes, maxlen: int | None = None) -> list[bytes]: ... def batch_proc(data: Iterable[bytes], cb_results: Combiner) -> bytes: for item in data: ... - def good_cb(*vals: bytes, maxlen: Optional[int] = None) -> list[bytes]: + def good_cb(*vals: bytes, maxlen: int | None = None) -> list[bytes]: ... - def bad_cb(*vals: bytes, maxitems: Optional[int]) -> list[bytes]: + def bad_cb(*vals: bytes, maxitems: int | None) -> list[bytes]: ... batch_proc([], good_cb) # OK @@ -559,9 +562,9 @@ contextlib.AbstractContextManager[T] def __enter__(self) -> T def __exit__(self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType]) -> Optional[bool] + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None) -> bool | None See also :py:class:`~contextlib.AbstractContextManager`. @@ -572,8 +575,8 @@ contextlib.AbstractAsyncContextManager[T] def __aenter__(self) -> Awaitable[T] def __aexit__(self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType]) -> Awaitable[Optional[bool]] + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None) -> Awaitable[bool | None] See also :py:class:`~contextlib.AbstractAsyncContextManager`. diff --git a/docs/source/runtime_troubles.rst b/docs/source/runtime_troubles.rst index 9e7df05e879e..d039db30f3fa 100644 --- a/docs/source/runtime_troubles.rst +++ b/docs/source/runtime_troubles.rst @@ -320,8 +320,8 @@ notes at :ref:`future annotations import`. Using X | Y syntax for Unions ----------------------------- -Starting with Python 3.10 (:pep:`604`), you can spell union types as ``x: int | -str``, instead of ``x: typing.Union[int, str]``. +Starting with Python 3.10 (:pep:`604`), you can spell union types as +``x: int | str``, instead of ``x: typing.Union[int, str]``. There is limited support for using this syntax in Python 3.7 and later as well: if you use ``from __future__ import annotations``, mypy will understand this diff --git a/docs/source/type_inference_and_annotations.rst b/docs/source/type_inference_and_annotations.rst index 6adb4e651224..318ca4cd9160 100644 --- a/docs/source/type_inference_and_annotations.rst +++ b/docs/source/type_inference_and_annotations.rst @@ -41,12 +41,10 @@ variable type annotation: .. code-block:: python - from typing import Union - - x: Union[int, str] = 1 + x: int | str = 1 Without the type annotation, the type of ``x`` would be just ``int``. We -use an annotation to give it a more general type ``Union[int, str]`` (this +use an annotation to give it a more general type ``int | str`` (this type means that the value can be either an ``int`` or a ``str``). The best way to think about this is that the type annotation sets the type of @@ -55,8 +53,8 @@ about the following code: .. code-block:: python - x: Union[int, str] = 1.1 # error: Incompatible types in assignment - # (expression has type "float", variable has type "Union[int, str]") + x: int | str = 1.1 # error: Incompatible types in assignment + # (expression has type "float", variable has type "int | str") .. note:: diff --git a/docs/source/type_narrowing.rst b/docs/source/type_narrowing.rst index 0f016497113e..231a7edccfe7 100644 --- a/docs/source/type_narrowing.rst +++ b/docs/source/type_narrowing.rst @@ -123,15 +123,14 @@ So, we know what ``callable()`` will return. For example: else: ... # Will never be executed and will raise error with `--warn-unreachable` -``callable`` function can even split ``Union`` type -for callable and non-callable parts: +The ``callable`` function can even split union types into +callable and non-callable parts: .. code-block:: python from collections.abc import Callable - from typing import Union - x: Union[int, Callable[[], int]] + x: int | Callable[[], int] if callable(x): reveal_type(x) # N: Revealed type is "def () -> builtins.int" From 7237d558acb1eff9b8c469cf5b5c1e7a45aa5203 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:39:24 +0200 Subject: [PATCH 157/247] Sync typeshed (#17833) Source commit: https://github.com/python/typeshed/commit/a94c927642eda093db226eb3e21c3a4681577f85 --- mypy/typeshed/stdlib/VERSIONS | 2 + mypy/typeshed/stdlib/importlib/abc.pyi | 4 +- .../stdlib/importlib/resources/__init__.pyi | 54 +++++-- .../stdlib/importlib/resources/_common.pyi | 42 +++++ .../importlib/resources/_functional.pyi | 30 ++++ mypy/typeshed/stdlib/nt.pyi | 2 + mypy/typeshed/stdlib/os/__init__.pyi | 11 +- mypy/typeshed/stdlib/tkinter/__init__.pyi | 148 +++++++++++++++--- mypy/typeshed/stdlib/typing.pyi | 2 +- 9 files changed, 254 insertions(+), 41 deletions(-) create mode 100644 mypy/typeshed/stdlib/importlib/resources/_common.pyi create mode 100644 mypy/typeshed/stdlib/importlib/resources/_functional.pyi diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index 66bf2bec7cb0..dfed62f694fc 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -161,6 +161,8 @@ importlib.metadata._meta: 3.10- importlib.metadata.diagnose: 3.13- importlib.readers: 3.10- importlib.resources: 3.7- +importlib.resources._common: 3.11- +importlib.resources._functional: 3.13- importlib.resources.abc: 3.11- importlib.resources.readers: 3.11- importlib.resources.simple: 3.11- diff --git a/mypy/typeshed/stdlib/importlib/abc.pyi b/mypy/typeshed/stdlib/importlib/abc.pyi index 3937481159dc..4a0a70d0930d 100644 --- a/mypy/typeshed/stdlib/importlib/abc.pyi +++ b/mypy/typeshed/stdlib/importlib/abc.pyi @@ -145,10 +145,10 @@ if sys.version_info >= (3, 9): # which is not the case. @overload @abstractmethod - def open(self, mode: Literal["r"] = "r", /, *, encoding: str | None = None, errors: str | None = None) -> IO[str]: ... + def open(self, mode: Literal["r"] = "r", *, encoding: str | None = None, errors: str | None = None) -> IO[str]: ... @overload @abstractmethod - def open(self, mode: Literal["rb"], /) -> IO[bytes]: ... + def open(self, mode: Literal["rb"]) -> IO[bytes]: ... @property @abstractmethod def name(self) -> str: ... diff --git a/mypy/typeshed/stdlib/importlib/resources/__init__.pyi b/mypy/typeshed/stdlib/importlib/resources/__init__.pyi index 8d656563772c..f82df8c591fa 100644 --- a/mypy/typeshed/stdlib/importlib/resources/__init__.pyi +++ b/mypy/typeshed/stdlib/importlib/resources/__init__.pyi @@ -7,10 +7,15 @@ from types import ModuleType from typing import Any, BinaryIO, TextIO from typing_extensions import TypeAlias +if sys.version_info >= (3, 11): + from importlib.resources._common import Package as Package +else: + Package: TypeAlias = str | ModuleType + if sys.version_info >= (3, 9): from importlib.abc import Traversable -__all__ = ["Package", "Resource", "contents", "is_resource", "open_binary", "open_text", "path", "read_binary", "read_text"] +__all__ = ["Package", "contents", "is_resource", "open_binary", "open_text", "path", "read_binary", "read_text"] if sys.version_info >= (3, 9): __all__ += ["as_file", "files"] @@ -18,26 +23,45 @@ if sys.version_info >= (3, 9): if sys.version_info >= (3, 10): __all__ += ["ResourceReader"] -Package: TypeAlias = str | ModuleType +if sys.version_info < (3, 13): + __all__ += ["Resource"] -if sys.version_info >= (3, 11): - Resource: TypeAlias = str -else: +if sys.version_info < (3, 11): Resource: TypeAlias = str | os.PathLike[Any] +elif sys.version_info < (3, 13): + Resource: TypeAlias = str -def open_binary(package: Package, resource: Resource) -> BinaryIO: ... -def open_text(package: Package, resource: Resource, encoding: str = "utf-8", errors: str = "strict") -> TextIO: ... -def read_binary(package: Package, resource: Resource) -> bytes: ... -def read_text(package: Package, resource: Resource, encoding: str = "utf-8", errors: str = "strict") -> str: ... -def path(package: Package, resource: Resource) -> AbstractContextManager[Path]: ... -def is_resource(package: Package, name: str) -> bool: ... -def contents(package: Package) -> Iterator[str]: ... +if sys.version_info >= (3, 13): + from importlib.resources._common import Anchor as Anchor -if sys.version_info >= (3, 9): + __all__ += ["Anchor"] + + from importlib.resources._functional import ( + contents as contents, + is_resource as is_resource, + open_binary as open_binary, + open_text as open_text, + path as path, + read_binary as read_binary, + read_text as read_text, + ) + +else: + def open_binary(package: Package, resource: Resource) -> BinaryIO: ... + def open_text(package: Package, resource: Resource, encoding: str = "utf-8", errors: str = "strict") -> TextIO: ... + def read_binary(package: Package, resource: Resource) -> bytes: ... + def read_text(package: Package, resource: Resource, encoding: str = "utf-8", errors: str = "strict") -> str: ... + def path(package: Package, resource: Resource) -> AbstractContextManager[Path]: ... + def is_resource(package: Package, name: str) -> bool: ... + def contents(package: Package) -> Iterator[str]: ... + +if sys.version_info >= (3, 11): + from importlib.resources._common import as_file as as_file +elif sys.version_info >= (3, 9): def as_file(path: Traversable) -> AbstractContextManager[Path]: ... -if sys.version_info >= (3, 12): - def files(anchor: Package | None = ...) -> Traversable: ... +if sys.version_info >= (3, 11): + from importlib.resources._common import files as files elif sys.version_info >= (3, 9): def files(package: Package) -> Traversable: ... diff --git a/mypy/typeshed/stdlib/importlib/resources/_common.pyi b/mypy/typeshed/stdlib/importlib/resources/_common.pyi new file mode 100644 index 000000000000..f04f70f25e23 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/resources/_common.pyi @@ -0,0 +1,42 @@ +import sys + +# Even though this file is 3.11+ only, Pyright will complain in stubtest for older versions. +if sys.version_info >= (3, 11): + import types + from collections.abc import Callable + from contextlib import AbstractContextManager + from importlib.abc import ResourceReader, Traversable + from pathlib import Path + from typing import overload + from typing_extensions import TypeAlias, deprecated + + Package: TypeAlias = str | types.ModuleType + + if sys.version_info >= (3, 12): + Anchor: TypeAlias = Package + + def package_to_anchor( + func: Callable[[Anchor | None], Traversable] + ) -> Callable[[Anchor | None, Anchor | None], Traversable]: ... + @overload + def files(anchor: Anchor | None = None) -> Traversable: ... + @overload + @deprecated("First parameter to files is renamed to 'anchor'") + def files(package: Anchor | None = None) -> Traversable: ... + + else: + def files(package: Package) -> Traversable: ... + + def get_resource_reader(package: types.ModuleType) -> ResourceReader | None: ... + + if sys.version_info >= (3, 12): + def resolve(cand: Anchor | None) -> types.ModuleType: ... + + else: + def resolve(cand: Package) -> types.ModuleType: ... + + if sys.version_info < (3, 12): + def get_package(package: Package) -> types.ModuleType: ... + + def from_package(package: types.ModuleType) -> Traversable: ... + def as_file(path: Traversable) -> AbstractContextManager[Path]: ... diff --git a/mypy/typeshed/stdlib/importlib/resources/_functional.pyi b/mypy/typeshed/stdlib/importlib/resources/_functional.pyi new file mode 100644 index 000000000000..97e46bdf0a53 --- /dev/null +++ b/mypy/typeshed/stdlib/importlib/resources/_functional.pyi @@ -0,0 +1,30 @@ +import sys + +# Even though this file is 3.13+ only, Pyright will complain in stubtest for older versions. +if sys.version_info >= (3, 13): + from _typeshed import StrPath + from collections.abc import Iterator + from contextlib import AbstractContextManager + from importlib.resources._common import Anchor + from io import TextIOWrapper + from pathlib import Path + from typing import BinaryIO, overload + from typing_extensions import Unpack + + def open_binary(anchor: Anchor, *path_names: StrPath) -> BinaryIO: ... + @overload + def open_text( + anchor: Anchor, *path_names: Unpack[tuple[StrPath]], encoding: str | None = "utf-8", errors: str | None = "strict" + ) -> TextIOWrapper: ... + @overload + def open_text(anchor: Anchor, *path_names: StrPath, encoding: str | None, errors: str | None = "strict") -> TextIOWrapper: ... + def read_binary(anchor: Anchor, *path_names: StrPath) -> bytes: ... + @overload + def read_text( + anchor: Anchor, *path_names: Unpack[tuple[StrPath]], encoding: str | None = "utf-8", errors: str | None = "strict" + ) -> str: ... + @overload + def read_text(anchor: Anchor, *path_names: StrPath, encoding: str | None, errors: str | None = "strict") -> str: ... + def path(anchor: Anchor, *path_names: StrPath) -> AbstractContextManager[Path]: ... + def is_resource(anchor: Anchor, *path_names: StrPath) -> bool: ... + def contents(anchor: Anchor, *path_names: StrPath) -> Iterator[str]: ... diff --git a/mypy/typeshed/stdlib/nt.pyi b/mypy/typeshed/stdlib/nt.pyi index 4066096f4c71..e1d57d09a9bd 100644 --- a/mypy/typeshed/stdlib/nt.pyi +++ b/mypy/typeshed/stdlib/nt.pyi @@ -107,5 +107,7 @@ if sys.platform == "win32": listvolumes as listvolumes, set_blocking as set_blocking, ) + if sys.version_info >= (3, 13): + from os import fchmod as fchmod, lchmod as lchmod environ: dict[str, str] diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index 700e0e9df310..d7bb4883a0f2 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -673,7 +673,6 @@ if sys.version_info >= (3, 12) or sys.platform != "win32": def set_blocking(fd: int, blocking: bool, /) -> None: ... if sys.platform != "win32": - def fchmod(fd: int, mode: int) -> None: ... def fchown(fd: int, uid: int, gid: int) -> None: ... def fpathconf(fd: int, name: str | int, /) -> int: ... def fstatvfs(fd: int, /) -> statvfs_result: ... @@ -754,7 +753,6 @@ def chmod(path: FileDescriptorOrPath, mode: int, *, dir_fd: int | None = None, f if sys.platform != "win32" and sys.platform != "linux": def chflags(path: StrOrBytesPath, flags: int, follow_symlinks: bool = True) -> None: ... # some flavors of Unix def lchflags(path: StrOrBytesPath, flags: int) -> None: ... - def lchmod(path: StrOrBytesPath, mode: int) -> None: ... if sys.platform != "win32": def chroot(path: StrOrBytesPath) -> None: ... @@ -1179,3 +1177,12 @@ if sys.version_info >= (3, 13) and sys.platform == "linux": def timerfd_settime_ns(fd: FileDescriptor, /, *, flags: int = 0, initial: int = 0, interval: int = 0) -> tuple[int, int]: ... def timerfd_gettime(fd: FileDescriptor, /) -> tuple[float, float]: ... def timerfd_gettime_ns(fd: FileDescriptor, /) -> tuple[int, int]: ... + +if sys.version_info >= (3, 13) or sys.platform != "win32": + # Added to Windows in 3.13. + def fchmod(fd: int, mode: int) -> None: ... + +if sys.platform != "linux": + if sys.version_info >= (3, 13) or sys.platform != "win32": + # Added to Windows in 3.13. + def lchmod(path: StrOrBytesPath, mode: int) -> None: ... diff --git a/mypy/typeshed/stdlib/tkinter/__init__.pyi b/mypy/typeshed/stdlib/tkinter/__init__.pyi index 2a42eb789731..4d25a04f8eb7 100644 --- a/mypy/typeshed/stdlib/tkinter/__init__.pyi +++ b/mypy/typeshed/stdlib/tkinter/__init__.pyi @@ -3025,27 +3025,133 @@ class Text(Widget, XView, YView): config = configure def bbox(self, index: _TextIndex) -> tuple[int, int, int, int] | None: ... # type: ignore[override] def compare(self, index1: _TextIndex, op: Literal["<", "<=", "==", ">=", ">", "!="], index2: _TextIndex) -> bool: ... - @overload - def count(self, index1: _TextIndex, index2: _TextIndex) -> tuple[int] | None: ... - @overload - def count(self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], /) -> tuple[int] | None: ... - @overload - def count(self, index1: _TextIndex, index2: _TextIndex, arg1: Literal["update"], arg2: _WhatToCount, /) -> int | None: ... - @overload - def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: Literal["update"], /) -> int | None: ... - @overload - def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /) -> tuple[int, int]: ... - @overload - def count( - self, - index1: _TextIndex, - index2: _TextIndex, - arg1: _WhatToCount | Literal["update"], - arg2: _WhatToCount | Literal["update"], - arg3: _WhatToCount | Literal["update"], - /, - *args: _WhatToCount | Literal["update"], - ) -> tuple[int, ...]: ... + if sys.version_info >= (3, 13): + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, *, return_ints: Literal[True]) -> int: ... + @overload + def count( + self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], /, *, return_ints: Literal[True] + ) -> int: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: Literal["update"], + arg2: _WhatToCount, + /, + *, + return_ints: Literal[True], + ) -> int: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount, + arg2: Literal["update"], + /, + *, + return_ints: Literal[True], + ) -> int: ... + @overload + def count( + self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /, *, return_ints: Literal[True] + ) -> tuple[int, int]: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount | Literal["update"], + arg2: _WhatToCount | Literal["update"], + arg3: _WhatToCount | Literal["update"], + /, + *args: _WhatToCount | Literal["update"], + return_ints: Literal[True], + ) -> tuple[int, ...]: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, *, return_ints: Literal[False] = False) -> tuple[int] | None: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg: _WhatToCount | Literal["update"], + /, + *, + return_ints: Literal[False] = False, + ) -> tuple[int] | None: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: Literal["update"], + arg2: _WhatToCount, + /, + *, + return_ints: Literal[False] = False, + ) -> int | None: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount, + arg2: Literal["update"], + /, + *, + return_ints: Literal[False] = False, + ) -> int | None: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount, + arg2: _WhatToCount, + /, + *, + return_ints: Literal[False] = False, + ) -> tuple[int, int]: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount | Literal["update"], + arg2: _WhatToCount | Literal["update"], + arg3: _WhatToCount | Literal["update"], + /, + *args: _WhatToCount | Literal["update"], + return_ints: Literal[False] = False, + ) -> tuple[int, ...]: ... + else: + @overload + def count(self, index1: _TextIndex, index2: _TextIndex) -> tuple[int] | None: ... + @overload + def count( + self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], / + ) -> tuple[int] | None: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, arg1: Literal["update"], arg2: _WhatToCount, /) -> int | None: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: Literal["update"], /) -> int | None: ... + @overload + def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /) -> tuple[int, int]: ... + @overload + def count( + self, + index1: _TextIndex, + index2: _TextIndex, + arg1: _WhatToCount | Literal["update"], + arg2: _WhatToCount | Literal["update"], + arg3: _WhatToCount | Literal["update"], + /, + *args: _WhatToCount | Literal["update"], + ) -> tuple[int, ...]: ... + @overload def debug(self, boolean: None = None) -> bool: ... @overload diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index cadd06358d4a..784f30ad7397 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -540,7 +540,7 @@ class AsyncIterator(AsyncIterable[_T_co], Protocol[_T_co]): def __aiter__(self) -> AsyncIterator[_T_co]: ... class AsyncGenerator(AsyncIterator[_YieldT_co], Generic[_YieldT_co, _SendT_contra]): - def __anext__(self) -> Awaitable[_YieldT_co]: ... + def __anext__(self) -> Coroutine[Any, Any, _YieldT_co]: ... @abstractmethod def asend(self, value: _SendT_contra, /) -> Coroutine[Any, Any, _YieldT_co]: ... @overload From 1c018589a621e38e4851a64f500a42ebe99ef21d Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Fri, 27 Sep 2024 02:32:59 -0400 Subject: [PATCH 158/247] Fix negative narrowing of tuples in match statement (#17817) Fixes #17328 ### Before Lines marked with `!!!` denote incorrect behavior. ([Playground link](https://mypy-play.net/?mypy=1.11.2&python=3.12&flags=strict%2Cwarn-unreachable&gist=7a7081c5fbc2fac9987f24e02421f24f)) ```python from typing import Literal m4: tuple[Literal[1], int] match m4: case (1, 5): reveal_type(m4) # N: Revealed type is "tuple[Literal[1], Literal[5]]" case (1, 6): reveal_type(m4) # !!! E: Statement is unreachable [unreachable] case _: reveal_type(m4) # !!! N: Revealed type is "tuple[Never, builtins.int]" m5: tuple[Literal[1, 2], Literal["a", "b"]] match m5: case (1, str()): reveal_type(m5) # N: Revealed type is "tuple[Literal[1], Union[Literal['a'], Literal['b']]]" case _: reveal_type(m5) # !!! N: Revealed type is "tuple[Literal[2], Never]" match m5: case (1, "a"): reveal_type(m5) # N: Revealed type is "tuple[Literal[1], Literal['a']]" case _: reveal_type(m5) # !!! N: Revealed type is "tuple[Literal[2], Literal['b']]" ``` ### After ```python from typing import Literal m4: tuple[Literal[1], int] match m4: case (1, 5): reveal_type(m4) # N: Revealed type is "tuple[Literal[1], Literal[5]]" case (1, 6): reveal_type(m4) # N: Revealed type is "tuple[Literal[1], Literal[6]]" case _: reveal_type(m4) # N: Revealed type is "tuple[Literal[1], builtins.int]" m5: tuple[Literal[1, 2], Literal["a", "b"]] match m5: case (1, str()): reveal_type(m5) # N: Revealed type is "tuple[Literal[1], Union[Literal['a'], Literal['b']]]" case _: reveal_type(m5) # N: Revealed type is "tuple[Literal[2], Union[Literal['a'], Literal['b']]]" match m5: case (1, "a"): reveal_type(m5) # N: Revealed type is "tuple[Literal[1], Literal['a']]" case _: reveal_type(m5) # N: Revealed type is "tuple[Union[Literal[1], Literal[2]], Union[Literal['a'], Literal['b']]]" ``` --- mypy/checkpattern.py | 12 +++++++++++- test-data/unit/check-python310.test | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index a23be464b825..cb3577ce2f6e 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -307,7 +307,7 @@ def visit_sequence_pattern(self, o: SequencePattern) -> PatternType: for inner_type, new_inner_type in zip(inner_types, new_inner_types): (narrowed_inner_type, inner_rest_type) = ( self.chk.conditional_types_with_intersection( - new_inner_type, [get_type_range(inner_type)], o, default=new_inner_type + inner_type, [get_type_range(new_inner_type)], o, default=inner_type ) ) narrowed_inner_types.append(narrowed_inner_type) @@ -320,6 +320,16 @@ def visit_sequence_pattern(self, o: SequencePattern) -> PatternType: if all(is_uninhabited(typ) for typ in inner_rest_types): # All subpatterns always match, so we can apply negative narrowing rest_type = TupleType(rest_inner_types, current_type.partial_fallback) + elif sum(not is_uninhabited(typ) for typ in inner_rest_types) == 1: + # Exactly one subpattern may conditionally match, the rest always match. + # We can apply negative narrowing to this one position. + rest_type = TupleType( + [ + curr if is_uninhabited(rest) else rest + for curr, rest in zip(inner_types, inner_rest_types) + ], + current_type.partial_fallback, + ) elif isinstance(current_type, TupleType): # For variadic tuples it is too tricky to match individual items like for fixed # tuples, so we instead try to narrow the entire type. diff --git a/test-data/unit/check-python310.test b/test-data/unit/check-python310.test index 5ecc69dc7c32..e7028a027e25 100644 --- a/test-data/unit/check-python310.test +++ b/test-data/unit/check-python310.test @@ -1424,6 +1424,7 @@ def f(value: Literal[1] | Literal[2]) -> int: [case testMatchSequencePatternNegativeNarrowing] from typing import Union, Sequence, Tuple +from typing_extensions import Literal m1: Sequence[int | str] @@ -1448,6 +1449,31 @@ match m3: reveal_type(m3) # N: Revealed type is "Tuple[Literal[1]]" case r2: reveal_type(m3) # N: Revealed type is "Tuple[Union[builtins.int, builtins.str]]" + +m4: Tuple[Literal[1], int] + +match m4: + case (1, 5): + reveal_type(m4) # N: Revealed type is "Tuple[Literal[1], Literal[5]]" + case (1, 6): + reveal_type(m4) # N: Revealed type is "Tuple[Literal[1], Literal[6]]" + case _: + reveal_type(m4) # N: Revealed type is "Tuple[Literal[1], builtins.int]" + +m5: Tuple[Literal[1, 2], Literal["a", "b"]] + +match m5: + case (1, str()): + reveal_type(m5) # N: Revealed type is "Tuple[Literal[1], Union[Literal['a'], Literal['b']]]" + case _: + reveal_type(m5) # N: Revealed type is "Tuple[Literal[2], Union[Literal['a'], Literal['b']]]" + +match m5: + case (1, "a"): + reveal_type(m5) # N: Revealed type is "Tuple[Literal[1], Literal['a']]" + case _: + reveal_type(m5) # N: Revealed type is "Tuple[Union[Literal[1], Literal[2]], Union[Literal['a'], Literal['b']]]" + [builtins fixtures/tuple.pyi] [case testMatchEnumSingleChoice] From 1995155ef80a9e4ccc434240a93843868afc4a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Fri, 27 Sep 2024 00:43:06 -0600 Subject: [PATCH 159/247] Document `--output=json` CLI option (#17611) Documents the new option added by https://github.com/python/mypy/pull/11396 --------- Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- docs/source/command_line.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 062739540ade..20fb3821438a 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -96,6 +96,10 @@ Optional arguments Show program's version number and exit. +.. option:: -O FORMAT, --output FORMAT {json} + + Set a custom output format. + .. _config-file-flag: Config file From 26a77f9d373a49fa3796082fb55a54517a876364 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 27 Sep 2024 17:03:59 +0100 Subject: [PATCH 160/247] Avoid type size explosion when expanding types (#17842) If TypedDict A has multiple items that refer to TypedDict B, don't duplicate the types representing B during type expansion (or generally when translating types). If TypedDicts are deeply nested, this could result in lot of redundant type objects. Example where this could matter (assume B is a big TypedDict): ``` class B(TypedDict): ... class A(TypedDict): a: B b: B c: B ... z: B ``` Also deduplicate large unions. It's common to have aliases that are defined as large unions, and again we want to avoid duplicating these unions. This may help with #17231, but this fix may not be sufficient. --- mypy/applytype.py | 1 + mypy/erasetype.py | 1 + mypy/expandtype.py | 19 +++++++++++++++++-- mypy/subtypes.py | 2 ++ mypy/type_visitor.py | 36 ++++++++++++++++++++++++++++++++++-- mypy/typeanal.py | 2 ++ mypy/types.py | 13 +++++++++---- 7 files changed, 66 insertions(+), 8 deletions(-) diff --git a/mypy/applytype.py b/mypy/applytype.py index 783748cd8a5e..e88947cc6430 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -215,6 +215,7 @@ def __init__( bound_tvars: frozenset[TypeVarLikeType] = frozenset(), seen_aliases: frozenset[TypeInfo] = frozenset(), ) -> None: + super().__init__() self.poly_tvars = set(poly_tvars) # This is a simplified version of TypeVarScope used during semantic analysis. self.bound_tvars = bound_tvars diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 5d95b221af15..222e7f2a6d7a 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -161,6 +161,7 @@ class TypeVarEraser(TypeTranslator): """Implementation of type erasure""" def __init__(self, erase_id: Callable[[TypeVarId], bool], replacement: Type) -> None: + super().__init__() self.erase_id = erase_id self.replacement = replacement diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 9336be54437b..b2040ec074c3 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -179,6 +179,7 @@ class ExpandTypeVisitor(TrivialSyntheticTypeTranslator): variables: Mapping[TypeVarId, Type] # TypeVar id -> TypeVar value def __init__(self, variables: Mapping[TypeVarId, Type]) -> None: + super().__init__() self.variables = variables self.recursive_tvar_guard: dict[TypeVarId, Type | None] = {} @@ -454,15 +455,25 @@ def visit_tuple_type(self, t: TupleType) -> Type: return t.copy_modified(items=items, fallback=fallback) def visit_typeddict_type(self, t: TypedDictType) -> Type: + if cached := self.get_cached(t): + return cached fallback = t.fallback.accept(self) assert isinstance(fallback, ProperType) and isinstance(fallback, Instance) - return t.copy_modified(item_types=self.expand_types(t.items.values()), fallback=fallback) + result = t.copy_modified(item_types=self.expand_types(t.items.values()), fallback=fallback) + self.set_cached(t, result) + return result def visit_literal_type(self, t: LiteralType) -> Type: # TODO: Verify this implementation is correct return t def visit_union_type(self, t: UnionType) -> Type: + # Use cache to avoid O(n**2) or worse expansion of types during translation + # (only for large unions, since caching adds overhead) + use_cache = len(t.items) > 3 + if use_cache and (cached := self.get_cached(t)): + return cached + expanded = self.expand_types(t.items) # After substituting for type variables in t.items, some resulting types # might be subtypes of others, however calling make_simplified_union() @@ -475,7 +486,11 @@ def visit_union_type(self, t: UnionType) -> Type: # otherwise a single item union of a type alias will break it. Note this should not # cause infinite recursion since pathological aliases like A = Union[A, B] are # banned at the semantic analysis level. - return get_proper_type(simplified) + result = get_proper_type(simplified) + + if use_cache: + self.set_cached(t, result) + return result def visit_partial_type(self, t: PartialType) -> Type: return t diff --git a/mypy/subtypes.py b/mypy/subtypes.py index c76b3569fdd4..608d098791a9 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -886,6 +886,8 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: if isinstance(right, Instance): return self._is_subtype(left.fallback, right) elif isinstance(right, TypedDictType): + if left == right: + return True # Fast path if not left.names_are_wider_than(right): return False for name, l, r in left.zip(right): diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 59e13d12485c..38e4c5ba0d01 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -181,8 +181,26 @@ class TypeTranslator(TypeVisitor[Type]): Subclass this and override some methods to implement a non-trivial transformation. + + We cache the results of certain translations to avoid + massively expanding the sizes of types. """ + def __init__(self, cache: dict[Type, Type] | None = None) -> None: + # For deduplication of results + self.cache = cache + + def get_cached(self, t: Type) -> Type | None: + if self.cache is None: + return None + return self.cache.get(t) + + def set_cached(self, orig: Type, new: Type) -> None: + if self.cache is None: + # Minor optimization: construct lazily + self.cache = {} + self.cache[orig] = new + def visit_unbound_type(self, t: UnboundType) -> Type: return t @@ -251,8 +269,11 @@ def visit_tuple_type(self, t: TupleType) -> Type: ) def visit_typeddict_type(self, t: TypedDictType) -> Type: + # Use cache to avoid O(n**2) or worse expansion of types during translation + if cached := self.get_cached(t): + return cached items = {item_name: item_type.accept(self) for (item_name, item_type) in t.items.items()} - return TypedDictType( + result = TypedDictType( items, t.required_keys, # TODO: This appears to be unsafe. @@ -260,6 +281,8 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: t.line, t.column, ) + self.set_cached(t, result) + return result def visit_literal_type(self, t: LiteralType) -> Type: fallback = t.fallback.accept(self) @@ -267,12 +290,21 @@ def visit_literal_type(self, t: LiteralType) -> Type: return LiteralType(value=t.value, fallback=fallback, line=t.line, column=t.column) def visit_union_type(self, t: UnionType) -> Type: - return UnionType( + # Use cache to avoid O(n**2) or worse expansion of types during translation + # (only for large unions, since caching adds overhead) + use_cache = len(t.items) > 3 + if use_cache and (cached := self.get_cached(t)): + return cached + + result = UnionType( self.translate_types(t.items), t.line, t.column, uses_pep604_syntax=t.uses_pep604_syntax, ) + if use_cache: + self.set_cached(t, result) + return result def translate_types(self, types: Iterable[Type]) -> list[Type]: return [t.accept(self) for t in types] diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 274b4b893a98..6c94390c23dc 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -2271,6 +2271,7 @@ def __init__( lookup: Callable[[str, Context], SymbolTableNode | None], scope: TypeVarLikeScope, ) -> None: + super().__init__() self.seen_nodes = seen_nodes self.lookup = lookup self.scope = scope @@ -2660,6 +2661,7 @@ class TypeVarDefaultTranslator(TrivialSyntheticTypeTranslator): def __init__( self, api: SemanticAnalyzerInterface, tvar_expr_name: str, context: Context ) -> None: + super().__init__() self.api = api self.tvar_expr_name = tvar_expr_name self.context = context diff --git a/mypy/types.py b/mypy/types.py index 78244d0f9cf4..b1e57b2f6a86 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -357,7 +357,7 @@ def _expand_once(self) -> Type: def _partial_expansion(self, nothing_args: bool = False) -> tuple[ProperType, bool]: # Private method mostly for debugging and testing. - unroller = UnrollAliasVisitor(set()) + unroller = UnrollAliasVisitor(set(), {}) if nothing_args: alias = self.copy_modified(args=[UninhabitedType()] * len(self.args)) else: @@ -2586,7 +2586,8 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, TypedDictType): return NotImplemented - + if self is other: + return True return ( frozenset(self.items.keys()) == frozenset(other.items.keys()) and all( @@ -3507,7 +3508,11 @@ def visit_type_list(self, t: TypeList) -> Type: class UnrollAliasVisitor(TrivialSyntheticTypeTranslator): - def __init__(self, initial_aliases: set[TypeAliasType]) -> None: + def __init__( + self, initial_aliases: set[TypeAliasType], cache: dict[Type, Type] | None + ) -> None: + assert cache is not None + super().__init__(cache) self.recursed = False self.initial_aliases = initial_aliases @@ -3519,7 +3524,7 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: # A = Tuple[B, B] # B = int # will not be detected as recursive on the second encounter of B. - subvisitor = UnrollAliasVisitor(self.initial_aliases | {t}) + subvisitor = UnrollAliasVisitor(self.initial_aliases | {t}, self.cache) result = get_proper_type(t).accept(subvisitor) if subvisitor.recursed: self.recursed = True From ca284252de6455444806ef9752d8a30d3a9d4c37 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sat, 28 Sep 2024 16:20:16 -0400 Subject: [PATCH 161/247] Fix get_member_expr_fullname returning strings with embedded "None" (#17848) Fixes #17847 ### Before ```python from mypy.nodes import CallExpr, MemberExpr, NameExpr, get_member_expr_fullname m3 = MemberExpr(MemberExpr(CallExpr(NameExpr("a"), [], [], []), "b"), "c") # a().b.c >>> get_member_expr_fullname(m3) 'None.c' ``` ### After ```python from mypy.nodes import CallExpr, MemberExpr, NameExpr, get_member_expr_fullname m3 = MemberExpr(MemberExpr(CallExpr(NameExpr("a"), [], [], []), "b"), "c") # a().b.c >>> get_member_expr_fullname(m3) is None True ``` --- mypy/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 4a5c7240fa83..39cbee3c8525 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -4090,7 +4090,7 @@ def get_member_expr_fullname(expr: MemberExpr) -> str | None: initial = expr.expr.name elif isinstance(expr.expr, MemberExpr): initial = get_member_expr_fullname(expr.expr) - else: + if initial is None: return None return f"{initial}.{expr.name}" From 1a2c8e2a4df21532e4952191cad74ae50083f4ad Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 28 Sep 2024 14:20:45 -0700 Subject: [PATCH 162/247] Fix tests on latest Python 3.13 (and 3.12) (#17849) Related to python/cpython@c6c3d970ba54f4ad4c4151bb262cef96d3299480 (python/cpython#121329), thanks to brianschubert for noticing --- mypyc/test-data/run-dicts.test | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/run-dicts.test b/mypyc/test-data/run-dicts.test index 58b862e3f303..d4f5b945309e 100644 --- a/mypyc/test-data/run-dicts.test +++ b/mypyc/test-data/run-dicts.test @@ -157,7 +157,11 @@ else: try: clear_during_iter(d) except RuntimeError as e: - assert str(e) == "OrderedDict changed size during iteration" + assert str(e) in ( + "OrderedDict changed size during iteration", + # Error message changed in Python 3.13 and some 3.12 patch version + "OrderedDict mutated during iteration", + ) else: assert False From 6726d77cd8414fccb5e52b084610aa47f5f189d3 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 30 Sep 2024 14:41:43 +0300 Subject: [PATCH 163/247] Add `ReadOnly` support for TypedDicts (#17644) Refs https://github.com/python/mypy/issues/17264 I will add docs in a separate PR. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/source/error_code_list.rst | 24 ++ mypy/checkexpr.py | 14 +- mypy/checkmember.py | 5 +- mypy/checkpattern.py | 2 +- mypy/copytype.py | 4 +- mypy/errorcodes.py | 3 + mypy/exprtotype.py | 2 +- mypy/fastparse.py | 2 +- mypy/join.py | 7 +- mypy/meet.py | 12 +- mypy/messages.py | 20 +- mypy/plugins/default.py | 22 +- mypy/plugins/proper_plugin.py | 1 + mypy/semanal.py | 8 +- mypy/semanal_shared.py | 2 +- mypy/semanal_typeddict.py | 105 ++++-- mypy/server/astdiff.py | 3 +- mypy/subtypes.py | 11 + mypy/type_visitor.py | 1 + mypy/typeanal.py | 50 ++- mypy/types.py | 56 ++- test-data/unit/check-typeddict.test | 338 +++++++++++++++++- test-data/unit/fine-grained.test | 28 ++ test-data/unit/fixtures/typing-typeddict.pyi | 5 + test-data/unit/lib-stub/typing_extensions.pyi | 1 + 25 files changed, 652 insertions(+), 74 deletions(-) diff --git a/docs/source/error_code_list.rst b/docs/source/error_code_list.rst index 2739894dd3ec..73171131bc8d 100644 --- a/docs/source/error_code_list.rst +++ b/docs/source/error_code_list.rst @@ -1217,6 +1217,30 @@ If the code being checked is not syntactically valid, mypy issues a syntax error. Most, but not all, syntax errors are *blocking errors*: they can't be ignored with a ``# type: ignore`` comment. +.. _code-typeddict-readonly-mutated: + +ReadOnly key of a TypedDict is mutated [typeddict-readonly-mutated] +------------------------------------------------------------------- + +Consider this example: + +.. code-block:: python + + from datetime import datetime + from typing import TypedDict + from typing_extensions import ReadOnly + + class User(TypedDict): + username: ReadOnly[str] + last_active: datetime + + user: User = {'username': 'foobar', 'last_active': datetime.now()} + user['last_active'] = datetime.now() # ok + user['username'] = 'other' # error: ReadOnly TypedDict key "key" TypedDict is mutated [typeddict-readonly-mutated] + +`PEP 705 `_ specifies +how ``ReadOnly`` special form works for ``TypedDict`` objects. + .. _code-misc: Miscellaneous checks [misc] diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 55c42335744d..98e6eb6a7fc3 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -986,6 +986,10 @@ def check_typeddict_call_with_kwargs( always_present_keys: set[str], ) -> Type: actual_keys = kwargs.keys() + if callee.to_be_mutated: + assigned_readonly_keys = actual_keys & callee.readonly_keys + if assigned_readonly_keys: + self.msg.readonly_keys_mutated(assigned_readonly_keys, context=context) if not ( callee.required_keys <= always_present_keys and actual_keys <= callee.items.keys() ): @@ -4349,7 +4353,7 @@ def visit_index_with_type( else: return self.nonliteral_tuple_index_helper(left_type, index) elif isinstance(left_type, TypedDictType): - return self.visit_typeddict_index_expr(left_type, e.index) + return self.visit_typeddict_index_expr(left_type, e.index)[0] elif isinstance(left_type, FunctionLike) and left_type.is_type_obj(): if left_type.type_object().is_enum: return self.visit_enum_index_expr(left_type.type_object(), e.index, e) @@ -4530,7 +4534,7 @@ def union_tuple_fallback_item(self, left_type: TupleType) -> Type: def visit_typeddict_index_expr( self, td_type: TypedDictType, index: Expression, setitem: bool = False - ) -> Type: + ) -> tuple[Type, set[str]]: if isinstance(index, StrExpr): key_names = [index.value] else: @@ -4553,17 +4557,17 @@ def visit_typeddict_index_expr( key_names.append(key_type.value) else: self.msg.typeddict_key_must_be_string_literal(td_type, index) - return AnyType(TypeOfAny.from_error) + return AnyType(TypeOfAny.from_error), set() value_types = [] for key_name in key_names: value_type = td_type.items.get(key_name) if value_type is None: self.msg.typeddict_key_not_found(td_type, key_name, index, setitem) - return AnyType(TypeOfAny.from_error) + return AnyType(TypeOfAny.from_error), set() else: value_types.append(value_type) - return make_simplified_union(value_types) + return make_simplified_union(value_types), set(key_names) def visit_enum_index_expr( self, enum_type: TypeInfo, index: Expression, context: Context diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 0f117f5475ed..8f99f96e2dd5 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -1185,9 +1185,12 @@ def analyze_typeddict_access( if isinstance(mx.context, IndexExpr): # Since we can get this during `a['key'] = ...` # it is safe to assume that the context is `IndexExpr`. - item_type = mx.chk.expr_checker.visit_typeddict_index_expr( + item_type, key_names = mx.chk.expr_checker.visit_typeddict_index_expr( typ, mx.context.index, setitem=True ) + assigned_readonly_keys = typ.readonly_keys & key_names + if assigned_readonly_keys: + mx.msg.readonly_keys_mutated(assigned_readonly_keys, context=mx.context) else: # It can also be `a.__setitem__(...)` direct call. # In this case `item_type` can be `Any`, diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index cb3577ce2f6e..fa23dfb5f453 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -498,7 +498,7 @@ def get_mapping_item_type( with self.msg.filter_errors() as local_errors: result: Type | None = self.chk.expr_checker.visit_typeddict_index_expr( mapping_type, key - ) + )[0] has_local_errors = local_errors.has_new_errors() # If we can't determine the type statically fall back to treating it as a normal # mapping diff --git a/mypy/copytype.py b/mypy/copytype.py index 465f06566f54..ecb1a89759b6 100644 --- a/mypy/copytype.py +++ b/mypy/copytype.py @@ -107,7 +107,9 @@ def visit_tuple_type(self, t: TupleType) -> ProperType: return self.copy_common(t, TupleType(t.items, t.partial_fallback, implicit=t.implicit)) def visit_typeddict_type(self, t: TypedDictType) -> ProperType: - return self.copy_common(t, TypedDictType(t.items, t.required_keys, t.fallback)) + return self.copy_common( + t, TypedDictType(t.items, t.required_keys, t.readonly_keys, t.fallback) + ) def visit_literal_type(self, t: LiteralType) -> ProperType: return self.copy_common(t, LiteralType(value=t.value, fallback=t.fallback)) diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index ad061b161af1..a170b5d4d65a 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -185,6 +185,9 @@ def __hash__(self) -> int: ANNOTATION_UNCHECKED = ErrorCode( "annotation-unchecked", "Notify about type annotations in unchecked functions", "General" ) +TYPEDDICT_READONLY_MUTATED = ErrorCode( + "typeddict-readonly-mutated", "TypedDict's ReadOnly key is mutated", "General" +) POSSIBLY_UNDEFINED: Final[ErrorCode] = ErrorCode( "possibly-undefined", "Warn about variables that are defined only in some execution paths", diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index c7df851668be..506194a4b285 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -244,7 +244,7 @@ def expr_to_unanalyzed_type( value, options, allow_new_syntax, expr ) result = TypedDictType( - items, set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column + items, set(), set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column ) result.extra_items_from = extra_items_from return result diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 18858b0fa0b8..bbbe2184738c 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -2130,7 +2130,7 @@ def visit_Dict(self, n: ast3.Dict) -> Type: continue return self.invalid_type(n) items[item_name.value] = self.visit(value) - result = TypedDictType(items, set(), _dummy_fallback, n.lineno, n.col_offset) + result = TypedDictType(items, set(), set(), _dummy_fallback, n.lineno, n.col_offset) result.extra_items_from = extra_items_from return result diff --git a/mypy/join.py b/mypy/join.py index 5284be7dd2a1..865dd073d081 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -631,10 +631,13 @@ def visit_typeddict_type(self, t: TypedDictType) -> ProperType: ) } fallback = self.s.create_anonymous_fallback() + all_keys = set(items.keys()) # We need to filter by items.keys() since some required keys present in both t and # self.s might be missing from the join if the types are incompatible. - required_keys = set(items.keys()) & t.required_keys & self.s.required_keys - return TypedDictType(items, required_keys, fallback) + required_keys = all_keys & t.required_keys & self.s.required_keys + # If one type has a key as readonly, we mark it as readonly for both: + readonly_keys = (t.readonly_keys | t.readonly_keys) & all_keys + return TypedDictType(items, required_keys, readonly_keys, fallback) elif isinstance(self.s, Instance): return join_types(self.s, t.fallback) else: diff --git a/mypy/meet.py b/mypy/meet.py index 91abf43c0877..9f5c2d72a8cb 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -1017,7 +1017,8 @@ def visit_typeddict_type(self, t: TypedDictType) -> ProperType: items = dict(item_list) fallback = self.s.create_anonymous_fallback() required_keys = t.required_keys | self.s.required_keys - return TypedDictType(items, required_keys, fallback) + readonly_keys = t.readonly_keys | self.s.readonly_keys + return TypedDictType(items, required_keys, readonly_keys, fallback) elif isinstance(self.s, Instance) and is_subtype(t, self.s): return t else: @@ -1139,6 +1140,9 @@ def typed_dict_mapping_overlap( - TypedDict(x=str, y=str, total=False) doesn't overlap with Dict[str, int] - TypedDict(x=int, y=str, total=False) overlaps with Dict[str, str] + * A TypedDict with at least one ReadOnly[] key does not overlap + with Dict or MutableMapping, because they assume mutable data. + As usual empty, dictionaries lie in a gray area. In general, List[str] and List[str] are considered non-overlapping despite empty list belongs to both. However, List[int] and List[Never] are considered overlapping. @@ -1159,6 +1163,12 @@ def typed_dict_mapping_overlap( assert isinstance(right, TypedDictType) typed, other = right, left + mutable_mapping = next( + (base for base in other.type.mro if base.fullname == "typing.MutableMapping"), None + ) + if mutable_mapping is not None and typed.readonly_keys: + return False + mapping = next(base for base in other.type.mro if base.fullname == "typing.Mapping") other = map_instance_to_supertype(other, mapping) key_type, value_type = get_proper_types(other.args) diff --git a/mypy/messages.py b/mypy/messages.py index 6567d9d96d0b..adf150eab50a 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -926,6 +926,17 @@ def invalid_index_type( code=code, ) + def readonly_keys_mutated(self, keys: set[str], context: Context) -> None: + if len(keys) == 1: + suffix = "is" + else: + suffix = "are" + self.fail( + "ReadOnly {} TypedDict {} mutated".format(format_key_list(sorted(keys)), suffix), + code=codes.TYPEDDICT_READONLY_MUTATED, + context=context, + ) + def too_few_arguments( self, callee: CallableType, context: Context, argument_names: Sequence[str | None] | None ) -> None: @@ -2613,10 +2624,13 @@ def format_literal_value(typ: LiteralType) -> str: return format(typ.fallback) items = [] for item_name, item_type in typ.items.items(): - modifier = "" if item_name in typ.required_keys else "?" + modifier = "" + if item_name not in typ.required_keys: + modifier += "?" + if item_name in typ.readonly_keys: + modifier += "=" items.append(f"{item_name!r}{modifier}: {format(item_type)}") - s = f"TypedDict({{{', '.join(items)}}})" - return s + return f"TypedDict({{{', '.join(items)}}})" elif isinstance(typ, LiteralType): return f"Literal[{format_literal_value(typ)}]" elif isinstance(typ, UnionType): diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 5139b9b82289..73c5742614ee 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -1,7 +1,7 @@ from __future__ import annotations from functools import partial -from typing import Callable +from typing import Callable, Final import mypy.errorcodes as codes from mypy import message_registry @@ -372,6 +372,10 @@ def typed_dict_setdefault_callback(ctx: MethodContext) -> Type: ) return AnyType(TypeOfAny.from_error) + assigned_readonly_keys = ctx.type.readonly_keys & set(keys) + if assigned_readonly_keys: + ctx.api.msg.readonly_keys_mutated(assigned_readonly_keys, context=ctx.context) + default_type = ctx.arg_types[1][0] value_types = [] @@ -415,13 +419,16 @@ def typed_dict_delitem_callback(ctx: MethodContext) -> Type: return AnyType(TypeOfAny.from_error) for key in keys: - if key in ctx.type.required_keys: + if key in ctx.type.required_keys or key in ctx.type.readonly_keys: ctx.api.msg.typeddict_key_cannot_be_deleted(ctx.type, key, ctx.context) elif key not in ctx.type.items: ctx.api.msg.typeddict_key_not_found(ctx.type, key, ctx.context) return ctx.default_return_type +_TP_DICT_MUTATING_METHODS: Final = frozenset({"update of TypedDict", "__ior__ of TypedDict"}) + + def typed_dict_update_signature_callback(ctx: MethodSigContext) -> CallableType: """Try to infer a better signature type for methods that update `TypedDict`. @@ -436,10 +443,19 @@ def typed_dict_update_signature_callback(ctx: MethodSigContext) -> CallableType: arg_type = arg_type.as_anonymous() arg_type = arg_type.copy_modified(required_keys=set()) if ctx.args and ctx.args[0]: - with ctx.api.msg.filter_errors(): + if signature.name in _TP_DICT_MUTATING_METHODS: + # If we want to mutate this object in place, we need to set this flag, + # it will trigger an extra check in TypedDict's checker. + arg_type.to_be_mutated = True + with ctx.api.msg.filter_errors( + filter_errors=lambda name, info: info.code != codes.TYPEDDICT_READONLY_MUTATED, + save_filtered_errors=True, + ): inferred = get_proper_type( ctx.api.get_expression_type(ctx.args[0][0], type_context=arg_type) ) + if arg_type.to_be_mutated: + arg_type.to_be_mutated = False # Done! possible_tds = [] if isinstance(inferred, TypedDictType): possible_tds = [inferred] diff --git a/mypy/plugins/proper_plugin.py b/mypy/plugins/proper_plugin.py index a1fd05272b65..f51685c80afa 100644 --- a/mypy/plugins/proper_plugin.py +++ b/mypy/plugins/proper_plugin.py @@ -106,6 +106,7 @@ def is_special_target(right: ProperType) -> bool: "mypy.types.ErasedType", "mypy.types.DeletedType", "mypy.types.RequiredType", + "mypy.types.ReadOnlyType", ): # Special case: these are not valid targets for a type alias and thus safe. # TODO: introduce a SyntheticType base to simplify this? diff --git a/mypy/semanal.py b/mypy/semanal.py index 0b654d6b145f..27abf2c1dc4c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -7169,7 +7169,7 @@ def type_analyzer( allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, allow_placeholder: bool = False, - allow_required: bool = False, + allow_typed_dict_special_forms: bool = False, allow_param_spec_literals: bool = False, allow_unpack: bool = False, report_invalid_types: bool = True, @@ -7188,7 +7188,7 @@ def type_analyzer( allow_tuple_literal=allow_tuple_literal, report_invalid_types=report_invalid_types, allow_placeholder=allow_placeholder, - allow_required=allow_required, + allow_typed_dict_special_forms=allow_typed_dict_special_forms, allow_param_spec_literals=allow_param_spec_literals, allow_unpack=allow_unpack, prohibit_self_type=prohibit_self_type, @@ -7211,7 +7211,7 @@ def anal_type( allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, allow_placeholder: bool = False, - allow_required: bool = False, + allow_typed_dict_special_forms: bool = False, allow_param_spec_literals: bool = False, allow_unpack: bool = False, report_invalid_types: bool = True, @@ -7246,7 +7246,7 @@ def anal_type( allow_unbound_tvars=allow_unbound_tvars, allow_tuple_literal=allow_tuple_literal, allow_placeholder=allow_placeholder, - allow_required=allow_required, + allow_typed_dict_special_forms=allow_typed_dict_special_forms, allow_param_spec_literals=allow_param_spec_literals, allow_unpack=allow_unpack, report_invalid_types=report_invalid_types, diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index db19f074911f..cb0bdebab724 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -181,7 +181,7 @@ def anal_type( tvar_scope: TypeVarLikeScope | None = None, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, - allow_required: bool = False, + allow_typed_dict_special_forms: bool = False, allow_placeholder: bool = False, report_invalid_types: bool = True, prohibit_self_type: str | None = None, diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 832530df55e5..d081898bf010 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -43,6 +43,7 @@ from mypy.types import ( TPDICT_NAMES, AnyType, + ReadOnlyType, RequiredType, Type, TypedDictType, @@ -102,13 +103,15 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N and defn.base_type_exprs[0].fullname in TPDICT_NAMES ): # Building a new TypedDict - fields, types, statements, required_keys = self.analyze_typeddict_classdef_fields(defn) + fields, types, statements, required_keys, readonly_keys = ( + self.analyze_typeddict_classdef_fields(defn) + ) if fields is None: return True, None # Defer if self.api.is_func_scope() and "@" not in defn.name: defn.name += "@" + str(defn.line) info = self.build_typeddict_typeinfo( - defn.name, fields, types, required_keys, defn.line, existing_info + defn.name, fields, types, required_keys, readonly_keys, defn.line, existing_info ) defn.analyzed = TypedDictExpr(info) defn.analyzed.line = defn.line @@ -154,10 +157,13 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N keys: list[str] = [] types = [] required_keys = set() + readonly_keys = set() # Iterate over bases in reverse order so that leftmost base class' keys take precedence for base in reversed(typeddict_bases): - self.add_keys_and_types_from_base(base, keys, types, required_keys, defn) - (new_keys, new_types, new_statements, new_required_keys) = ( + self.add_keys_and_types_from_base( + base, keys, types, required_keys, readonly_keys, defn + ) + (new_keys, new_types, new_statements, new_required_keys, new_readonly_keys) = ( self.analyze_typeddict_classdef_fields(defn, keys) ) if new_keys is None: @@ -165,8 +171,9 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N keys.extend(new_keys) types.extend(new_types) required_keys.update(new_required_keys) + readonly_keys.update(new_readonly_keys) info = self.build_typeddict_typeinfo( - defn.name, keys, types, required_keys, defn.line, existing_info + defn.name, keys, types, required_keys, readonly_keys, defn.line, existing_info ) defn.analyzed = TypedDictExpr(info) defn.analyzed.line = defn.line @@ -180,6 +187,7 @@ def add_keys_and_types_from_base( keys: list[str], types: list[Type], required_keys: set[str], + readonly_keys: set[str], ctx: Context, ) -> None: base_args: list[Type] = [] @@ -221,6 +229,7 @@ def add_keys_and_types_from_base( keys.extend(valid_items.keys()) types.extend(valid_items.values()) required_keys.update(base_typed_dict.required_keys) + readonly_keys.update(base_typed_dict.readonly_keys) def analyze_base_args(self, base: IndexExpr, ctx: Context) -> list[Type] | None: """Analyze arguments of base type expressions as types. @@ -241,7 +250,9 @@ def analyze_base_args(self, base: IndexExpr, ctx: Context) -> list[Type] | None: self.fail("Invalid TypedDict type argument", ctx) return None analyzed = self.api.anal_type( - type, allow_required=True, allow_placeholder=not self.api.is_func_scope() + type, + allow_typed_dict_special_forms=True, + allow_placeholder=not self.api.is_func_scope(), ) if analyzed is None: return None @@ -270,7 +281,7 @@ def map_items_to_base( def analyze_typeddict_classdef_fields( self, defn: ClassDef, oldfields: list[str] | None = None - ) -> tuple[list[str] | None, list[Type], list[Statement], set[str]]: + ) -> tuple[list[str] | None, list[Type], list[Statement], set[str], set[str]]: """Analyze fields defined in a TypedDict class definition. This doesn't consider inherited fields (if any). Also consider totality, @@ -316,17 +327,15 @@ def analyze_typeddict_classdef_fields( else: analyzed = self.api.anal_type( stmt.unanalyzed_type, - allow_required=True, + allow_typed_dict_special_forms=True, allow_placeholder=not self.api.is_func_scope(), prohibit_self_type="TypedDict item type", ) if analyzed is None: - return None, [], [], set() # Need to defer + return None, [], [], set(), set() # Need to defer types.append(analyzed) if not has_placeholder(analyzed): - stmt.type = ( - analyzed.item if isinstance(analyzed, RequiredType) else analyzed - ) + stmt.type = self.extract_meta_info(analyzed, stmt)[0] # ...despite possible minor failures that allow further analysis. if stmt.type is None or hasattr(stmt, "new_syntax") and not stmt.new_syntax: self.fail(TPDICT_CLASS_ERROR, stmt) @@ -342,17 +351,49 @@ def analyze_typeddict_classdef_fields( if key == "total": continue self.msg.unexpected_keyword_argument_for_function(for_function, key, defn) - required_keys = { - field - for (field, t) in zip(fields, types) - if (total or (isinstance(t, RequiredType) and t.required)) - and not (isinstance(t, RequiredType) and not t.required) - } - types = [ # unwrap Required[T] to just T - t.item if isinstance(t, RequiredType) else t for t in types - ] - - return fields, types, statements, required_keys + + res_types = [] + readonly_keys = set() + required_keys = set() + for field, t in zip(fields, types): + typ, required, readonly = self.extract_meta_info(t) + res_types.append(typ) + if (total or required is True) and required is not False: + required_keys.add(field) + if readonly: + readonly_keys.add(field) + + return fields, res_types, statements, required_keys, readonly_keys + + def extract_meta_info( + self, typ: Type, context: Context | None = None + ) -> tuple[Type, bool | None, bool]: + """Unwrap all metadata types.""" + is_required = None # default, no modification + readonly = False # by default all is mutable + + seen_required = False + seen_readonly = False + while isinstance(typ, (RequiredType, ReadOnlyType)): + if isinstance(typ, RequiredType): + if context is not None and seen_required: + self.fail( + '"{}" type cannot be nested'.format( + "Required[]" if typ.required else "NotRequired[]" + ), + context, + code=codes.VALID_TYPE, + ) + is_required = typ.required + seen_required = True + typ = typ.item + if isinstance(typ, ReadOnlyType): + if context is not None and seen_readonly: + self.fail('"ReadOnly[]" type cannot be nested', context, code=codes.VALID_TYPE) + readonly = True + seen_readonly = True + typ = typ.item + return typ, is_required, readonly def check_typeddict( self, node: Expression, var_name: str | None, is_func_scope: bool @@ -391,7 +432,7 @@ def check_typeddict( name += "@" + str(call.line) else: name = var_name = "TypedDict@" + str(call.line) - info = self.build_typeddict_typeinfo(name, [], [], set(), call.line, None) + info = self.build_typeddict_typeinfo(name, [], [], set(), set(), call.line, None) else: if var_name is not None and name != var_name: self.fail( @@ -410,8 +451,11 @@ def check_typeddict( if (total or (isinstance(t, RequiredType) and t.required)) and not (isinstance(t, RequiredType) and not t.required) } - types = [ # unwrap Required[T] to just T - t.item if isinstance(t, RequiredType) else t for t in types + readonly_keys = { + field for (field, t) in zip(items, types) if isinstance(t, ReadOnlyType) + } + types = [ # unwrap Required[T] or ReadOnly[T] to just T + t.item if isinstance(t, (RequiredType, ReadOnlyType)) else t for t in types ] # Perform various validations after unwrapping. @@ -428,7 +472,7 @@ def check_typeddict( if isinstance(node.analyzed, TypedDictExpr): existing_info = node.analyzed.info info = self.build_typeddict_typeinfo( - name, items, types, required_keys, call.line, existing_info + name, items, types, required_keys, readonly_keys, call.line, existing_info ) info.line = node.line # Store generated TypeInfo under both names, see semanal_namedtuple for more details. @@ -514,7 +558,7 @@ def parse_typeddict_fields_with_types( return [], [], False analyzed = self.api.anal_type( type, - allow_required=True, + allow_typed_dict_special_forms=True, allow_placeholder=not self.api.is_func_scope(), prohibit_self_type="TypedDict item type", ) @@ -535,6 +579,7 @@ def build_typeddict_typeinfo( items: list[str], types: list[Type], required_keys: set[str], + readonly_keys: set[str], line: int, existing_info: TypeInfo | None, ) -> TypeInfo: @@ -546,7 +591,9 @@ def build_typeddict_typeinfo( ) assert fallback is not None info = existing_info or self.api.basic_new_typeinfo(name, fallback, line) - typeddict_type = TypedDictType(dict(zip(items, types)), required_keys, fallback) + typeddict_type = TypedDictType( + dict(zip(items, types)), required_keys, readonly_keys, fallback + ) if info.special_alias and has_placeholder(info.special_alias.target): self.api.process_placeholder( None, "TypedDict item", info, force_progress=typeddict_type != info.typeddict_type diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index f8a874005adb..fc868d288b4d 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -475,7 +475,8 @@ def visit_tuple_type(self, typ: TupleType) -> SnapshotItem: def visit_typeddict_type(self, typ: TypedDictType) -> SnapshotItem: items = tuple((key, snapshot_type(item_type)) for key, item_type in typ.items.items()) required = tuple(sorted(typ.required_keys)) - return ("TypedDictType", items, required) + readonly = tuple(sorted(typ.readonly_keys)) + return ("TypedDictType", items, required, readonly) def visit_literal_type(self, typ: LiteralType) -> SnapshotItem: return ("LiteralType", snapshot_type(typ.fallback), typ.value) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 608d098791a9..3b775a06bd6e 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -914,6 +914,17 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: # lands so here we are anticipating that change. if (name in left.required_keys) != (name in right.required_keys): return False + # Readonly fields check: + # + # A = TypedDict('A', {'x': ReadOnly[int]}) + # B = TypedDict('A', {'x': int}) + # def reset_x(b: B) -> None: + # b['x'] = 0 + # + # So, `A` cannot be a subtype of `B`, while `B` can be a subtype of `A`, + # because you can use `B` everywhere you use `A`, but not the other way around. + if name in left.readonly_keys and name not in right.readonly_keys: + return False # (NOTE: Fallbacks don't matter.) return True else: diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 38e4c5ba0d01..8aac7e5c2bbd 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -276,6 +276,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: result = TypedDictType( items, t.required_keys, + t.readonly_keys, # TODO: This appears to be unsafe. cast(Any, t.fallback.accept(self)), t.line, diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6c94390c23dc..0a6b7689136e 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -83,6 +83,7 @@ PlaceholderType, ProperType, RawExpressionType, + ReadOnlyType, RequiredType, SyntheticTypeVisitor, TrivialSyntheticTypeTranslator, @@ -219,7 +220,7 @@ def __init__( allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, allow_placeholder: bool = False, - allow_required: bool = False, + allow_typed_dict_special_forms: bool = False, allow_param_spec_literals: bool = False, allow_unpack: bool = False, report_invalid_types: bool = True, @@ -253,7 +254,7 @@ def __init__( # If false, record incomplete ref if we generate PlaceholderType. self.allow_placeholder = allow_placeholder # Are we in a context where Required[] is allowed? - self.allow_required = allow_required + self.allow_typed_dict_special_forms = allow_typed_dict_special_forms # Are we in a context where ParamSpec literals are allowed? self.allow_param_spec_literals = allow_param_spec_literals # Are we in context where literal "..." specifically is allowed? @@ -684,7 +685,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ return AnyType(TypeOfAny.from_error) return self.anal_type(t.args[0]) elif fullname in ("typing_extensions.Required", "typing.Required"): - if not self.allow_required: + if not self.allow_typed_dict_special_forms: self.fail( "Required[] can be only used in a TypedDict definition", t, @@ -696,9 +697,11 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ "Required[] must have exactly one type argument", t, code=codes.VALID_TYPE ) return AnyType(TypeOfAny.from_error) - return RequiredType(self.anal_type(t.args[0]), required=True) + return RequiredType( + self.anal_type(t.args[0], allow_typed_dict_special_forms=True), required=True + ) elif fullname in ("typing_extensions.NotRequired", "typing.NotRequired"): - if not self.allow_required: + if not self.allow_typed_dict_special_forms: self.fail( "NotRequired[] can be only used in a TypedDict definition", t, @@ -710,7 +713,23 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ "NotRequired[] must have exactly one type argument", t, code=codes.VALID_TYPE ) return AnyType(TypeOfAny.from_error) - return RequiredType(self.anal_type(t.args[0]), required=False) + return RequiredType( + self.anal_type(t.args[0], allow_typed_dict_special_forms=True), required=False + ) + elif fullname in ("typing_extensions.ReadOnly", "typing.ReadOnly"): + if not self.allow_typed_dict_special_forms: + self.fail( + "ReadOnly[] can be only used in a TypedDict definition", + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) + if len(t.args) != 1: + self.fail( + '"ReadOnly[]" must have exactly one type argument', t, code=codes.VALID_TYPE + ) + return AnyType(TypeOfAny.from_error) + return ReadOnlyType(self.anal_type(t.args[0], allow_typed_dict_special_forms=True)) elif ( self.anal_type_guard_arg(t, fullname) is not None or self.anal_type_is_arg(t, fullname) is not None @@ -1223,9 +1242,11 @@ def visit_tuple_type(self, t: TupleType) -> Type: def visit_typeddict_type(self, t: TypedDictType) -> Type: req_keys = set() + readonly_keys = set() items = {} for item_name, item_type in t.items.items(): - analyzed = self.anal_type(item_type, allow_required=True) + # TODO: rework + analyzed = self.anal_type(item_type, allow_typed_dict_special_forms=True) if isinstance(analyzed, RequiredType): if analyzed.required: req_keys.add(item_name) @@ -1233,6 +1254,9 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: else: # Keys are required by default. req_keys.add(item_name) + if isinstance(analyzed, ReadOnlyType): + readonly_keys.add(item_name) + analyzed = analyzed.item items[item_name] = analyzed if t.fallback.type is MISSING_FALLBACK: # anonymous/inline TypedDict if INLINE_TYPEDDICT not in self.options.enable_incomplete_feature: @@ -1257,10 +1281,12 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: items[sub_item_name] = sub_item_type if sub_item_name in p_analyzed.required_keys: req_keys.add(sub_item_name) + if sub_item_name in p_analyzed.readonly_keys: + readonly_keys.add(sub_item_name) else: required_keys = t.required_keys fallback = t.fallback - return TypedDictType(items, required_keys, fallback, t.line, t.column) + return TypedDictType(items, required_keys, readonly_keys, fallback, t.line, t.column) def visit_raw_expression_type(self, t: RawExpressionType) -> Type: # We should never see a bare Literal. We synthesize these raw literals @@ -1811,12 +1837,12 @@ def anal_type( allow_param_spec: bool = False, allow_unpack: bool = False, allow_ellipsis: bool = False, - allow_required: bool = False, + allow_typed_dict_special_forms: bool = False, ) -> Type: if nested: self.nesting_level += 1 - old_allow_required = self.allow_required - self.allow_required = allow_required + old_allow_typed_dict_special_forms = self.allow_typed_dict_special_forms + self.allow_typed_dict_special_forms = allow_typed_dict_special_forms old_allow_ellipsis = self.allow_ellipsis self.allow_ellipsis = allow_ellipsis old_allow_unpack = self.allow_unpack @@ -1826,7 +1852,7 @@ def anal_type( finally: if nested: self.nesting_level -= 1 - self.allow_required = old_allow_required + self.allow_typed_dict_special_forms = old_allow_typed_dict_special_forms self.allow_ellipsis = old_allow_ellipsis self.allow_unpack = old_allow_unpack if ( diff --git a/mypy/types.py b/mypy/types.py index b1e57b2f6a86..dff7e2c0c829 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -476,6 +476,20 @@ def accept(self, visitor: TypeVisitor[T]) -> T: return self.item.accept(visitor) +class ReadOnlyType(Type): + """ReadOnly[T] Only usable at top-level of a TypedDict definition.""" + + def __init__(self, item: Type) -> None: + super().__init__(line=item.line, column=item.column) + self.item = item + + def __repr__(self) -> str: + return f"ReadOnly[{self.item}]" + + def accept(self, visitor: TypeVisitor[T]) -> T: + return self.item.accept(visitor) + + class ProperType(Type): """Not a type alias. @@ -2554,17 +2568,28 @@ class TypedDictType(ProperType): TODO: The fallback structure is perhaps overly complicated. """ - __slots__ = ("items", "required_keys", "fallback", "extra_items_from") + __slots__ = ( + "items", + "required_keys", + "readonly_keys", + "fallback", + "extra_items_from", + "to_be_mutated", + ) items: dict[str, Type] # item_name -> item_type required_keys: set[str] + readonly_keys: set[str] fallback: Instance + extra_items_from: list[ProperType] # only used during semantic analysis + to_be_mutated: bool # only used in a plugin for `.update`, `|=`, etc def __init__( self, items: dict[str, Type], required_keys: set[str], + readonly_keys: set[str], fallback: Instance, line: int = -1, column: int = -1, @@ -2572,16 +2597,25 @@ def __init__( super().__init__(line, column) self.items = items self.required_keys = required_keys + self.readonly_keys = readonly_keys self.fallback = fallback self.can_be_true = len(self.items) > 0 self.can_be_false = len(self.required_keys) == 0 self.extra_items_from = [] + self.to_be_mutated = False def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_typeddict_type(self) def __hash__(self) -> int: - return hash((frozenset(self.items.items()), self.fallback, frozenset(self.required_keys))) + return hash( + ( + frozenset(self.items.items()), + self.fallback, + frozenset(self.required_keys), + frozenset(self.readonly_keys), + ) + ) def __eq__(self, other: object) -> bool: if not isinstance(other, TypedDictType): @@ -2596,6 +2630,7 @@ def __eq__(self, other: object) -> bool: ) and self.fallback == other.fallback and self.required_keys == other.required_keys + and self.readonly_keys == other.readonly_keys ) def serialize(self) -> JsonDict: @@ -2603,6 +2638,7 @@ def serialize(self) -> JsonDict: ".class": "TypedDictType", "items": [[n, t.serialize()] for (n, t) in self.items.items()], "required_keys": sorted(self.required_keys), + "readonly_keys": sorted(self.readonly_keys), "fallback": self.fallback.serialize(), } @@ -2612,6 +2648,7 @@ def deserialize(cls, data: JsonDict) -> TypedDictType: return TypedDictType( {n: deserialize_type(t) for (n, t) in data["items"]}, set(data["required_keys"]), + set(data["readonly_keys"]), Instance.deserialize(data["fallback"]), ) @@ -2635,6 +2672,7 @@ def copy_modified( item_types: list[Type] | None = None, item_names: list[str] | None = None, required_keys: set[str] | None = None, + readonly_keys: set[str] | None = None, ) -> TypedDictType: if fallback is None: fallback = self.fallback @@ -2644,10 +2682,12 @@ def copy_modified( items = dict(zip(self.items, item_types)) if required_keys is None: required_keys = self.required_keys + if readonly_keys is None: + readonly_keys = self.readonly_keys if item_names is not None: items = {k: v for (k, v) in items.items() if k in item_names} required_keys &= set(item_names) - return TypedDictType(items, required_keys, fallback, self.line, self.column) + return TypedDictType(items, required_keys, readonly_keys, fallback, self.line, self.column) def create_anonymous_fallback(self) -> Instance: anonymous = self.as_anonymous() @@ -3421,10 +3461,12 @@ def visit_tuple_type(self, t: TupleType) -> str: def visit_typeddict_type(self, t: TypedDictType) -> str: def item_str(name: str, typ: str) -> str: - if name in t.required_keys: - return f"{name!r}: {typ}" - else: - return f"{name!r}?: {typ}" + modifier = "" + if name not in t.required_keys: + modifier += "?" + if name in t.readonly_keys: + modifier += "=" + return f"{name!r}{modifier}: {typ}" s = ( "{" diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index dc1929751977..e1797421636e 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -2384,10 +2384,12 @@ class ForceDeferredEval: pass [case testTypedDictRequiredUnimportedAny] # flags: --disallow-any-unimported -from typing import NotRequired, TypedDict +from typing import NotRequired, TypedDict, ReadOnly from nonexistent import Foo # type: ignore[import-not-found] class Bar(TypedDict): foo: NotRequired[Foo] # E: Type of variable becomes "Any" due to an unfollowed import + bar: ReadOnly[Foo] # E: Type of variable becomes "Any" due to an unfollowed import + baz: NotRequired[ReadOnly[Foo]] # E: Type of variable becomes "Any" due to an unfollowed import [typing fixtures/typing-typeddict.pyi] -- Required[] @@ -3631,6 +3633,16 @@ y = {"one": 1} # E: Expected TypedDict keys ("one", "other") but found only key [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] +[case testTypedDictInlineReadOnly] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing import ReadOnly + +x: {"one": int, "other": ReadOnly[int]} +x["one"] = 1 # ok +x["other"] = 1 # E: ReadOnly TypedDict key "other" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + [case testTypedDictInlineNestedSchema] # flags: --enable-incomplete-feature=InlineTypedDict def nested() -> {"one": str, "other": {"a": int, "b": int}}: @@ -3652,3 +3664,327 @@ x: {"a": int, **X[str], "b": int} reveal_type(x) # N: Revealed type is "TypedDict({'a': builtins.int, 'b': builtins.int, 'item': builtins.str})" [builtins fixtures/dict.pyi] [typing fixtures/typing-full.pyi] + + +# ReadOnly +# See: https://peps.python.org/pep-0705 + +[case testTypedDictReadOnly] +# flags: --show-error-codes +from typing import ReadOnly, TypedDict + +class TP(TypedDict): + one: int + other: ReadOnly[str] + +x: TP +reveal_type(x["one"]) # N: Revealed type is "builtins.int" +reveal_type(x["other"]) # N: Revealed type is "builtins.str" +x["one"] = 1 # ok +x["other"] = "a" # E: ReadOnly TypedDict key "other" TypedDict is mutated [typeddict-readonly-mutated] +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyCreation] +from typing import ReadOnly, TypedDict + +class TD(TypedDict): + x: ReadOnly[int] + y: int + +# Ok: +x = TD({"x": 1, "y": 2}) +y = TD(x=1, y=2) +z: TD = {"x": 1, "y": 2} + +# Error: +x2 = TD({"x": "a", "y": 2}) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +y2 = TD(x="a", y=2) # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +z2: TD = {"x": "a", "y": 2} # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyDel] +from typing import ReadOnly, TypedDict, NotRequired + +class TP(TypedDict): + required_key: ReadOnly[str] + optional_key: ReadOnly[NotRequired[str]] + +x: TP +del x["required_key"] # E: Key "required_key" of TypedDict "TP" cannot be deleted +del x["optional_key"] # E: Key "optional_key" of TypedDict "TP" cannot be deleted +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyMutateMethods] +from typing import ReadOnly, TypedDict + +class TP(TypedDict): + key: ReadOnly[str] + other: ReadOnly[int] + mutable: bool + +x: TP +reveal_type(x.pop("key")) # E: Key "key" of TypedDict "TP" cannot be deleted \ + # N: Revealed type is "builtins.str" + +x.update({"key": "abc", "other": 1, "mutable": True}) # E: ReadOnly TypedDict keys ("key", "other") TypedDict are mutated +x.setdefault("key", "abc") # E: ReadOnly TypedDict key "key" TypedDict is mutated +x.setdefault("other", 1) # E: ReadOnly TypedDict key "other" TypedDict is mutated +x.setdefault("mutable", False) # ok +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictFromTypingExtensionsReadOnlyMutateMethods] +from typing_extensions import ReadOnly, TypedDict + +class TP(TypedDict): + key: ReadOnly[str] + +x: TP +x.update({"key": "abc"}) # E: ReadOnly TypedDict key "key" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictFromMypyExtensionsReadOnlyMutateMethods] +from mypy_extensions import TypedDict +from typing_extensions import ReadOnly + +class TP(TypedDict): + key: ReadOnly[str] + +x: TP +x.update({"key": "abc"}) # E: ReadOnly TypedDict key "key" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyMutate__ior__Statements] +from typing_extensions import ReadOnly, TypedDict + +class TP(TypedDict): + key: ReadOnly[str] + other: ReadOnly[int] + mutable: bool + +x: TP +x |= {"mutable": True} # ok +x |= {"key": "a"} # E: ReadOnly TypedDict key "key" TypedDict is mutated +x |= {"key": "a", "other": 1, "mutable": True} # E: ReadOnly TypedDict keys ("key", "other") TypedDict are mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict-iror.pyi] + +[case testTypedDictReadOnlyMutate__or__Statements] +from typing_extensions import ReadOnly, TypedDict + +class TP(TypedDict): + key: ReadOnly[str] + other: ReadOnly[int] + mutable: bool + +x: TP +# These are new objects, not mutation: +x = x | {"mutable": True} +x = x | {"key": "a"} +x = x | {"key": "a", "other": 1, "mutable": True} +y1 = x | {"mutable": True} +y2 = x | {"key": "a"} +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict-iror.pyi] + +[case testTypedDictReadOnlyMutateWithOtherDicts] +from typing import ReadOnly, TypedDict, Dict + +class TP(TypedDict): + key: ReadOnly[str] + mutable: bool + +class Mutable(TypedDict): + mutable: bool + +class Regular(TypedDict): + key: str + +m: Mutable +r: Regular +d: Dict[str, object] + +# Creating new objects is ok: +tp: TP = {**r, **m} +tp1: TP = {**tp, **m} +tp2: TP = {**r, **m} +tp3: TP = {**tp, **r} +tp4: TP = {**tp, **d} # E: Unsupported type "Dict[str, object]" for ** expansion in TypedDict +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictGenericReadOnly] +from typing import ReadOnly, TypedDict, TypeVar, Generic + +T = TypeVar('T') + +class TP(TypedDict, Generic[T]): + key: ReadOnly[T] + +x: TP[int] +reveal_type(x["key"]) # N: Revealed type is "builtins.int" +x["key"] = 1 # E: ReadOnly TypedDict key "key" TypedDict is mutated +x["key"] = "a" # E: ReadOnly TypedDict key "key" TypedDict is mutated \ + # E: Value of "key" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyOtherTypedDict] +from typing import ReadOnly, TypedDict + +class First(TypedDict): + field: int + +class TP(TypedDict): + key: ReadOnly[First] + +x: TP +reveal_type(x["key"]["field"]) # N: Revealed type is "builtins.int" +x["key"]["field"] = 1 # ok +x["key"] = {"field": 2} # E: ReadOnly TypedDict key "key" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyInheritance] +from typing import ReadOnly, TypedDict + +class Base(TypedDict): + a: ReadOnly[str] + +class Child(Base): + b: ReadOnly[int] + +base: Base +reveal_type(base["a"]) # N: Revealed type is "builtins.str" +base["a"] = "x" # E: ReadOnly TypedDict key "a" TypedDict is mutated +base["b"] # E: TypedDict "Base" has no key "b" + +child: Child +reveal_type(child["a"]) # N: Revealed type is "builtins.str" +reveal_type(child["b"]) # N: Revealed type is "builtins.int" +child["a"] = "x" # E: ReadOnly TypedDict key "a" TypedDict is mutated +child["b"] = 1 # E: ReadOnly TypedDict key "b" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlySubtyping] +from typing import ReadOnly, TypedDict + +class A(TypedDict): + key: ReadOnly[str] + +class B(TypedDict): + key: str + +a: A +b: B + +def accepts_A(d: A): ... +def accepts_B(d: B): ... + +accepts_A(a) +accepts_A(b) +accepts_B(a) # E: Argument 1 to "accepts_B" has incompatible type "A"; expected "B" +accepts_B(b) +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyCall] +from typing import ReadOnly, TypedDict + +TP = TypedDict("TP", {"one": int, "other": ReadOnly[str]}) + +x: TP +reveal_type(x["one"]) # N: Revealed type is "builtins.int" +reveal_type(x["other"]) # N: Revealed type is "builtins.str" +x["one"] = 1 # ok +x["other"] = "a" # E: ReadOnly TypedDict key "other" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyABCSubtypes] +from typing import ReadOnly, TypedDict, Mapping, Dict, MutableMapping + +class TP(TypedDict): + one: int + other: ReadOnly[int] + +def accepts_mapping(m: Mapping[str, object]): ... +def accepts_mutable_mapping(mm: MutableMapping[str, object]): ... +def accepts_dict(d: Dict[str, object]): ... + +x: TP +accepts_mapping(x) +accepts_mutable_mapping(x) # E: Argument 1 to "accepts_mutable_mapping" has incompatible type "TP"; expected "MutableMapping[str, object]" +accepts_dict(x) # E: Argument 1 to "accepts_dict" has incompatible type "TP"; expected "Dict[str, object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyAndNotRequired] +from typing import ReadOnly, TypedDict, NotRequired + +class TP(TypedDict): + one: ReadOnly[NotRequired[int]] + two: NotRequired[ReadOnly[str]] + +x: TP +reveal_type(x) # N: Revealed type is "TypedDict('__main__.TP', {'one'?=: builtins.int, 'two'?=: builtins.str})" +reveal_type(x.get("one")) # N: Revealed type is "Union[builtins.int, None]" +reveal_type(x.get("two")) # N: Revealed type is "Union[builtins.str, None]" +x["one"] = 1 # E: ReadOnly TypedDict key "one" TypedDict is mutated +x["two"] = "a" # E: ReadOnly TypedDict key "two" TypedDict is mutated +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testMeetOfTypedDictsWithReadOnly] +from typing import TypeVar, Callable, TypedDict, ReadOnly +XY = TypedDict('XY', {'x': ReadOnly[int], 'y': int}) +YZ = TypedDict('YZ', {'y': int, 'z': ReadOnly[int]}) +T = TypeVar('T') +def f(x: Callable[[T, T], None]) -> T: pass +def g(x: XY, y: YZ) -> None: pass +reveal_type(f(g)) # N: Revealed type is "TypedDict({'x'=: builtins.int, 'y': builtins.int, 'z'=: builtins.int})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyUnpack] +from typing_extensions import TypedDict, Unpack, ReadOnly + +class TD(TypedDict): + x: ReadOnly[int] + y: str + +def func(**kwargs: Unpack[TD]): + kwargs["x"] = 1 # E: ReadOnly TypedDict key "x" TypedDict is mutated + kwargs["y" ] = "a" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testIncorrectTypedDictSpecialFormsUsage] +from typing import ReadOnly, TypedDict, NotRequired, Required + +x: ReadOnly[int] # E: ReadOnly[] can be only used in a TypedDict definition +y: Required[int] # E: Required[] can be only used in a TypedDict definition +z: NotRequired[int] # E: NotRequired[] can be only used in a TypedDict definition + +class TP(TypedDict): + a: ReadOnly[ReadOnly[int]] # E: "ReadOnly[]" type cannot be nested + b: ReadOnly[NotRequired[ReadOnly[str]]] # E: "ReadOnly[]" type cannot be nested + c: NotRequired[Required[int]] # E: "Required[]" type cannot be nested + d: Required[NotRequired[int]] # E: "NotRequired[]" type cannot be nested + e: Required[ReadOnly[NotRequired[int]]] # E: "NotRequired[]" type cannot be nested + f: ReadOnly[ReadOnly[ReadOnly[int]]] # E: "ReadOnly[]" type cannot be nested + g: Required[Required[int]] # E: "Required[]" type cannot be nested + h: NotRequired[NotRequired[int]] # E: "NotRequired[]" type cannot be nested + + j: NotRequired[ReadOnly[Required[ReadOnly[int]]]] # E: "Required[]" type cannot be nested \ + # E: "ReadOnly[]" type cannot be nested + + k: ReadOnly # E: "ReadOnly[]" must have exactly one type argument +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 2ad31311a402..faedd890922d 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3705,6 +3705,34 @@ def foo() -> None: == b.py:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") +[case testTypedDictUpdateReadOnly] +import b +[file a.py] +from typing_extensions import TypedDict, ReadOnly +Point = TypedDict('Point', {'x': int, 'y': int}) +p = Point(x=1, y=2) +[file a.py.2] +from typing_extensions import TypedDict, ReadOnly +class Point(TypedDict): + x: int + y: ReadOnly[int] +p = Point(x=1, y=2) +[file a.py.3] +from typing_extensions import TypedDict, ReadOnly +Point = TypedDict('Point', {'x': ReadOnly[int], 'y': int}) +p = Point(x=1, y=2) +[file b.py] +from a import Point +def foo(x: Point) -> None: + x['x'] = 1 + x['y'] = 2 +[builtins fixtures/dict.pyi] +[out] +== +b.py:4: error: ReadOnly TypedDict key "y" TypedDict is mutated +== +b.py:3: error: ReadOnly TypedDict key "x" TypedDict is mutated + [case testBasicAliasUpdate] import b [file a.py] diff --git a/test-data/unit/fixtures/typing-typeddict.pyi b/test-data/unit/fixtures/typing-typeddict.pyi index d136ac4ab8be..7e9c642cf261 100644 --- a/test-data/unit/fixtures/typing-typeddict.pyi +++ b/test-data/unit/fixtures/typing-typeddict.pyi @@ -26,6 +26,7 @@ TypedDict = 0 NoReturn = 0 Required = 0 NotRequired = 0 +ReadOnly = 0 Self = 0 T = TypeVar('T') @@ -59,6 +60,10 @@ class Mapping(Iterable[T], Generic[T, T_co], metaclass=ABCMeta): def __len__(self) -> int: ... def __contains__(self, arg: object) -> int: pass +class MutableMapping(Mapping[T, T_co], Generic[T, T_co], metaclass=ABCMeta): + # Other methods are not used in tests. + def clear(self) -> None: ... + # Fallback type for all typed dicts (does not exist at runtime). class _TypedDict(Mapping[str, object]): # Needed to make this class non-abstract. It is explicitly declared abstract in diff --git a/test-data/unit/lib-stub/typing_extensions.pyi b/test-data/unit/lib-stub/typing_extensions.pyi index b5bfc1ab3f20..d9d7067efe0f 100644 --- a/test-data/unit/lib-stub/typing_extensions.pyi +++ b/test-data/unit/lib-stub/typing_extensions.pyi @@ -41,6 +41,7 @@ TypeVarTuple: _SpecialForm Unpack: _SpecialForm Required: _SpecialForm NotRequired: _SpecialForm +ReadOnly: _SpecialForm @final class TypeAliasType: From 7dc23fe83700babfe297e368b4019266e403c7bf Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Mon, 30 Sep 2024 19:35:25 -0400 Subject: [PATCH 164/247] Enable negative narrowing of Union TypeVar upper bounds (#17850) Fixes #15235 ### Before ```python from typing import TypeVar class A: pass class B: b: int T = TypeVar("T", bound=A | B) def foo(x: T) -> T: if isinstance(x, A): reveal_type(x) # N: Revealed type is "__main__.A" else: reveal_type(x) # N: Revealed type is "T`-1" x.b # E: Item "A" of the upper bound "A | B" of type variable "T" has no attribute "b" return x ``` ### After ```python from typing import TypeVar class A: pass class B: b: int T = TypeVar("T", bound=A | B) def foo(x: T) -> T: if isinstance(x, A): reveal_type(x) # N: Revealed type is "__main__.A" else: reveal_type(x) # N: Revealed type is "T`-1" x.b # ok! Upper bound of T narrowed to B return x ``` --- mypy/subtypes.py | 2 ++ test-data/unit/check-isinstance.test | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 3b775a06bd6e..787d5cb89b0a 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1941,6 +1941,8 @@ def restrict_subtype_away(t: Type, s: Type) -> Type: if (isinstance(get_proper_type(item), AnyType) or not covers_at_runtime(item, s)) ] return UnionType.make_union(new_items) + elif isinstance(p_t, TypeVarType): + return p_t.copy_modified(upper_bound=restrict_subtype_away(p_t.upper_bound, s)) elif covers_at_runtime(t, s): return UninhabitedType() else: diff --git a/test-data/unit/check-isinstance.test b/test-data/unit/check-isinstance.test index b7ee38b69d00..8fa1bc1ca1ac 100644 --- a/test-data/unit/check-isinstance.test +++ b/test-data/unit/check-isinstance.test @@ -1833,6 +1833,30 @@ def f(x: T) -> None: reveal_type(x) # N: Revealed type is "T`-1" [builtins fixtures/isinstance.pyi] +[case testIsinstanceAndNegativeNarrowTypeVariableWithUnionBound] +from typing import Union, TypeVar + +class A: + a: int +class B: + b: int + +T = TypeVar("T", bound=Union[A, B]) + +def f(x: T) -> T: + if isinstance(x, A): + reveal_type(x) # N: Revealed type is "__main__.A" + x.a + x.b # E: "A" has no attribute "b" + else: + reveal_type(x) # N: Revealed type is "T`-1" + x.a # E: "T" has no attribute "a" + x.b + x.a # E: Item "B" of the upper bound "Union[A, B]" of type variable "T" has no attribute "a" + x.b # E: Item "A" of the upper bound "Union[A, B]" of type variable "T" has no attribute "b" + return x +[builtins fixtures/isinstance.pyi] + [case testIsinstanceAndTypeType] from typing import Type def f(x: Type[int]) -> None: From 1855bb9ad71cad1c6b0ebefd2808c82273a35a3a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 19:11:22 -0700 Subject: [PATCH 165/247] Sync typeshed (#17855) Source commit: https://github.com/python/typeshed/commit/91a58b07cdd807b1d965e04ba85af2adab8bf924 --- mypy/typeshed/stdlib/cProfile.pyi | 6 +++--- mypy/typeshed/stdlib/calendar.pyi | 6 +++--- mypy/typeshed/stdlib/profile.pyi | 6 +++--- mypy/typeshed/stdlib/typing.pyi | 14 +++++++------- mypy/typeshed/stdlib/typing_extensions.pyi | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/mypy/typeshed/stdlib/cProfile.pyi b/mypy/typeshed/stdlib/cProfile.pyi index 0cf6e34ec99e..e921584d4390 100644 --- a/mypy/typeshed/stdlib/cProfile.pyi +++ b/mypy/typeshed/stdlib/cProfile.pyi @@ -1,6 +1,6 @@ import _lsprof from _typeshed import StrOrBytesPath, Unused -from collections.abc import Callable +from collections.abc import Callable, Mapping from types import CodeType from typing import Any, TypeVar from typing_extensions import ParamSpec, Self, TypeAlias @@ -9,7 +9,7 @@ __all__ = ["run", "runctx", "Profile"] def run(statement: str, filename: str | None = None, sort: str | int = -1) -> None: ... def runctx( - statement: str, globals: dict[str, Any], locals: dict[str, Any], filename: str | None = None, sort: str | int = -1 + statement: str, globals: dict[str, Any], locals: Mapping[str, Any], filename: str | None = None, sort: str | int = -1 ) -> None: ... _T = TypeVar("_T") @@ -23,7 +23,7 @@ class Profile(_lsprof.Profiler): def create_stats(self) -> None: ... def snapshot_stats(self) -> None: ... def run(self, cmd: str) -> Self: ... - def runctx(self, cmd: str, globals: dict[str, Any], locals: dict[str, Any]) -> Self: ... + def runctx(self, cmd: str, globals: dict[str, Any], locals: Mapping[str, Any]) -> Self: ... def runcall(self, func: Callable[_P, _T], /, *args: _P.args, **kw: _P.kwargs) -> _T: ... def __enter__(self) -> Self: ... def __exit__(self, *exc_info: Unused) -> None: ... diff --git a/mypy/typeshed/stdlib/calendar.pyi b/mypy/typeshed/stdlib/calendar.pyi index 39312d0b2523..cabf3b881c30 100644 --- a/mypy/typeshed/stdlib/calendar.pyi +++ b/mypy/typeshed/stdlib/calendar.pyi @@ -79,9 +79,9 @@ class Calendar: def monthdatescalendar(self, year: int, month: int) -> list[list[datetime.date]]: ... def monthdays2calendar(self, year: int, month: int) -> list[list[tuple[int, int]]]: ... def monthdayscalendar(self, year: int, month: int) -> list[list[int]]: ... - def yeardatescalendar(self, year: int, width: int = 3) -> list[list[int]]: ... - def yeardays2calendar(self, year: int, width: int = 3) -> list[list[tuple[int, int]]]: ... - def yeardayscalendar(self, year: int, width: int = 3) -> list[list[int]]: ... + def yeardatescalendar(self, year: int, width: int = 3) -> list[list[list[list[datetime.date]]]]: ... + def yeardays2calendar(self, year: int, width: int = 3) -> list[list[list[list[tuple[int, int]]]]]: ... + def yeardayscalendar(self, year: int, width: int = 3) -> list[list[list[list[int]]]]: ... def itermonthdays3(self, year: int, month: int) -> Iterable[tuple[int, int, int]]: ... def itermonthdays4(self, year: int, month: int) -> Iterable[tuple[int, int, int, int]]: ... diff --git a/mypy/typeshed/stdlib/profile.pyi b/mypy/typeshed/stdlib/profile.pyi index 73eba36344fe..696193d9dc16 100644 --- a/mypy/typeshed/stdlib/profile.pyi +++ b/mypy/typeshed/stdlib/profile.pyi @@ -1,5 +1,5 @@ from _typeshed import StrOrBytesPath -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Any, TypeVar from typing_extensions import ParamSpec, Self, TypeAlias @@ -7,7 +7,7 @@ __all__ = ["run", "runctx", "Profile"] def run(statement: str, filename: str | None = None, sort: str | int = -1) -> None: ... def runctx( - statement: str, globals: dict[str, Any], locals: dict[str, Any], filename: str | None = None, sort: str | int = -1 + statement: str, globals: dict[str, Any], locals: Mapping[str, Any], filename: str | None = None, sort: str | int = -1 ) -> None: ... _T = TypeVar("_T") @@ -26,6 +26,6 @@ class Profile: def create_stats(self) -> None: ... def snapshot_stats(self) -> None: ... def run(self, cmd: str) -> Self: ... - def runctx(self, cmd: str, globals: dict[str, Any], locals: dict[str, Any]) -> Self: ... + def runctx(self, cmd: str, globals: dict[str, Any], locals: Mapping[str, Any]) -> Self: ... def runcall(self, func: Callable[_P, _T], /, *args: _P.args, **kw: _P.kwargs) -> _T: ... def calibrate(self, m: int, verbose: int = 0) -> float: ... diff --git a/mypy/typeshed/stdlib/typing.pyi b/mypy/typeshed/stdlib/typing.pyi index 784f30ad7397..ce16d9adff99 100644 --- a/mypy/typeshed/stdlib/typing.pyi +++ b/mypy/typeshed/stdlib/typing.pyi @@ -861,13 +861,13 @@ if sys.version_info >= (3, 9): def get_type_hints( obj: _get_type_hints_obj_allowed_types, globalns: dict[str, Any] | None = None, - localns: dict[str, Any] | None = None, + localns: Mapping[str, Any] | None = None, include_extras: bool = False, ) -> dict[str, Any]: ... else: def get_type_hints( - obj: _get_type_hints_obj_allowed_types, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None + obj: _get_type_hints_obj_allowed_types, globalns: dict[str, Any] | None = None, localns: Mapping[str, Any] | None = None ) -> dict[str, Any]: ... def get_args(tp: Any) -> tuple[Any, ...]: ... @@ -995,13 +995,13 @@ class ForwardRef: "that references a PEP 695 type parameter. It will be disallowed in Python 3.15." ) def _evaluate( - self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None, *, recursive_guard: frozenset[str] + self, globalns: dict[str, Any] | None, localns: Mapping[str, Any] | None, *, recursive_guard: frozenset[str] ) -> Any | None: ... @overload def _evaluate( self, globalns: dict[str, Any] | None, - localns: dict[str, Any] | None, + localns: Mapping[str, Any] | None, type_params: tuple[TypeVar | ParamSpec | TypeVarTuple, ...], *, recursive_guard: frozenset[str], @@ -1010,17 +1010,17 @@ class ForwardRef: def _evaluate( self, globalns: dict[str, Any] | None, - localns: dict[str, Any] | None, + localns: Mapping[str, Any] | None, type_params: tuple[TypeVar | ParamSpec | TypeVarTuple, ...] | None = None, *, recursive_guard: frozenset[str], ) -> Any | None: ... elif sys.version_info >= (3, 9): def _evaluate( - self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None, recursive_guard: frozenset[str] + self, globalns: dict[str, Any] | None, localns: Mapping[str, Any] | None, recursive_guard: frozenset[str] ) -> Any | None: ... else: - def _evaluate(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None) -> Any | None: ... + def _evaluate(self, globalns: dict[str, Any] | None, localns: Mapping[str, Any] | None) -> Any | None: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... diff --git a/mypy/typeshed/stdlib/typing_extensions.pyi b/mypy/typeshed/stdlib/typing_extensions.pyi index 1e4f90a0a722..3240eff0f5e9 100644 --- a/mypy/typeshed/stdlib/typing_extensions.pyi +++ b/mypy/typeshed/stdlib/typing_extensions.pyi @@ -261,7 +261,7 @@ OrderedDict = _Alias() def get_type_hints( obj: Callable[..., Any], globalns: dict[str, Any] | None = None, - localns: dict[str, Any] | None = None, + localns: Mapping[str, Any] | None = None, include_extras: bool = False, ) -> dict[str, Any]: ... def get_args(tp: Any) -> tuple[Any, ...]: ... From 87b005ab2fc290002ec0ad7eda6058ab15e659b4 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Tue, 1 Oct 2024 23:33:24 -0400 Subject: [PATCH 166/247] .git-blame-ignore-revs: add #15059 and #16847 (#17862) Noticed these while walking through some git blames. Adds some recent black upgrades to `.git-blame-ignore-revs`, #15059 and #16847. --- .git-blame-ignore-revs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index be8582a6b221..8d89ec6d6043 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -6,3 +6,7 @@ f98f78216ba9d6ab68c8e69c19e9f3c7926c5efe # run pyupgrade (#12711) fc335cb16315964b923eb1927e3aad1516891c28 +# update black to 23.3.0 (#15059) +4276308be01ea498d946a79554b4a10b1cf13ccb +# Update black to 24.1.1 (#16847) +8107e53158d83d30bb04d290ac10d8d3ccd344f8 From 329e38e6f23c92d6be7c7839d3b44bf7e2b517dc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 2 Oct 2024 15:44:28 +0100 Subject: [PATCH 167/247] Revert "Narrow based on collection containment (#17344)" (#17865) This reverts commit ed0cd4acba02ed19b1cf18ac0ac416dc251d7714. See #17864 for context. The implementation has some issues, and I'm reverting the PR to unblock the 1.12 release. Closes #17864. Closes #17841. --- mypy/checker.py | 15 ++-- test-data/unit/check-narrowing.test | 112 +------------------------- test-data/unit/fixtures/narrowing.pyi | 9 +-- 3 files changed, 8 insertions(+), 128 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 9c4f4ce88690..a089576e6ffe 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -6023,16 +6023,11 @@ def has_no_custom_eq_checks(t: Type) -> bool: if_map, else_map = {}, {} if left_index in narrowable_operand_index_to_hash: - collection_item_type = get_proper_type(builtin_item_type(iterable_type)) - # Narrow if the collection is a subtype - if ( - collection_item_type is not None - and collection_item_type != item_type - and is_subtype(collection_item_type, item_type) - ): - if_map[operands[left_index]] = collection_item_type - # Try and narrow away 'None' - elif is_overlapping_none(item_type): + # We only try and narrow away 'None' for now + if is_overlapping_none(item_type): + collection_item_type = get_proper_type( + builtin_item_type(iterable_type) + ) if ( collection_item_type is not None and not is_overlapping_none(collection_item_type) diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 0cb4bf8e4a19..21a1523580c2 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -1392,13 +1392,13 @@ else: reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" if val in (None,): - reveal_type(val) # N: Revealed type is "None" + reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" else: reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" if val not in (None,): reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" else: - reveal_type(val) # N: Revealed type is "None" + reveal_type(val) # N: Revealed type is "Union[__main__.A, None]" [builtins fixtures/primitives.pyi] [case testNarrowingWithTupleOfTypes] @@ -2130,111 +2130,3 @@ else: [typing fixtures/typing-medium.pyi] [builtins fixtures/ops.pyi] - - -[case testTypeNarrowingStringInLiteralUnion] -from typing import Literal, Tuple -typ: Tuple[Literal['a', 'b'], ...] = ('a', 'b') -x: str = "hi!" -if x in typ: - reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" -else: - reveal_type(x) # N: Revealed type is "builtins.str" -[builtins fixtures/tuple.pyi] -[typing fixtures/typing-medium.pyi] - -[case testTypeNarrowingStringInLiteralUnionSubset] -from typing import Literal, Tuple -typeAlpha: Tuple[Literal['a', 'b', 'c'], ...] = ('a', 'b') -strIn: str = "b" -strOut: str = "c" -if strIn in typeAlpha: - reveal_type(strIn) # N: Revealed type is "Union[Literal['a'], Literal['b'], Literal['c']]" -else: - reveal_type(strIn) # N: Revealed type is "builtins.str" -if strOut in typeAlpha: - reveal_type(strOut) # N: Revealed type is "Union[Literal['a'], Literal['b'], Literal['c']]" -else: - reveal_type(strOut) # N: Revealed type is "builtins.str" -[builtins fixtures/primitives.pyi] -[typing fixtures/typing-medium.pyi] - -[case testNarrowingStringNotInLiteralUnion] -from typing import Literal, Tuple -typeAlpha: Tuple[Literal['a', 'b', 'c'],...] = ('a', 'b', 'c') -strIn: str = "c" -strOut: str = "d" -if strIn not in typeAlpha: - reveal_type(strIn) # N: Revealed type is "builtins.str" -else: - reveal_type(strIn) # N: Revealed type is "Union[Literal['a'], Literal['b'], Literal['c']]" -if strOut in typeAlpha: - reveal_type(strOut) # N: Revealed type is "Union[Literal['a'], Literal['b'], Literal['c']]" -else: - reveal_type(strOut) # N: Revealed type is "builtins.str" -[builtins fixtures/primitives.pyi] -[typing fixtures/typing-medium.pyi] - -[case testNarrowingStringInLiteralUnionDontExpand] -from typing import Literal, Tuple -typeAlpha: Tuple[Literal['a', 'b', 'c'], ...] = ('a', 'b', 'c') -strIn: Literal['c'] = "c" -reveal_type(strIn) # N: Revealed type is "Literal['c']" -#Check we don't expand a Literal into the Union type -if strIn not in typeAlpha: - reveal_type(strIn) # N: Revealed type is "Literal['c']" -else: - reveal_type(strIn) # N: Revealed type is "Literal['c']" -[builtins fixtures/primitives.pyi] -[typing fixtures/typing-medium.pyi] - -[case testTypeNarrowingStringInMixedUnion] -from typing import Literal, Tuple -typ: Tuple[Literal['a', 'b'], ...] = ('a', 'b') -x: str = "hi!" -if x in typ: - reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" -else: - reveal_type(x) # N: Revealed type is "builtins.str" -[builtins fixtures/tuple.pyi] -[typing fixtures/typing-medium.pyi] - -[case testTypeNarrowingStringInSet] -from typing import Literal, Set -typ: Set[Literal['a', 'b']] = {'a', 'b'} -x: str = "hi!" -if x in typ: - reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" -else: - reveal_type(x) # N: Revealed type is "builtins.str" -if x not in typ: - reveal_type(x) # N: Revealed type is "builtins.str" -else: - reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" -[builtins fixtures/narrowing.pyi] -[typing fixtures/typing-medium.pyi] - -[case testTypeNarrowingStringInList] -from typing import Literal, List -typ: List[Literal['a', 'b']] = ['a', 'b'] -x: str = "hi!" -if x in typ: - reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" -else: - reveal_type(x) # N: Revealed type is "builtins.str" -if x not in typ: - reveal_type(x) # N: Revealed type is "builtins.str" -else: - reveal_type(x) # N: Revealed type is "Union[Literal['a'], Literal['b']]" -[builtins fixtures/narrowing.pyi] -[typing fixtures/typing-medium.pyi] - -[case testTypeNarrowingUnionStringFloat] -from typing import Union -def foobar(foo: Union[str, float]): - if foo in ['a', 'b']: - reveal_type(foo) # N: Revealed type is "builtins.str" - else: - reveal_type(foo) # N: Revealed type is "Union[builtins.str, builtins.float]" -[builtins fixtures/primitives.pyi] -[typing fixtures/typing-medium.pyi] diff --git a/test-data/unit/fixtures/narrowing.pyi b/test-data/unit/fixtures/narrowing.pyi index a36ac7f29bd2..89ee011c1c80 100644 --- a/test-data/unit/fixtures/narrowing.pyi +++ b/test-data/unit/fixtures/narrowing.pyi @@ -1,5 +1,5 @@ # Builtins stub used in check-narrowing test cases. -from typing import Generic, Sequence, Tuple, Type, TypeVar, Union, Iterable +from typing import Generic, Sequence, Tuple, Type, TypeVar, Union Tco = TypeVar('Tco', covariant=True) @@ -15,13 +15,6 @@ class function: pass class ellipsis: pass class int: pass class str: pass -class float: pass class dict(Generic[KT, VT]): pass def isinstance(x: object, t: Union[Type[object], Tuple[Type[object], ...]]) -> bool: pass - -class list(Sequence[Tco]): - def __contains__(self, other: object) -> bool: pass -class set(Iterable[Tco], Generic[Tco]): - def __init__(self, iterable: Iterable[Tco] = ...) -> None: ... - def __contains__(self, item: object) -> bool: pass From aa7733a1e2dd5fc6014da839b3151b04f9533707 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 2 Oct 2024 16:45:56 +0100 Subject: [PATCH 168/247] Don't use equality to narrow when value is IntEnum/StrEnum (#17866) IntEnum/StrEnum values compare equal to the corresponding int/str values, which breaks the logic we use for narrowing based on equality to a literal value. Special case IntEnum/StrEnum to avoid the incorrect behavior. Fix #17860. --- mypy/typeops.py | 8 +++ test-data/unit/check-narrowing.test | 76 +++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/mypy/typeops.py b/mypy/typeops.py index 7f530d13d4e2..221976ce02b3 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -1022,6 +1022,14 @@ def custom_special_method(typ: Type, name: str, check_all: bool = False) -> bool """ typ = get_proper_type(typ) if isinstance(typ, Instance): + if ( + typ.type.is_enum + and name in ("__eq__", "__ne__") + and any(base.fullname in ("enum.IntEnum", "enum.StrEnum") for base in typ.type.mro) + ): + # IntEnum and StrEnum values have non-straightfoward equality, so treat them + # as if they had custom __eq__ and __ne__ + return True method = typ.type.get(name) if method and isinstance(method.node, (SYMBOL_FUNCBASE_TYPES, Decorator, Var)): if method.node.info: diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 21a1523580c2..169523e61be7 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2130,3 +2130,79 @@ else: [typing fixtures/typing-medium.pyi] [builtins fixtures/ops.pyi] + +[case testNarrowingWithIntEnum] +# mypy: strict-equality +from __future__ import annotations +from typing import Any +from enum import IntEnum, StrEnum + +class IE(IntEnum): + X = 1 + Y = 2 + +def f1(x: int) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # N: Revealed type is "builtins.int" + if x != IE.X: + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # N: Revealed type is "builtins.int" + +def f2(x: IE) -> None: + if x == 1: + reveal_type(x) # N: Revealed type is "__main__.IE" + else: + reveal_type(x) # N: Revealed type is "__main__.IE" + +def f3(x: object) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "builtins.object" + else: + reveal_type(x) # N: Revealed type is "builtins.object" + +def f4(x: int | Any) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Union[builtins.int, Any]" + else: + reveal_type(x) # N: Revealed type is "Union[builtins.int, Any]" + +def f5(x: int) -> None: + if x is IE.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + else: + reveal_type(x) # N: Revealed type is "builtins.int" + if x is not IE.X: + reveal_type(x) # N: Revealed type is "builtins.int" + else: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingWithStrEnum] +# mypy: strict-equality +from enum import StrEnum + +class SE(StrEnum): + A = 'a' + B = 'b' + +def f1(x: str) -> None: + if x == SE.A: + reveal_type(x) # N: Revealed type is "builtins.str" + else: + reveal_type(x) # N: Revealed type is "builtins.str" + +def f2(x: SE) -> None: + if x == 'a': + reveal_type(x) # N: Revealed type is "__main__.SE" + else: + reveal_type(x) # N: Revealed type is "__main__.SE" + +def f3(x: object) -> None: + if x == SE.A: + reveal_type(x) # N: Revealed type is "builtins.object" + else: + reveal_type(x) # N: Revealed type is "builtins.object" +[builtins fixtures/primitives.pyi] From ac98ab59f7811a4b7272161610abc21958a528b2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 2 Oct 2024 16:46:12 +0100 Subject: [PATCH 169/247] Fix typos in .github/worflows/test.yml (#17867) --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e29feef7b00..39fbb14bd3b7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,14 +71,14 @@ jobs: tox_extra_args: "-n 4" test_mypyc: true - - name: Test suit with py313-dev-ubuntu, mypyc-compiled + - name: Test suite with py313-dev-ubuntu, mypyc-compiled python: '3.13-dev' arch: x64 os: ubuntu-latest toxenv: py tox_extra_args: "-n 4" test_mypyc: true - # - name: Test suit with py314-dev-ubuntu + # - name: Test suite with py314-dev-ubuntu # python: '3.14-dev' # arch: x64 # os: ubuntu-latest From 3c09b3241d22e21d7fa160ae444a93c9487a927f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 3 Oct 2024 16:12:52 +0100 Subject: [PATCH 170/247] Filter overload items based on self type during type inference (#17873) Fix type argument inference for overloaded functions with explicit self types. Filter out the overload items based on the declared and actual types of self. The implementation is best effort and does the filtering only in simple cases, to reduce the risk of regressions (primarily performance, but I worry also about infinite recursion). I added a fast path for the typical case, since without it the filtering was quite expensive. Note that the overload item filtering already worked in many contexts. This only improves it in specific contexts -- at least when inferring generic protocol compatibility. This is a more localized (and thus lower-risk) fix compared to #14975 (thanks @tyralla!). #14975 might still be a good idea, but I'm not comfortable merging it now, and I want a quick fix to unblock the mypy 1.12 release. Fixes #15031. Fixes #17863. Co-authored by @tyralla. --- mypy/typeops.py | 62 ++++++++++++++++++- test-data/unit/check-overloading.test | 18 ++++++ test-data/unit/check-protocols.test | 88 +++++++++++++++++++++++++++ 3 files changed, 165 insertions(+), 3 deletions(-) diff --git a/mypy/typeops.py b/mypy/typeops.py index 221976ce02b3..efb5d8fa505f 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -14,6 +14,7 @@ from mypy.expandtype import expand_type, expand_type_by_instance from mypy.maptype import map_instance_to_supertype from mypy.nodes import ( + ARG_OPT, ARG_POS, ARG_STAR, ARG_STAR2, @@ -305,9 +306,27 @@ class B(A): pass """ if isinstance(method, Overloaded): - items = [ - bind_self(c, original_type, is_classmethod, ignore_instances) for c in method.items - ] + items = [] + original_type = get_proper_type(original_type) + for c in method.items: + if isinstance(original_type, Instance): + # Filter based on whether declared self type can match actual object type. + # For example, if self has type C[int] and method is accessed on a C[str] value, + # omit this item. This is best effort since bind_self can be called in many + # contexts, and doing complete validation might trigger infinite recursion. + # + # Note that overload item filtering normally happens elsewhere. This is needed + # at least during constraint inference. + keep = is_valid_self_type_best_effort(c, original_type) + else: + keep = True + if keep: + items.append(bind_self(c, original_type, is_classmethod, ignore_instances)) + if len(items) == 0: + # If no item matches, returning all items helps avoid some spurious errors + items = [ + bind_self(c, original_type, is_classmethod, ignore_instances) for c in method.items + ] return cast(F, Overloaded(items)) assert isinstance(method, CallableType) func = method @@ -379,6 +398,43 @@ class B(A): pass return cast(F, res) +def is_valid_self_type_best_effort(c: CallableType, self_type: Instance) -> bool: + """Quickly check if self_type might match the self in a callable. + + Avoid performing any complex type operations. This is performance-critical. + + Default to returning True if we don't know (or it would be too expensive). + """ + if ( + self_type.args + and c.arg_types + and isinstance((arg_type := get_proper_type(c.arg_types[0])), Instance) + and c.arg_kinds[0] in (ARG_POS, ARG_OPT) + and arg_type.args + and self_type.type.fullname != "functools._SingleDispatchCallable" + ): + if self_type.type is not arg_type.type: + # We can't map to supertype, since it could trigger expensive checks for + # protocol types, so we consevatively assume this is fine. + return True + + # Fast path: no explicit annotation on self + if all( + ( + type(arg) is TypeVarType + and type(arg.upper_bound) is Instance + and arg.upper_bound.type.fullname == "builtins.object" + ) + for arg in arg_type.args + ): + return True + + from mypy.meet import is_overlapping_types + + return is_overlapping_types(self_type, c.arg_types[0]) + return True + + def erase_to_bound(t: Type) -> Type: # TODO: use value restrictions to produce a union? t = get_proper_type(t) diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 48d5996b226f..e414c1c9b0b6 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -6750,3 +6750,21 @@ def foo(x: object) -> str: ... def bar(x: int) -> int: ... @overload def bar(x: Any) -> str: ... + +[case testOverloadOnInvalidTypeArgument] +from typing import TypeVar, Self, Generic, overload + +class C: pass + +T = TypeVar("T", bound=C) + +class D(Generic[T]): + @overload + def f(self, x: int) -> int: ... + @overload + def f(self, x: str) -> str: ... + def f(Self, x): ... + +a: D[str] # E: Type argument "str" of "D" must be a subtype of "C" +reveal_type(a.f(1)) # N: Revealed type is "builtins.int" +reveal_type(a.f("x")) # N: Revealed type is "builtins.str" diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index ee7556461fd3..5ed2351e33e6 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -4127,3 +4127,91 @@ class P(Protocol): class C(P): ... C(0) # OK + +[case testTypeVarValueConstraintAgainstGenericProtocol] +from typing import TypeVar, Generic, Protocol, overload + +T_contra = TypeVar("T_contra", contravariant=True) +AnyStr = TypeVar("AnyStr", str, bytes) + +class SupportsWrite(Protocol[T_contra]): + def write(self, s: T_contra, /) -> None: ... + +class Buffer: ... + +class IO(Generic[AnyStr]): + @overload + def write(self: IO[bytes], s: Buffer, /) -> None: ... + @overload + def write(self, s: AnyStr, /) -> None: ... + def write(self, s): ... + +def foo(fdst: SupportsWrite[AnyStr]) -> None: ... + +x: IO[str] +foo(x) + +[case testTypeVarValueConstraintAgainstGenericProtocol2] +from typing import Generic, Protocol, TypeVar, overload + +AnyStr = TypeVar("AnyStr", str, bytes) +T_co = TypeVar("T_co", covariant=True) +T_contra = TypeVar("T_contra", contravariant=True) + +class SupportsRead(Generic[T_co]): + def read(self) -> T_co: ... + +class SupportsWrite(Protocol[T_contra]): + def write(self, s: T_contra) -> object: ... + +def copyfileobj(fsrc: SupportsRead[AnyStr], fdst: SupportsWrite[AnyStr]) -> None: ... + +class WriteToMe(Generic[AnyStr]): + @overload + def write(self: WriteToMe[str], s: str) -> int: ... + @overload + def write(self: WriteToMe[bytes], s: bytes) -> int: ... + def write(self, s): ... + +class WriteToMeOrReadFromMe(WriteToMe[AnyStr], SupportsRead[AnyStr]): ... + +copyfileobj(WriteToMeOrReadFromMe[bytes](), WriteToMe[bytes]()) + +[case testOverloadedMethodWithExplictSelfTypes] +from typing import Generic, overload, Protocol, TypeVar, Union + +AnyStr = TypeVar("AnyStr", str, bytes) +T_co = TypeVar("T_co", covariant=True) +T_contra = TypeVar("T_contra", contravariant=True) + +class SupportsRead(Protocol[T_co]): + def read(self) -> T_co: ... + +class SupportsWrite(Protocol[T_contra]): + def write(self, s: T_contra) -> int: ... + +class Input(Generic[AnyStr]): + def read(self) -> AnyStr: ... + +class Output(Generic[AnyStr]): + @overload + def write(self: Output[str], s: str) -> int: ... + @overload + def write(self: Output[bytes], s: bytes) -> int: ... + def write(self, s: Union[str, bytes]) -> int: ... + +def f(src: SupportsRead[AnyStr], dst: SupportsWrite[AnyStr]) -> None: ... + +def g1(a: Input[bytes], b: Output[bytes]) -> None: + f(a, b) + +def g2(a: Input[bytes], b: Output[bytes]) -> None: + f(a, b) + +def g3(a: Input[str], b: Output[bytes]) -> None: + f(a, b) # E: Cannot infer type argument 1 of "f" + +def g4(a: Input[bytes], b: Output[str]) -> None: + f(a, b) # E: Cannot infer type argument 1 of "f" + +[builtins fixtures/tuple.pyi] From 9330193cded6a872a6f8202ee197a5a378e17673 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 4 Oct 2024 12:52:01 +0100 Subject: [PATCH 171/247] Fix narrowing of IntEnum and StrEnum types (#17874) Fix regression introduced in #17866. It should still be possible to narrow IntEnum and StrEnum types, but only when types match or are disjoint. Add more logic to rule out narrowing when types are ambigous. --- mypy/checker.py | 46 +++++++++++++++++- mypy/typeops.py | 8 ---- test-data/unit/check-narrowing.test | 73 ++++++++++++++++++++++++++++- 3 files changed, 117 insertions(+), 10 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index a089576e6ffe..0d8ae3da1c9f 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5984,7 +5984,9 @@ def has_no_custom_eq_checks(t: Type) -> bool: coerce_only_in_literal_context = True expr_types = [operand_types[i] for i in expr_indices] - should_narrow_by_identity = all(map(has_no_custom_eq_checks, expr_types)) + should_narrow_by_identity = all( + map(has_no_custom_eq_checks, expr_types) + ) and not is_ambiguous_mix_of_enums(expr_types) if_map: TypeMap = {} else_map: TypeMap = {} @@ -8604,3 +8606,45 @@ def visit_starred_pattern(self, p: StarredPattern) -> None: self.lvalue = True p.capture.accept(self) self.lvalue = False + + +def is_ambiguous_mix_of_enums(types: list[Type]) -> bool: + """Do types have IntEnum/StrEnum types that are potentially overlapping with other types? + + If True, we shouldn't attempt type narrowing based on enum values, as it gets + too ambiguous. + + For example, return True if there's an 'int' type together with an IntEnum literal. + However, IntEnum together with a literal of the same IntEnum type is not ambiguous. + """ + # We need these things for this to be ambiguous: + # (1) an IntEnum or StrEnum type + # (2) either a different IntEnum/StrEnum type or a non-enum type ("") + # + # It would be slightly more correct to calculate this separately for IntEnum and + # StrEnum related types, as an IntEnum can't be confused with a StrEnum. + return len(_ambiguous_enum_variants(types)) > 1 + + +def _ambiguous_enum_variants(types: list[Type]) -> set[str]: + result = set() + for t in types: + t = get_proper_type(t) + if isinstance(t, UnionType): + result.update(_ambiguous_enum_variants(t.items)) + elif isinstance(t, Instance): + if t.last_known_value: + result.update(_ambiguous_enum_variants([t.last_known_value])) + elif t.type.is_enum and any( + base.fullname in ("enum.IntEnum", "enum.StrEnum") for base in t.type.mro + ): + result.add(t.type.fullname) + elif not t.type.is_enum: + # These might compare equal to IntEnum/StrEnum types (e.g. Decimal), so + # let's be conservative + result.add("") + elif isinstance(t, LiteralType): + result.update(_ambiguous_enum_variants([t.fallback])) + else: + result.add("") + return result diff --git a/mypy/typeops.py b/mypy/typeops.py index efb5d8fa505f..0699cda53cfa 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -1078,14 +1078,6 @@ def custom_special_method(typ: Type, name: str, check_all: bool = False) -> bool """ typ = get_proper_type(typ) if isinstance(typ, Instance): - if ( - typ.type.is_enum - and name in ("__eq__", "__ne__") - and any(base.fullname in ("enum.IntEnum", "enum.StrEnum") for base in typ.type.mro) - ): - # IntEnum and StrEnum values have non-straightfoward equality, so treat them - # as if they had custom __eq__ and __ne__ - return True method = typ.type.get(name) if method and isinstance(method.node, (SYMBOL_FUNCBASE_TYPES, Decorator, Var)): if method.node.info: diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 169523e61be7..11bbf0a95084 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2135,7 +2135,7 @@ else: # mypy: strict-equality from __future__ import annotations from typing import Any -from enum import IntEnum, StrEnum +from enum import IntEnum class IE(IntEnum): X = 1 @@ -2178,6 +2178,71 @@ def f5(x: int) -> None: reveal_type(x) # N: Revealed type is "builtins.int" else: reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + +def f6(x: IE) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + else: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.Y]" +[builtins fixtures/primitives.pyi] + +[case testNarrowingWithIntEnum2] +# mypy: strict-equality +from __future__ import annotations +from typing import Any +from enum import IntEnum, Enum + +class MyDecimal: ... + +class IE(IntEnum): + X = 1 + Y = 2 + +class IE2(IntEnum): + X = 1 + Y = 2 + +class E(Enum): + X = 1 + Y = 2 + +def f1(x: IE | MyDecimal) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, __main__.MyDecimal]" + else: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, __main__.MyDecimal]" + +def f2(x: E | bytes) -> None: + if x == E.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.E.X]" + else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.E.Y], builtins.bytes]" + +def f3(x: IE | IE2) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, __main__.IE2]" + else: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, __main__.IE2]" + +def f4(x: IE | E) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + elif x == E.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.E.X]" + else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.IE.Y], Literal[__main__.E.Y]]" + +def f5(x: E | str | int) -> None: + if x == E.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.E.X]" + else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.E.Y], builtins.str, builtins.int]" + +def f6(x: IE | Any) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, Any]" + else: + reveal_type(x) # N: Revealed type is "Union[__main__.IE, Any]" [builtins fixtures/primitives.pyi] [case testNarrowingWithStrEnum] @@ -2205,4 +2270,10 @@ def f3(x: object) -> None: reveal_type(x) # N: Revealed type is "builtins.object" else: reveal_type(x) # N: Revealed type is "builtins.object" + +def f4(x: SE) -> None: + if x == SE.A: + reveal_type(x) # N: Revealed type is "Literal[__main__.SE.A]" + else: + reveal_type(x) # N: Revealed type is "Literal[__main__.SE.B]" [builtins fixtures/primitives.pyi] From 012d52c36cbade3b0677f422a86a213e90788f18 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 4 Oct 2024 14:33:32 +0100 Subject: [PATCH 172/247] Don't consider None vs IntEnum comparison ambiguous (#17877) We can assume that None doesn't compare equal to an enum. --- mypy/checker.py | 2 ++ test-data/unit/check-narrowing.test | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 0d8ae3da1c9f..8d77bb02eeb2 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -8645,6 +8645,8 @@ def _ambiguous_enum_variants(types: list[Type]) -> set[str]: result.add("") elif isinstance(t, LiteralType): result.update(_ambiguous_enum_variants([t.fallback])) + elif isinstance(t, NoneType): + pass else: result.add("") return result diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 11bbf0a95084..60fc39dd817b 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2243,6 +2243,20 @@ def f6(x: IE | Any) -> None: reveal_type(x) # N: Revealed type is "Union[__main__.IE, Any]" else: reveal_type(x) # N: Revealed type is "Union[__main__.IE, Any]" + +def f7(x: IE | None) -> None: + if x == IE.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + else: + reveal_type(x) # N: Revealed type is "Union[Literal[__main__.IE.Y], None]" + +def f8(x: IE | None) -> None: + if x is None: + reveal_type(x) # N: Revealed type is "None" + elif x == IE.X: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.X]" + else: + reveal_type(x) # N: Revealed type is "Literal[__main__.IE.Y]" [builtins fixtures/primitives.pyi] [case testNarrowingWithStrEnum] From bc8119150e49895f7a496ae7ae7362a2828e7e9e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 4 Oct 2024 15:26:48 +0100 Subject: [PATCH 173/247] [PEP 696] Report error if using unsupported type parameter defaults (#17876) We don't yet support default types for type parameters when using the new type parameter syntax. Generate an error instead of just ignoring them. Also make it possible to write Python 3.13 specific tests. --- mypy/fastparse.py | 7 +++++++ mypy/message_registry.py | 5 +++++ mypy/test/helpers.py | 13 +++---------- mypy/test/testcheck.py | 2 ++ test-data/unit/check-python313.test | 11 +++++++++++ 5 files changed, 28 insertions(+), 10 deletions(-) create mode 100644 test-data/unit/check-python313.test diff --git a/mypy/fastparse.py b/mypy/fastparse.py index bbbe2184738c..726397adb849 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1198,6 +1198,13 @@ def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: for p in type_params: bound = None values: list[Type] = [] + if sys.version_info >= (3, 13) and p.default_value is not None: + self.fail( + message_registry.TYPE_PARAM_DEFAULT_NOT_SUPPORTED, + p.lineno, + p.col_offset, + blocker=False, + ) if isinstance(p, ast_ParamSpec): # type: ignore[misc] explicit_type_params.append(TypeParam(p.name, PARAM_SPEC_KIND, None, [])) elif isinstance(p, ast_TypeVarTuple): # type: ignore[misc] diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 29d539faaed6..507af6fdad74 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -362,3 +362,8 @@ def with_additional_msg(self, info: str) -> ErrorMessage: TYPE_ALIAS_WITH_AWAIT_EXPRESSION: Final = ErrorMessage( "Await expression cannot be used within a type alias", codes.SYNTAX ) + +TYPE_PARAM_DEFAULT_NOT_SUPPORTED: Final = ErrorMessage( + "Type parameter default types not supported when using Python 3.12 type parameter syntax", + codes.MISC, +) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index f532e77b82d3..4a80207d3ec7 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -256,16 +256,9 @@ def local_sys_path_set() -> Iterator[None]: def testfile_pyversion(path: str) -> tuple[int, int]: - if path.endswith("python312.test"): - return 3, 12 - elif path.endswith("python311.test"): - return 3, 11 - elif path.endswith("python310.test"): - return 3, 10 - elif path.endswith("python39.test"): - return 3, 9 - elif path.endswith("python38.test"): - return 3, 8 + m = re.search(r"python3([0-9]+)\.test$", path) + if m: + return 3, int(m.group(1)) else: return defaults.PYTHON3_VERSION diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 5fba6192dcaf..330e191af252 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -45,6 +45,8 @@ typecheck_files.remove("check-python311.test") if sys.version_info < (3, 12): typecheck_files.remove("check-python312.test") +if sys.version_info < (3, 13): + typecheck_files.remove("check-python313.test") # Special tests for platforms with case-insensitive filesystems. if sys.platform not in ("darwin", "win32"): diff --git a/test-data/unit/check-python313.test b/test-data/unit/check-python313.test new file mode 100644 index 000000000000..0ba64ad67c91 --- /dev/null +++ b/test-data/unit/check-python313.test @@ -0,0 +1,11 @@ +[case testPEP695TypeParameterDefaultNotSupported] +class C[T = None]: # E: Type parameter default types not supported when using Python 3.12 type parameter syntax + pass + +def f[T = list[int]]() -> None: # E: Type parameter default types not supported when using Python 3.12 type parameter syntax + pass + +def g[**P = [int, str]]() -> None: # E: Type parameter default types not supported when using Python 3.12 type parameter syntax + pass + +type A[T, S = int, U = str] = list[T] # E: Type parameter default types not supported when using Python 3.12 type parameter syntax From 16ba7f41a4a9786b55658dd89d23f74a9d9a1cb0 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Fri, 4 Oct 2024 21:35:03 +0200 Subject: [PATCH 174/247] Bump version to 1.13.0+dev (#17879) The release branch has been cut: https://github.com/python/mypy/tree/release-1.12 --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index 8e00b4cce702..e0671c9feb06 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.12.0+dev" +__version__ = "1.13.0+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From 4e4826fa8006c2413486ffa55747790fa7484644 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 4 Oct 2024 18:33:07 -0700 Subject: [PATCH 175/247] Add regression test cases (#17869) https://github.com/python/mypy/issues/17864 --- test-data/unit/check-narrowing.test | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index 60fc39dd817b..d740708991d0 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2291,3 +2291,45 @@ def f4(x: SE) -> None: else: reveal_type(x) # N: Revealed type is "Literal[__main__.SE.B]" [builtins fixtures/primitives.pyi] + +[case testConsistentNarrowingEqAndIn] +# flags: --python-version 3.10 + +# https://github.com/python/mypy/issues/17864 +def f(x: str | int) -> None: + if x == "x": + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" + y = x + + if x in ["x"]: + # TODO: we should fix this reveal https://github.com/python/mypy/issues/3229 + reveal_type(x) # N: Revealed type is "Union[builtins.str, builtins.int]" + y = x + z = x + z = y +[builtins fixtures/primitives.pyi] + +[case testConsistentNarrowingInWithCustomEq] +# flags: --python-version 3.10 + +# https://github.com/python/mypy/issues/17864 +class C: + def __init__(self, x: int) -> None: + self.x = x + + def __eq__(self, other: object) -> bool: + raise + # Example implementation: + # if isinstance(other, C) and other.x == self.x: + # return True + # return NotImplemented + +class D(C): + pass + +def f(x: C) -> None: + if x in [D(5)]: + reveal_type(x) # D # N: Revealed type is "__main__.C" + +f(C(5)) +[builtins fixtures/primitives.pyi] From dfb7be19b6fc2b3fe20ee07851ae3759cf18033d Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sun, 6 Oct 2024 05:56:44 -0400 Subject: [PATCH 176/247] Include CHANGELOG.md in sdists (#17882) Refs #17880 --- MANIFEST.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index c18b83cc0088..c2399d2b00b6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -41,8 +41,8 @@ include runtests.py include pytest.ini include tox.ini -include LICENSE mypyc/README.md -exclude .gitmodules CONTRIBUTING.md CREDITS ROADMAP.md CHANGELOG.md action.yml .editorconfig +include LICENSE mypyc/README.md CHANGELOG.md +exclude .gitmodules CONTRIBUTING.md CREDITS ROADMAP.md action.yml .editorconfig exclude .git-blame-ignore-revs .pre-commit-config.yaml global-exclude *.py[cod] From b10d781e004339c4d592b0ad5372f8e014130f7f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 6 Oct 2024 14:25:54 +0100 Subject: [PATCH 177/247] Fix re-processing cross-reference when node kind changes (#17883) This is quite a bad bug. Currently we rely on `SymbolNode` being updated in-place for all indirect references, but this is not the case when node kind (`FuncDef`, `Decorator`, etc.) changes, in this case a _new_ `SymbolNode` is created. I fix this by forcing reprocessing if the node kind changes. This currently blocks support for PEP 702, see https://github.com/python/mypy/pull/17476, so I will not wait for long before merging. --- mypy/server/astdiff.py | 4 +- test-data/unit/fine-grained.test | 98 ++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index fc868d288b4d..d5a303128126 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -219,7 +219,9 @@ def snapshot_symbol_table(name_prefix: str, table: SymbolTable) -> dict[str, Sym assert symbol.kind != UNBOUND_IMPORTED if node and get_prefix(node.fullname) != name_prefix: # This is a cross-reference to a node defined in another module. - result[name] = ("CrossRef", common) + # Include the node kind (FuncDef, Decorator, TypeInfo, ...), so that we will + # reprocess when a *new* node is created instead of merging an existing one. + result[name] = ("CrossRef", common, type(node).__name__) else: result[name] = snapshot_definition(node, common) return result diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index faedd890922d..15e47ff296ea 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10573,3 +10573,101 @@ m.py:9: error: Argument 1 to "foo" has incompatible type "int"; expected "str" m.py:9: error: Argument 2 to "foo" has incompatible type "str"; expected "int" m.py:10: error: Unexpected keyword argument "a" for "foo" partial.py:4: note: "foo" defined here + +[case testReplaceFunctionWithDecoratedFunctionIndirect] +from b import f +x: int = f() +import b +y: int = b.f() + +[file b.py] +from a import f + +[file a.py] +def f() -> int: ... + +[file a.py.2] +from typing import Callable +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@d +def f() -> str: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testReplaceFunctionWithDecoratedFunctionIndirect2] +from c import f +x: int = f() +import c +y: int = c.f() + +[file c.py] +from b import f + +[file b.py] +from a import f + +[file a.py] +def f() -> int: ... + +[file a.py.2] +from typing import Callable +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@d +def f() -> str: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: Incompatible types in assignment (expression has type "str", variable has type "int") +main:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + +[case testReplaceFunctionWithClassIndirect] +from b import f +x: int = f() +import b +y: int = b.f() + +[file b.py] +from a import f + +[file a.py] +def f() -> int: ... + +[file a.py.2] +class f: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: Incompatible types in assignment (expression has type "f", variable has type "int") +main:4: error: Incompatible types in assignment (expression has type "f", variable has type "int") + +[case testReplaceFunctionWithClassIndirect2] +from c import f +x: int = f() +import c +y: int = c.f() + +[file c.py] +from b import f + +[file b.py] +from a import f + +[file a.py] +def f() -> int: ... + +[file a.py.2] +class f: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:2: error: Incompatible types in assignment (expression has type "f", variable has type "int") +main:4: error: Incompatible types in assignment (expression has type "f", variable has type "int") From 13d6738f5295eef58aab2b660763baa6609bb690 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Sun, 6 Oct 2024 09:49:18 -0400 Subject: [PATCH 178/247] Add is_lvalue attribute to AttributeContext (#17881) Refs #17878 --- docs/source/extending_mypy.rst | 2 +- mypy/checkmember.py | 14 +++++++++++--- mypy/plugin.py | 1 + test-data/unit/check-custom-plugin.test | 3 +++ test-data/unit/plugins/attrhook2.py | 9 +++++++++ 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/source/extending_mypy.rst b/docs/source/extending_mypy.rst index bbbec2ad3880..bded07319b64 100644 --- a/docs/source/extending_mypy.rst +++ b/docs/source/extending_mypy.rst @@ -179,7 +179,7 @@ mypy will call ``get_method_signature_hook("ctypes.Array.__setitem__")`` so that the plugin can mimic the :py:mod:`ctypes` auto-convert behavior. **get_attribute_hook()** overrides instance member field lookups and property -access (not assignments, and not method calls). This hook is only called for +access (not method calls). This hook is only called for fields which already exist on the class. *Exception:* if :py:meth:`__getattr__ ` or :py:meth:`__getattribute__ ` is a method on the class, the hook is called for all fields which do not refer to methods. diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 8f99f96e2dd5..adfd5c2a97d6 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -572,7 +572,11 @@ def analyze_member_var_access( if hook: result = hook( AttributeContext( - get_proper_type(mx.original_type), result, mx.context, mx.chk + get_proper_type(mx.original_type), + result, + mx.is_lvalue, + mx.context, + mx.chk, ) ) return result @@ -829,7 +833,9 @@ def analyze_var( result = analyze_descriptor_access(result, mx) if hook: result = hook( - AttributeContext(get_proper_type(mx.original_type), result, mx.context, mx.chk) + AttributeContext( + get_proper_type(mx.original_type), result, mx.is_lvalue, mx.context, mx.chk + ) ) return result @@ -1148,7 +1154,9 @@ def apply_class_attr_hook( ) -> Type | None: if hook: result = hook( - AttributeContext(get_proper_type(mx.original_type), result, mx.context, mx.chk) + AttributeContext( + get_proper_type(mx.original_type), result, mx.is_lvalue, mx.context, mx.chk + ) ) return result diff --git a/mypy/plugin.py b/mypy/plugin.py index a1af7fa76350..a4dc0052ec79 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -495,6 +495,7 @@ class MethodContext(NamedTuple): class AttributeContext(NamedTuple): type: ProperType # Type of object with attribute default_attr_type: Type # Original attribute type + is_lvalue: bool # Whether the attribute is the target of an assignment context: Context # Relevant location context (e.g. for error messages) api: CheckerPluginInterface diff --git a/test-data/unit/check-custom-plugin.test b/test-data/unit/check-custom-plugin.test index 2b3b3f4a8695..666bf9680405 100644 --- a/test-data/unit/check-custom-plugin.test +++ b/test-data/unit/check-custom-plugin.test @@ -232,6 +232,8 @@ reveal_type(magic.non_magic_method()) # N: Revealed type is "builtins.int" reveal_type(magic.non_magic_field) # N: Revealed type is "builtins.int" magic.nonexistent_field # E: Field does not exist reveal_type(magic.fallback_example) # N: Revealed type is "Any" +reveal_type(magic.no_assignment_field) # N: Revealed type is "builtins.float" +magic.no_assignment_field = "bad" # E: Cannot assign to field derived = DerivedMagic() reveal_type(derived.magic_field) # N: Revealed type is "builtins.str" @@ -250,6 +252,7 @@ class Magic: def __getattr__(self, x: Any) -> Any: ... def non_magic_method(self) -> int: ... non_magic_field: int + no_assignment_field: float class DerivedMagic(Magic): ... [file mypy.ini] diff --git a/test-data/unit/plugins/attrhook2.py b/test-data/unit/plugins/attrhook2.py index 2d41a0fdf52f..1ce318d2057b 100644 --- a/test-data/unit/plugins/attrhook2.py +++ b/test-data/unit/plugins/attrhook2.py @@ -12,6 +12,8 @@ def get_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type return magic_field_callback if fullname == "m.Magic.nonexistent_field": return nonexistent_field_callback + if fullname == "m.Magic.no_assignment_field": + return no_assignment_field_callback return None @@ -24,5 +26,12 @@ def nonexistent_field_callback(ctx: AttributeContext) -> Type: return AnyType(TypeOfAny.from_error) +def no_assignment_field_callback(ctx: AttributeContext) -> Type: + if ctx.is_lvalue: + ctx.api.fail(f"Cannot assign to field", ctx.context) + return AnyType(TypeOfAny.from_error) + return ctx.default_attr_type + + def plugin(version: str) -> type[AttrPlugin]: return AttrPlugin From 4da779bea0e89f4945e3cd216c9bdf4a4e68537b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 6 Oct 2024 22:07:15 +0100 Subject: [PATCH 179/247] [mypyc] Make C unit tests faster by compiling with -O0 (#17884) Most time is spent in compilation, so runtime performance doesn't really matter. --- mypyc/test/test_external.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypyc/test/test_external.py b/mypyc/test/test_external.py index 22eb8019133c..010c74dee42e 100644 --- a/mypyc/test/test_external.py +++ b/mypyc/test/test_external.py @@ -20,7 +20,9 @@ def test_c_unit_test(self) -> None: cppflags: list[str] = [] env = os.environ.copy() if sys.platform == "darwin": - cppflags += ["-mmacosx-version-min=10.10", "-stdlib=libc++"] + cppflags += ["-O0", "-mmacosx-version-min=10.10", "-stdlib=libc++"] + elif sys.platform == "linux": + cppflags += ["-O0"] env["CPPFLAGS"] = " ".join(cppflags) # Build Python wrapper for C unit tests. From 94c49a8215c9c8d3a4ec146696213dd2b9ebfb42 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Mon, 7 Oct 2024 00:25:02 +0200 Subject: [PATCH 180/247] Add basic support for PEP 702 (@deprecated). (#17476) Closes #16111 This PR provides only basic support. Many special cases might need additional attention (descriptors, some special methods like `__int__`, etc.). Other open issues are code comments, eventual documentation updates, the deprecation message style, etc.). But I wanted to offer these first steps before going on vacation (so I cannot respond to possible reviews too soon). Maybe someone wants to extend the list of (test) cases the basic support should address? --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ivan Levkivskyi --- docs/source/command_line.rst | 6 + docs/source/error_code_list2.rst | 38 +++ mypy/checker.py | 46 ++++ mypy/checkexpr.py | 39 ++- mypy/checkmember.py | 13 +- mypy/errorcodes.py | 6 + mypy/errors.py | 5 +- mypy/main.py | 7 + mypy/nodes.py | 17 +- mypy/options.py | 3 + mypy/semanal.py | 64 ++++- mypy/server/astdiff.py | 1 + test-data/unit/check-deprecated.test | 396 +++++++++++++++++++++++++++ test-data/unit/fine-grained.test | 234 ++++++++++++++++ test-data/unit/pythoneval.test | 2 +- 15 files changed, 860 insertions(+), 17 deletions(-) create mode 100644 test-data/unit/check-deprecated.test diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 20fb3821438a..92db5d59d0ee 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -537,6 +537,12 @@ potentially problematic or redundant in some way. This limitation will be removed in future releases of mypy. +.. option:: --report-deprecated-as-error + + By default, mypy emits notes if your code imports or uses deprecated + features. This flag converts such notes to errors, causing mypy to + eventually finish with a non-zero exit code. Features are considered + deprecated when decorated with ``warnings.deprecated``. .. _miscellaneous-strictness-flags: diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst index 6d50e217a77d..eb18d76e2f2f 100644 --- a/docs/source/error_code_list2.rst +++ b/docs/source/error_code_list2.rst @@ -231,6 +231,44 @@ incorrect control flow or conditional checks that are accidentally always true o # Error: Statement is unreachable [unreachable] print('unreachable') +.. _code-deprecated: + +Check that imported or used feature is deprecated [deprecated] +-------------------------------------------------------------- + +By default, mypy generates a note if your code imports a deprecated feature explicitly with a +``from mod import depr`` statement or uses a deprecated feature imported otherwise or defined +locally. Features are considered deprecated when decorated with ``warnings.deprecated``, as +specified in `PEP 702 `_. You can silence single notes via +``# type: ignore[deprecated]`` or turn off this check completely via ``--disable-error-code=deprecated``. +Use the :option:`--report-deprecated-as-error ` option for +more strictness, which turns all such notes into errors. + +.. note:: + + The ``warnings`` module provides the ``@deprecated`` decorator since Python 3.13. + To use it with older Python versions, import it from ``typing_extensions`` instead. + +Examples: + +.. code-block:: python + + # mypy: report-deprecated-as-error + + # Error: abc.abstractproperty is deprecated: Deprecated, use 'property' with 'abstractmethod' instead + from abc import abstractproperty + + from typing_extensions import deprecated + + @deprecated("use new_function") + def old_function() -> None: + print("I am old") + + # Error: __main__.old_function is deprecated: use new_function + old_function() + old_function() # type: ignore[deprecated] + + .. _code-redundant-expr: Check that expression is redundant [redundant-expr] diff --git a/mypy/checker.py b/mypy/checker.py index 8d77bb02eeb2..4bbd49cd7198 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2838,6 +2838,9 @@ def check_metaclass_compatibility(self, typ: TypeInfo) -> None: ) def visit_import_from(self, node: ImportFrom) -> None: + for name, _ in node.names: + if (sym := self.globals.get(name)) is not None: + self.warn_deprecated(sym.node, node) self.check_import(node) def visit_import_all(self, node: ImportAll) -> None: @@ -2926,6 +2929,16 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: Handle all kinds of assignment statements (simple, indexed, multiple). """ + + if isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs: + for lvalue in s.lvalues: + if ( + isinstance(lvalue, NameExpr) + and isinstance(var := lvalue.node, Var) + and isinstance(instance := get_proper_type(var.type), Instance) + ): + self.check_deprecated(instance.type, s) + # Avoid type checking type aliases in stubs to avoid false # positives about modern type syntax available in stubs such # as X | Y. @@ -4671,6 +4684,16 @@ def visit_operator_assignment_stmt(self, s: OperatorAssignmentStmt) -> None: if inplace: # There is __ifoo__, treat as x = x.__ifoo__(y) rvalue_type, method_type = self.expr_checker.check_op(method, lvalue_type, s.rvalue, s) + if isinstance(inst := get_proper_type(lvalue_type), Instance) and isinstance( + defn := inst.type.get_method(method), OverloadedFuncDef + ): + for item in defn.items: + if ( + isinstance(item, Decorator) + and isinstance(typ := item.func.type, CallableType) + and (bind_self(typ) == method_type) + ): + self.warn_deprecated(item.func, s) if not is_subtype(rvalue_type, lvalue_type): self.msg.incompatible_operator_assignment(s.op, s) else: @@ -7535,6 +7558,29 @@ def has_valid_attribute(self, typ: Type, name: str) -> bool: def get_expression_type(self, node: Expression, type_context: Type | None = None) -> Type: return self.expr_checker.accept(node, type_context=type_context) + def check_deprecated(self, node: SymbolNode | None, context: Context) -> None: + """Warn if deprecated and not directly imported with a `from` statement.""" + if isinstance(node, Decorator): + node = node.func + if isinstance(node, (FuncDef, OverloadedFuncDef, TypeInfo)) and ( + node.deprecated is not None + ): + for imp in self.tree.imports: + if isinstance(imp, ImportFrom) and any(node.name == n[0] for n in imp.names): + break + else: + self.warn_deprecated(node, context) + + def warn_deprecated(self, node: SymbolNode | None, context: Context) -> None: + """Warn if deprecated.""" + if isinstance(node, Decorator): + node = node.func + if isinstance(node, (FuncDef, OverloadedFuncDef, TypeInfo)) and ( + (deprecated := node.deprecated) is not None + ): + warn = self.msg.fail if self.options.report_deprecated_as_error else self.msg.note + warn(deprecated, context, code=codes.DEPRECATED) + class CollectArgTypeVarTypes(TypeTraverserVisitor): """Collects the non-nested argument types in a set.""" diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 98e6eb6a7fc3..0754b1db7141 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -126,6 +126,7 @@ validate_instance, ) from mypy.typeops import ( + bind_self, callable_type, custom_special_method, erase_to_union_or_bound, @@ -354,7 +355,9 @@ def visit_name_expr(self, e: NameExpr) -> Type: """ self.chk.module_refs.update(extract_refexpr_names(e)) result = self.analyze_ref_expr(e) - return self.narrow_type_from_binder(e, result) + narrowed = self.narrow_type_from_binder(e, result) + self.chk.check_deprecated(e.node, e) + return narrowed def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: result: Type | None = None @@ -1479,6 +1482,10 @@ def check_call_expr_with_callee_type( object_type=object_type, ) proper_callee = get_proper_type(callee_type) + if isinstance(e.callee, NameExpr) and isinstance(e.callee.node, OverloadedFuncDef): + for item in e.callee.node.items: + if isinstance(item, Decorator) and (item.func.type == callee_type): + self.chk.check_deprecated(item.func, e) if isinstance(e.callee, RefExpr) and isinstance(proper_callee, CallableType): # Cache it for find_isinstance_check() if proper_callee.type_guard is not None: @@ -3267,7 +3274,9 @@ def visit_member_expr(self, e: MemberExpr, is_lvalue: bool = False) -> Type: """Visit member expression (of form e.id).""" self.chk.module_refs.update(extract_refexpr_names(e)) result = self.analyze_ordinary_member_access(e, is_lvalue) - return self.narrow_type_from_binder(e, result) + narrowed = self.narrow_type_from_binder(e, result) + self.chk.warn_deprecated(e.node, e) + return narrowed def analyze_ordinary_member_access(self, e: MemberExpr, is_lvalue: bool) -> Type: """Analyse member expression or member lvalue.""" @@ -3956,7 +3965,7 @@ def lookup_definer(typ: Instance, attr_name: str) -> str | None: # This is the case even if the __add__ method is completely missing and the __radd__ # method is defined. - variants_raw = [(left_op, left_type, right_expr)] + variants_raw = [(op_name, left_op, left_type, right_expr)] elif ( is_subtype(right_type, left_type) and isinstance(left_type, Instance) @@ -3977,19 +3986,25 @@ def lookup_definer(typ: Instance, attr_name: str) -> str | None: # As a special case, the alt_promote check makes sure that we don't use the # __radd__ method of int if the LHS is a native int type. - variants_raw = [(right_op, right_type, left_expr), (left_op, left_type, right_expr)] + variants_raw = [ + (rev_op_name, right_op, right_type, left_expr), + (op_name, left_op, left_type, right_expr), + ] else: # In all other cases, we do the usual thing and call __add__ first and # __radd__ second when doing "A() + B()". - variants_raw = [(left_op, left_type, right_expr), (right_op, right_type, left_expr)] + variants_raw = [ + (op_name, left_op, left_type, right_expr), + (rev_op_name, right_op, right_type, left_expr), + ] # STEP 3: # We now filter out all non-existent operators. The 'variants' list contains # all operator methods that are actually present, in the order that Python # attempts to invoke them. - variants = [(op, obj, arg) for (op, obj, arg) in variants_raw if op is not None] + variants = [(na, op, obj, arg) for (na, op, obj, arg) in variants_raw if op is not None] # STEP 4: # We now try invoking each one. If an operation succeeds, end early and return @@ -3998,13 +4013,23 @@ def lookup_definer(typ: Instance, attr_name: str) -> str | None: errors = [] results = [] - for method, obj, arg in variants: + for name, method, obj, arg in variants: with self.msg.filter_errors(save_filtered_errors=True) as local_errors: result = self.check_method_call(op_name, obj, method, [arg], [ARG_POS], context) if local_errors.has_new_errors(): errors.append(local_errors.filtered_errors()) results.append(result) else: + if isinstance(obj, Instance) and isinstance( + defn := obj.type.get_method(name), OverloadedFuncDef + ): + for item in defn.items: + if ( + isinstance(item, Decorator) + and isinstance(typ := item.func.type, CallableType) + and bind_self(typ) == result[1] + ): + self.chk.check_deprecated(item.func, context) return result # We finish invoking above operators and no early return happens. Therefore, diff --git a/mypy/checkmember.py b/mypy/checkmember.py index adfd5c2a97d6..4448072d52c3 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -316,9 +316,12 @@ def analyze_instance_member_access( if method.is_property: assert isinstance(method, OverloadedFuncDef) - first_item = method.items[0] - assert isinstance(first_item, Decorator) - return analyze_var(name, first_item.var, typ, info, mx) + getter = method.items[0] + assert isinstance(getter, Decorator) + if mx.is_lvalue and (len(items := method.items) > 1): + mx.chk.warn_deprecated(items[1], mx.context) + return analyze_var(name, getter.var, typ, info, mx) + if mx.is_lvalue: mx.msg.cant_assign_to_method(mx.context) if not isinstance(method, OverloadedFuncDef): @@ -493,6 +496,8 @@ def analyze_member_var_access( # It was not a method. Try looking up a variable. v = lookup_member_var_or_accessor(info, name, mx.is_lvalue) + mx.chk.warn_deprecated(v, mx.context) + vv = v if isinstance(vv, Decorator): # The associated Var node of a decorator contains the type. @@ -1010,6 +1015,8 @@ def analyze_class_attribute_access( # on the class object itself rather than the instance. return None + mx.chk.warn_deprecated(node.node, mx.context) + is_decorated = isinstance(node.node, Decorator) is_method = is_decorated or isinstance(node.node, FuncBase) if mx.is_lvalue: diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index a170b5d4d65a..b835f27bbad9 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -304,5 +304,11 @@ def __hash__(self) -> int: "General", ) +DEPRECATED: Final = ErrorCode( + "deprecated", + "Warn when importing or using deprecated (overloaded) functions, methods or classes", + "General", +) + # This copy will not include any error codes defined later in the plugins. mypy_error_codes = error_codes.copy() diff --git a/mypy/errors.py b/mypy/errors.py index d6dcd4e49e13..13452b14a237 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -20,7 +20,7 @@ # Show error codes for some note-level messages (these usually appear alone # and not as a comment for a previous error-level message). -SHOW_NOTE_CODES: Final = {codes.ANNOTATION_UNCHECKED} +SHOW_NOTE_CODES: Final = {codes.ANNOTATION_UNCHECKED, codes.DEPRECATED} # Do not add notes with links to error code docs to errors with these codes. # We can tweak this set as we get more experience about what is helpful and what is not. @@ -194,6 +194,9 @@ def on_error(self, file: str, info: ErrorInfo) -> bool: Return True to filter out the error, preventing it from being seen by other ErrorWatcher further down the stack and from being recorded by Errors """ + if info.code == codes.DEPRECATED: + return False + self._has_new_errors = True if isinstance(self._filter, bool): should_filter = self._filter diff --git a/mypy/main.py b/mypy/main.py index f177bb1c2062..0674e3b7e79b 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -799,6 +799,13 @@ def add_invertible_flag( help="Warn about statements or expressions inferred to be unreachable", group=lint_group, ) + add_invertible_flag( + "--report-deprecated-as-error", + default=False, + strict_flag=False, + help="Report importing or using deprecated features as errors instead of notes", + group=lint_group, + ) # Note: this group is intentionally added here even though we don't add # --strict to this group near the end. diff --git a/mypy/nodes.py b/mypy/nodes.py index 39cbee3c8525..c4d23ca7bff8 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -561,17 +561,19 @@ class OverloadedFuncDef(FuncBase, SymbolNode, Statement): Overloaded variants must be consecutive in the source file. """ - __slots__ = ("items", "unanalyzed_items", "impl") + __slots__ = ("items", "unanalyzed_items", "impl", "deprecated") items: list[OverloadPart] unanalyzed_items: list[OverloadPart] impl: OverloadPart | None + deprecated: str | None def __init__(self, items: list[OverloadPart]) -> None: super().__init__() self.items = items self.unanalyzed_items = items.copy() self.impl = None + self.deprecated = None if items: # TODO: figure out how to reliably set end position (we don't know the impl here). self.set_line(items[0].line, items[0].column) @@ -596,6 +598,7 @@ def serialize(self) -> JsonDict: "fullname": self._fullname, "impl": None if self.impl is None else self.impl.serialize(), "flags": get_flags(self, FUNCBASE_FLAGS), + "deprecated": self.deprecated, } @classmethod @@ -615,6 +618,7 @@ def deserialize(cls, data: JsonDict) -> OverloadedFuncDef: res.type = typ res._fullname = data["fullname"] set_flags(res, data["flags"]) + res.deprecated = data["deprecated"] # NOTE: res.info will be set in the fixup phase. return res @@ -781,6 +785,7 @@ class FuncDef(FuncItem, SymbolNode, Statement): # Present only when a function is decorated with @typing.datasclass_transform or similar "dataclass_transform_spec", "docstring", + "deprecated", ) __match_args__ = ("name", "arguments", "type", "body") @@ -810,6 +815,7 @@ def __init__( self.is_mypy_only = False self.dataclass_transform_spec: DataclassTransformSpec | None = None self.docstring: str | None = None + self.deprecated: str | None = None @property def name(self) -> str: @@ -840,6 +846,7 @@ def serialize(self) -> JsonDict: if self.dataclass_transform_spec is None else self.dataclass_transform_spec.serialize() ), + "deprecated": self.deprecated, } @classmethod @@ -867,6 +874,7 @@ def deserialize(cls, data: JsonDict) -> FuncDef: if data["dataclass_transform_spec"] is not None else None ) + ret.deprecated = data["deprecated"] # Leave these uninitialized so that future uses will trigger an error del ret.arguments del ret.max_pos @@ -2942,6 +2950,7 @@ class is generic then it will be a type constructor of higher kind. "self_type", "dataclass_transform_spec", "is_type_check_only", + "deprecated", ) _fullname: str # Fully qualified name @@ -3095,6 +3104,9 @@ class is generic then it will be a type constructor of higher kind. # Is set to `True` when class is decorated with `@typing.type_check_only` is_type_check_only: bool + # The type's deprecation message (in case it is deprecated) + deprecated: str | None + FLAGS: Final = [ "is_abstract", "is_enum", @@ -3152,6 +3164,7 @@ def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None self.self_type = None self.dataclass_transform_spec = None self.is_type_check_only = False + self.deprecated = None def add_type_vars(self) -> None: self.has_type_var_tuple_type = False @@ -3374,6 +3387,7 @@ def serialize(self) -> JsonDict: if self.dataclass_transform_spec is not None else None ), + "deprecated": self.deprecated, } return data @@ -3441,6 +3455,7 @@ def deserialize(cls, data: JsonDict) -> TypeInfo: ti.dataclass_transform_spec = DataclassTransformSpec.deserialize( data["dataclass_transform_spec"] ) + ti.deprecated = data.get("deprecated") return ti diff --git a/mypy/options.py b/mypy/options.py index 56bd92957b41..d315d297e023 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -173,6 +173,9 @@ def __init__(self) -> None: # declared with a precise type self.warn_return_any = False + # Report importing or using deprecated features as errors instead of notes. + self.report_deprecated_as_error = False + # Warn about unused '# type: ignore' comments self.warn_unused_ignores = False diff --git a/mypy/semanal.py b/mypy/semanal.py index 27abf2c1dc4c..37aafc3b2647 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -249,6 +249,7 @@ from mypy.types import ( ASSERT_TYPE_NAMES, DATACLASS_TRANSFORM_NAMES, + DEPRECATED_TYPE_NAMES, FINAL_DECORATOR_NAMES, FINAL_TYPE_NAMES, IMPORTED_REVEAL_TYPE_NAMES, @@ -1260,10 +1261,51 @@ def analyze_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: return # We know this is an overload def. Infer properties and perform some checks. + self.process_deprecated_overload(defn) self.process_final_in_overload(defn) self.process_static_or_class_method_in_overload(defn) self.process_overload_impl(defn) + def process_deprecated_overload(self, defn: OverloadedFuncDef) -> None: + if defn.is_property: + return + + if isinstance(impl := defn.impl, Decorator) and ( + (deprecated := impl.func.deprecated) is not None + ): + defn.deprecated = deprecated + for item in defn.items: + if isinstance(item, Decorator): + item.func.deprecated = deprecated + + for item in defn.items: + deprecation = False + if isinstance(item, Decorator): + for d in item.decorators: + if deprecation and refers_to_fullname(d, OVERLOAD_NAMES): + self.msg.note("@overload should be placed before @deprecated", d) + elif (deprecated := self.get_deprecated(d)) is not None: + deprecation = True + if isinstance(typ := item.func.type, CallableType): + typestr = f" {typ} " + else: + typestr = " " + item.func.deprecated = ( + f"overload{typestr}of function {defn.fullname} is deprecated: " + f"{deprecated}" + ) + + @staticmethod + def get_deprecated(expression: Expression) -> str | None: + if ( + isinstance(expression, CallExpr) + and refers_to_fullname(expression.callee, DEPRECATED_TYPE_NAMES) + and (len(args := expression.args) >= 1) + and isinstance(deprecated := args[0], StrExpr) + ): + return deprecated.value + return None + def process_overload_impl(self, defn: OverloadedFuncDef) -> None: """Set flags for an overload implementation. @@ -1440,15 +1482,17 @@ def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) - for i, item in enumerate(items[1:]): if isinstance(item, Decorator): if len(item.decorators) >= 1: - node = item.decorators[0] - if isinstance(node, MemberExpr): - if node.name == "setter": + first_node = item.decorators[0] + if isinstance(first_node, MemberExpr): + if first_node.name == "setter": # The first item represents the entire property. first_item.var.is_settable_property = True # Get abstractness from the original definition. item.func.abstract_status = first_item.func.abstract_status - if node.name == "deleter": + if first_node.name == "deleter": item.func.abstract_status = first_item.func.abstract_status + for other_node in item.decorators[1:]: + other_node.accept(self) else: self.fail( f"Only supported top decorator is @{first_item.func.name}.setter", item @@ -1460,6 +1504,14 @@ def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) - for i in reversed(deleted_items): del items[i] + for item in items[1:]: + if isinstance(item, Decorator): + for d in item.decorators: + if (deprecated := self.get_deprecated(d)) is not None: + item.func.deprecated = ( + f"function {item.fullname} is deprecated: {deprecated}" + ) + def add_function_to_symbol_table(self, func: FuncDef | OverloadedFuncDef) -> None: if self.is_class_scope(): assert self.type is not None @@ -1663,6 +1715,8 @@ def visit_decorator(self, dec: Decorator) -> None: d.callee, DATACLASS_TRANSFORM_NAMES ): dec.func.dataclass_transform_spec = self.parse_dataclass_transform_spec(d) + elif (deprecated := self.get_deprecated(d)) is not None: + dec.func.deprecated = f"function {dec.fullname} is deprecated: {deprecated}" elif not dec.var.is_property: # We have seen a "non-trivial" decorator before seeing @property, if # we will see a @property later, give an error, as we don't support this. @@ -2099,6 +2153,8 @@ def analyze_class_decorator_common( info.is_final = True elif refers_to_fullname(decorator, TYPE_CHECK_ONLY_NAMES): info.is_type_check_only = True + elif (deprecated := self.get_deprecated(decorator)) is not None: + info.deprecated = f"class {defn.fullname} is deprecated: {deprecated}" def clean_up_bases_and_infer_type_variables( self, defn: ClassDef, base_type_exprs: list[Expression], context: Context diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index d5a303128126..131a13ffd62d 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -256,6 +256,7 @@ def snapshot_definition(node: SymbolNode | None, common: SymbolSnapshot) -> Symb signature, is_trivial_body, dataclass_transform_spec.serialize() if dataclass_transform_spec is not None else None, + node.deprecated if isinstance(node, FuncDef) else None, ) elif isinstance(node, Var): return ("Var", common, snapshot_optional_type(node.type), node.is_final) diff --git a/test-data/unit/check-deprecated.test b/test-data/unit/check-deprecated.test new file mode 100644 index 000000000000..f587034d8059 --- /dev/null +++ b/test-data/unit/check-deprecated.test @@ -0,0 +1,396 @@ +-- Type checker test cases for reporting deprecations. + + +[case testDeprecatedDisableNotes] +# flags: --disable-error-code=deprecated + +from typing_extensions import deprecated + +@deprecated("use f2 instead") +def f() -> None: ... + +f() + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedAsNoteWithErrorCode] +# flags: --show-error-codes + +from typing_extensions import deprecated + +@deprecated("use f2 instead") +def f() -> None: ... + +f() # type: ignore[deprecated] +f() # N: function __main__.f is deprecated: use f2 instead [deprecated] + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedAsErrorWithErrorCode] +# flags: --report-deprecated-as-error --show-error-codes + +from typing_extensions import deprecated + +@deprecated("use f2 instead") +def f() -> None: ... + +f() # type: ignore[deprecated] +f() # E: function __main__.f is deprecated: use f2 instead [deprecated] + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedFunction] + +from typing_extensions import deprecated + +@deprecated("use f2 instead") +def f() -> None: ... + +f # N: function __main__.f is deprecated: use f2 instead # type: ignore[deprecated] +f(1) # N: function __main__.f is deprecated: use f2 instead \ + # E: Too many arguments for "f" +f[1] # N: function __main__.f is deprecated: use f2 instead \ + # E: Value of type "Callable[[], None]" is not indexable +g = f # N: function __main__.f is deprecated: use f2 instead +g() +t = (f, f, g) # N: function __main__.f is deprecated: use f2 instead + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedFunctionDifferentModule] + +import m +import p.s +import m as n +import p.s as ps +from m import f # N: function m.f is deprecated: use f2 instead +from p.s import g # N: function p.s.g is deprecated: use g2 instead +from k import * + +m.f() # N: function m.f is deprecated: use f2 instead +p.s.g() # N: function p.s.g is deprecated: use g2 instead +n.f() # N: function m.f is deprecated: use f2 instead +ps.g() # N: function p.s.g is deprecated: use g2 instead +f() +g() +h() # N: function k.h is deprecated: use h2 instead + +[file m.py] +from typing_extensions import deprecated + +@deprecated("use f2 instead") +def f() -> None: ... + +[file p/s.py] +from typing_extensions import deprecated + +@deprecated("use g2 instead") +def g() -> None: ... + +[file k.py] +from typing_extensions import deprecated + +@deprecated("use h2 instead") +def h() -> None: ... + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedClass] + +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: ... + +c: C # N: class __main__.C is deprecated: use C2 instead +C() # N: class __main__.C is deprecated: use C2 instead +C.missing() # N: class __main__.C is deprecated: use C2 instead \ + # E: "Type[C]" has no attribute "missing" +C.__init__(c) # N: class __main__.C is deprecated: use C2 instead +C(1) # N: class __main__.C is deprecated: use C2 instead \ + # E: Too many arguments for "C" +D = C # N: class __main__.C is deprecated: use C2 instead +D() +t = (C, C, D) # N: class __main__.C is deprecated: use C2 instead + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedClassDifferentModule] + +import m +import p.s +import m as n +import p.s as ps +from m import C # N: class m.C is deprecated: use C2 instead +from p.s import D # N: class p.s.D is deprecated: use D2 instead +from k import * + +m.C() # N: class m.C is deprecated: use C2 instead +p.s.D() # N: class p.s.D is deprecated: use D2 instead +n.C() # N: class m.C is deprecated: use C2 instead +ps.D() # N: class p.s.D is deprecated: use D2 instead +C() +D() +E() # N: class k.E is deprecated: use E2 instead + +[file m.py] +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: ... + +[file p/s.py] +from typing_extensions import deprecated + +@deprecated("use D2 instead") +class D: ... + +[file k.py] +from typing_extensions import deprecated + +@deprecated("use E2 instead") +class E: ... + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedClassInitMethod] + +from typing_extensions import deprecated + +@deprecated("use C2 instead") +class C: + def __init__(self) -> None: ... + +c: C # N: class __main__.C is deprecated: use C2 instead +C() # N: class __main__.C is deprecated: use C2 instead +C.__init__(c) # N: class __main__.C is deprecated: use C2 instead + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedSpecialMethods] + +from typing import Iterator +from typing_extensions import deprecated + +class A: + @deprecated("no A + int") + def __add__(self, v: int) -> None: ... + + @deprecated("no int + A") + def __radd__(self, v: int) -> None: ... + + @deprecated("no A = A + int") + def __iadd__(self, v: int) -> A: ... + + @deprecated("no iteration") + def __iter__(self) -> Iterator[int]: ... + + @deprecated("no in") + def __contains__(self, v: int) -> int: ... + + @deprecated("no integer") + def __int__(self) -> int: ... + + @deprecated("no inversion") + def __invert__(self) -> A: ... + +class B: + @deprecated("still no in") + def __contains__(self, v: int) -> int: ... + +a = A() +b = B() +a + 1 # N: function __main__.A.__add__ is deprecated: no A + int +1 + a # N: function __main__.A.__radd__ is deprecated: no int + A +a += 1 # N: function __main__.A.__iadd__ is deprecated: no A = A + int +for i in a: # N: function __main__.A.__iter__ is deprecated: no iteration + reveal_type(i) # N: Revealed type is "builtins.int" +1 in a # N: function __main__.A.__contains__ is deprecated: no in +1 in b # N: function __main__.B.__contains__ is deprecated: still no in +~a # N: function __main__.A.__invert__ is deprecated: no inversion + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedOverloadedSpecialMethods] + +from typing import Iterator, Union +from typing_extensions import deprecated, overload + +class A: + @overload + @deprecated("no A + int") + def __add__(self, v: int) -> None: ... + @overload + def __add__(self, v: str) -> None: ... + def __add__(self, v: Union[int, str]) -> None: ... + + @overload + def __radd__(self, v: int) -> None: ... + @overload + @deprecated("no str + A") + def __radd__(self, v: str) -> None: ... + def __radd__(self, v: Union[int, str]) -> None: ... + + @overload + def __iadd__(self, v: int) -> A: ... + @overload + def __iadd__(self, v: str) -> A: ... + @deprecated("no A += Any") + def __iadd__(self, v: Union[int, str]) -> A: ... + +a = A() +a + 1 # N: overload def (__main__.A, builtins.int) of function __main__.A.__add__ is deprecated: no A + int +a + "x" +1 + a +"x" + a # N: overload def (__main__.A, builtins.str) of function __main__.A.__radd__ is deprecated: no str + A +a += 1 # N: function __main__.A.__iadd__ is deprecated: no A += Any +a += "x" # N: function __main__.A.__iadd__ is deprecated: no A += Any + +[builtins fixtures/tuple.pyi] + + +[case testDeprecatedMethod] + +from typing_extensions import deprecated + +class C: + @deprecated("use g instead") + def f(self) -> None: ... + + def g(self) -> None: ... + + @staticmethod + @deprecated("use g instead") + def h() -> None: ... + + @deprecated("use g instead") + @staticmethod + def k() -> None: ... + +C.f # N: function __main__.C.f is deprecated: use g instead +C().f # N: function __main__.C.f is deprecated: use g instead +C().f() # N: function __main__.C.f is deprecated: use g instead +C().f(1) # N: function __main__.C.f is deprecated: use g instead \ + # E: Too many arguments for "f" of "C" +f = C().f # N: function __main__.C.f is deprecated: use g instead +f() +t = (C.f, C.f, C.g) # N: function __main__.C.f is deprecated: use g instead + +C().g() +C().h() # N: function __main__.C.h is deprecated: use g instead +C().k() # N: function __main__.C.k is deprecated: use g instead + +[builtins fixtures/callable.pyi] + + +[case testDeprecatedClassWithDeprecatedMethod] + +from typing_extensions import deprecated + +@deprecated("use D instead") +class C: + @deprecated("use g instead") + def f(self) -> None: ... + def g(self) -> None: ... + +C().f() # N: class __main__.C is deprecated: use D instead \ + # N: function __main__.C.f is deprecated: use g instead +C().g() # N: class __main__.C is deprecated: use D instead + +[builtins fixtures/callable.pyi] + + +[case testDeprecatedProperty] + +from typing_extensions import deprecated + +class C: + @property + @deprecated("use f2 instead") + def f(self) -> int: ... + + @property + def g(self) -> int: ... + @g.setter + @deprecated("use g2 instead") + def g(self, v: int) -> None: ... + + +C.f # N: function __main__.C.f is deprecated: use f2 instead +C().f # N: function __main__.C.f is deprecated: use f2 instead +C().f() # N: function __main__.C.f is deprecated: use f2 instead \ + # E: "int" not callable +C().f = 1 # N: function __main__.C.f is deprecated: use f2 instead \ + # E: Property "f" defined in "C" is read-only + + +C.g +C().g +C().g = 1 # N: function __main__.C.g is deprecated: use g2 instead +C().g = "x" # N: function __main__.C.g is deprecated: use g2 instead \ + # E: Incompatible types in assignment (expression has type "str", variable has type "int") + +[builtins fixtures/property.pyi] + + +[case testDeprecatedOverloadedFunction] + +from typing import Union +from typing_extensions import deprecated, overload + +@overload +def f(x: int) -> int: ... +@overload +def f(x: str) -> str: ... +@deprecated("use f2 instead") +def f(x: Union[int, str]) -> Union[int, str]: ... + +f # N: function __main__.f is deprecated: use f2 instead +f(1) # N: function __main__.f is deprecated: use f2 instead +f("x") # N: function __main__.f is deprecated: use f2 instead +f(1.0) # N: function __main__.f is deprecated: use f2 instead \ + # E: No overload variant of "f" matches argument type "float" \ + # N: Possible overload variants: \ + # N: def f(x: int) -> int \ + # N: def f(x: str) -> str + +@overload +@deprecated("work with str instead") +def g(x: int) -> int: ... +@overload +def g(x: str) -> str: ... +def g(x: Union[int, str]) -> Union[int, str]: ... + +g +g(1) # N: overload def (x: builtins.int) -> builtins.int of function __main__.g is deprecated: work with str instead +g("x") +g(1.0) # E: No overload variant of "g" matches argument type "float" \ + # N: Possible overload variants: \ + # N: def g(x: int) -> int \ + # N: def g(x: str) -> str + +@overload +def h(x: int) -> int: ... +@deprecated("work with int instead") +@overload # N: @overload should be placed before @deprecated +def h(x: str) -> str: ... +def h(x: Union[int, str]) -> Union[int, str]: ... + +h +h(1) +h("x") # N: overload def (x: builtins.str) -> builtins.str of function __main__.h is deprecated: work with int instead +h(1.0) # E: No overload variant of "h" matches argument type "float" \ + # N: Possible overload variants: \ + # N: def h(x: int) -> int \ + # N: def h(x: str) -> str + +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 15e47ff296ea..d4c61cbf1d5b 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10671,3 +10671,237 @@ class f: ... == main:2: error: Incompatible types in assignment (expression has type "f", variable has type "int") main:4: error: Incompatible types in assignment (expression has type "f", variable has type "int") + + +[case testDeprecatedAddKeepChangeAndRemoveFunctionDeprecation] +from a import f +f() +import a +a.f() + +[file a.py] +def f() -> None: ... + +[file a.py.2] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> None: ... + +[file a.py.3] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> None: ... + +[file a.py.4] +from typing_extensions import deprecated +@deprecated("use f3 instead") +def f() -> None: ... + +[file a.py.5] +def f() -> None: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:1: note: function a.f is deprecated: use f2 instead +main:4: note: function a.f is deprecated: use f2 instead +== +main:1: note: function a.f is deprecated: use f2 instead +main:4: note: function a.f is deprecated: use f2 instead +== +main:1: note: function a.f is deprecated: use f3 instead +main:4: note: function a.f is deprecated: use f3 instead +== + + +[case testDeprecatedRemoveFunctionDeprecation] +from a import f +f() +import a +a.f() + +[file a.py] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> None: ... + +[file a.py.2] +def f() -> None: ... + +[builtins fixtures/tuple.pyi] +[out] +main:1: note: function a.f is deprecated: use f2 instead +main:4: note: function a.f is deprecated: use f2 instead +== + +[case testDeprecatedKeepFunctionDeprecation] +from a import f +f() +import a +a.f() + +[file a.py] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> None: ... + +[file a.py.2] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> None: ... + +[builtins fixtures/tuple.pyi] +[out] +main:1: note: function a.f is deprecated: use f2 instead +main:4: note: function a.f is deprecated: use f2 instead +== +main:1: note: function a.f is deprecated: use f2 instead +main:4: note: function a.f is deprecated: use f2 instead + + +[case testDeprecatedAddFunctionDeprecationIndirectImport] +from b import f +f() +import b +b.f() + +[file b.py] +from a import f + +[file a.py] +def f() -> int: ... + +[file a.py.2] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> int: ... + +[builtins fixtures/tuple.pyi] +[out] +== +b.py:1: note: function a.f is deprecated: use f2 instead +main:1: note: function a.f is deprecated: use f2 instead +main:4: note: function a.f is deprecated: use f2 instead + + +[case testDeprecatedChangeFunctionDeprecationIndirectImport] +from b import f +f() +import b +b.f() + +[file b.py] +from a import f + +[file a.py] +from typing_extensions import deprecated +@deprecated("use f1 instead") +def f() -> int: ... + +[file a.py.2] +from typing_extensions import deprecated +@deprecated("use f2 instead") +def f() -> int: ... + +[builtins fixtures/tuple.pyi] +[out] +b.py:1: note: function a.f is deprecated: use f1 instead +main:1: note: function a.f is deprecated: use f1 instead +main:4: note: function a.f is deprecated: use f1 instead +== +b.py:1: note: function a.f is deprecated: use f2 instead +main:1: note: function a.f is deprecated: use f2 instead +main:4: note: function a.f is deprecated: use f2 instead + +[case testDeprecatedRemoveFunctionDeprecationIndirectImport] +from b import f +f() +import b +b.f() + +[file b.py] +from a import f + +[file a.py] +from typing_extensions import deprecated +@deprecated("use f1 instead") +def f() -> int: ... + +[file a.py.2] +def f() -> int: ... + +[builtins fixtures/tuple.pyi] +[out] +b.py:1: note: function a.f is deprecated: use f1 instead +main:1: note: function a.f is deprecated: use f1 instead +main:4: note: function a.f is deprecated: use f1 instead +== + +[case testDeprecatedFunctionAlreadyDecorated1-only_when_cache] +from b import f +x: str = f() +import b +y: str = b.f() + +[file b.py] +from a import f + +[file a.py] +from typing import Callable + +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@d +def f() -> str: ... + +[file a.py.2] +from typing import Callable +from typing_extensions import deprecated + +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@deprecated("deprecated decorated function") +@d +def f() -> str: ... + +[builtins fixtures/tuple.pyi] +[out] +== +b.py:1: note: function a.f is deprecated: deprecated decorated function +main:1: note: function a.f is deprecated: deprecated decorated function +main:4: note: function a.f is deprecated: deprecated decorated function + + +[case testDeprecatedFunctionAlreadyDecorated2-only_when_nocache] +from b import f +x: str = f() +import b +y: str = b.f() + +[file b.py] +from a import f + +[file a.py] +from typing import Callable + +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@d +def f() -> str: ... + +[file a.py.2] +from typing import Callable +from typing_extensions import deprecated + +def d(t: Callable[[], str]) -> Callable[[], str]: ... + +@deprecated("deprecated decorated function") +@d +def f() -> str: ... + +[builtins fixtures/tuple.pyi] +[out] +== +main:1: note: function a.f is deprecated: deprecated decorated function +main:4: note: function a.f is deprecated: deprecated decorated function +b.py:1: note: function a.f is deprecated: deprecated decorated function diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 89f01bff963e..174ba8e98c3f 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -954,7 +954,7 @@ a = [] # type: List[Dict[str, str]] sorted(a, key=lambda y: y['']) [case testAbstractProperty] -from abc import abstractproperty, ABCMeta +from abc import abstractproperty, ABCMeta # type: ignore[deprecated] class A(metaclass=ABCMeta): @abstractproperty def x(self) -> int: pass From 7255ece5b0f2ab63baa4b8b9cd1a16d90cbedc1c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Oct 2024 09:47:31 +0100 Subject: [PATCH 181/247] Add Python 3.13 to classifiers (#17891) Mypy now supports 3.13 (though not all features yet). --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 160e2b054b0e..a50afde4ce6b 100644 --- a/setup.py +++ b/setup.py @@ -190,6 +190,7 @@ def run(self): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Software Development", "Typing :: Typed", ] From 82d04257eced45a2692f94999026497321619134 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Oct 2024 10:12:50 +0100 Subject: [PATCH 182/247] Add changelog for mypy 1.12 (#17889) Related to #17815. --------- Co-authored-by: Jelle Zijlstra --- CHANGELOG.md | 286 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 286 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9632cb39a8b1..c664121f097d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,292 @@ ## Next release +## Mypy 1.12 + +We’ve just uploaded mypy 1.12 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type +checker for Python. This release includes new features, performance improvements and bug fixes. +You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +### Support Python 3.12 Syntax for Generics (PEP 695) + +Support for the new type parameter syntax introduced in Python 3.12 is now enabled by default, +documented, and no longer experimental. It was available through a feature flag in +mypy 1.11 as an experimental feature. + +This example demonstrates the new syntax: + +```python +# Generic function +def f[T](x: T) -> T: ... + +reveal_type(f(1)) # Revealed type is 'int' + +# Generic class +class C[T]: + def __init__(self, x: T) -> None: + self.x = x + +c = C('a') +reveal_type(c.x) # Revealed type is 'str' + +# Type alias +type A[T] = C[list[T]] +``` + +For more information, refer to the [documentation](https://mypy.readthedocs.io/en/latest/generics.html). + +These improvements are included: + + * Document Python 3.12 type parameter syntax (Jukka Lehtosalo, PR [17816](https://github.com/python/mypy/pull/17816)) + * Further documentation updates (Jukka Lehtosalo, PR [17826](https://github.com/python/mypy/pull/17826)) + * Allow Self return types with contravariance (Jukka Lehtosalo, PR [17786](https://github.com/python/mypy/pull/17786)) + * Enable new type parameter syntax by default (Jukka Lehtosalo, PR [17798](https://github.com/python/mypy/pull/17798)) + * Generate error if new-style type alias used as base class (Jukka Lehtosalo, PR [17789](https://github.com/python/mypy/pull/17789)) + * Inherit variance if base class has explicit variance (Jukka Lehtosalo, PR [17787](https://github.com/python/mypy/pull/17787)) + * Fix crash on invalid type var reference (Jukka Lehtosalo, PR [17788](https://github.com/python/mypy/pull/17788)) + * Fix covariance of frozen dataclasses (Jukka Lehtosalo, PR [17783](https://github.com/python/mypy/pull/17783)) + * Allow covariance with attribute that has "`_`" name prefix (Jukka Lehtosalo, PR [17782](https://github.com/python/mypy/pull/17782)) + * Support `Annotated[...]` in new-style type aliases (Jukka Lehtosalo, PR [17777](https://github.com/python/mypy/pull/17777)) + * Fix nested generic classes (Jukka Lehtosalo, PR [17776](https://github.com/python/mypy/pull/17776)) + * Add detection and error reporting for the use of incorrect expressions within the scope of a type parameter and a type alias (Kirill Podoprigora, PR [17560](https://github.com/python/mypy/pull/17560)) + +### Basic Support for Python 3.13 + +This release adds partial support for Python 3.13 features and compiled binaries for +Python 3.13. Mypyc now also supports Python 3.13. + +In particular, these features are supported: + * Various new stdlib features and changes (through typeshed stub improvements) + * `typing.ReadOnly` (see below for more) + * `typing.TypeIs` (added in mypy 1.10, [PEP 742](https://peps.python.org/pep-0742/)) + * Type parameter defaults when using the legacy syntax ([PEP 696](https://peps.python.org/pep-0696/)) + +These features are not supported yet: + * `warnings.deprecated` ([PEP 702](https://peps.python.org/pep-0702/)) + * Type parameter defaults when using Python 3.12 type parameter syntax + +### Mypyc Support for Python 3.13 + +Mypyc now supports Python 3.13. This was contributed by Marc Mueller, with additional +fixes by Jukka Lehtosalo. Free threaded Python 3.13 builds are not supported yet. + +List of changes: + + * Add additional includes for Python 3.13 (Marc Mueller, PR [17506](https://github.com/python/mypy/pull/17506)) + * Add another include for Python 3.13 (Marc Mueller, PR [17509](https://github.com/python/mypy/pull/17509)) + * Fix ManagedDict functions for Python 3.13 (Marc Mueller, PR [17507](https://github.com/python/mypy/pull/17507)) + * Update mypyc test output for Python 3.13 (Marc Mueller, PR [17508](https://github.com/python/mypy/pull/17508)) + * Fix `PyUnicode` functions for Python 3.13 (Marc Mueller, PR [17504](https://github.com/python/mypy/pull/17504)) + * Fix `_PyObject_LookupAttrId` for Python 3.13 (Marc Mueller, PR [17505](https://github.com/python/mypy/pull/17505)) + * Fix `_PyList_Extend` for Python 3.13 (Marc Mueller, PR [17503](https://github.com/python/mypy/pull/17503)) + * Fix `gen_is_coroutine` for Python 3.13 (Marc Mueller, PR [17501](https://github.com/python/mypy/pull/17501)) + * Fix `_PyObject_FastCall` for Python 3.13 (Marc Mueller, PR [17502](https://github.com/python/mypy/pull/17502)) + * Avoid uses of `_PyObject_CallMethodOneArg` on 3.13 (Jukka Lehtosalo, PR [17526](https://github.com/python/mypy/pull/17526)) + * Don't rely on `_PyType_CalculateMetaclass` on 3.13 (Jukka Lehtosalo, PR [17525](https://github.com/python/mypy/pull/17525)) + * Don't use `_PyUnicode_FastCopyCharacters` on 3.13 (Jukka Lehtosalo, PR [17524](https://github.com/python/mypy/pull/17524)) + * Don't use `_PyUnicode_EQ` on 3.13, as it's no longer exported (Jukka Lehtosalo, PR [17523](https://github.com/python/mypy/pull/17523)) + +### Inferring Unions for Conditional Expressions + +Mypy now always tries to infer a union type for a conditional expression if left and right +operand types are different. This results in more precise inferred types and lets mypy detect +more issues. Example: + +```python +s = "foo" if cond() else 1 +# Type of "s" is now "str | int" (it used to be "object") +``` + +Notably, if one of the operands has type `Any`, the type of a conditional expression is +now ` | Any`. Previously the inferred type was just `Any`. The new type essentially +indicates that the value can be of type ``, and potentially of some (unknown) type. +Most operations performed on the result must also be valid for ``. +Example where this is relevant: + +```python +from typing import Any + +def func(a: Any, b: bool) -> None: + x = a if b else None + # Type of x is "Any | None" + print(x.y) # Error: None has no attribute "y" +``` + +This feature was contributed by Ivan Levkivskyi (PR [17427](https://github.com/python/mypy/pull/17427)). + +### ReadOnly Support for TypedDict (PEP 705) + +You can now use `typing.ReadOnly` to specity TypedDict items as +read-only ([PEP 705](https://peps.python.org/pep-0705/)): + +```python +from typing import TypedDict + +# Or "from typing ..." on Python 3.13 +from typing_extensions import ReadOnly + +class TD(TypedDict): + a: int + b: ReadOnly[int] + +d: TD = {"a": 1, "b": 2} +d["a"] = 3 # OK +d["b"] = 5 # Error: "b" is ReadOnly +``` + +This feature was contributed by Nikita Sobolev (PR [17644](https://github.com/python/mypy/pull/17644)). + +### Python 3.8 End of Life Approaching + +We are planning to drop support for Python 3.8 in the next mypy feature release or the +one after that. Python 3.8 reaches end of life in October 2024. + +### Planned Changes to Defaults + +We are planning to enable `--local-partial-types` by default in mypy 2.0. This will +often require at least minor code changes. This option is implicitly enabled by mypy +daemon, so this makes the behavior of daemon and non-daemon modes consistent. + +We recommend that mypy users start using local partial types soon (or to explicitly disable +them) to prepare for the change. + +This can also be configured in a mypy configuration file: + +``` +local_partial_types = True +``` + +For more information, refer to the +[documentation](https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-local-partial-types). + +### Documentation Updates + +Mypy documentation now uses modern syntax variants and imports in many examples. Some +examples no longer work on Python 3.8, which is the earliest Python version that mypy supports. + +Notably, `Iterable` and other protocols/ABCs are imported from `collections.abc` instead of +`typing`: +```python +from collections.abc import Iterable, Callable +``` + +Examples also avoid the upper-case aliases to built-in types: `list[str]` is used instead +of `List[str]`. The `X | Y` union type syntax introduced in Python 3.10 is also now prevalent. + +List of documentation updates: + + * Document `--output=json` CLI option (Edgar Ramírez Mondragón, PR [17611](https://github.com/python/mypy/pull/17611)) + * Update various references to deprecated type aliases in docs (Jukka Lehtosalo, PR [17829](https://github.com/python/mypy/pull/17829)) + * Make "X | Y" union syntax more prominent in documentation (Jukka Lehtosalo, PR [17835](https://github.com/python/mypy/pull/17835)) + * Discuss upper bounds before self types in documentation (Jukka Lehtosalo, PR [17827](https://github.com/python/mypy/pull/17827)) + * Make changelog visible in mypy documentation (quinn-sasha, PR [17742](https://github.com/python/mypy/pull/17742)) + * List all incomplete features in `--enable-incomplete-feature` docs (sobolevn, PR [17633](https://github.com/python/mypy/pull/17633)) + * Remove the explicit setting of a pygments theme (Pradyun Gedam, PR [17571](https://github.com/python/mypy/pull/17571)) + +### Experimental Inline TypedDict Syntax + +Mypy now supports a non-standard, experimental syntax for defining anonymous TypedDicts. +Example: + +```python +def func(n: str, y: int) -> {"name": str, "year": int}: + return {"name": n, "year": y} +``` + +The feature is disabled by default. Use `--enable-incomplete-feature=InlineTypedDict` to +enable it. *We might remove this feature in a future release.* + +This feature was contributed by Ivan Levkivskyi (PR [17457](https://github.com/python/mypy/pull/17457)). + +### Stubgen Improvements + + * Fix crash on literal class-level keywords (sobolevn, PR [17663](https://github.com/python/mypy/pull/17663)) + * Stubgen add `--version` (sobolevn, PR [17662](https://github.com/python/mypy/pull/17662)) + * Fix `stubgen --no-analysis/--parse-only` docs (sobolevn, PR [17632](https://github.com/python/mypy/pull/17632)) + * Include keyword only args when generating signatures in stubgenc (Eric Mark Martin, PR [17448](https://github.com/python/mypy/pull/17448)) + * Add support for detecting `Literal` types when extracting types from docstrings (Michael Carlstrom, PR [17441](https://github.com/python/mypy/pull/17441)) + * Use `Generator` type var defaults (Sebastian Rittau, PR [17670](https://github.com/python/mypy/pull/17670)) + +### Stubtest Improvements + * Add support for `cached_property` (Ali Hamdan, PR [17626](https://github.com/python/mypy/pull/17626)) + * Add `enable_incomplete_feature` validation to `stubtest` (sobolevn, PR [17635](https://github.com/python/mypy/pull/17635)) + * Fix error code handling in `stubtest` with `--mypy-config-file` (sobolevn, PR [17629](https://github.com/python/mypy/pull/17629)) + +### Other Notables Fixes and Improvements + + * Report error if using unsupported type parameter defaults (Jukka Lehtosalo, PR [17876](https://github.com/python/mypy/pull/17876)) + * Fix re-processing cross-reference in mypy daemon when node kind changes (Ivan Levkivskyi, PR [17883](https://github.com/python/mypy/pull/17883)) + * Don't use equality to narrow when value is IntEnum/StrEnum (Jukka Lehtosalo, PR [17866](https://github.com/python/mypy/pull/17866)) + * Don't consider None vs IntEnum comparison ambiguous (Jukka Lehtosalo, PR [17877](https://github.com/python/mypy/pull/17877)) + * Fix narrowing of IntEnum and StrEnum types (Jukka Lehtosalo, PR [17874](https://github.com/python/mypy/pull/17874)) + * Filter overload items based on self type during type inference (Jukka Lehtosalo, PR [17873](https://github.com/python/mypy/pull/17873)) + * Enable negative narrowing of union TypeVar upper bounds (Brian Schubert, PR [17850](https://github.com/python/mypy/pull/17850)) + * Fix issue with member expression formatting (Brian Schubert, PR [17848](https://github.com/python/mypy/pull/17848)) + * Avoid type size explosion when expanding types (Jukka Lehtosalo, PR [17842](https://github.com/python/mypy/pull/17842)) + * Fix negative narrowing of tuples in match statement (Brian Schubert, PR [17817](https://github.com/python/mypy/pull/17817)) + * Narrow falsey str/bytes/int to literal type (Brian Schubert, PR [17818](https://github.com/python/mypy/pull/17818)) + * Test against latest Python 3.13, make testing 3.14 easy (Shantanu, PR [17812](https://github.com/python/mypy/pull/17812)) + * Reject ParamSpec-typed callables calls with insufficient arguments (Stanislav Terliakov, PR [17323](https://github.com/python/mypy/pull/17323)) + * Fix crash when passing too many type arguments to generic base class accepting single ParamSpec (Brian Schubert, PR [17770](https://github.com/python/mypy/pull/17770)) + * Fix TypeVar upper bounds sometimes not being displayed in pretty callables (Brian Schubert, PR [17802](https://github.com/python/mypy/pull/17802)) + * Added error code for overlapping function signatures (Katrina Connors, PR [17597](https://github.com/python/mypy/pull/17597)) + * Check for `truthy-bool` in `not ...` unary expressions (sobolevn, PR [17773](https://github.com/python/mypy/pull/17773)) + * Add missing lines-covered and lines-valid attributes (Soubhik Kumar Mitra, PR [17738](https://github.com/python/mypy/pull/17738)) + * Fix another crash scenario with recursive tuple types (Ivan Levkivskyi, PR [17708](https://github.com/python/mypy/pull/17708)) + * Resolve TypeVar upper bounds in `functools.partial` (Shantanu, PR [17660](https://github.com/python/mypy/pull/17660)) + * Always reset binder when checking deferred nodes (Ivan Levkivskyi, PR [17643](https://github.com/python/mypy/pull/17643)) + * Fix crash on a callable attribute with single unpack (Ivan Levkivskyi, PR [17641](https://github.com/python/mypy/pull/17641)) + * Fix mismatched signature between checker plugin API and implementation (bzoracler, PR [17343](https://github.com/python/mypy/pull/17343)) + * Indexing a type also produces a GenericAlias (Shantanu, PR [17546](https://github.com/python/mypy/pull/17546)) + * Fix crash on self-type in callable protocol (Ivan Levkivskyi, PR [17499](https://github.com/python/mypy/pull/17499)) + * Fix crash on NamedTuple with method and error in function (Ivan Levkivskyi, PR [17498](https://github.com/python/mypy/pull/17498)) + * Add `__replace__` for dataclasses in 3.13 (Max Muoto, PR [17469](https://github.com/python/mypy/pull/17469)) + * Fix help message for `--no-namespace-packages` (Raphael Krupinski, PR [17472](https://github.com/python/mypy/pull/17472)) + * Fix typechecking for async generators (Danny Yang, PR [17452](https://github.com/python/mypy/pull/17452)) + * Fix strict optional handling in attrs plugin (Ivan Levkivskyi, PR [17451](https://github.com/python/mypy/pull/17451)) + * Allow mixing ParamSpec and TypeVarTuple in Generic (Ivan Levkivskyi, PR [17450](https://github.com/python/mypy/pull/17450)) + +### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=91a58b07cdd807b1d965e04ba85af2adab8bf924+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + +### Acknowledgements +Thanks to all mypy contributors who contributed to this release: + +- Ali Hamdan +- Anders Kaseorg +- Bénédikt Tran +- Brian Schubert +- bzoracler +- Danny Yang +- Edgar Ramírez Mondragón +- Eric Mark Martin +- InSync +- Ivan Levkivskyi +- Jordandev678 +- Katrina Connors +- Kirill Podoprigora +- Marc Mueller +- Max Muoto +- Max Murin +- Michael Carlstrom +- Michael I Chen +- Pradyun Gedam +- quinn-sasha +- Raphael Krupinski +- Sebastian Rittau +- Shantanu +- sobolevn +- Soubhik Kumar Mitra +- Stanislav Terliakov +- wyattscarpenter + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 1.11 From 1d4b45bd576d7e34ea7a471646dc1ed0aabe3d48 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 8 Oct 2024 12:25:44 -0700 Subject: [PATCH 183/247] stubtest: Stop telling people to use double underscores (#17897) Python 3.7 is long dead. --- mypy/stubtest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index c54f83f33b00..65730828ad9f 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -133,7 +133,7 @@ def is_missing_stub(self) -> bool: def is_positional_only_related(self) -> bool: """Whether or not the error is for something being (or not being) positional-only.""" # TODO: This is hacky, use error codes or something more resilient - return "leading double underscore" in self.message + return "should be positional" in self.message def get_description(self, concise: bool = False) -> str: """Returns a description of the error. @@ -909,7 +909,7 @@ def _verify_signature( ): yield ( f'stub argument "{stub_arg.variable.name}" should be positional-only ' - f'(rename with a leading double underscore, i.e. "__{runtime_arg.name}")' + f'(add "/", e.g. "{runtime_arg.name}, /")' ) if ( runtime_arg.kind != inspect.Parameter.POSITIONAL_ONLY @@ -919,7 +919,7 @@ def _verify_signature( ): yield ( f'stub argument "{stub_arg.variable.name}" should be positional or keyword ' - "(remove leading double underscore)" + '(remove "/")' ) # Check unmatched positional args From b0db694cc7c6c359a9ce7b036727d0a6f3336f58 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 8 Oct 2024 23:47:41 +0200 Subject: [PATCH 184/247] Use 3.13.0 for ci tests (#17900) --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39fbb14bd3b7..296f6fde1523 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,8 +71,8 @@ jobs: tox_extra_args: "-n 4" test_mypyc: true - - name: Test suite with py313-dev-ubuntu, mypyc-compiled - python: '3.13-dev' + - name: Test suite with py313-ubuntu, mypyc-compiled + python: '3.13' arch: x64 os: ubuntu-latest toxenv: py From 621c7aa7088faa19aa6588ab7ad60561ec2fce53 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 9 Oct 2024 01:56:59 -0700 Subject: [PATCH 185/247] Improvements to functools.partial of types (#17898) Fixes https://github.com/python/mypy/issues/17556 , fixes https://github.com/python/mypy/issues/17659 --- mypy/checker.py | 8 ++--- test-data/unit/check-functools.test | 36 +++++++++++++++++++ test-data/unit/lib-stub/typing_extensions.pyi | 2 ++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 4bbd49cd7198..e0c2578df095 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -686,10 +686,10 @@ def extract_callable_type(self, inner_type: Type | None, ctx: Context) -> Callab if isinstance(inner_type, TypeVarLikeType): inner_type = get_proper_type(inner_type.upper_bound) if isinstance(inner_type, TypeType): - if isinstance(inner_type.item, Instance): - inner_type = expand_type_by_instance( - type_object_type(inner_type.item.type, self.named_type), inner_type.item - ) + inner_type = get_proper_type( + self.expr_checker.analyze_type_type_callee(inner_type.item, ctx) + ) + if isinstance(inner_type, CallableType): outer_type = inner_type elif isinstance(inner_type, Instance): diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index 9f8fbd42440b..50de3789ebd2 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -557,3 +557,39 @@ def bar(f: S) -> S: g = functools.partial(f, "foo") return f [builtins fixtures/primitives.pyi] + + +[case testFunctoolsPartialAbstractType] +# flags: --python-version 3.9 +from abc import ABC, abstractmethod +from functools import partial + +class A(ABC): + def __init__(self) -> None: ... + @abstractmethod + def method(self) -> None: ... + +def f1(cls: type[A]) -> None: + cls() + partial_cls = partial(cls) + partial_cls() + +def f2() -> None: + A() # E: Cannot instantiate abstract class "A" with abstract attribute "method" + partial_cls = partial(A) # E: Cannot instantiate abstract class "A" with abstract attribute "method" + partial_cls() # E: Cannot instantiate abstract class "A" with abstract attribute "method" +[builtins fixtures/tuple.pyi] + + +[case testFunctoolsPartialSelfType] +from functools import partial +from typing_extensions import Self + +class A: + def __init__(self, ts: float, msg: str) -> None: ... + + @classmethod + def from_msg(cls, msg: str) -> Self: + factory = partial(cls, ts=0) + return factory(msg=msg) +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/lib-stub/typing_extensions.pyi b/test-data/unit/lib-stub/typing_extensions.pyi index d9d7067efe0f..cb054b0e6b4f 100644 --- a/test-data/unit/lib-stub/typing_extensions.pyi +++ b/test-data/unit/lib-stub/typing_extensions.pyi @@ -43,6 +43,8 @@ Required: _SpecialForm NotRequired: _SpecialForm ReadOnly: _SpecialForm +Self: _SpecialForm + @final class TypeAliasType: def __init__( From 9e24b56e86fe10b2d89631854770bbd139bcf55d Mon Sep 17 00:00:00 2001 From: Chelsea Durazo Date: Wed, 9 Oct 2024 02:05:51 -0700 Subject: [PATCH 186/247] documentation for TypeIs (#17821) Fixes #17156. As requested in the issue, added documentation for the desired behaviour of TypeIs. --- docs/source/type_narrowing.rst | 166 +++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/docs/source/type_narrowing.rst b/docs/source/type_narrowing.rst index 231a7edccfe7..230c40b30894 100644 --- a/docs/source/type_narrowing.rst +++ b/docs/source/type_narrowing.rst @@ -389,3 +389,169 @@ or rewrite the function to be slightly more verbose: elif b is not None: return b return C() + + +.. _typeis: + +TypeIs +------ + +Mypy supports TypeIs (:pep:`742`). + +A `TypeIs narrowing function `_ +allows you to define custom type checks that can narrow the type of a variable +in `both the if and else _` +branches of a conditional, similar to how the built-in isinstance() function works. + +TypeIs is new in Python 3.13 — for use in older Python versions, use the backport +from `typing_extensions _` + +Consider the following example using TypeIs: + +.. code-block:: python + + from typing import TypeIs + + def is_str(x: object) -> TypeIs[str]: + return isinstance(x, str) + + def process(x: int | str) -> None: + if is_str(x): + reveal_type(x) # Revealed type is 'str' + print(x.upper()) # Valid: x is str + else: + reveal_type(x) # Revealed type is 'int' + print(x + 1) # Valid: x is int + +In this example, the function is_str is a type narrowing function +that returns TypeIs[str]. When used in an if statement, x is narrowed +to str in the if branch and to int in the else branch. + +Key points: + + +- The function must accept at least one positional argument. + +- The return type is annotated as ``TypeIs[T]``, where ``T`` is the type you + want to narrow to. + +- The function must return a ``bool`` value. + +- In the ``if`` branch (when the function returns ``True``), the type of the + argument is narrowed to the intersection of its original type and ``T``. + +- In the ``else`` branch (when the function returns ``False``), the type of + the argument is narrowed to the intersection of its original type and the + complement of ``T``. + + +TypeIs vs TypeGuard +~~~~~~~~~~~~~~~~~~~ + +While both TypeIs and TypeGuard allow you to define custom type narrowing +functions, they differ in important ways: + +- **Type narrowing behavior**: TypeIs narrows the type in both the if and else branches, + whereas TypeGuard narrows only in the if branch. + +- **Compatibility requirement**: TypeIs requires that the narrowed type T be + compatible with the input type of the function. TypeGuard does not have this restriction. + +- **Type inference**: With TypeIs, the type checker may infer a more precise type by + combining existing type information with T. + +Here's an example demonstrating the behavior with TypeGuard: + +.. code-block:: python + + from typing import TypeGuard, reveal_type + + def is_str(x: object) -> TypeGuard[str]: + return isinstance(x, str) + + def process(x: int | str) -> None: + if is_str(x): + reveal_type(x) # Revealed type is "builtins.str" + print(x.upper()) # ok: x is str + else: + reveal_type(x) # Revealed type is "Union[builtins.int, builtins.str]" + print(x + 1) # ERROR: Unsupported operand types for + ("str" and "int") [operator] + +Generic TypeIs +~~~~~~~~~~~~~~ + +``TypeIs`` functions can also work with generic types: + +.. code-block:: python + + from typing import TypeVar, TypeIs + + T = TypeVar('T') + + def is_two_element_tuple(val: tuple[T, ...]) -> TypeIs[tuple[T, T]]: + return len(val) == 2 + + def process(names: tuple[str, ...]) -> None: + if is_two_element_tuple(names): + reveal_type(names) # Revealed type is 'tuple[str, str]' + else: + reveal_type(names) # Revealed type is 'tuple[str, ...]' + + +TypeIs with Additional Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +TypeIs functions can accept additional parameters beyond the first. +The type narrowing applies only to the first argument. + +.. code-block:: python + + from typing import Any, TypeVar, reveal_type, TypeIs + + T = TypeVar('T') + + def is_instance_of(val: Any, typ: type[T]) -> TypeIs[T]: + return isinstance(val, typ) + + def process(x: Any) -> None: + if is_instance_of(x, int): + reveal_type(x) # Revealed type is 'int' + print(x + 1) # ok + else: + reveal_type(x) # Revealed type is 'Any' + +TypeIs in Methods +~~~~~~~~~~~~~~~~~ + +A method can also serve as a ``TypeIs`` function. Note that in instance or +class methods, the type narrowing applies to the second parameter +(after ``self`` or ``cls``). + +.. code-block:: python + + class Validator: + def is_valid(self, instance: object) -> TypeIs[str]: + return isinstance(instance, str) + + def process(self, to_validate: object) -> None: + if Validator().is_valid(to_validate): + reveal_type(to_validate) # Revealed type is 'str' + print(to_validate.upper()) # ok: to_validate is str + + +Assignment Expressions with TypeIs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can use the assignment expression operator ``:=`` with ``TypeIs`` to create a new variable and narrow its type simultaneously. + +.. code-block:: python + + from typing import TypeIs, reveal_type + + def is_float(x: object) -> TypeIs[float]: + return isinstance(x, float) + + def main(a: object) -> None: + if is_float(x := a): + reveal_type(x) # Revealed type is 'float' + # x is narrowed to float in this block + print(x + 1.0) From e126ba02fe32918ada61cf191b8a23047281ec9f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 9 Oct 2024 14:48:09 +0100 Subject: [PATCH 187/247] Make ReadOnly TypedDict items covariant (#17904) Fixes #17901. --- mypy/subtypes.py | 23 +++++++++------- test-data/unit/check-typeddict.test | 41 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 787d5cb89b0a..a63db93fd9cb 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -892,15 +892,20 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: return False for name, l, r in left.zip(right): # TODO: should we pass on the full subtype_context here and below? - if self.proper_subtype: - check = is_same_type(l, r) + right_readonly = name in right.readonly_keys + if not right_readonly: + if self.proper_subtype: + check = is_same_type(l, r) + else: + check = is_equivalent( + l, + r, + ignore_type_params=self.subtype_context.ignore_type_params, + options=self.options, + ) else: - check = is_equivalent( - l, - r, - ignore_type_params=self.subtype_context.ignore_type_params, - options=self.options, - ) + # Read-only items behave covariantly + check = self._is_subtype(l, r) if not check: return False # Non-required key is not compatible with a required key since @@ -917,7 +922,7 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: # Readonly fields check: # # A = TypedDict('A', {'x': ReadOnly[int]}) - # B = TypedDict('A', {'x': int}) + # B = TypedDict('B', {'x': int}) # def reset_x(b: B) -> None: # b['x'] = 0 # diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index e1797421636e..affa472bb640 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -3988,3 +3988,44 @@ class TP(TypedDict): k: ReadOnly # E: "ReadOnly[]" must have exactly one type argument [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + +[case testTypedDictReadOnlyCovariant] +from typing import ReadOnly, TypedDict, Union + +class A(TypedDict): + a: ReadOnly[Union[int, str]] + +class A2(TypedDict): + a: ReadOnly[int] + +class B(TypedDict): + a: int + +class B2(TypedDict): + a: Union[int, str] + +class B3(TypedDict): + a: int + +def fa(a: A) -> None: ... +def fa2(a: A2) -> None: ... + +b: B = {"a": 1} +fa(b) +fa2(b) +b2: B2 = {"a": 1} +fa(b2) +fa2(b2) # E: Argument 1 to "fa2" has incompatible type "B2"; expected "A2" + +class C(TypedDict): + a: ReadOnly[Union[int, str]] + b: Union[str, bytes] + +class D(TypedDict): + a: int + b: str + +d: D = {"a": 1, "b": "x"} +c: C = d # E: Incompatible types in assignment (expression has type "D", variable has type "C") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] From a1fa1c483febf8907fdb71f04e9e9a1d6be7809a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 9 Oct 2024 14:57:09 +0100 Subject: [PATCH 188/247] Document ReadOnly (PEP 705) (#17905) Add basic documentation for mypy 1.12 release. --------- Co-authored-by: sobolevn --- docs/source/typed_dict.rst | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/source/typed_dict.rst b/docs/source/typed_dict.rst index e69b3895c668..bbb10a12abe8 100644 --- a/docs/source/typed_dict.rst +++ b/docs/source/typed_dict.rst @@ -236,6 +236,46 @@ another ``TypedDict`` if all required keys in the other ``TypedDict`` are requir first ``TypedDict``, and all non-required keys of the other ``TypedDict`` are also non-required keys in the first ``TypedDict``. +Read-only items +--------------- + +You can use ``typing.ReadOnly``, introduced in Python 3.13, or +``typing_extensions.ReadOnly`` to mark TypedDict items as read-only (:pep:`705`): + +.. code-block:: python + + from typing import TypedDict + + # Or "from typing ..." on Python 3.13+ + from typing_extensions import ReadOnly + + class Movie(TypedDict): + name: ReadOnly[str] + num_watched: int + + m: Movie = {"name": "Jaws", "num_watched": 1} + m["name"] = "The Godfather" # Error: "name" is read-only + m["num_watched"] += 1 # OK + +A TypedDict with a mutable item can be assigned to a TypedDict +with a corresponding read-only item, and the type of the item can +vary :ref:`covariantly `: + +.. code-block:: python + + class Entry(TypedDict): + name: ReadOnly[str | None] + year: ReadOnly[int] + + class Movie(TypedDict): + name: str + year: int + + def process_entry(i: Entry) -> None: ... + + m: Movie = {"name": "Jaws", "year": 1975} + process_entry(m) # OK + Unions of TypedDicts -------------------- From 62a971c868e43d940769d9b9da341b4ed175ef15 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Wed, 9 Oct 2024 10:19:57 -0400 Subject: [PATCH 189/247] Emit error for "raise NotImplemented" (#17890) Refs #5710 Adds special-case handling for raising `NotImplemented`: ```python raise NotImplemented # E: Exception must be derived from BaseException; did you mean "NotImplementedError"? ``` Per the linked issue, there's some debate as to how to best handle `NotImplemented`. This PR special-cases its behavior in `raise` statements, whereas the leaning in the issue (at the time it was opened) was towards updating its definition in typeshed and possibly special-casing its use in the relevant dunder methods. Going the typeshed/special-dunder route may still happen, but it hasn't in the six years since the issue was opened. It would be nice to at least catch errors from raising `NotImplemented` in the interim. Making this change also uncovered a regression introduced in python/typeshed#4222. Previously, `NotImplemented` was annotated as `Any` in typeshed, so returning `NotImplemented` from a non-dunder would emit an error when `--warn-return-any` was used: ```python class A: def some(self) -> bool: return NotImplemented # E: Returning Any from function declared to return "bool" ``` However, in python/typeshed#4222, the type of `NotImplemented` was updated to be a subclass of `Any`. This broke the handling of `--warn-return-any`, but it wasn't caught since the definition of `NotImplemented` in `fixtures/notimplemented.pyi` wasn't updated along with the definition in typeshed. As a result, current mypy doesn't emit an error here ([playground](https://mypy-play.net/?mypy=1.11.2&python=3.12&flags=strict%2Cwarn-return-any&gist=8a78e3eb68b0b738f73fdd326f0bfca1)), despite having a test case saying that it should. As a bandaid, this PR add special handling for `NotImplemented` in return statements to treat it like an explicit `Any`, restoring the pre- python/typeshed#4222 behavior. --- mypy/checker.py | 15 +++++++++++++++ test-data/unit/check-statements.test | 7 +++++++ test-data/unit/fixtures/notimplemented.pyi | 10 +++++++--- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e0c2578df095..3483f1b029c8 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4581,6 +4581,13 @@ def check_return_stmt(self, s: ReturnStmt) -> None: s.expr, return_type, allow_none_return=allow_none_func_call ) ) + # Treat NotImplemented as having type Any, consistent with its + # definition in typeshed prior to python/typeshed#4222. + if ( + isinstance(typ, Instance) + and typ.type.fullname == "builtins._NotImplementedType" + ): + typ = AnyType(TypeOfAny.special_form) if defn.is_async_generator: self.fail(message_registry.RETURN_IN_ASYNC_GENERATOR, s) @@ -4746,6 +4753,14 @@ def type_check_raise(self, e: Expression, s: RaiseStmt, optional: bool = False) # https://github.com/python/mypy/issues/11089 self.expr_checker.check_call(typ, [], [], e) + if isinstance(typ, Instance) and typ.type.fullname == "builtins._NotImplementedType": + self.fail( + message_registry.INVALID_EXCEPTION.with_additional_msg( + '; did you mean "NotImplementedError"?' + ), + s, + ) + def visit_try_stmt(self, s: TryStmt) -> None: """Type check a try statement.""" # Our enclosing frame will get the result if the try/except falls through. diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index d1464423e90f..44880cf35204 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -519,6 +519,13 @@ if object(): raise BaseException from f # E: Exception must be derived from BaseException [builtins fixtures/exception.pyi] +[case testRaiseNotImplementedFails] +if object(): + raise NotImplemented # E: Exception must be derived from BaseException; did you mean "NotImplementedError"? +if object(): + raise NotImplemented() # E: NotImplemented? not callable +[builtins fixtures/notimplemented.pyi] + [case testTryFinallyStatement] import typing try: diff --git a/test-data/unit/fixtures/notimplemented.pyi b/test-data/unit/fixtures/notimplemented.pyi index 2ca376ea0760..92edf84a7fd1 100644 --- a/test-data/unit/fixtures/notimplemented.pyi +++ b/test-data/unit/fixtures/notimplemented.pyi @@ -1,6 +1,5 @@ # builtins stub used in NotImplemented related cases. -from typing import Any, cast - +from typing import Any class object: def __init__(self) -> None: pass @@ -10,5 +9,10 @@ class function: pass class bool: pass class int: pass class str: pass -NotImplemented = cast(Any, None) class dict: pass + +class _NotImplementedType(Any): + __call__: NotImplemented # type: ignore +NotImplemented: _NotImplementedType + +class BaseException: pass From 706a5468f078d27c9783db420755a3ebc86937d3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 9 Oct 2024 15:21:54 +0100 Subject: [PATCH 190/247] [mypyc] Add "runtests.py mypyc-fast" for running fast mypyc tests (#17906) Many tests are a bit slow and but don't increase test coverage by a lot. Running these fast tests should find the vast majority of issues while being significantly faster than running all mypyc tests. --- mypyc/doc/dev-intro.md | 4 ++++ runtests.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mypyc/doc/dev-intro.md b/mypyc/doc/dev-intro.md index d11df7068e91..461a19d37121 100644 --- a/mypyc/doc/dev-intro.md +++ b/mypyc/doc/dev-intro.md @@ -202,6 +202,10 @@ general overview of how things work. Test cases live under -q mypyc`. If you don't make changes to code under `mypy/`, it's not important to regularly run mypy tests during development. +You can use `python runtests.py mypyc-fast` to run a subset of mypyc +tests that covers most functionality but runs significantly quicker +than the entire test suite. + When you create a PR, we have Continuous Integration jobs set up that compile mypy using mypyc and run the mypy test suite using the compiled mypy. This will sometimes catch additional issues not caught diff --git a/runtests.py b/runtests.py index 80ef8d814ee1..9863e8491500 100755 --- a/runtests.py +++ b/runtests.py @@ -17,6 +17,8 @@ MYPYC_RUN_MULTI = "TestRunMultiFile" MYPYC_EXTERNAL = "TestExternal" MYPYC_COMMAND_LINE = "TestCommandLine" +MYPYC_SEPARATE = "TestRunSeparate" +MYPYC_MULTIMODULE = "multimodule" # Subset of mypyc run tests that are slow ERROR_STREAM = "ErrorStreamSuite" @@ -31,6 +33,7 @@ MYPYC_RUN_MULTI, MYPYC_EXTERNAL, MYPYC_COMMAND_LINE, + MYPYC_SEPARATE, ERROR_STREAM, ] @@ -40,7 +43,10 @@ # These must be enabled by explicitly including 'mypyc-extra' on the command line. -MYPYC_OPT_IN = [MYPYC_RUN, MYPYC_RUN_MULTI] +MYPYC_OPT_IN = [MYPYC_RUN, MYPYC_RUN_MULTI, MYPYC_SEPARATE] + +# These mypyc test filters cover most slow test cases +MYPYC_SLOW = [MYPYC_RUN_MULTI, MYPYC_COMMAND_LINE, MYPYC_SEPARATE, MYPYC_MULTIMODULE] # We split the pytest run into three parts to improve test @@ -77,6 +83,7 @@ "-k", " or ".join([DAEMON, MYPYC_EXTERNAL, MYPYC_COMMAND_LINE, ERROR_STREAM]), ], + "mypyc-fast": ["pytest", "-q", "mypyc", "-k", f"not ({' or '.join(MYPYC_SLOW)})"], # Test cases that might take minutes to run "pytest-extra": ["pytest", "-q", "-k", " or ".join(PYTEST_OPT_IN)], # Mypyc tests that aren't run by default, since they are slow and rarely @@ -87,7 +94,7 @@ # Stop run immediately if these commands fail FAST_FAIL = ["self", "lint"] -EXTRA_COMMANDS = ("pytest-extra", "mypyc-extra") +EXTRA_COMMANDS = ("pytest-extra", "mypyc-fast", "mypyc-extra") DEFAULT_COMMANDS = [cmd for cmd in cmds if cmd not in EXTRA_COMMANDS] assert all(cmd in cmds for cmd in FAST_FAIL) From eca206d3c96bf4082a0a087c2614deb5af8c4afd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 9 Oct 2024 19:52:47 +0100 Subject: [PATCH 191/247] Fix union callees with functools.partial (#17903) Fixes #17741. --- mypy/plugins/functools.py | 15 ++++++++++++++- test-data/unit/check-functools.test | 21 +++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index 6650af637519..f09ea88f7162 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -18,6 +18,7 @@ Type, TypeOfAny, UnboundType, + UnionType, get_proper_type, ) @@ -130,7 +131,19 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: if isinstance(get_proper_type(ctx.arg_types[0][0]), Overloaded): # TODO: handle overloads, just fall back to whatever the non-plugin code does return ctx.default_return_type - fn_type = ctx.api.extract_callable_type(ctx.arg_types[0][0], ctx=ctx.default_return_type) + return handle_partial_with_callee(ctx, callee=ctx.arg_types[0][0]) + + +def handle_partial_with_callee(ctx: mypy.plugin.FunctionContext, callee: Type) -> Type: + if not isinstance(ctx.api, mypy.checker.TypeChecker): # use internals + return ctx.default_return_type + + if isinstance(callee_proper := get_proper_type(callee), UnionType): + return UnionType.make_union( + [handle_partial_with_callee(ctx, item) for item in callee_proper.items] + ) + + fn_type = ctx.api.extract_callable_type(callee, ctx=ctx.default_return_type) if fn_type is None: return ctx.default_return_type diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index 50de3789ebd2..bee30931a92b 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -346,15 +346,32 @@ fn1: Union[Callable[[int], int], Callable[[int], int]] reveal_type(functools.partial(fn1, 2)()) # N: Revealed type is "builtins.int" fn2: Union[Callable[[int], int], Callable[[int], str]] -reveal_type(functools.partial(fn2, 2)()) # N: Revealed type is "builtins.object" +reveal_type(functools.partial(fn2, 2)()) # N: Revealed type is "Union[builtins.int, builtins.str]" fn3: Union[Callable[[int], int], str] reveal_type(functools.partial(fn3, 2)()) # E: "str" not callable \ - # E: "Union[Callable[[int], int], str]" not callable \ # N: Revealed type is "builtins.int" \ # E: Argument 1 to "partial" has incompatible type "Union[Callable[[int], int], str]"; expected "Callable[..., int]" [builtins fixtures/tuple.pyi] +[case testFunctoolsPartialUnionOfTypeAndCallable] +import functools +from typing import Callable, Union, Type +from typing_extensions import TypeAlias + +class FooBar: + def __init__(self, arg1: str) -> None: + pass + +def f1(t: Union[Type[FooBar], Callable[..., 'FooBar']]) -> None: + val = functools.partial(t) + +FooBarFunc: TypeAlias = Callable[..., 'FooBar'] + +def f2(t: Union[Type[FooBar], FooBarFunc]) -> None: + val = functools.partial(t) +[builtins fixtures/tuple.pyi] + [case testFunctoolsPartialExplicitType] from functools import partial from typing import Type, TypeVar, Callable From 33717e5ae0805076a98c1a9f2c45b032299c3f90 Mon Sep 17 00:00:00 2001 From: Joe Gordon Date: Thu, 10 Oct 2024 15:40:13 -0700 Subject: [PATCH 192/247] Better error for `mypy -p package` without py.typed (#17908) Improve the error message when running `mypy -p packagename` with a package that doesn't have py.typed set. > package 'example' is skipping due to missing py.typed marker. See https://mypy.readthedocs.io/en/stable/installed_packages.html for more details Fix #17048 --------- Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- mypy/main.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 0674e3b7e79b..3f25ced16106 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -22,7 +22,14 @@ from mypy.errors import CompileError from mypy.find_sources import InvalidSourceList, create_source_list from mypy.fscache import FileSystemCache -from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, get_search_dirs, mypy_path +from mypy.modulefinder import ( + BuildSource, + FindModuleCache, + ModuleNotFoundReason, + SearchPaths, + get_search_dirs, + mypy_path, +) from mypy.options import INCOMPLETE_FEATURES, BuildType, Options from mypy.split_namespace import SplitNamespace from mypy.version import __version__ @@ -1413,7 +1420,15 @@ def set_strict_flags() -> None: fail(f"Package name '{p}' cannot have a slash in it.", stderr, options) p_targets = cache.find_modules_recursive(p) if not p_targets: - fail(f"Can't find package '{p}'", stderr, options) + reason = cache.find_module(p) + if reason is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS: + fail( + f"Package '{p}' cannot be type checked due to missing py.typed marker. See https://mypy.readthedocs.io/en/stable/installed_packages.html for more details", + stderr, + options, + ) + else: + fail(f"Can't find package '{p}'", stderr, options) targets.extend(p_targets) for m in special_opts.modules: targets.append(BuildSource(None, m, None)) From bd9200bda5595fc71c01fe0dff9debbce3467a84 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 10 Oct 2024 20:02:27 -0700 Subject: [PATCH 193/247] Remove debug build for now (#17917) It's started failing on master due to agent changes and also it never tested debug builds in the first place, see issues linked https://github.com/python/mypy/actions/runs/11283075094/job/31381791947 --- .github/workflows/test.yml | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 296f6fde1523..6ec8954bbe3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,7 +70,6 @@ jobs: toxenv: py tox_extra_args: "-n 4" test_mypyc: true - - name: Test suite with py313-ubuntu, mypyc-compiled python: '3.13' arch: x64 @@ -78,6 +77,7 @@ jobs: toxenv: py tox_extra_args: "-n 4" test_mypyc: true + # - name: Test suite with py314-dev-ubuntu # python: '3.14-dev' # arch: x64 @@ -94,13 +94,16 @@ jobs: os: macos-13 toxenv: py tox_extra_args: "-n 3 mypyc/test/test_run.py mypyc/test/test_external.py" - - name: mypyc runtime tests with py38-debug-build-ubuntu - python: '3.8.17' - arch: x64 - os: ubuntu-latest - toxenv: py - tox_extra_args: "-n 4 mypyc/test/test_run.py mypyc/test/test_external.py" - debug_build: true + # This is broken. See + # - https://github.com/python/mypy/issues/17819 + # - https://github.com/python/mypy/pull/17822 + # - name: mypyc runtime tests with py38-debug-build-ubuntu + # python: '3.8.17' + # arch: x64 + # os: ubuntu-latest + # toxenv: py + # tox_extra_args: "-n 4 mypyc/test/test_run.py mypyc/test/test_external.py" + # debug_build: true - name: Type check our own code (py38-ubuntu) python: '3.8' @@ -148,17 +151,17 @@ jobs: ./misc/build-debug-python.sh $PYTHONVERSION $PYTHONDIR $VENV # TODO: does this do anything? env vars aren't passed to the next step right source $VENV/bin/activate - - name: Latest Dev build + - name: Latest dev build if: ${{ endsWith(matrix.python, '-dev') }} run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - build-essential gdb lcov libbz2-dev libffi-dev libgdbm-dev liblzma-dev libncurses5-dev \ - libreadline6-dev libsqlite3-dev libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g-dev git clone --depth 1 https://github.com/python/cpython.git /tmp/cpython --branch $( echo ${{ matrix.python }} | sed 's/-dev//' ) cd /tmp/cpython echo git rev-parse HEAD; git rev-parse HEAD git show --no-patch + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential gdb lcov libbz2-dev libffi-dev libgdbm-dev liblzma-dev libncurses5-dev \ + libreadline6-dev libsqlite3-dev libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g-dev ./configure --prefix=/opt/pythondev make -j$(nproc) sudo make install @@ -190,7 +193,7 @@ jobs: - name: Setup tox environment run: | - tox run -e ${{ matrix.toxenv }} --notes + tox run -e ${{ matrix.toxenv }} --notest - name: Test run: tox run -e ${{ matrix.toxenv }} --skip-pkg-install -- ${{ matrix.tox_extra_args }} continue-on-error: ${{ matrix.allow_failure == 'true' }} From 46c108e35c215aef53b638b1f9e34e4665759ec4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Oct 2024 08:47:26 +0100 Subject: [PATCH 194/247] Add latest 1.12 changes to changelog (#17921) These were cherry-picked to the release branch. --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c664121f097d..ded92b58daaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -187,6 +187,8 @@ List of documentation updates: * Make changelog visible in mypy documentation (quinn-sasha, PR [17742](https://github.com/python/mypy/pull/17742)) * List all incomplete features in `--enable-incomplete-feature` docs (sobolevn, PR [17633](https://github.com/python/mypy/pull/17633)) * Remove the explicit setting of a pygments theme (Pradyun Gedam, PR [17571](https://github.com/python/mypy/pull/17571)) + * Document ReadOnly with TypedDict (Jukka Lehtosalo, PR [17905](https://github.com/python/mypy/pull/17905)) + * Document TypeIs (Chelsea Durazo, PR [17821](https://github.com/python/mypy/pull/17821)) ### Experimental Inline TypedDict Syntax @@ -250,6 +252,9 @@ This feature was contributed by Ivan Levkivskyi (PR [17457](https://github.com/p * Fix typechecking for async generators (Danny Yang, PR [17452](https://github.com/python/mypy/pull/17452)) * Fix strict optional handling in attrs plugin (Ivan Levkivskyi, PR [17451](https://github.com/python/mypy/pull/17451)) * Allow mixing ParamSpec and TypeVarTuple in Generic (Ivan Levkivskyi, PR [17450](https://github.com/python/mypy/pull/17450)) + * Improvements to `functools.partial` of types (Shantanu, PR [17898](https://github.com/python/mypy/pull/17898)) + * Make ReadOnly TypedDict items covariant (Jukka Lehtosalo, PR [17904](https://github.com/python/mypy/pull/17904)) + * Fix union callees with `functools.partial` (Jukka Lehtosalo, PR [17903](https://github.com/python/mypy/pull/17903)) ### Typeshed Updates @@ -263,6 +268,7 @@ Thanks to all mypy contributors who contributed to this release: - Bénédikt Tran - Brian Schubert - bzoracler +- Chelsea Durazo - Danny Yang - Edgar Ramírez Mondragón - Eric Mark Martin From 54f495421e3aaf7d3fede9ff899ce78c1cd6cf66 Mon Sep 17 00:00:00 2001 From: Max Chang <62986841+changhoetyng@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:15:58 +0100 Subject: [PATCH 195/247] [PEP 695] Fix multiple nested classes don't work (#17820) This PR modifies the `lookup_fully_qualified_or_none` method to support multiple nested classes. Fixes #17780 --- mypy/semanal.py | 50 ++++++++++---- test-data/unit/check-python312.test | 76 ++++++++++++++++++++++ test-data/unit/fine-grained-python312.test | 20 ++++++ 3 files changed, 135 insertions(+), 11 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 37aafc3b2647..95efe2b0f30c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -6461,18 +6461,46 @@ def lookup_fully_qualified_or_none(self, fullname: str) -> SymbolTableNode | Non Note that this can't be used for names nested in class namespaces. """ # TODO: unify/clean-up/simplify lookup methods, see #4157. - # TODO: support nested classes (but consider performance impact, - # we might keep the module level only lookup for thing like 'builtins.int'). - assert "." in fullname module, name = fullname.rsplit(".", maxsplit=1) - if module not in self.modules: - return None - filenode = self.modules[module] - result = filenode.names.get(name) - if result is None and self.is_incomplete_namespace(module): - # TODO: More explicit handling of incomplete refs? - self.record_incomplete_ref() - return result + + if module in self.modules: + # If the module exists, look up the name in the module. + # This is the common case. + filenode = self.modules[module] + result = filenode.names.get(name) + if result is None and self.is_incomplete_namespace(module): + # TODO: More explicit handling of incomplete refs? + self.record_incomplete_ref() + return result + else: + # Else, try to find the longest prefix of the module name that is in the modules dictionary. + splitted_modules = fullname.split(".") + names = [] + + while splitted_modules and ".".join(splitted_modules) not in self.modules: + names.append(splitted_modules.pop()) + + if not splitted_modules or not names: + # If no module or name is found, return None. + return None + + # Reverse the names list to get the correct order of names. + names.reverse() + + module = ".".join(splitted_modules) + filenode = self.modules[module] + result = filenode.names.get(names[0]) + + if result is None and self.is_incomplete_namespace(module): + # TODO: More explicit handling of incomplete refs? + self.record_incomplete_ref() + + for part in names[1:]: + if result is not None and isinstance(result.node, TypeInfo): + result = result.node.names.get(part) + else: + return None + return result def object_type(self) -> Instance: return self.named_type("builtins.object") diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 085cc052705d..c5c8ada1aae1 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1873,3 +1873,79 @@ d1: Multi[int, str] = Multi[float, str]() # E: Incompatible types in assignment d2: Multi[float, str] = Multi[int, str]() # E: Incompatible types in assignment (expression has type "Multi[int, str]", variable has type "Multi[float, str]") d3: Multi[str, int] = Multi[str, float]() d4: Multi[str, float] = Multi[str, int]() # E: Incompatible types in assignment (expression has type "Multi[str, int]", variable has type "Multi[str, float]") + +[case testPEP695MultipleNestedGenericClass1] +# flags: --enable-incomplete-feature=NewGenericSyntax +class A: + class B: + class C: + class D[Q]: + def g(self, x: Q): ... + d: D[str] + +x: A.B.C.D[int] +x.g('a') # E: Argument 1 to "g" of "D" has incompatible type "str"; expected "int" +reveal_type(x) # N: Revealed type is "__main__.A.B.C.D[builtins.int]" +reveal_type(A.B.C.d) # N: Revealed type is "__main__.A.B.C.D[builtins.str]" + +[case testPEP695MultipleNestedGenericClass2] +# flags: --enable-incomplete-feature=NewGenericSyntax +class A: + class B: + def m(self) -> None: + class C[T]: + def f(self) -> T: ... + x: C[int] + reveal_type(x.f()) # N: Revealed type is "builtins.int" + self.a = C[str]() + +reveal_type(A().B().a) # N: Revealed type is "__main__.C@5[builtins.str]" + +[case testPEP695MultipleNestedGenericClass3] +# flags: --enable-incomplete-feature=NewGenericSyntax +class A: + class C[T]: + def f(self) -> T: ... + class D[S]: + x: T # E: Name "T" is not defined + def g(self) -> S: ... + +a: A.C[int] +reveal_type(a.f()) # N: Revealed type is "builtins.int" +b: A.C.D[str] +reveal_type(b.g()) # N: Revealed type is "builtins.str" + +class B: + class E[T]: + class F[T]: # E: "T" already defined as a type parameter + x: T + +c: B.E.F[int] + +[case testPEP695MultipleNestedGenericClass4] +# flags: --enable-incomplete-feature=NewGenericSyntax +class Z: + class A: + class B[T]: + def __get__(self, instance: Z.A, owner: type[Z.A]) -> T: + return None # E: Incompatible return value type (got "None", expected "T") + f = B[int]() + +a = Z.A() +v = a.f + +[case testPEP695MultipleNestedGenericClass5] +# flags: --enable-incomplete-feature=NewGenericSyntax +from a.b.c import d +x: d.D.E.F.G[int] +x.g('a') # E: Argument 1 to "g" of "G" has incompatible type "str"; expected "int" +reveal_type(x) # N: Revealed type is "a.b.c.d.D.E.F.G[builtins.int]" +reveal_type(d.D.E.F.d) # N: Revealed type is "a.b.c.d.D.E.F.G[builtins.str]" + +[file a/b/c/d.py] +class D: + class E: + class F: + class G[Q]: + def g(self, x: Q): ... + d: G[str] diff --git a/test-data/unit/fine-grained-python312.test b/test-data/unit/fine-grained-python312.test index 0e438ca06574..2cb2148a66fe 100644 --- a/test-data/unit/fine-grained-python312.test +++ b/test-data/unit/fine-grained-python312.test @@ -95,3 +95,23 @@ def f(x: int) -> None: pass [out] == main:7: error: Missing positional argument "x" in call to "f" + +[case testPEP695MultipleNestedGenericClassMethodUpdated] +from a import f + +class A: + class C: + class D[T]: + x: T + def m(self) -> T: + f() + return self.x + +[file a.py] +def f() -> None: pass + +[file a.py.2] +def f(x: int) -> None: pass +[out] +== +main:8: error: Missing positional argument "x" in call to "f" From b1701e516ad019a8d1dfc463541a0fa850221543 Mon Sep 17 00:00:00 2001 From: Max Chang <62986841+changhoetyng@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:16:20 +0100 Subject: [PATCH 196/247] fix crash issue when using shadowfile with pretty #17853 (#17894) - Fix crash issue when using --pretty with --shadow-file Example Scenario: a.py ```python b: bytes ``` b.py ```python a: int = "" b: bytes = 1 ``` output ``` $ mypy a.py --pretty --shadow-file a.py b.py a.py:1: error: Incompatible types in assignment (expression has type "str", variable has type "int") a: int = "" ^~ a.py:2: error: Incompatible types in assignment (expression has type "int", variable has type "bytes") b: bytes = 1 ^ ``` Fixes #17853 --- mypy/errors.py | 18 +++++++++++++++++- test-data/unit/cmdline.test | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/mypy/errors.py b/mypy/errors.py index 13452b14a237..1b3f485d19c0 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -922,9 +922,25 @@ def file_messages(self, path: str, formatter: ErrorFormatter | None = None) -> l self.flushed_files.add(path) source_lines = None if self.options.pretty and self.read_source: - source_lines = self.read_source(path) + # Find shadow file mapping and read source lines if a shadow file exists for the given path. + # If shadow file mapping is not found, read source lines + mapped_path = self.find_shadow_file_mapping(path) + if mapped_path: + source_lines = self.read_source(mapped_path) + else: + source_lines = self.read_source(path) return self.format_messages(error_tuples, source_lines) + def find_shadow_file_mapping(self, path: str) -> str | None: + """Return the shadow file path for a given source file path or None.""" + if self.options.shadow_file is None: + return None + + for i in self.options.shadow_file: + if i[0] == path: + return i[1] + return None + def new_messages(self) -> list[str]: """Return a string list of new error messages. diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 2262b7e7280c..38ea83cdbcf4 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -914,6 +914,23 @@ s4.py:2: error: Incompatible return value type (got "int", expected "str") s3.py:2: error: Incompatible return value type (got "List[int]", expected "int") s1.py:2: error: Incompatible return value type (got "int", expected "str") +[case testShadowFileWithPretty] +# cmd: mypy a.py --pretty --shadow-file a.py b.py +[file a.py] +b: bytes +[file b.py] +a: int = "" +b: bytes = 1 +[out] +a.py:1: error: Incompatible types in assignment (expression has type "str", +variable has type "int") + a: int = "" + ^~ +a.py:2: error: Incompatible types in assignment (expression has type "int", +variable has type "bytes") + b: bytes = 1 + ^ + [case testConfigWarnUnusedSection1] # cmd: mypy foo.py quux.py spam/eggs.py [file mypy.ini] From 0c10dc3b609f6afc9c6147a70adda54d93bcca86 Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Fri, 11 Oct 2024 23:24:16 +0200 Subject: [PATCH 197/247] PEP 702 (@deprecated): improve the handling of explicit type annotations of assignment statements (#17899) Two improvements of the current PEP 702 implementation (deprecated): * Analyse the explicit type annotations of "normal" assignment statements (that have an rvalue). * Dive into nested type annotations of assignment statements. (I intend to continue working on PEP 702 and would prefer to do so in small steps if this is okay for the maintainers/reviewers.) --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/checker.py | 18 ++++++++++++--- test-data/unit/check-deprecated.test | 33 +++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 3483f1b029c8..ca35144456fe 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -287,6 +287,18 @@ class PartialTypeScope(NamedTuple): is_local: bool +class InstanceDeprecatedVisitor(TypeTraverserVisitor): + """Visitor that recursively checks for deprecations in nested instances.""" + + def __init__(self, typechecker: TypeChecker, context: Context) -> None: + self.typechecker = typechecker + self.context = context + + def visit_instance(self, t: Instance) -> None: + super().visit_instance(t) + self.typechecker.check_deprecated(t.type, self.context) + + class TypeChecker(NodeVisitor[None], CheckerPluginInterface): """Mypy type checker. @@ -2930,14 +2942,14 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: Handle all kinds of assignment statements (simple, indexed, multiple). """ - if isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs: + if s.unanalyzed_type is not None: for lvalue in s.lvalues: if ( isinstance(lvalue, NameExpr) and isinstance(var := lvalue.node, Var) - and isinstance(instance := get_proper_type(var.type), Instance) + and (var.type is not None) ): - self.check_deprecated(instance.type, s) + var.type.accept(InstanceDeprecatedVisitor(typechecker=self, context=s)) # Avoid type checking type aliases in stubs to avoid false # positives about modern type syntax available in stubs such diff --git a/test-data/unit/check-deprecated.test b/test-data/unit/check-deprecated.test index f587034d8059..13cebc85513e 100644 --- a/test-data/unit/check-deprecated.test +++ b/test-data/unit/check-deprecated.test @@ -102,7 +102,8 @@ def h() -> None: ... [case testDeprecatedClass] -from typing_extensions import deprecated +from typing import Callable, List, Optional, Tuple, Union +from typing_extensions import deprecated, TypeAlias, TypeVar @deprecated("use C2 instead") class C: ... @@ -114,10 +115,40 @@ C.missing() # N: class __main__.C is deprecated: use C2 instead \ C.__init__(c) # N: class __main__.C is deprecated: use C2 instead C(1) # N: class __main__.C is deprecated: use C2 instead \ # E: Too many arguments for "C" + D = C # N: class __main__.C is deprecated: use C2 instead D() t = (C, C, D) # N: class __main__.C is deprecated: use C2 instead +u1: Union[C, int] = 1 # N: class __main__.C is deprecated: use C2 instead +u1 = 1 +u2 = 1 # type: Union[C, int] # N: class __main__.C is deprecated: use C2 instead +u2 = 1 + +c1 = c2 = C() # N: class __main__.C is deprecated: use C2 instead +i, c3 = 1, C() # N: class __main__.C is deprecated: use C2 instead + +class E: ... + +x1: Optional[C] # N: class __main__.C is deprecated: use C2 instead +x2: Union[D, C, E] # N: class __main__.C is deprecated: use C2 instead +x3: Union[D, Optional[C], E] # N: class __main__.C is deprecated: use C2 instead +x4: Tuple[D, C, E] # N: class __main__.C is deprecated: use C2 instead +x5: Tuple[Tuple[D, C], E] # N: class __main__.C is deprecated: use C2 instead +x6: List[C] # N: class __main__.C is deprecated: use C2 instead +x7: List[List[C]] # N: class __main__.C is deprecated: use C2 instead +x8: List[Optional[Tuple[Union[List[C], int]]]] # N: class __main__.C is deprecated: use C2 instead +x9: Callable[[int], C] # N: class __main__.C is deprecated: use C2 instead +x10: Callable[[int, C, int], int] # N: class __main__.C is deprecated: use C2 instead + +T = TypeVar("T") +A1: TypeAlias = Optional[C] # ToDo +x11: A1 +A2: TypeAlias = List[Union[A2, C]] # ToDo +x12: A2 +A3: TypeAlias = List[Optional[T]] +x13: A3[C] # N: class __main__.C is deprecated: use C2 instead + [builtins fixtures/tuple.pyi] From ca97d9667da1cf0bf22886ecfc2280958b56ac68 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 14 Oct 2024 02:42:11 +0200 Subject: [PATCH 198/247] [mypyc] Sync pythoncapi_compat.h (#17929) Use functions added for Python 3.14 * `PyBytes_Join` added in https://github.com/python/cpython/issues/121645 * `PyUnicode_Equal` added in https://github.com/python/cpython/issues/124502 --- mypyc/lib-rt/bytes_ops.c | 2 +- mypyc/lib-rt/getargsfast.c | 9 +- mypyc/lib-rt/pythoncapi_compat.h | 340 ++++++++++++++++++++++++++++++- mypyc/lib-rt/pythonsupport.h | 5 +- 4 files changed, 341 insertions(+), 15 deletions(-) diff --git a/mypyc/lib-rt/bytes_ops.c b/mypyc/lib-rt/bytes_ops.c index 4da62be11571..0cb2f300d507 100644 --- a/mypyc/lib-rt/bytes_ops.c +++ b/mypyc/lib-rt/bytes_ops.c @@ -99,7 +99,7 @@ PyObject *CPyBytes_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end) { // (mostly commonly, for bytearrays) PyObject *CPyBytes_Join(PyObject *sep, PyObject *iter) { if (PyBytes_CheckExact(sep)) { - return _PyBytes_Join(sep, iter); + return PyBytes_Join(sep, iter); } else { _Py_IDENTIFIER(join); return _PyObject_CallMethodIdOneArg(sep, &PyId_join, iter); diff --git a/mypyc/lib-rt/getargsfast.c b/mypyc/lib-rt/getargsfast.c index 62d0dfed0a6d..e5667e22efe3 100644 --- a/mypyc/lib-rt/getargsfast.c +++ b/mypyc/lib-rt/getargsfast.c @@ -271,16 +271,9 @@ find_keyword(PyObject *kwnames, PyObject *const *kwstack, PyObject *key) for (i = 0; i < nkwargs; i++) { PyObject *kwname = PyTuple_GET_ITEM(kwnames, i); assert(PyUnicode_Check(kwname)); -#if CPY_3_13_FEATURES - if (_PyUnicode_Equal(kwname, key)) { + if (PyUnicode_Equal(kwname, key)) { return kwstack[i]; } -#else - if (_PyUnicode_EQ(kwname, key)) { - return kwstack[i]; - } -#endif - } return NULL; } diff --git a/mypyc/lib-rt/pythoncapi_compat.h b/mypyc/lib-rt/pythoncapi_compat.h index 1b59f93de7ec..34a84c969b6a 100644 --- a/mypyc/lib-rt/pythoncapi_compat.h +++ b/mypyc/lib-rt/pythoncapi_compat.h @@ -45,6 +45,13 @@ extern "C" { # define _PyObject_CAST(op) _Py_CAST(PyObject*, op) #endif +#ifndef Py_BUILD_ASSERT +# define Py_BUILD_ASSERT(cond) \ + do { \ + (void)sizeof(char [1 - 2 * !(cond)]); \ + } while(0) +#endif + // bpo-42262 added Py_NewRef() to Python 3.10.0a3 #if PY_VERSION_HEX < 0x030A00A3 && !defined(Py_NewRef) @@ -1338,9 +1345,169 @@ PyDict_SetDefaultRef(PyObject *d, PyObject *key, PyObject *default_value, } #endif +#if PY_VERSION_HEX < 0x030D00B3 +# define Py_BEGIN_CRITICAL_SECTION(op) { +# define Py_END_CRITICAL_SECTION() } +# define Py_BEGIN_CRITICAL_SECTION2(a, b) { +# define Py_END_CRITICAL_SECTION2() } +#endif + +#if PY_VERSION_HEX < 0x030E0000 && PY_VERSION_HEX >= 0x03060000 && !defined(PYPY_VERSION) +typedef struct PyUnicodeWriter PyUnicodeWriter; + +static inline void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) +{ + _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); + PyMem_Free(writer); +} + +static inline PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length) +{ + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length must be positive"); + return NULL; + } + + const size_t size = sizeof(_PyUnicodeWriter); + PyUnicodeWriter *pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + if (pub_writer == _Py_NULL) { + PyErr_NoMemory(); + return _Py_NULL; + } + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + _PyUnicodeWriter_Init(writer); + if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { + PyUnicodeWriter_Discard(pub_writer); + return NULL; + } + writer->overallocate = 1; + return pub_writer; +} + +static inline PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) +{ + PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); + assert(((_PyUnicodeWriter*)writer)->buffer == NULL); + PyMem_Free(writer); + return str; +} + +static inline int +PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) +{ + if (ch > 0x10ffff) { + PyErr_SetString(PyExc_ValueError, + "character must be in range(0x110000)"); + return -1; + } + + return _PyUnicodeWriter_WriteChar((_PyUnicodeWriter*)writer, ch); +} + +static inline int +PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Str(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + +static inline int +PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyObject *str = PyObject_Repr(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + +static inline int +PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, + const char *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)strlen(str); + } + + PyObject *str_obj = PyUnicode_FromStringAndSize(str, size); + if (str_obj == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str_obj); + Py_DECREF(str_obj); + return res; +} + +static inline int +PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, + const wchar_t *str, Py_ssize_t size) +{ + if (size < 0) { + size = (Py_ssize_t)wcslen(str); + } + + PyObject *str_obj = PyUnicode_FromWideChar(str, size); + if (str_obj == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str_obj); + Py_DECREF(str_obj); + return res; +} + +static inline int +PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + if (!PyUnicode_Check(str)) { + PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + return -1; + } + if (start < 0 || start > end) { + PyErr_Format(PyExc_ValueError, "invalid start argument"); + return -1; + } + if (end > PyUnicode_GET_LENGTH(str)) { + PyErr_Format(PyExc_ValueError, "invalid end argument"); + return -1; + } + + return _PyUnicodeWriter_WriteSubstring((_PyUnicodeWriter*)writer, str, + start, end); +} + +static inline int +PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + PyObject *str = PyUnicode_FromFormatV(format, vargs); + va_end(vargs); + if (str == _Py_NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} +#endif // PY_VERSION_HEX < 0x030E0000 -// gh-116560 added PyLong_GetSign() to Python 3.14a4 -#if PY_VERSION_HEX < 0x030E00A1 +// gh-116560 added PyLong_GetSign() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 static inline int PyLong_GetSign(PyObject *obj, int *sign) { if (!PyLong_Check(obj)) { @@ -1354,6 +1521,175 @@ static inline int PyLong_GetSign(PyObject *obj, int *sign) #endif +// gh-124502 added PyUnicode_Equal() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 +static inline int PyUnicode_Equal(PyObject *str1, PyObject *str2) +{ + if (!PyUnicode_Check(str1)) { + PyErr_Format(PyExc_TypeError, "first argument must be str, not %s", + Py_TYPE(str1)->tp_name); + return -1; + } + if (!PyUnicode_Check(str2)) { + PyErr_Format(PyExc_TypeError, "second argument must be str, not %s", + Py_TYPE(str2)->tp_name); + return -1; + } + +#if PY_VERSION_HEX >= 0x030d0000 && !defined(PYPY_VERSION) + extern int _PyUnicode_Equal(PyObject *str1, PyObject *str2); + + return _PyUnicode_Equal(str1, str2); +#elif PY_VERSION_HEX >= 0x03060000 && !defined(PYPY_VERSION) + return _PyUnicode_EQ(str1, str2); +#elif PY_VERSION_HEX >= 0x03090000 && defined(PYPY_VERSION) + return _PyUnicode_EQ(str1, str2); +#else + return (PyUnicode_Compare(str1, str2) == 0); +#endif +} +#endif + + +// gh-121645 added PyBytes_Join() to Python 3.14.0a0 +#if PY_VERSION_HEX < 0x030E00A0 +static inline PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) +{ + return _PyBytes_Join(sep, iterable); +} +#endif + + +#if PY_VERSION_HEX < 0x030E00A0 +static inline Py_hash_t Py_HashBuffer(const void *ptr, Py_ssize_t len) +{ +#if PY_VERSION_HEX >= 0x03000000 && !defined(PYPY_VERSION) + extern Py_hash_t _Py_HashBytes(const void *src, Py_ssize_t len); + + return _Py_HashBytes(ptr, len); +#else + Py_hash_t hash; + PyObject *bytes = PyBytes_FromStringAndSize((const char*)ptr, len); + if (bytes == NULL) { + return -1; + } + hash = PyObject_Hash(bytes); + Py_DECREF(bytes); + return hash; +#endif +} +#endif + + +#if PY_VERSION_HEX < 0x030E00A0 +static inline int PyIter_NextItem(PyObject *iter, PyObject **item) +{ + iternextfunc tp_iternext; + + assert(iter != NULL); + assert(item != NULL); + + tp_iternext = Py_TYPE(iter)->tp_iternext; + if (tp_iternext == NULL) { + *item = NULL; + PyErr_Format(PyExc_TypeError, "expected an iterator, got '%s'", + Py_TYPE(iter)->tp_name); + return -1; + } + + if ((*item = tp_iternext(iter))) { + return 1; + } + if (!PyErr_Occurred()) { + return 0; + } + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Clear(); + return 0; + } + return -1; +} +#endif + + +#if PY_VERSION_HEX < 0x030E00A0 +static inline PyObject* PyLong_FromInt32(int32_t value) +{ + Py_BUILD_ASSERT(sizeof(long) >= 4); + return PyLong_FromLong(value); +} + +static inline PyObject* PyLong_FromInt64(int64_t value) +{ + Py_BUILD_ASSERT(sizeof(long long) >= 8); + return PyLong_FromLongLong(value); +} + +static inline PyObject* PyLong_FromUInt32(uint32_t value) +{ + Py_BUILD_ASSERT(sizeof(unsigned long) >= 4); + return PyLong_FromUnsignedLong(value); +} + +static inline PyObject* PyLong_FromUInt64(uint64_t value) +{ + Py_BUILD_ASSERT(sizeof(unsigned long long) >= 8); + return PyLong_FromUnsignedLongLong(value); +} + +static inline int PyLong_AsInt32(PyObject *obj, int32_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(int) == 4); + int value = PyLong_AsInt(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + *pvalue = (int32_t)value; + return 0; +} + +static inline int PyLong_AsInt64(PyObject *obj, int64_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(long long) == 8); + long long value = PyLong_AsLongLong(obj); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + *pvalue = (int64_t)value; + return 0; +} + +static inline int PyLong_AsUInt32(PyObject *obj, uint32_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(long) >= 4); + unsigned long value = PyLong_AsUnsignedLong(obj); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return -1; + } +#if SIZEOF_LONG > 4 + if ((unsigned long)UINT32_MAX < value) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C uint32_t"); + return -1; + } +#endif + *pvalue = (uint32_t)value; + return 0; +} + +static inline int PyLong_AsUInt64(PyObject *obj, uint64_t *pvalue) +{ + Py_BUILD_ASSERT(sizeof(long long) == 8); + unsigned long long value = PyLong_AsUnsignedLongLong(obj); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return -1; + } + *pvalue = (uint64_t)value; + return 0; +} +#endif + + #ifdef __cplusplus } #endif diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index bf7e5203758d..5d595dba1a12 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -17,13 +17,10 @@ #ifndef Py_BUILD_CORE #define Py_BUILD_CORE #endif -#include "internal/pycore_bytesobject.h" // _PyBytes_Join -#include "internal/pycore_call.h" // _PyObject_CallMethodIdNoArgs, _PyObject_CallMethodIdObjArgs, _PyObject_CallMethodIdOneArg +#include "internal/pycore_call.h" // _PyObject_CallMethodIdNoArgs, _PyObject_CallMethodIdOneArg #include "internal/pycore_genobject.h" // _PyGen_FetchStopIterationValue -#include "internal/pycore_object.h" // _PyType_CalculateMetaclass #include "internal/pycore_pyerrors.h" // _PyErr_FormatFromCause, _PyErr_SetKeyError #include "internal/pycore_setobject.h" // _PySet_Update -#include "internal/pycore_unicodeobject.h" // _PyUnicode_EQ, _PyUnicode_FastCopyCharacters #endif #if CPY_3_12_FEATURES From c32d11e60f04a0798292c438186199ccdec0db61 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 13 Oct 2024 17:50:15 -0700 Subject: [PATCH 199/247] Significantly speed up file handling error paths (#17920) This can have a huge overall impact on mypy performance when search paths are long --- mypy/build.py | 17 +++++++------- mypy/fscache.py | 59 +++++++++++++++++++++-------------------------- mypy/fswatcher.py | 13 +++++------ 3 files changed, 40 insertions(+), 49 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 733f0685792e..964da5aac8b0 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -736,8 +736,8 @@ def maybe_swap_for_shadow_path(self, path: str) -> str: shadow_file = self.shadow_equivalence_map.get(path) return shadow_file if shadow_file else path - def get_stat(self, path: str) -> os.stat_result: - return self.fscache.stat(self.maybe_swap_for_shadow_path(path)) + def get_stat(self, path: str) -> os.stat_result | None: + return self.fscache.stat_or_none(self.maybe_swap_for_shadow_path(path)) def getmtime(self, path: str) -> int: """Return a file's mtime; but 0 in bazel mode. @@ -1394,9 +1394,9 @@ def validate_meta( if bazel: # Normalize path under bazel to make sure it isn't absolute path = normpath(path, manager.options) - try: - st = manager.get_stat(path) - except OSError: + + st = manager.get_stat(path) + if st is None: return None if not stat.S_ISDIR(st.st_mode) and not stat.S_ISREG(st.st_mode): manager.log(f"Metadata abandoned for {id}: file or directory {path} does not exist") @@ -1572,10 +1572,9 @@ def write_cache( plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False)) # Obtain and set up metadata - try: - st = manager.get_stat(path) - except OSError as err: - manager.log(f"Cannot get stat for {path}: {err}") + st = manager.get_stat(path) + if st is None: + manager.log(f"Cannot get stat for {path}") # Remove apparently-invalid cache files. # (This is purely an optimization.) for filename in [data_json, meta_json]: diff --git a/mypy/fscache.py b/mypy/fscache.py index 15679ad03e85..8251f4bd9488 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -51,8 +51,8 @@ def set_package_root(self, package_root: list[str]) -> None: def flush(self) -> None: """Start another transaction and empty all caches.""" - self.stat_cache: dict[str, os.stat_result] = {} - self.stat_error_cache: dict[str, OSError] = {} + self.stat_or_none_cache: dict[str, os.stat_result | None] = {} + self.listdir_cache: dict[str, list[str]] = {} self.listdir_error_cache: dict[str, OSError] = {} self.isfile_case_cache: dict[str, bool] = {} @@ -62,24 +62,21 @@ def flush(self) -> None: self.hash_cache: dict[str, str] = {} self.fake_package_cache: set[str] = set() - def stat(self, path: str) -> os.stat_result: - if path in self.stat_cache: - return self.stat_cache[path] - if path in self.stat_error_cache: - raise copy_os_error(self.stat_error_cache[path]) + def stat_or_none(self, path: str) -> os.stat_result | None: + if path in self.stat_or_none_cache: + return self.stat_or_none_cache[path] + + st = None try: st = os.stat(path) - except OSError as err: + except OSError: if self.init_under_package_root(path): try: - return self._fake_init(path) + st = self._fake_init(path) except OSError: pass - # Take a copy to get rid of associated traceback and frame objects. - # Just assigning to __traceback__ doesn't free them. - self.stat_error_cache[path] = copy_os_error(err) - raise err - self.stat_cache[path] = st + + self.stat_or_none_cache[path] = st return st def init_under_package_root(self, path: str) -> bool: @@ -112,9 +109,9 @@ def init_under_package_root(self, path: str) -> bool: if not os.path.basename(dirname).isidentifier(): # Can't put an __init__.py in a place that's not an identifier return False - try: - st = self.stat(dirname) - except OSError: + + st = self.stat_or_none(dirname) + if st is None: return False else: if not stat.S_ISDIR(st.st_mode): @@ -145,7 +142,7 @@ def _fake_init(self, path: str) -> os.stat_result: assert basename == "__init__.py", path assert not os.path.exists(path), path # Not cached! dirname = os.path.normpath(dirname) - st = self.stat(dirname) # May raise OSError + st = os.stat(dirname) # May raise OSError # Get stat result as a list so we can modify it. seq: list[float] = list(st) seq[stat.ST_MODE] = stat.S_IFREG | 0o444 @@ -153,7 +150,6 @@ def _fake_init(self, path: str) -> os.stat_result: seq[stat.ST_NLINK] = 1 seq[stat.ST_SIZE] = 0 st = os.stat_result(seq) - self.stat_cache[path] = st # Make listdir() and read() also pretend this file exists. self.fake_package_cache.add(dirname) return st @@ -181,9 +177,8 @@ def listdir(self, path: str) -> list[str]: return results def isfile(self, path: str) -> bool: - try: - st = self.stat(path) - except OSError: + st = self.stat_or_none(path) + if st is None: return False return stat.S_ISREG(st.st_mode) @@ -248,18 +243,14 @@ def exists_case(self, path: str, prefix: str) -> bool: return res def isdir(self, path: str) -> bool: - try: - st = self.stat(path) - except OSError: + st = self.stat_or_none(path) + if st is None: return False return stat.S_ISDIR(st.st_mode) def exists(self, path: str) -> bool: - try: - self.stat(path) - except FileNotFoundError: - return False - return True + st = self.stat_or_none(path) + return st is not None def read(self, path: str) -> bytes: if path in self.read_cache: @@ -269,7 +260,7 @@ def read(self, path: str) -> bytes: # Need to stat first so that the contents of file are from no # earlier instant than the mtime reported by self.stat(). - self.stat(path) + self.stat_or_none(path) dirname, basename = os.path.split(path) dirname = os.path.normpath(dirname) @@ -294,8 +285,10 @@ def hash_digest(self, path: str) -> str: return self.hash_cache[path] def samefile(self, f1: str, f2: str) -> bool: - s1 = self.stat(f1) - s2 = self.stat(f2) + s1 = self.stat_or_none(f1) + s2 = self.stat_or_none(f2) + if s1 is None or s2 is None: + return False return os.path.samestat(s1, s2) diff --git a/mypy/fswatcher.py b/mypy/fswatcher.py index a574a36a0cc5..97a62ca9f9f7 100644 --- a/mypy/fswatcher.py +++ b/mypy/fswatcher.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os from typing import AbstractSet, Iterable, NamedTuple from mypy.fscache import FileSystemCache @@ -56,8 +57,7 @@ def remove_watched_paths(self, paths: Iterable[str]) -> None: del self._file_data[path] self._paths -= set(paths) - def _update(self, path: str) -> None: - st = self.fs.stat(path) + def _update(self, path: str, st: os.stat_result) -> None: hash_digest = self.fs.hash_digest(path) self._file_data[path] = FileData(st.st_mtime, st.st_size, hash_digest) @@ -65,9 +65,8 @@ def _find_changed(self, paths: Iterable[str]) -> AbstractSet[str]: changed = set() for path in paths: old = self._file_data[path] - try: - st = self.fs.stat(path) - except FileNotFoundError: + st = self.fs.stat_or_none(path) + if st is None: if old is not None: # File was deleted. changed.add(path) @@ -76,13 +75,13 @@ def _find_changed(self, paths: Iterable[str]) -> AbstractSet[str]: if old is None: # File is new. changed.add(path) - self._update(path) + self._update(path, st) # Round mtimes down, to match the mtimes we write to meta files elif st.st_size != old.st_size or int(st.st_mtime) != int(old.st_mtime): # Only look for changes if size or mtime has changed as an # optimization, since calculating hash is expensive. new_hash = self.fs.hash_digest(path) - self._update(path) + self._update(path, st) if st.st_size != old.st_size or new_hash != old.hash: # Changed file. changed.add(path) From 1a074b6b2434ae2f7bbf5f12d3b0db208136dca8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 14 Oct 2024 02:50:03 +0100 Subject: [PATCH 200/247] Better handling of generic functions in partial plugin (#17925) Fixes https://github.com/python/mypy/issues/17411 The fix is that we remove type variables that can never be inferred from the initial `check_call()` call. Actual diff is tiny, I just moved a bunch of code, since I need formal to actual mapping sooner now. --- mypy/plugins/functools.py | 62 ++++++++++++++++++----------- test-data/unit/check-functools.test | 31 ++++++++++++++- 2 files changed, 68 insertions(+), 25 deletions(-) diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index f09ea88f7162..4dfeb752b5d2 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -10,6 +10,7 @@ from mypy.argmap import map_actuals_to_formals from mypy.nodes import ARG_POS, ARG_STAR2, ArgKind, Argument, CallExpr, FuncItem, Var from mypy.plugins.common import add_method_to_class +from mypy.typeops import get_all_type_vars from mypy.types import ( AnyType, CallableType, @@ -17,6 +18,7 @@ Overloaded, Type, TypeOfAny, + TypeVarType, UnboundType, UnionType, get_proper_type, @@ -164,21 +166,6 @@ def handle_partial_with_callee(ctx: mypy.plugin.FunctionContext, callee: Type) - ctx.api.type_context[-1] = None wrapped_return = False - defaulted = fn_type.copy_modified( - arg_kinds=[ - ( - ArgKind.ARG_OPT - if k == ArgKind.ARG_POS - else (ArgKind.ARG_NAMED_OPT if k == ArgKind.ARG_NAMED else k) - ) - for k in fn_type.arg_kinds - ], - ret_type=ret_type, - ) - if defaulted.line < 0: - # Make up a line number if we don't have one - defaulted.set_line(ctx.default_return_type) - # Flatten actual to formal mapping, since this is what check_call() expects. actual_args = [] actual_arg_kinds = [] @@ -199,6 +186,43 @@ def handle_partial_with_callee(ctx: mypy.plugin.FunctionContext, callee: Type) - actual_arg_names.append(ctx.arg_names[i][j]) actual_types.append(ctx.arg_types[i][j]) + formal_to_actual = map_actuals_to_formals( + actual_kinds=actual_arg_kinds, + actual_names=actual_arg_names, + formal_kinds=fn_type.arg_kinds, + formal_names=fn_type.arg_names, + actual_arg_type=lambda i: actual_types[i], + ) + + # We need to remove any type variables that appear only in formals that have + # no actuals, to avoid eagerly binding them in check_call() below. + can_infer_ids = set() + for i, arg_type in enumerate(fn_type.arg_types): + if not formal_to_actual[i]: + continue + can_infer_ids.update({tv.id for tv in get_all_type_vars(arg_type)}) + + defaulted = fn_type.copy_modified( + arg_kinds=[ + ( + ArgKind.ARG_OPT + if k == ArgKind.ARG_POS + else (ArgKind.ARG_NAMED_OPT if k == ArgKind.ARG_NAMED else k) + ) + for k in fn_type.arg_kinds + ], + ret_type=ret_type, + variables=[ + tv + for tv in fn_type.variables + # Keep TypeVarTuple/ParamSpec to avoid spurious errors on empty args. + if tv.id in can_infer_ids or not isinstance(tv, TypeVarType) + ], + ) + if defaulted.line < 0: + # Make up a line number if we don't have one + defaulted.set_line(ctx.default_return_type) + # Create a valid context for various ad-hoc inspections in check_call(). call_expr = CallExpr( callee=ctx.args[0][0], @@ -231,14 +255,6 @@ def handle_partial_with_callee(ctx: mypy.plugin.FunctionContext, callee: Type) - return ctx.default_return_type bound = bound.copy_modified(ret_type=ret_type.args[0]) - formal_to_actual = map_actuals_to_formals( - actual_kinds=actual_arg_kinds, - actual_names=actual_arg_names, - formal_kinds=fn_type.arg_kinds, - formal_names=fn_type.arg_names, - actual_arg_type=lambda i: actual_types[i], - ) - partial_kinds = [] partial_types = [] partial_names = [] diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index bee30931a92b..ea98a902d14b 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -575,7 +575,6 @@ def bar(f: S) -> S: return f [builtins fixtures/primitives.pyi] - [case testFunctoolsPartialAbstractType] # flags: --python-version 3.9 from abc import ABC, abstractmethod @@ -597,7 +596,6 @@ def f2() -> None: partial_cls() # E: Cannot instantiate abstract class "A" with abstract attribute "method" [builtins fixtures/tuple.pyi] - [case testFunctoolsPartialSelfType] from functools import partial from typing_extensions import Self @@ -610,3 +608,32 @@ class A: factory = partial(cls, ts=0) return factory(msg=msg) [builtins fixtures/tuple.pyi] + +[case testFunctoolsPartialTypeVarValues] +from functools import partial +from typing import TypeVar + +T = TypeVar("T", int, str) + +def f(x: int, y: T) -> T: + return y + +def g(x: T, y: int) -> T: + return x + +def h(x: T, y: T) -> T: + return x + +fp = partial(f, 1) +reveal_type(fp(1)) # N: Revealed type is "builtins.int" +reveal_type(fp("a")) # N: Revealed type is "builtins.str" +fp(object()) # E: Value of type variable "T" of "f" cannot be "object" + +gp = partial(g, 1) +reveal_type(gp(1)) # N: Revealed type is "builtins.int" +gp("a") # E: Argument 1 to "g" has incompatible type "str"; expected "int" + +hp = partial(h, 1) +reveal_type(hp(1)) # N: Revealed type is "builtins.int" +hp("a") # E: Argument 1 to "h" has incompatible type "str"; expected "int" +[builtins fixtures/tuple.pyi] From 395108d287f1d143423a830e22b313e405b872b3 Mon Sep 17 00:00:00 2001 From: Newbyte Date: Mon, 14 Oct 2024 10:24:46 +0200 Subject: [PATCH 201/247] [mypyc] Support ellipsis (...) expressions in class bodies (#17923) This can be used to declare concise custom exceptions, e.g. class UnknownReleaseError(ValueError): ... which otherwise probably would be written class UnknownReleaseError(ValueError): pass and is supported by CPython. Closes https://github.com/mypyc/mypyc/issues/1069 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: hauntsaninja --- mypyc/irbuild/classdef.py | 5 ++++- mypyc/test-data/run-classes.test | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 7e0a842b1b41..bcc9594adcb9 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -12,6 +12,7 @@ CallExpr, ClassDef, Decorator, + EllipsisExpr, ExpressionStmt, FuncDef, Lvalue, @@ -145,7 +146,9 @@ def transform_class_def(builder: IRBuilder, cdef: ClassDef) -> None: continue with builder.catch_errors(stmt.line): cls_builder.add_method(get_func_def(stmt)) - elif isinstance(stmt, PassStmt): + elif isinstance(stmt, PassStmt) or ( + isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, EllipsisExpr) + ): continue elif isinstance(stmt, AssignmentStmt): if len(stmt.lvalues) != 1: diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 59617714f7e7..7c2998874f78 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -3,8 +3,13 @@ class Empty: pass def f(e: Empty) -> Empty: return e + +class EmptyEllipsis: ... + +def g(e: EmptyEllipsis) -> EmptyEllipsis: + return e [file driver.py] -from native import Empty, f +from native import Empty, EmptyEllipsis, f, g print(isinstance(Empty, type)) print(Empty) @@ -12,11 +17,22 @@ print(str(Empty())[:20]) e = Empty() print(f(e) is e) + +print(isinstance(EmptyEllipsis, type)) +print(EmptyEllipsis) +print(str(EmptyEllipsis())[:28]) + +e2 = EmptyEllipsis() +print(g(e2) is e2) [out] True + Date: Mon, 14 Oct 2024 03:51:41 -0500 Subject: [PATCH 202/247] [mypyc] Optimize calls to final classes (#17886) Fixes #9612 This change allows to gain more efficiency where classes are annotated with `@final` bypassing entirely the vtable for method calls and property accessors. For example: In ```python @final class Vector: __slots__ = ("_x", "_y") def __init__(self, x: i32, y: i32) -> None: self._x = x self._y = y @property def y(self) -> i32: return self._y def test_vector() -> None: v3 = Vector(1, 2) assert v3.y == 2 ``` The call will produce: ```c ... cpy_r_r6 = CPyDef_Vector___y(cpy_r_r0); ... ``` Instead of: ```c ... cpy_r_r1 = CPY_GET_ATTR(cpy_r_r0, CPyType_Vector, 2, farm_rush___engine___vectors2___VectorObject, int32_t); /* y */ ... ``` (which uses vtable) --- mypyc/codegen/emitclass.py | 1 + mypyc/codegen/emitfunc.py | 42 ++++++++++-------- mypyc/ir/class_ir.py | 10 +++-- mypyc/ir/rtypes.py | 2 +- mypyc/irbuild/ll_builder.py | 4 +- mypyc/irbuild/prepare.py | 6 ++- mypyc/irbuild/util.py | 11 ++++- mypyc/test-data/run-classes.test | 75 ++++++++++++++++++++++++++++++++ 8 files changed, 125 insertions(+), 26 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index ad95a1b0f323..3ab6932546a6 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -571,6 +571,7 @@ def generate_setup_for_class( emitter.emit_line("}") else: emitter.emit_line(f"self->vtable = {vtable_name};") + for i in range(0, len(cl.bitmap_attrs), BITMAP_BITS): field = emitter.bitmap_field(i) emitter.emit_line(f"self->{field} = 0;") diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index d945a28d8481..6088fb06dd32 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -72,6 +72,7 @@ from mypyc.ir.pprint import generate_names_for_ir from mypyc.ir.rtypes import ( RArray, + RInstance, RStruct, RTuple, RType, @@ -362,20 +363,23 @@ def visit_get_attr(self, op: GetAttr) -> None: prefer_method = cl.is_trait and attr_rtype.error_overlap if cl.get_method(op.attr, prefer_method=prefer_method): # Properties are essentially methods, so use vtable access for them. - version = "_TRAIT" if cl.is_trait else "" - self.emit_line( - "%s = CPY_GET_ATTR%s(%s, %s, %d, %s, %s); /* %s */" - % ( - dest, - version, - obj, - self.emitter.type_struct_name(rtype.class_ir), - rtype.getter_index(op.attr), - rtype.struct_name(self.names), - self.ctype(rtype.attr_type(op.attr)), - op.attr, + if cl.is_method_final(op.attr): + self.emit_method_call(f"{dest} = ", op.obj, op.attr, []) + else: + version = "_TRAIT" if cl.is_trait else "" + self.emit_line( + "%s = CPY_GET_ATTR%s(%s, %s, %d, %s, %s); /* %s */" + % ( + dest, + version, + obj, + self.emitter.type_struct_name(rtype.class_ir), + rtype.getter_index(op.attr), + rtype.struct_name(self.names), + self.ctype(rtype.attr_type(op.attr)), + op.attr, + ) ) - ) else: # Otherwise, use direct or offset struct access. attr_expr = self.get_attr_expr(obj, op, decl_cl) @@ -529,11 +533,13 @@ def visit_call(self, op: Call) -> None: def visit_method_call(self, op: MethodCall) -> None: """Call native method.""" dest = self.get_dest_assign(op) - obj = self.reg(op.obj) + self.emit_method_call(dest, op.obj, op.method, op.args) - rtype = op.receiver_type + def emit_method_call(self, dest: str, op_obj: Value, name: str, op_args: list[Value]) -> None: + obj = self.reg(op_obj) + rtype = op_obj.type + assert isinstance(rtype, RInstance) class_ir = rtype.class_ir - name = op.method method = rtype.class_ir.get_method(name) assert method is not None @@ -547,7 +553,7 @@ def visit_method_call(self, op: MethodCall) -> None: if method.decl.kind == FUNC_STATICMETHOD else [f"(PyObject *)Py_TYPE({obj})"] if method.decl.kind == FUNC_CLASSMETHOD else [obj] ) - args = ", ".join(obj_args + [self.reg(arg) for arg in op.args]) + args = ", ".join(obj_args + [self.reg(arg) for arg in op_args]) mtype = native_function_type(method, self.emitter) version = "_TRAIT" if rtype.class_ir.is_trait else "" if is_direct: @@ -567,7 +573,7 @@ def visit_method_call(self, op: MethodCall) -> None: rtype.struct_name(self.names), mtype, args, - op.method, + name, ) ) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index 18f3cbcff987..94bf714b28d4 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -93,6 +93,7 @@ def __init__( is_generated: bool = False, is_abstract: bool = False, is_ext_class: bool = True, + is_final_class: bool = False, ) -> None: self.name = name self.module_name = module_name @@ -100,6 +101,7 @@ def __init__( self.is_generated = is_generated self.is_abstract = is_abstract self.is_ext_class = is_ext_class + self.is_final_class = is_final_class # An augmented class has additional methods separate from what mypyc generates. # Right now the only one is dataclasses. self.is_augmented = False @@ -199,7 +201,8 @@ def __repr__(self) -> str: "ClassIR(" "name={self.name}, module_name={self.module_name}, " "is_trait={self.is_trait}, is_generated={self.is_generated}, " - "is_abstract={self.is_abstract}, is_ext_class={self.is_ext_class}" + "is_abstract={self.is_abstract}, is_ext_class={self.is_ext_class}, " + "is_final_class={self.is_final_class}" ")".format(self=self) ) @@ -248,8 +251,7 @@ def has_method(self, name: str) -> bool: def is_method_final(self, name: str) -> bool: subs = self.subclasses() if subs is None: - # TODO: Look at the final attribute! - return False + return self.is_final_class if self.has_method(name): method_decl = self.method_decl(name) @@ -349,6 +351,7 @@ def serialize(self) -> JsonDict: "is_abstract": self.is_abstract, "is_generated": self.is_generated, "is_augmented": self.is_augmented, + "is_final_class": self.is_final_class, "inherits_python": self.inherits_python, "has_dict": self.has_dict, "allow_interpreted_subclasses": self.allow_interpreted_subclasses, @@ -404,6 +407,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: ir.is_abstract = data["is_abstract"] ir.is_ext_class = data["is_ext_class"] ir.is_augmented = data["is_augmented"] + ir.is_final_class = data["is_final_class"] ir.inherits_python = data["inherits_python"] ir.has_dict = data["has_dict"] ir.allow_interpreted_subclasses = data["allow_interpreted_subclasses"] diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index fecfaee5ef77..53e3cee74e56 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -64,7 +64,7 @@ class RType: @abstractmethod def accept(self, visitor: RTypeVisitor[T]) -> T: - raise NotImplementedError + raise NotImplementedError() def short_name(self) -> str: return short_name(self.name) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 0c9310e6a5ca..c98136ce06d2 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -1889,7 +1889,7 @@ def primitive_op( # Does this primitive map into calling a Python C API # or an internal mypyc C API function? if desc.c_function_name: - # TODO: Generate PrimitiOps here and transform them into CallC + # TODO: Generate PrimitiveOps here and transform them into CallC # ops only later in the lowering pass c_desc = CFunctionDescription( desc.name, @@ -1908,7 +1908,7 @@ def primitive_op( ) return self.call_c(c_desc, args, line, result_type) - # This primitve gets transformed in a lowering pass to + # This primitive gets transformed in a lowering pass to # lower-level IR ops using a custom transform function. coerced = [] diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 29e06439abdd..40a40b79df49 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -81,7 +81,11 @@ def build_type_map( # references even if there are import cycles. for module, cdef in classes: class_ir = ClassIR( - cdef.name, module.fullname, is_trait(cdef), is_abstract=cdef.info.is_abstract + cdef.name, + module.fullname, + is_trait(cdef), + is_abstract=cdef.info.is_abstract, + is_final_class=cdef.info.is_final, ) class_ir.is_ext_class = is_extension_class(cdef) if class_ir.is_ext_class: diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index ed01a59d1214..e27e509ad7fa 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -27,10 +27,16 @@ UnaryExpr, Var, ) +from mypy.semanal import refers_to_fullname +from mypy.types import FINAL_DECORATOR_NAMES DATACLASS_DECORATORS = {"dataclasses.dataclass", "attr.s", "attr.attrs"} +def is_final_decorator(d: Expression) -> bool: + return refers_to_fullname(d, FINAL_DECORATOR_NAMES) + + def is_trait_decorator(d: Expression) -> bool: return isinstance(d, RefExpr) and d.fullname == "mypy_extensions.trait" @@ -119,7 +125,10 @@ def get_mypyc_attrs(stmt: ClassDef | Decorator) -> dict[str, Any]: def is_extension_class(cdef: ClassDef) -> bool: if any( - not is_trait_decorator(d) and not is_dataclass_decorator(d) and not get_mypyc_attr_call(d) + not is_trait_decorator(d) + and not is_dataclass_decorator(d) + and not get_mypyc_attr_call(d) + and not is_final_decorator(d) for d in cdef.decorators ): return False diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 7c2998874f78..49eb3028c9ee 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -2519,3 +2519,78 @@ class C: def test_final_attribute() -> None: assert C.A == -1 assert C.a == [-1] + +[case testClassWithFinalDecorator] +from typing import final + +@final +class C: + def a(self) -> int: + return 1 + +def test_class_final_attribute() -> None: + assert C().a() == 1 + + +[case testClassWithFinalDecoratorCtor] +from typing import final + +@final +class C: + def __init__(self) -> None: + self.a = 1 + + def b(self) -> int: + return 2 + + @property + def c(self) -> int: + return 3 + +def test_class_final_attribute() -> None: + assert C().a == 1 + assert C().b() == 2 + assert C().c == 3 + +[case testClassWithFinalDecoratorInheritedWithProperties] +from typing import final + +class B: + def a(self) -> int: + return 2 + + @property + def b(self) -> int: + return self.a() + 2 + + @property + def c(self) -> int: + return 3 + +def test_class_final_attribute_basic() -> None: + assert B().a() == 2 + assert B().b == 4 + assert B().c == 3 + +@final +class C(B): + def a(self) -> int: + return 1 + + @property + def b(self) -> int: + return self.a() + 1 + +def fn(cl: B) -> int: + return cl.a() + +def test_class_final_attribute_inherited() -> None: + assert C().a() == 1 + assert fn(C()) == 1 + assert B().a() == 2 + assert fn(B()) == 2 + + assert B().b == 4 + assert C().b == 2 + assert B().c == 3 + assert C().c == 3 From 706680f9cadfebe77208f20be965e2714e2b39df Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 14 Oct 2024 10:26:32 +0100 Subject: [PATCH 203/247] Add one more 1.12 changelog item (#17936) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ded92b58daaf..dfefc690195c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -255,6 +255,7 @@ This feature was contributed by Ivan Levkivskyi (PR [17457](https://github.com/p * Improvements to `functools.partial` of types (Shantanu, PR [17898](https://github.com/python/mypy/pull/17898)) * Make ReadOnly TypedDict items covariant (Jukka Lehtosalo, PR [17904](https://github.com/python/mypy/pull/17904)) * Fix union callees with `functools.partial` (Jukka Lehtosalo, PR [17903](https://github.com/python/mypy/pull/17903)) + * Improve handling of generic functions with `functools.partial` (Ivan Levkivskyi, PR [17925](https://github.com/python/mypy/pull/17925)) ### Typeshed Updates From e6ced4866e825bea01d9158c96b12376defd408e Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:07:23 +0200 Subject: [PATCH 204/247] [mypyc] Use PyGen_GetCode in gen_is_coroutine (#17931) Instead of copying the implementation of `_PyGen_GetCode` every time it changes in cpython, use the public `PyGen_GetCode` function. The current implementation would break for Python 3.14 as it has been changed upstream in https://github.com/python/cpython/pull/120835. --- mypyc/lib-rt/pythonsupport.h | 34 ++-------------------------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 5d595dba1a12..61929f512608 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -402,45 +402,15 @@ _CPyObject_HasAttrId(PyObject *v, _Py_Identifier *name) { PyObject_CallMethodObjArgs((self), (name), (arg), NULL) #endif -#if CPY_3_13_FEATURES - -// These are copied from genobject.c in Python 3.13 - -/* Returns a borrowed reference */ -static inline PyCodeObject * -_PyGen_GetCode(PyGenObject *gen) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); - return _PyFrame_GetCode(frame); -} - -static int -gen_is_coroutine(PyObject *o) -{ - if (PyGen_CheckExact(o)) { - PyCodeObject *code = _PyGen_GetCode((PyGenObject*)o); - if (code->co_flags & CO_ITERABLE_COROUTINE) { - return 1; - } - } - return 0; -} - -#elif CPY_3_12_FEATURES +#if CPY_3_12_FEATURES // These are copied from genobject.c in Python 3.12 -/* Returns a borrowed reference */ -static inline PyCodeObject * -_PyGen_GetCode(PyGenObject *gen) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); - return frame->f_code; -} - static int gen_is_coroutine(PyObject *o) { if (PyGen_CheckExact(o)) { - PyCodeObject *code = _PyGen_GetCode((PyGenObject*)o); + PyCodeObject *code = PyGen_GetCode((PyGenObject*)o); if (code->co_flags & CO_ITERABLE_COROUTINE) { return 1; } From c8ebc75bf36494d99afe731e216d08bbb4c06375 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:18:01 +0200 Subject: [PATCH 205/247] Update black to 24.8.0 (#17939) Update to 24.8.0 the last release with support for Python 3.8. https://github.com/psf/black/blob/24.8.0/CHANGES.md --- .pre-commit-config.yaml | 2 +- test-requirements.in | 2 +- test-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7ff48051aad..14b6dc17b6d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.1 # must match test-requirements.txt + rev: 24.8.0 # must match test-requirements.txt hooks: - id: black exclude: '^(test-data/)' diff --git a/test-requirements.in b/test-requirements.in index 5a888811bfcd..686dbba3b4de 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -4,7 +4,7 @@ -r mypy-requirements.txt -r build-requirements.txt attrs>=18.0 -black==24.3.0 # must match version in .pre-commit-config.yaml +black==24.8.0 # must match version in .pre-commit-config.yaml filelock>=3.3.0 # lxml 4.9.3 switched to manylinux_2_28, the wheel builder still uses manylinux2014 lxml>=4.9.1,<4.9.3; (python_version<'3.11' or sys_platform!='win32') and python_version<'3.12' diff --git a/test-requirements.txt b/test-requirements.txt index f4fb4a20cce7..6e89e473bcb2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # attrs==23.1.0 # via -r test-requirements.in -black==24.3.0 +black==24.8.0 # via -r test-requirements.in click==8.1.7 # via black From 5fba078a21952d6d21f18456cfbcce9a2d2a2062 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:18:59 +0200 Subject: [PATCH 206/247] Update ruff to 0.6.9 (#17940) https://github.com/astral-sh/ruff/blob/0.6.9/CHANGELOG.md --- .pre-commit-config.yaml | 2 +- mypy/fastparse.py | 2 +- mypy/ipc.py | 3 +-- mypy/server/astmerge.py | 2 +- mypy/stubtest.py | 2 +- mypy/util.py | 3 +-- pyproject.toml | 2 ++ test-requirements.in | 2 +- test-requirements.txt | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 14b6dc17b6d8..4efed772396e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: black exclude: '^(test-data/)' - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.0 # must match test-requirements.txt + rev: v0.6.9 # must match test-requirements.txt hooks: - id: ruff args: [--exit-non-zero-on-fix] diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 726397adb849..bc5b1ba8e57a 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -2033,7 +2033,7 @@ def visit_UnaryOp(self, n: UnaryOp) -> Type: if ( isinstance(typ, RawExpressionType) # Use type() because we do not want to allow bools. - and type(typ.literal_value) is int # noqa: E721 + and type(typ.literal_value) is int ): if isinstance(n.op, USub): typ.literal_value *= -1 diff --git a/mypy/ipc.py b/mypy/ipc.py index ab01f1b79e7d..991f9ac56652 100644 --- a/mypy/ipc.py +++ b/mypy/ipc.py @@ -17,9 +17,8 @@ if sys.platform == "win32": # This may be private, but it is needed for IPC on Windows, and is basically stable - import ctypes - import _winapi + import ctypes _IPCHandle = int diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 174c2922c767..5dc254422328 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -160,7 +160,7 @@ def replacement_map_from_symbol_table( ): new_node = new[name] if ( - type(new_node.node) == type(node.node) # noqa: E721 + type(new_node.node) == type(node.node) and new_node.node and node.node and new_node.node.fullname == node.node.fullname diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 65730828ad9f..756f90dccf2e 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -700,7 +700,7 @@ def _verify_arg_default_value( stub_default != runtime_arg.default # We want the types to match exactly, e.g. in case the stub has # True and the runtime has 1 (or vice versa). - or type(stub_default) is not type(runtime_arg.default) # noqa: E721 + or type(stub_default) is not type(runtime_arg.default) ) ): yield ( diff --git a/mypy/util.py b/mypy/util.py index 4b1b918b92e6..d2cba9b7a662 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -15,9 +15,8 @@ from typing_extensions import Literal try: - import curses - import _curses # noqa: F401 + import curses CURSES_ENABLED = True except ImportError: diff --git a/pyproject.toml b/pyproject.toml index 12a0dc109cd5..7e1556357998 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,8 +66,10 @@ ignore = [ "E2", # conflicts with black "E402", # module level import not at top of file "E501", # conflicts with black + "E721", # Use `is` and `is not` for type comparisons, or `isinstance()` for isinstance checks "E731", # Do not assign a `lambda` expression, use a `def` "E741", # Ambiguous variable name + "UP031", # Use format specifiers instead of percent format "UP032", # 'f-string always preferable to format' is controversial "C416", # There are a few cases where it's nice to have names for the dict items ] diff --git a/test-requirements.in b/test-requirements.in index 686dbba3b4de..b008522edebc 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -12,6 +12,6 @@ psutil>=4.0 pytest>=8.1.0 pytest-xdist>=1.34.0 pytest-cov>=2.10.0 -ruff==0.2.0 # must match version in .pre-commit-config.yaml +ruff==0.6.9 # must match version in .pre-commit-config.yaml setuptools>=65.5.1 tomli>=1.1.0 # needed even on py311+ so the self check passes with --python-version 3.8 diff --git a/test-requirements.txt b/test-requirements.txt index 6e89e473bcb2..56cebf95bfde 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -45,7 +45,7 @@ pytest-cov==4.1.0 # via -r test-requirements.in pytest-xdist==3.3.1 # via -r test-requirements.in -ruff==0.2.0 +ruff==0.6.9 # via -r test-requirements.in tomli==2.0.1 # via -r test-requirements.in From fc75ca76366d43b3083aabfc2bcde848adb44bfd Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 14 Oct 2024 17:16:06 +0200 Subject: [PATCH 207/247] [mypyc] Fix wheel build for cp313-win (#17941) https://github.com/python/mypy/pull/17929 seems to have broken the wheel builder but only on Windows with Python 3.13. ``` C:\Users\runneradmin\AppData\Local\pypa\cibuildwheel\Cache\nuget-cpython\python.3.13.0\tools\include\internal\pycore_unicodeobject.h(262): error C2375: '_PyUnicode_Equal': redefinition; different linkage D:\a\mypy_mypyc-wheels\mypy_mypyc-wheels\mypy\mypyc\lib-rt\pythoncapi_compat.h(1540): note: see declaration of '_PyUnicode_Equal' C:\Users\runneradmin\AppData\Local\pypa\cibuildwheel\Cache\nuget-cpython\python.3.13.0\tools\include\internal\pycore_pyhash.h(24): error C2375: '_Py_HashBytes': redefinition; different linkage D:\a\mypy_mypyc-wheels\mypy_mypyc-wheels\mypy\mypyc\lib-rt\pythoncapi_compat.h(1567): note: see declaration of '_Py_HashBytes' ``` https://github.com/mypyc/mypy_mypyc-wheels/actions/runs/11327195039/job/31497783023#step:4:3766 MSVC seems to require that the signature matches the upstream one exactly. --- mypyc/lib-rt/pythoncapi_compat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/pythoncapi_compat.h b/mypyc/lib-rt/pythoncapi_compat.h index 34a84c969b6a..acaadf34bf2e 100644 --- a/mypyc/lib-rt/pythoncapi_compat.h +++ b/mypyc/lib-rt/pythoncapi_compat.h @@ -1537,7 +1537,7 @@ static inline int PyUnicode_Equal(PyObject *str1, PyObject *str2) } #if PY_VERSION_HEX >= 0x030d0000 && !defined(PYPY_VERSION) - extern int _PyUnicode_Equal(PyObject *str1, PyObject *str2); + PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *str1, PyObject *str2); return _PyUnicode_Equal(str1, str2); #elif PY_VERSION_HEX >= 0x03060000 && !defined(PYPY_VERSION) @@ -1564,7 +1564,7 @@ static inline PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) static inline Py_hash_t Py_HashBuffer(const void *ptr, Py_ssize_t len) { #if PY_VERSION_HEX >= 0x03000000 && !defined(PYPY_VERSION) - extern Py_hash_t _Py_HashBytes(const void *src, Py_ssize_t len); + PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void *src, Py_ssize_t len); return _Py_HashBytes(ptr, len); #else From f4f16673b9f6cb3c7752d8a01e52e0e371353b6b Mon Sep 17 00:00:00 2001 From: Gaurav Giri <64427471+gaurovgiri@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:21:20 +0545 Subject: [PATCH 208/247] fix: make pre_commit installable via test-requirements.txt (#17911) Fixes #17902 --- test-requirements.in | 1 + test-requirements.txt | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index b008522edebc..cb4c7704c482 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -15,3 +15,4 @@ pytest-cov>=2.10.0 ruff==0.6.9 # must match version in .pre-commit-config.yaml setuptools>=65.5.1 tomli>=1.1.0 # needed even on py311+ so the self check passes with --python-version 3.8 +pre_commit>=3.5.0 diff --git a/test-requirements.txt b/test-requirements.txt index 56cebf95bfde..26159b0b68eb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,14 +8,22 @@ attrs==23.1.0 # via -r test-requirements.in black==24.8.0 # via -r test-requirements.in +cfgv==3.4.0 + # via pre-commit click==8.1.7 # via black coverage==7.3.2 # via pytest-cov +distlib==0.3.9 + # via virtualenv execnet==2.0.2 # via pytest-xdist filelock==3.12.4 - # via -r test-requirements.in + # via + # -r test-requirements.in + # virtualenv +identify==2.6.1 + # via pre-commit iniconfig==2.0.0 # via pytest lxml==4.9.2 ; (python_version < "3.11" or sys_platform != "win32") and python_version < "3.12" @@ -24,6 +32,8 @@ mypy-extensions==1.0.0 # via # -r mypy-requirements.txt # black +nodeenv==1.9.1 + # via pre-commit packaging==23.2 # via # black @@ -31,9 +41,13 @@ packaging==23.2 pathspec==0.11.2 # via black platformdirs==3.11.0 - # via black + # via + # black + # virtualenv pluggy==1.4.0 # via pytest +pre-commit==3.5.0 + # via -r test-requirements.in psutil==5.9.6 # via -r test-requirements.in pytest==8.1.1 @@ -45,6 +59,8 @@ pytest-cov==4.1.0 # via -r test-requirements.in pytest-xdist==3.3.1 # via -r test-requirements.in +pyyaml==6.0.2 + # via pre-commit ruff==0.6.9 # via -r test-requirements.in tomli==2.0.1 @@ -55,6 +71,8 @@ types-setuptools==68.2.0.0 # via -r build-requirements.txt typing-extensions==4.12.2 # via -r mypy-requirements.txt +virtualenv==20.26.6 + # via pre-commit # The following packages are considered to be unsafe in a requirements file: setuptools==70.0.0 From 740292ad2614c184dbcbfabbae994c17553abf16 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:21:52 +0200 Subject: [PATCH 209/247] Update test requirements (#17938) --- test-requirements.txt | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 26159b0b68eb..4b10b0e55197 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --allow-unsafe --output-file=test-requirements.txt --strip-extras test-requirements.in # -attrs==23.1.0 +attrs==24.2.0 # via -r test-requirements.in black==24.8.0 # via -r test-requirements.in @@ -12,13 +12,13 @@ cfgv==3.4.0 # via pre-commit click==8.1.7 # via black -coverage==7.3.2 +coverage==7.6.1 # via pytest-cov distlib==0.3.9 # via virtualenv -execnet==2.0.2 +execnet==2.1.1 # via pytest-xdist -filelock==3.12.4 +filelock==3.16.1 # via # -r test-requirements.in # virtualenv @@ -34,38 +34,38 @@ mypy-extensions==1.0.0 # black nodeenv==1.9.1 # via pre-commit -packaging==23.2 +packaging==24.1 # via # black # pytest -pathspec==0.11.2 +pathspec==0.12.1 # via black -platformdirs==3.11.0 +platformdirs==4.3.6 # via # black # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via pytest pre-commit==3.5.0 # via -r test-requirements.in -psutil==5.9.6 +psutil==6.0.0 # via -r test-requirements.in -pytest==8.1.1 +pytest==8.3.3 # via # -r test-requirements.in # pytest-cov # pytest-xdist -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r test-requirements.in -pytest-xdist==3.3.1 +pytest-xdist==3.6.1 # via -r test-requirements.in pyyaml==6.0.2 # via pre-commit ruff==0.6.9 # via -r test-requirements.in -tomli==2.0.1 +tomli==2.0.2 # via -r test-requirements.in -types-psutil==5.9.5.17 +types-psutil==6.0.0.20241011 # via -r build-requirements.txt types-setuptools==68.2.0.0 # via -r build-requirements.txt From 80b59139e11faf6ab3867dc8969db327f8861694 Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Tue, 15 Oct 2024 01:53:15 +0530 Subject: [PATCH 210/247] [docs] fix the edit page buttton link in docs (#17933) Fixes #17870 The current docs `edit` the page button gives a 404. This PR edits the sphinx `config.py` and configures the edit button url to the `master` branch of this repository. --- docs/source/conf.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index f8faa03a09b2..7f5fe9d43d5f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -107,6 +107,12 @@ # a list of builtin themes. html_theme = "furo" +html_theme_options = { + "source_repository": "https://github.com/python/mypy", + "source_branch": "master", + "source_directory": "docs/source", +} + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. From 970428a18704ba938c7bf0ddef60321cc785e402 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 15 Oct 2024 00:00:29 +0200 Subject: [PATCH 211/247] Update setuptools to 75.1.0 (#17943) https://setuptools.pypa.io/en/stable/history.html#v75-1-0 Since setuptools `70.1` the `bdist_wheel` command is shipped with setuptools directly. It's no longer necessary to specify `wheel` as a build system requirement. --- .github/workflows/test.yml | 4 ++-- pyproject.toml | 3 +-- test-requirements.in | 2 +- test-requirements.txt | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6ec8954bbe3e..9c35903f5f4a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -183,7 +183,7 @@ jobs: echo debug build; python -c 'import sysconfig; print(bool(sysconfig.get_config_var("Py_DEBUG")))' echo os.cpu_count; python -c 'import os; print(os.cpu_count())' echo os.sched_getaffinity; python -c 'import os; print(len(getattr(os, "sched_getaffinity", lambda *args: [])(0)))' - pip install setuptools==68.2.2 tox==4.11.0 + pip install setuptools==75.1.0 tox==4.11.0 - name: Compiled with mypyc if: ${{ matrix.test_mypyc }} @@ -243,7 +243,7 @@ jobs: default: 3.11.1 command: python -c "import platform; print(f'{platform.architecture()=} {platform.machine()=}');" - name: Install tox - run: pip install setuptools==68.2.2 tox==4.11.0 + run: pip install setuptools==75.1.0 tox==4.11.0 - name: Setup tox environment run: tox run -e py --notest - name: Test diff --git a/pyproject.toml b/pyproject.toml index 7e1556357998..06f83506ae4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,7 @@ requires = [ # NOTE: this needs to be kept in sync with mypy-requirements.txt # and build-requirements.txt, because those are both needed for # self-typechecking :/ - "setuptools >= 40.6.2", - "wheel >= 0.30.0", + "setuptools >= 75.1.0", # the following is from mypy-requirements.txt/setup.py "typing_extensions>=4.6.0", "mypy_extensions>=1.0.0", diff --git a/test-requirements.in b/test-requirements.in index cb4c7704c482..e702da28acf1 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -13,6 +13,6 @@ pytest>=8.1.0 pytest-xdist>=1.34.0 pytest-cov>=2.10.0 ruff==0.6.9 # must match version in .pre-commit-config.yaml -setuptools>=65.5.1 +setuptools>=75.1.0 tomli>=1.1.0 # needed even on py311+ so the self check passes with --python-version 3.8 pre_commit>=3.5.0 diff --git a/test-requirements.txt b/test-requirements.txt index 4b10b0e55197..ab3884b99f3b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -67,7 +67,7 @@ tomli==2.0.2 # via -r test-requirements.in types-psutil==6.0.0.20241011 # via -r build-requirements.txt -types-setuptools==68.2.0.0 +types-setuptools==75.1.0.20241014 # via -r build-requirements.txt typing-extensions==4.12.2 # via -r mypy-requirements.txt @@ -75,5 +75,5 @@ virtualenv==20.26.6 # via pre-commit # The following packages are considered to be unsafe in a requirements file: -setuptools==70.0.0 +setuptools==75.1.0 # via -r test-requirements.in From 676ed066e9cee31bc104910f7785cac005dda7f7 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 15 Oct 2024 01:17:33 +0200 Subject: [PATCH 212/247] Update tox to 4.21.2 (#17946) https://tox.wiki/en/latest/changelog.html#v4-21-2-2024-10-03 --- .github/workflows/docs.yml | 2 +- .github/workflows/test.yml | 8 ++++---- tox.ini | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c6ed3cf1a08d..923d74a02f71 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -38,7 +38,7 @@ jobs: with: python-version: '3.12' - name: Install tox - run: pip install tox==4.11.0 + run: pip install tox==4.21.2 - name: Setup tox environment run: tox run -e ${{ env.TOXENV }} --notest - name: Test diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c35903f5f4a..4150d1968bfb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -128,8 +128,8 @@ jobs: name: ${{ matrix.name }} env: TOX_SKIP_MISSING_INTERPRETERS: False - # Rich (pip) - FORCE_COLOR: 1 + # Rich (pip) -- Disable color for windows + pytest + FORCE_COLOR: ${{ !(startsWith(matrix.os, 'windows-') && startsWith(matrix.toxenv, 'py')) && 1 || 0 }} # Tox PY_COLORS: 1 # Mypy (see https://github.com/python/mypy/issues/7771) @@ -183,7 +183,7 @@ jobs: echo debug build; python -c 'import sysconfig; print(bool(sysconfig.get_config_var("Py_DEBUG")))' echo os.cpu_count; python -c 'import os; print(os.cpu_count())' echo os.sched_getaffinity; python -c 'import os; print(len(getattr(os, "sched_getaffinity", lambda *args: [])(0)))' - pip install setuptools==75.1.0 tox==4.11.0 + pip install setuptools==75.1.0 tox==4.21.2 - name: Compiled with mypyc if: ${{ matrix.test_mypyc }} @@ -243,7 +243,7 @@ jobs: default: 3.11.1 command: python -c "import platform; print(f'{platform.architecture()=} {platform.machine()=}');" - name: Install tox - run: pip install setuptools==75.1.0 tox==4.11.0 + run: pip install setuptools==75.1.0 tox==4.21.2 - name: Setup tox environment run: tox run -e py --notest - name: Test diff --git a/tox.ini b/tox.ini index c2abd05d7b6c..7f89b7d4f7ef 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ envlist = py310, py311, py312, + py313, docs, lint, type, From 284677e99071d783686978ca3bc6f40e1b7f25b8 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 15 Oct 2024 01:48:34 -0700 Subject: [PATCH 213/247] Use sha1 for hashing (#17953) This is a pretty small win, it's below the noise floor on macrobenchmark, but if you time the hashing specifically it saves about 100ms (0.5%) on `python -m mypy -c 'import torch' --no-incremental`. blake2b is slower --- mypy/util.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mypy/util.py b/mypy/util.py index d2cba9b7a662..4b19adc7ab1f 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -534,9 +534,7 @@ def hash_digest(data: bytes) -> str: accidental collision, but we don't really care about any of the cryptographic properties. """ - # Once we drop Python 3.5 support, we should consider using - # blake2b, which is faster. - return hashlib.sha256(data).hexdigest() + return hashlib.sha1(data).hexdigest() def parse_gray_color(cup: bytes) -> str: From fea947a7b0f35d5fbe7b189bd25600c80fb7e8b1 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 15 Oct 2024 02:01:08 -0700 Subject: [PATCH 214/247] Let mypyc optimise os.path.join (#17949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/python/mypy/issues/17948 There's one call site which has varargs that I leave as os.path.join, it doesn't show up on my profile. I do see the `endswith` on the profile, we could try `path[-1] == '/'` instead (could save a few dozen milliseconds) In my work environment, this is about a 10% speedup: ``` λ hyperfine -w 1 -M 3 '/tmp/mypy_primer/timer_mypy_6eddd3ab1/venv/bin/mypy -c "import torch" --no-incremental --python-executable /opt/oai/bin/python' Benchmark 1: /tmp/mypy_primer/timer_mypy_6eddd3ab1/venv/bin/mypy -c "import torch" --no-incremental --python-executable /opt/oai/bin/python Time (mean ± σ): 30.842 s ± 0.119 s [User: 26.383 s, System: 4.396 s] Range (min … max): 30.706 s … 30.927 s 3 runs ``` Compared to: ``` λ hyperfine -w 1 -M 3 '/tmp/mypy_primer/timer_mypy_88ae62b4a/venv/bin/mypy -c "import torch" --no-incremental --python-executable /opt/oai/bin/python' Benchmark 1: /tmp/mypy_primer/timer_mypy_88ae62b4a/venv/bin/mypy -c "import torch" --no-incremental --python-executable /opt/oai/bin/python Time (mean ± σ): 34.161 s ± 0.163 s [User: 29.818 s, System: 4.289 s] Range (min … max): 34.013 s … 34.336 s 3 runs ``` In the toy "long" environment mentioned in the issue, this is about a 7% speedup: ``` λ hyperfine -w 1 -M 3 '/tmp/mypy_primer/timer_mypy_6eddd3ab1/venv/bin/mypy -c "import torch" --no-incremental --python-executable long/bin/python' Benchmark 1: /tmp/mypy_primer/timer_mypy_6eddd3ab1/venv/bin/mypy -c "import torch" --no-incremental --python-executable long/bin/python Time (mean ± σ): 23.177 s ± 0.317 s [User: 20.265 s, System: 2.873 s] Range (min … max): 22.815 s … 23.407 s 3 runs ``` Compared to: ``` λ hyperfine -w 1 -M 3 '/tmp/mypy_primer/timer_mypy_88ae62b4a/venv/bin/mypy -c "import torch" --python-executable=long/bin/python --no-incremental' Benchmark 1: /tmp/mypy_primer/timer_mypy_88ae62b4a/venv/bin/mypy -c "import torch" --python-executable=long/bin/python --no-incremental Time (mean ± σ): 24.838 s ± 0.237 s [User: 22.038 s, System: 2.750 s] Range (min … max): 24.598 s … 25.073 s 3 runs ``` In the "clean" environment, this is a 1% speedup, but below the noise floor. --- mypy/modulefinder.py | 31 ++++++++++++++++--------------- mypy/util.py | 17 +++++++++++++++++ 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 452cfef20f4c..0cfe8f3b9d2f 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -22,6 +22,7 @@ from mypy.nodes import MypyFile from mypy.options import Options from mypy.stubinfo import approved_stub_package_exists +from mypy.util import os_path_join # Paths to be searched in find_module(). @@ -205,7 +206,7 @@ def find_module_via_source_set(self, id: str) -> ModuleSearchResult | None: d = os.path.dirname(p) for _ in range(id.count(".")): if not any( - self.fscache.isfile(os.path.join(d, "__init__" + x)) for x in PYTHON_EXTENSIONS + self.fscache.isfile(os_path_join(d, "__init__" + x)) for x in PYTHON_EXTENSIONS ): return None d = os.path.dirname(d) @@ -249,7 +250,7 @@ def find_lib_path_dirs(self, id: str, lib_path: tuple[str, ...]) -> PackageDirs: dirs = [] for pathitem in self.get_toplevel_possibilities(lib_path, components[0]): # e.g., '/usr/lib/python3.4/foo/bar' - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + dir = os.path.normpath(os_path_join(pathitem, dir_chain)) if self.fscache.isdir(dir): dirs.append((dir, True)) return dirs @@ -320,8 +321,8 @@ def _find_module_non_stub_helper( plausible_match = False dir_path = pkg_dir for index, component in enumerate(components): - dir_path = os.path.join(dir_path, component) - if self.fscache.isfile(os.path.join(dir_path, "py.typed")): + dir_path = os_path_join(dir_path, component) + if self.fscache.isfile(os_path_join(dir_path, "py.typed")): return os.path.join(pkg_dir, *components[:-1]), index == 0 elif not plausible_match and ( self.fscache.isdir(dir_path) or self.fscache.isfile(dir_path + ".py") @@ -418,9 +419,9 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: # Third-party stub/typed packages for pkg_dir in self.search_paths.package_path: stub_name = components[0] + "-stubs" - stub_dir = os.path.join(pkg_dir, stub_name) + stub_dir = os_path_join(pkg_dir, stub_name) if fscache.isdir(stub_dir): - stub_typed_file = os.path.join(stub_dir, "py.typed") + stub_typed_file = os_path_join(stub_dir, "py.typed") stub_components = [stub_name] + components[1:] path = os.path.join(pkg_dir, *stub_components[:-1]) if fscache.isdir(path): @@ -430,7 +431,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: # Partial here means that mypy should look at the runtime # package if installed. if fscache.read(stub_typed_file).decode().strip() == "partial": - runtime_path = os.path.join(pkg_dir, dir_chain) + runtime_path = os_path_join(pkg_dir, dir_chain) third_party_inline_dirs.append((runtime_path, True)) # if the package is partial, we don't verify the module, as # the partial stub package may not have a __init__.pyi @@ -580,7 +581,7 @@ def find_modules_recursive(self, module: str) -> list[BuildSource]: # Skip certain names altogether if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."): continue - subpath = os.path.join(package_path, name) + subpath = os_path_join(package_path, name) if self.options and matches_exclude( subpath, self.options.exclude, self.fscache, self.options.verbosity >= 2 @@ -590,8 +591,8 @@ def find_modules_recursive(self, module: str) -> list[BuildSource]: if self.fscache.isdir(subpath): # Only recurse into packages if (self.options and self.options.namespace_packages) or ( - self.fscache.isfile(os.path.join(subpath, "__init__.py")) - or self.fscache.isfile(os.path.join(subpath, "__init__.pyi")) + self.fscache.isfile(os_path_join(subpath, "__init__.py")) + or self.fscache.isfile(os_path_join(subpath, "__init__.pyi")) ): seen.add(name) sources.extend(self.find_modules_recursive(module + "." + name)) @@ -636,7 +637,7 @@ def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> for i in range(id.count(".")): path = os.path.dirname(path) if not any( - fscache.isfile_case(os.path.join(path, f"__init__{extension}"), prefix) + fscache.isfile_case(os_path_join(path, f"__init__{extension}"), prefix) for extension in PYTHON_EXTENSIONS ): return False @@ -651,7 +652,7 @@ def highest_init_level(fscache: FileSystemCache, id: str, path: str, prefix: str for i in range(id.count(".")): path = os.path.dirname(path) if any( - fscache.isfile_case(os.path.join(path, f"__init__{extension}"), prefix) + fscache.isfile_case(os_path_join(path, f"__init__{extension}"), prefix) for extension in PYTHON_EXTENSIONS ): level = i + 1 @@ -842,11 +843,11 @@ def load_stdlib_py_versions(custom_typeshed_dir: str | None) -> StdlibVersions: None means there is no maximum version. """ - typeshed_dir = custom_typeshed_dir or os.path.join(os.path.dirname(__file__), "typeshed") - stdlib_dir = os.path.join(typeshed_dir, "stdlib") + typeshed_dir = custom_typeshed_dir or os_path_join(os.path.dirname(__file__), "typeshed") + stdlib_dir = os_path_join(typeshed_dir, "stdlib") result = {} - versions_path = os.path.join(stdlib_dir, "VERSIONS") + versions_path = os_path_join(stdlib_dir, "VERSIONS") assert os.path.isfile(versions_path), (custom_typeshed_dir, versions_path, __file__) with open(versions_path) as f: for line in f: diff --git a/mypy/util.py b/mypy/util.py index 4b19adc7ab1f..74bf08c9d6de 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -416,6 +416,23 @@ def is_sub_path(path1: str, path2: str) -> bool: return pathlib.Path(path2) in pathlib.Path(path1).parents +if sys.platform == "linux" or sys.platform == "darwin": + + def os_path_join(path: str, b: str) -> str: + # Based off of os.path.join, but simplified to str-only, 2 args and mypyc can compile it. + if b.startswith("/") or not path: + return b + elif path.endswith("/"): + return path + b + else: + return path + "/" + b + +else: + + def os_path_join(a: str, p: str) -> str: + return os.path.join(a, p) + + def hard_exit(status: int = 0) -> None: """Kill the current process without fully cleaning up. From 7f081542fb0e0c0d0e05dbcfa80d8e20b0fd460f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 15 Oct 2024 02:24:54 -0700 Subject: [PATCH 215/247] Update changelog for 1.11.* releases (#17957) --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfefc690195c..4522465a484d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -503,6 +503,15 @@ Mypyc now supports the new syntax for generics introduced in Python 3.12 (see ab Please see [git log](https://github.com/python/typeshed/commits/main?after=6dda799d8ad1d89e0f8aad7ac41d2d34bd838ace+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. +### Mypy 1.11.1 + * Fix `RawExpressionType.accept` crash with `--cache-fine-grained` (Anders Kaseorg, PR [17588](https://github.com/python/mypy/pull/17588)) + * Fix PEP 604 isinstance caching (Shantanu, PR [17563](https://github.com/python/mypy/pull/17563)) + * Fix `typing.TypeAliasType` being undefined on python < 3.12 (Nikita Sobolev, PR [17558](https://github.com/python/mypy/pull/17558)) + * Fix `types.GenericAlias` lookup crash (Shantanu, PR [17543](https://github.com/python/mypy/pull/17543)) + +### Mypy 1.11.2 + * Alternative fix for a union-like literal string (Ivan Levkivskyi, PR [17639](https://github.com/python/mypy/pull/17639)) + * Unwrap `TypedDict` item types before storing (Ivan Levkivskyi, PR [17640](https://github.com/python/mypy/pull/17640)) ### Acknowledgements Thanks to all mypy contributors who contributed to this release: From eb816b05ce70f8bf3c86ecf85a6ac64c10a17aeb Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 15 Oct 2024 07:58:42 -0700 Subject: [PATCH 216/247] Use fast path in modulefinder more often (#17950) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/python/mypy/issues/17948 This is about 1.06x faster on `mypy -c 'import torch'` (in both the clean and openai environments) - 19.094 -> 17.896 - 34.161 -> 32.214 ``` λ hyperfine -w 1 -M 3 '/tmp/mypy_primer/timer_mypy_36738b392/venv/bin/mypy -c "import torch" --no-incremental --python-executable clean/bin/python' Benchmark 1: /tmp/mypy_primer/timer_mypy_36738b392/venv/bin/mypy -c "import torch" --no-incremental --python-executable clean/bin/python Time (mean ± σ): 17.896 s ± 0.130 s [User: 16.472 s, System: 1.408 s] Range (min … max): 17.757 s … 18.014 s 3 runs λ hyperfine -w 1 -M 3 '/tmp/mypy_primer/timer_mypy_36738b392/venv/bin/mypy -c "import torch" --no-incremental --python-executable /opt/oai/bin/python' Benchmark 1: /tmp/mypy_primer/timer_mypy_36738b392/venv/bin/mypy -c "import torch" --no-incremental --python-executable /opt/oai/bin/python Time (mean ± σ): 32.214 s ± 0.106 s [User: 29.468 s, System: 2.722 s] Range (min … max): 32.098 s … 32.305 s 3 runs ``` --- mypy/build.py | 8 +++++--- mypy/modulefinder.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 964da5aac8b0..043b52f0a241 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -664,7 +664,7 @@ def __init__( for module in CORE_BUILTIN_MODULES: if options.use_builtins_fixtures: continue - path = self.find_module_cache.find_module(module) + path = self.find_module_cache.find_module(module, fast_path=True) if not isinstance(path, str): raise CompileError( [f"Failed to find builtin module {module}, perhaps typeshed is broken?"] @@ -2725,7 +2725,9 @@ def exist_added_packages(suppressed: list[str], manager: BuildManager, options: def find_module_simple(id: str, manager: BuildManager) -> str | None: """Find a filesystem path for module `id` or `None` if not found.""" - x = find_module_with_reason(id, manager) + t0 = time.time() + x = manager.find_module_cache.find_module(id, fast_path=True) + manager.add_stats(find_module_time=time.time() - t0, find_module_calls=1) if isinstance(x, ModuleNotFoundReason): return None return x @@ -2734,7 +2736,7 @@ def find_module_simple(id: str, manager: BuildManager) -> str | None: def find_module_with_reason(id: str, manager: BuildManager) -> ModuleSearchResult: """Find a filesystem path for module `id` or the reason it can't be found.""" t0 = time.time() - x = manager.find_module_cache.find_module(id) + x = manager.find_module_cache.find_module(id, fast_path=False) manager.add_stats(find_module_time=time.time() - t0, find_module_calls=1) return x diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 0cfe8f3b9d2f..59a71025f71e 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -557,7 +557,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: return ModuleNotFoundReason.NOT_FOUND def find_modules_recursive(self, module: str) -> list[BuildSource]: - module_path = self.find_module(module) + module_path = self.find_module(module, fast_path=True) if isinstance(module_path, ModuleNotFoundReason): return [] sources = [BuildSource(module_path, module, None)] From 10f3ce5584cc4053fc9c7fe1eee1b34c7d13f0ed Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:47:19 -0700 Subject: [PATCH 217/247] Fix generator comprehension in meet.py (#17969) Since mypyc will treat these as list comprehensions --- mypy/meet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/meet.py b/mypy/meet.py index 9f5c2d72a8cb..d614ecc45a57 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -243,8 +243,8 @@ def is_enum_overlapping_union(x: ProperType, y: ProperType) -> bool: and x.type.is_enum and isinstance(y, UnionType) and any( - isinstance(p, LiteralType) and x.type == p.fallback.type - for p in (get_proper_type(z) for z in y.relevant_items()) + isinstance(p := get_proper_type(z), LiteralType) and x.type == p.fallback.type + for z in y.relevant_items() ) ) From fbae4325e5fa2b4cf8502402ee0440e7870f933d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 16 Oct 2024 15:36:09 +0100 Subject: [PATCH 218/247] Fix crash when showing partially analyzed type in error message (#17961) Fixes https://github.com/python/mypy/issues/17954 People say something about cache invalidation being one of the hardest problems... --- mypy/semanal.py | 8 ++++++-- test-data/unit/check-recursive-types.test | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 95efe2b0f30c..5332c98c8f0d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4014,8 +4014,10 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: # so we need to replace it with non-explicit Anys. res = make_any_non_explicit(res) if self.options.disallow_any_unimported and has_any_from_unimported_type(res): - self.msg.unimported_type_becomes_any("Type alias target", res, s) - res = make_any_non_unimported(res) + # Only show error message once, when the type is fully analyzed. + if not has_placeholder(res): + self.msg.unimported_type_becomes_any("Type alias target", res, s) + res = make_any_non_unimported(res) # Note: with the new (lazy) type alias representation we only need to set no_args to True # if the expected number of arguments is non-zero, so that aliases like `A = List` work # but not aliases like `A = TypeAliasType("A", List)` as these need explicit type params. @@ -4069,6 +4071,8 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: existing.node.alias_tvars = alias_tvars existing.node.no_args = no_args updated = True + # Invalidate recursive status cache in case it was previously set. + existing.node._is_recursive = None else: # Otherwise just replace existing placeholder with type alias. existing.node = alias_node diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index ac1ea0c0035a..4d7af98204fb 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -1006,3 +1006,11 @@ ta: Tuple[A] p: Proto p = ta [builtins fixtures/tuple.pyi] + +[case testRecursiveAliasesWithAnyUnimported] +# flags: --disallow-any-unimported +from typing import Callable +from bogus import Foo # type: ignore + +A = Callable[[Foo, "B"], Foo] # E: Type alias target becomes "Callable[[Any, B], Any]" due to an unfollowed import +B = Callable[[Foo, A], Foo] # E: Type alias target becomes "Callable[[Any, A], Any]" due to an unfollowed import From f63fdb391ac9314dc093f3750c4747c4a63c41f2 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:52:16 +0200 Subject: [PATCH 219/247] [mypyc] Replace deprecated _PyDict_GetItemStringWithError (#17930) `_PyDict_GetItemStringWithError ` was deprecated for Python 3.14 in https://github.com/python/cpython/pull/119855. Use `PyDict_GetItemStringRef` instead. It was added in 3.13 but is available via `pythoncapi_compat.h`. https://docs.python.org/3/c-api/dict.html#c.PyDict_GetItemStringRef --- mypyc/lib-rt/getargs.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mypyc/lib-rt/getargs.c b/mypyc/lib-rt/getargs.c index 1bc2f5b02ba8..4f2f8aa0be83 100644 --- a/mypyc/lib-rt/getargs.c +++ b/mypyc/lib-rt/getargs.c @@ -247,14 +247,14 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, #endif if (!skip) { if (i < nargs && i < max) { - current_arg = PyTuple_GET_ITEM(args, i); + current_arg = Py_NewRef(PyTuple_GET_ITEM(args, i)); } else if (nkwargs && i >= pos) { - current_arg = _PyDict_GetItemStringWithError(kwargs, kwlist[i]); - if (current_arg) { + int res = PyDict_GetItemStringRef(kwargs, kwlist[i], ¤t_arg); + if (res == 1) { --nkwargs; } - else if (PyErr_Occurred()) { + else if (res == -1) { return 0; } } @@ -265,6 +265,7 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, if (current_arg) { PyObject **p = va_arg(*p_va, PyObject **); *p = current_arg; + Py_DECREF(current_arg); format++; continue; } @@ -370,8 +371,11 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, Py_ssize_t j; /* make sure there are no arguments given by name and position */ for (i = pos; i < bound_pos_args && i < len; i++) { - current_arg = _PyDict_GetItemStringWithError(kwargs, kwlist[i]); - if (unlikely(current_arg != NULL)) { + int res = PyDict_GetItemStringRef(kwargs, kwlist[i], ¤t_arg); + if (res == 1) { + Py_DECREF(current_arg); + } + else if (unlikely(res == 0)) { /* arg present in tuple and in dict */ PyErr_Format(PyExc_TypeError, "argument for %.200s%s given by name ('%s') " @@ -381,7 +385,7 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, kwlist[i], i+1); goto latefail; } - else if (unlikely(PyErr_Occurred() != NULL)) { + else if (unlikely(res == -1)) { goto latefail; } } From c1f2db34fbec3b8347f9037703a804a4cd427c33 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:03:25 -0700 Subject: [PATCH 220/247] Use orjson instead of json, when available (#17955) For `mypy -c 'import torch'`, the cache load time goes from 0.44s to 0.25s as measured by manager's data_json_load_time. If I time dump times specifically, I see a saving of 0.65s to 0.07s. Overall, a pretty reasonable perf win -- should we make it a required dependency? See also https://github.com/python/mypy/issues/3456 --- misc/apply-cache-diff.py | 12 +++++----- misc/diff-cache.py | 14 ++++++------ mypy/build.py | 45 +++++++++++++++---------------------- mypy/metastore.py | 39 +++++++++++++------------------- mypy/util.py | 31 ++++++++++++++++++++++++- mypyc/codegen/emitmodule.py | 8 +++---- 6 files changed, 81 insertions(+), 68 deletions(-) diff --git a/misc/apply-cache-diff.py b/misc/apply-cache-diff.py index 29c55247de92..8ede9766bd06 100644 --- a/misc/apply-cache-diff.py +++ b/misc/apply-cache-diff.py @@ -8,13 +8,13 @@ from __future__ import annotations import argparse -import json import os import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from mypy.metastore import FilesystemMetadataStore, MetadataStore, SqliteMetadataStore +from mypy.util import json_dumps, json_loads def make_cache(input_dir: str, sqlite: bool) -> MetadataStore: @@ -26,10 +26,10 @@ def make_cache(input_dir: str, sqlite: bool) -> MetadataStore: def apply_diff(cache_dir: str, diff_file: str, sqlite: bool = False) -> None: cache = make_cache(cache_dir, sqlite) - with open(diff_file) as f: - diff = json.load(f) + with open(diff_file, "rb") as f: + diff = json_loads(f.read()) - old_deps = json.loads(cache.read("@deps.meta.json")) + old_deps = json_loads(cache.read("@deps.meta.json")) for file, data in diff.items(): if data is None: @@ -37,10 +37,10 @@ def apply_diff(cache_dir: str, diff_file: str, sqlite: bool = False) -> None: else: cache.write(file, data) if file.endswith(".meta.json") and "@deps" not in file: - meta = json.loads(data) + meta = json_loads(data) old_deps["snapshot"][meta["id"]] = meta["hash"] - cache.write("@deps.meta.json", json.dumps(old_deps)) + cache.write("@deps.meta.json", json_dumps(old_deps)) cache.commit() diff --git a/misc/diff-cache.py b/misc/diff-cache.py index 15d3e5a83983..8441caf81304 100644 --- a/misc/diff-cache.py +++ b/misc/diff-cache.py @@ -8,7 +8,6 @@ from __future__ import annotations import argparse -import json import os import sys from collections import defaultdict @@ -17,6 +16,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from mypy.metastore import FilesystemMetadataStore, MetadataStore, SqliteMetadataStore +from mypy.util import json_dumps, json_loads def make_cache(input_dir: str, sqlite: bool) -> MetadataStore: @@ -33,7 +33,7 @@ def merge_deps(all: dict[str, set[str]], new: dict[str, set[str]]) -> None: def load(cache: MetadataStore, s: str) -> Any: data = cache.read(s) - obj = json.loads(data) + obj = json_loads(data) if s.endswith(".meta.json"): # For meta files, zero out the mtimes and sort the # dependencies to avoid spurious conflicts @@ -73,7 +73,7 @@ def main() -> None: type_misses: dict[str, int] = defaultdict(int) type_hits: dict[str, int] = defaultdict(int) - updates: dict[str, str | None] = {} + updates: dict[str, bytes | None] = {} deps1: dict[str, set[str]] = {} deps2: dict[str, set[str]] = {} @@ -96,7 +96,7 @@ def main() -> None: # so we can produce a much smaller direct diff of them. if ".deps." not in s: if obj2 is not None: - updates[s] = json.dumps(obj2) + updates[s] = json_dumps(obj2) else: updates[s] = None elif obj2: @@ -122,7 +122,7 @@ def main() -> None: merge_deps(new_deps, root_deps) new_deps_json = {k: list(v) for k, v in new_deps.items() if v} - updates["@root.deps.json"] = json.dumps(new_deps_json) + updates["@root.deps.json"] = json_dumps(new_deps_json) # Drop updates to deps.meta.json for size reasons. The diff # applier will manually fix it up. @@ -136,8 +136,8 @@ def main() -> None: print("hits", type_hits) print("misses", type_misses) - with open(args.output, "w") as f: - json.dump(updates, f) + with open(args.output, "wb") as f: + f.write(json_dumps(updates)) if __name__ == "__main__": diff --git a/mypy/build.py b/mypy/build.py index 043b52f0a241..fd1c63868a69 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -95,6 +95,7 @@ from mypy.stubinfo import legacy_bundled_packages, non_bundled_packages, stub_distribution_name from mypy.types import Type from mypy.typestate import reset_global_state, type_state +from mypy.util import json_dumps, json_loads from mypy.version import __version__ # Switch to True to produce debug output related to fine-grained incremental @@ -858,7 +859,7 @@ def load_fine_grained_deps(self, id: str) -> dict[str, set[str]]: t0 = time.time() if id in self.fg_deps_meta: # TODO: Assert deps file wasn't changed. - deps = json.loads(self.metastore.read(self.fg_deps_meta[id]["path"])) + deps = json_loads(self.metastore.read(self.fg_deps_meta[id]["path"])) else: deps = {} val = {k: set(v) for k, v in deps.items()} @@ -911,8 +912,8 @@ def stats_summary(self) -> Mapping[str, object]: return self.stats -def deps_to_json(x: dict[str, set[str]]) -> str: - return json.dumps({k: list(v) for k, v in x.items()}, separators=(",", ":")) +def deps_to_json(x: dict[str, set[str]]) -> bytes: + return json_dumps({k: list(v) for k, v in x.items()}) # File for storing metadata about all the fine-grained dependency caches @@ -980,7 +981,7 @@ def write_deps_cache( meta = {"snapshot": meta_snapshot, "deps_meta": fg_deps_meta} - if not metastore.write(DEPS_META_FILE, json.dumps(meta, separators=(",", ":"))): + if not metastore.write(DEPS_META_FILE, json_dumps(meta)): manager.log(f"Error writing fine-grained deps meta JSON file {DEPS_META_FILE}") error = True @@ -1048,7 +1049,7 @@ def generate_deps_for_cache(manager: BuildManager, graph: Graph) -> dict[str, di def write_plugins_snapshot(manager: BuildManager) -> None: """Write snapshot of versions and hashes of currently active plugins.""" - snapshot = json.dumps(manager.plugins_snapshot, separators=(",", ":")) + snapshot = json_dumps(manager.plugins_snapshot) if not manager.metastore.write(PLUGIN_SNAPSHOT_FILE, snapshot): manager.errors.set_file(_cache_dir_prefix(manager.options), None, manager.options) manager.errors.report(0, 0, "Error writing plugins snapshot", blocker=True) @@ -1079,8 +1080,8 @@ def read_quickstart_file( # just ignore it. raw_quickstart: dict[str, Any] = {} try: - with open(options.quickstart_file) as f: - raw_quickstart = json.load(f) + with open(options.quickstart_file, "rb") as f: + raw_quickstart = json_loads(f.read()) quickstart = {} for file, (x, y, z) in raw_quickstart.items(): @@ -1148,10 +1149,10 @@ def _load_json_file( manager.add_stats(metastore_read_time=time.time() - t0) # Only bother to compute the log message if we are logging it, since it could be big if manager.verbosity() >= 2: - manager.trace(log_success + data.rstrip()) + manager.trace(log_success + data.rstrip().decode()) try: t1 = time.time() - result = json.loads(data) + result = json_loads(data) manager.add_stats(data_json_load_time=time.time() - t1) except json.JSONDecodeError: manager.errors.set_file(file, None, manager.options) @@ -1343,8 +1344,8 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No # So that plugins can return data with tuples in it without # things silently always invalidating modules, we round-trip # the config data. This isn't beautiful. - plugin_data = json.loads( - json.dumps(manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=True))) + plugin_data = json_loads( + json_dumps(manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=True))) ) if m.plugin_data != plugin_data: manager.log(f"Metadata abandoned for {id}: plugin configuration differs") @@ -1478,10 +1479,7 @@ def validate_meta( "ignore_all": meta.ignore_all, "plugin_data": meta.plugin_data, } - if manager.options.debug_cache: - meta_str = json.dumps(meta_dict, indent=2, sort_keys=True) - else: - meta_str = json.dumps(meta_dict, separators=(",", ":")) + meta_bytes = json_dumps(meta_dict, manager.options.debug_cache) meta_json, _, _ = get_cache_names(id, path, manager.options) manager.log( "Updating mtime for {}: file {}, meta {}, mtime {}".format( @@ -1489,7 +1487,7 @@ def validate_meta( ) ) t1 = time.time() - manager.metastore.write(meta_json, meta_str) # Ignore errors, just an optimization. + manager.metastore.write(meta_json, meta_bytes) # Ignore errors, just an optimization. manager.add_stats(validate_update_time=time.time() - t1, validate_munging_time=t1 - t0) return meta @@ -1507,13 +1505,6 @@ def compute_hash(text: str) -> str: return hash_digest(text.encode("utf-8")) -def json_dumps(obj: Any, debug_cache: bool) -> str: - if debug_cache: - return json.dumps(obj, indent=2, sort_keys=True) - else: - return json.dumps(obj, sort_keys=True, separators=(",", ":")) - - def write_cache( id: str, path: str, @@ -1566,8 +1557,8 @@ def write_cache( # Serialize data and analyze interface data = tree.serialize() - data_str = json_dumps(data, manager.options.debug_cache) - interface_hash = compute_hash(data_str) + data_bytes = json_dumps(data, manager.options.debug_cache) + interface_hash = hash_digest(data_bytes) plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False)) @@ -1591,7 +1582,7 @@ def write_cache( manager.trace(f"Interface for {id} is unchanged") else: manager.trace(f"Interface for {id} has changed") - if not metastore.write(data_json, data_str): + if not metastore.write(data_json, data_bytes): # Most likely the error is the replace() call # (see https://github.com/python/mypy/issues/3215). manager.log(f"Error writing data JSON file {data_json}") @@ -3568,4 +3559,4 @@ def write_undocumented_ref_info( assert not ref_info_file.startswith(".") deps_json = get_undocumented_ref_info_json(state.tree, type_map) - metastore.write(ref_info_file, json.dumps(deps_json, separators=(",", ":"))) + metastore.write(ref_info_file, json_dumps(deps_json)) diff --git a/mypy/metastore.py b/mypy/metastore.py index 4caa7d7f0534..c9139045e3b1 100644 --- a/mypy/metastore.py +++ b/mypy/metastore.py @@ -33,14 +33,14 @@ def getmtime(self, name: str) -> float: """ @abstractmethod - def read(self, name: str) -> str: + def read(self, name: str) -> bytes: """Read the contents of a metadata entry. Raises FileNotFound if the entry does not exist. """ @abstractmethod - def write(self, name: str, data: str, mtime: float | None = None) -> bool: + def write(self, name: str, data: bytes, mtime: float | None = None) -> bool: """Write a metadata entry. If mtime is specified, set it as the mtime of the entry. Otherwise, @@ -86,16 +86,16 @@ def getmtime(self, name: str) -> float: return int(os.path.getmtime(os.path.join(self.cache_dir_prefix, name))) - def read(self, name: str) -> str: + def read(self, name: str) -> bytes: assert os.path.normpath(name) != os.path.abspath(name), "Don't use absolute paths!" if not self.cache_dir_prefix: raise FileNotFoundError() - with open(os.path.join(self.cache_dir_prefix, name)) as f: + with open(os.path.join(self.cache_dir_prefix, name), "rb") as f: return f.read() - def write(self, name: str, data: str, mtime: float | None = None) -> bool: + def write(self, name: str, data: bytes, mtime: float | None = None) -> bool: assert os.path.normpath(name) != os.path.abspath(name), "Don't use absolute paths!" if not self.cache_dir_prefix: @@ -105,7 +105,7 @@ def write(self, name: str, data: str, mtime: float | None = None) -> bool: tmp_filename = path + "." + random_string() try: os.makedirs(os.path.dirname(path), exist_ok=True) - with open(tmp_filename, "w") as f: + with open(tmp_filename, "wb") as f: f.write(data) os.replace(tmp_filename, path) if mtime is not None: @@ -135,15 +135,13 @@ def list_all(self) -> Iterable[str]: SCHEMA = """ -CREATE TABLE IF NOT EXISTS files ( +CREATE TABLE IF NOT EXISTS files2 ( path TEXT UNIQUE NOT NULL, mtime REAL, - data TEXT + data BLOB ); -CREATE INDEX IF NOT EXISTS path_idx on files(path); +CREATE INDEX IF NOT EXISTS path_idx on files2(path); """ -# No migrations yet -MIGRATIONS: list[str] = [] def connect_db(db_file: str) -> sqlite3.Connection: @@ -151,11 +149,6 @@ def connect_db(db_file: str) -> sqlite3.Connection: db = sqlite3.dbapi2.connect(db_file) db.executescript(SCHEMA) - for migr in MIGRATIONS: - try: - db.executescript(migr) - except sqlite3.OperationalError: - pass return db @@ -176,7 +169,7 @@ def _query(self, name: str, field: str) -> Any: if not self.db: raise FileNotFoundError() - cur = self.db.execute(f"SELECT {field} FROM files WHERE path = ?", (name,)) + cur = self.db.execute(f"SELECT {field} FROM files2 WHERE path = ?", (name,)) results = cur.fetchall() if not results: raise FileNotFoundError() @@ -188,12 +181,12 @@ def getmtime(self, name: str) -> float: assert isinstance(mtime, float) return mtime - def read(self, name: str) -> str: + def read(self, name: str) -> bytes: data = self._query(name, "data") - assert isinstance(data, str) + assert isinstance(data, bytes) return data - def write(self, name: str, data: str, mtime: float | None = None) -> bool: + def write(self, name: str, data: bytes, mtime: float | None = None) -> bool: import sqlite3 if not self.db: @@ -202,7 +195,7 @@ def write(self, name: str, data: str, mtime: float | None = None) -> bool: if mtime is None: mtime = time.time() self.db.execute( - "INSERT OR REPLACE INTO files(path, mtime, data) VALUES(?, ?, ?)", + "INSERT OR REPLACE INTO files2(path, mtime, data) VALUES(?, ?, ?)", (name, mtime, data), ) except sqlite3.OperationalError: @@ -213,7 +206,7 @@ def remove(self, name: str) -> None: if not self.db: raise FileNotFoundError() - self.db.execute("DELETE FROM files WHERE path = ?", (name,)) + self.db.execute("DELETE FROM files2 WHERE path = ?", (name,)) def commit(self) -> None: if self.db: @@ -221,5 +214,5 @@ def commit(self) -> None: def list_all(self) -> Iterable[str]: if self.db: - for row in self.db.execute("SELECT path FROM files"): + for row in self.db.execute("SELECT path FROM files2"): yield row[0] diff --git a/mypy/util.py b/mypy/util.py index 74bf08c9d6de..8ec979af27e1 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -4,6 +4,7 @@ import hashlib import io +import json import os import pathlib import re @@ -11,9 +12,15 @@ import sys import time from importlib import resources as importlib_resources -from typing import IO, Callable, Container, Final, Iterable, Sequence, Sized, TypeVar +from typing import IO, Any, Callable, Container, Final, Iterable, Sequence, Sized, TypeVar from typing_extensions import Literal +orjson: Any +try: + import orjson # type: ignore[import-not-found, no-redef, unused-ignore] +except ImportError: + orjson = None + try: import _curses # noqa: F401 import curses @@ -888,3 +895,25 @@ def quote_docstring(docstr: str) -> str: return f"''{docstr_repr}''" else: return f'""{docstr_repr}""' + + +def json_dumps(obj: object, debug: bool = False) -> bytes: + if orjson is not None: + if debug: + return orjson.dumps(obj, option=orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS) # type: ignore[no-any-return] + else: + # TODO: If we don't sort keys here, testIncrementalInternalScramble fails + # We should document exactly what is going on there + return orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) # type: ignore[no-any-return] + + if debug: + return json.dumps(obj, indent=2, sort_keys=True).encode("utf-8") + else: + # See above for sort_keys comment + return json.dumps(obj, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def json_loads(data: bytes) -> Any: + if orjson is not None: + return orjson.loads(data) + return json.loads(data) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 1d8708912de5..b5e0a37f0cca 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -24,7 +24,7 @@ from mypy.nodes import MypyFile from mypy.options import Options from mypy.plugin import Plugin, ReportConfigContext -from mypy.util import hash_digest +from mypy.util import hash_digest, json_dumps from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import Emitter, EmitterContext, HeaderDeclaration, c_array_initializer from mypyc.codegen.emitclass import generate_class, generate_class_type_decl @@ -154,7 +154,7 @@ def report_config_data(self, ctx: ReportConfigContext) -> tuple[str | None, list ir_data = json.loads(ir_json) # Check that the IR cache matches the metadata cache - if compute_hash(meta_json) != ir_data["meta_hash"]: + if hash_digest(meta_json) != ir_data["meta_hash"]: return None # Check that all of the source files are present and as @@ -369,11 +369,11 @@ def write_cache( newpath = get_state_ir_cache_name(st) ir_data = { "ir": module.serialize(), - "meta_hash": compute_hash(meta_data), + "meta_hash": hash_digest(meta_data), "src_hashes": hashes[group_map[id]], } - result.manager.metastore.write(newpath, json.dumps(ir_data, separators=(",", ":"))) + result.manager.metastore.write(newpath, json_dumps(ir_data)) result.manager.metastore.commit() From 6728848d8f360fe4c763ee51c16fa524815444fd Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:21:45 -0700 Subject: [PATCH 221/247] Fix cache-convert (#17974) Tested via: ``` set -x rm -rf .mypy_cache time python -m mypy --config-file mypy_self_check.ini -p mypy python misc/convert-cache.py --to-sqlite .mypy_cache/3.8 time python -m mypy --config-file mypy_self_check.ini -p mypy --sqlite-cache time python -m mypy --config-file mypy_self_check.ini -p mypy --sqlite-cache ``` --- mypy/metastore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/metastore.py b/mypy/metastore.py index c9139045e3b1..21fb8730f2c9 100644 --- a/mypy/metastore.py +++ b/mypy/metastore.py @@ -131,7 +131,7 @@ def list_all(self) -> Iterable[str]: for dir, _, files in os.walk(self.cache_dir_prefix): dir = os.path.relpath(dir, self.cache_dir_prefix) for file in files: - yield os.path.join(dir, file) + yield os.path.normpath(os.path.join(dir, file)) SCHEMA = """ From ddeb5bdd643d8f2b1c222b95dcb4f89e3ccc6958 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:12:22 -0700 Subject: [PATCH 222/247] Use kw-only args for member access booleans (#17975) Gets hard to know what's what at call site --- mypy/checker.py | 8 ++++---- mypy/checkexpr.py | 32 ++++++++++++++++---------------- mypy/checkmember.py | 35 ++++++++++++++++++----------------- mypy/checkpattern.py | 16 ++++++++-------- 4 files changed, 46 insertions(+), 45 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index ca35144456fe..c646ebf3736b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -7570,10 +7570,10 @@ def has_valid_attribute(self, typ: Type, name: str) -> bool: name, typ, TempNode(AnyType(TypeOfAny.special_form)), - False, - False, - False, - self.msg, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, original_type=typ, chk=self, # This is not a real attribute lookup so don't mess with deferring nodes. diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 0754b1db7141..8d1833a772e2 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1504,10 +1504,10 @@ def check_union_call_expr(self, e: CallExpr, object_type: UnionType, member: str member, typ, e, - False, - False, - False, - self.msg, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, original_type=object_type, chk=self.chk, in_literal_context=self.is_literal_context(), @@ -3302,10 +3302,10 @@ def analyze_ordinary_member_access(self, e: MemberExpr, is_lvalue: bool) -> Type e.name, original_type, e, - is_lvalue, - False, - False, - self.msg, + is_lvalue=is_lvalue, + is_super=False, + is_operator=False, + msg=self.msg, original_type=original_type, chk=self.chk, in_literal_context=self.is_literal_context(), @@ -3326,10 +3326,10 @@ def analyze_external_member_access( member, base_type, context, - False, - False, - False, - self.msg, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, original_type=base_type, chk=self.chk, in_literal_context=self.is_literal_context(), @@ -3809,10 +3809,10 @@ def check_method_call_by_name( method, base_type, context, - False, - False, - True, - self.msg, + is_lvalue=False, + is_super=False, + is_operator=True, + msg=self.msg, original_type=original_type, chk=self.chk, in_literal_context=self.is_literal_context(), diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 4448072d52c3..e6f7570c3df2 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -87,6 +87,7 @@ class MemberContext: def __init__( self, + *, is_lvalue: bool, is_super: bool, is_operator: bool, @@ -126,16 +127,16 @@ def copy_modified( original_type: Type | None = None, ) -> MemberContext: mx = MemberContext( - self.is_lvalue, - self.is_super, - self.is_operator, - self.original_type, - self.context, - self.msg, - self.chk, - self.self_type, - self.module_symbol_table, - self.no_deferral, + is_lvalue=self.is_lvalue, + is_super=self.is_super, + is_operator=self.is_operator, + original_type=self.original_type, + context=self.context, + msg=self.msg, + chk=self.chk, + self_type=self.self_type, + module_symbol_table=self.module_symbol_table, + no_deferral=self.no_deferral, ) if messages is not None: mx.msg = messages @@ -152,11 +153,11 @@ def analyze_member_access( name: str, typ: Type, context: Context, + *, is_lvalue: bool, is_super: bool, is_operator: bool, msg: MessageBuilder, - *, original_type: Type, chk: mypy.checker.TypeChecker, override_info: TypeInfo | None = None, @@ -190,12 +191,12 @@ def analyze_member_access( are not available via the type object directly) """ mx = MemberContext( - is_lvalue, - is_super, - is_operator, - original_type, - context, - msg, + is_lvalue=is_lvalue, + is_super=is_super, + is_operator=is_operator, + original_type=original_type, + context=context, + msg=msg, chk=chk, self_type=self_type, module_symbol_table=module_symbol_table, diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index fa23dfb5f453..6b4fa35f9c49 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -594,10 +594,10 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType: "__match_args__", typ, o, - False, - False, - False, - self.msg, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, original_type=typ, chk=self.chk, ) @@ -660,10 +660,10 @@ def visit_class_pattern(self, o: ClassPattern) -> PatternType: keyword, narrowed_type, pattern, - False, - False, - False, - self.msg, + is_lvalue=False, + is_super=False, + is_operator=False, + msg=self.msg, original_type=new_type, chk=self.chk, ) From 1be3a8b40153d28414fee8049b435d69db90e325 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:21:47 -0700 Subject: [PATCH 223/247] Speed up stubs suggestions (#17965) See https://github.com/python/mypy/issues/17948 This is starting to show up on profiles - 1.01x faster on clean (below noise) - 1.02x faster on long - 1.02x faster on openai - 1.01x faster on openai incremental I had a dumb bug that was preventing the optimisation for a while, I'll see if I can make it even faster. Currently it's a small improvement We could also get rid of the legacy stuff in mypy 2.0 --- mypy/build.py | 21 +++++--------- mypy/modulefinder.py | 5 ++-- mypy/stubinfo.py | 60 ++++++++++++++++++++++++++++++--------- mypy/test/teststubinfo.py | 42 ++++++++++++++++++++++++--- 4 files changed, 93 insertions(+), 35 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index fd1c63868a69..52c11e065b63 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -92,7 +92,7 @@ from mypy.plugins.default import DefaultPlugin from mypy.renaming import LimitedVariableRenameVisitor, VariableRenameVisitor from mypy.stats import dump_type_stats -from mypy.stubinfo import legacy_bundled_packages, non_bundled_packages, stub_distribution_name +from mypy.stubinfo import is_module_from_legacy_bundled_package, stub_distribution_name from mypy.types import Type from mypy.typestate import reset_global_state, type_state from mypy.util import json_dumps, json_loads @@ -2658,17 +2658,13 @@ def find_module_and_diagnose( ignore_missing_imports = options.ignore_missing_imports - id_components = id.split(".") # Don't honor a global (not per-module) ignore_missing_imports # setting for modules that used to have bundled stubs, as # otherwise updating mypy can silently result in new false # negatives. (Unless there are stubs but they are incomplete.) global_ignore_missing_imports = manager.options.ignore_missing_imports if ( - any( - ".".join(id_components[:i]) in legacy_bundled_packages - for i in range(len(id_components), 0, -1) - ) + is_module_from_legacy_bundled_package(id) and global_ignore_missing_imports and not options.ignore_missing_imports_per_module and result is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED @@ -2789,18 +2785,15 @@ def module_not_found( code = codes.IMPORT errors.report(line, 0, msg.format(module=target), code=code) - components = target.split(".") - for i in range(len(components), 0, -1): - module = ".".join(components[:i]) - if module in legacy_bundled_packages or module in non_bundled_packages: - break - + dist = stub_distribution_name(target) for note in notes: if "{stub_dist}" in note: - note = note.format(stub_dist=stub_distribution_name(module)) + assert dist is not None + note = note.format(stub_dist=dist) errors.report(line, 0, note, severity="note", only_once=True, code=code) if reason is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED: - manager.missing_stub_packages.add(stub_distribution_name(module)) + assert dist is not None + manager.missing_stub_packages.add(dist) errors.set_import_context(save_import_context) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 59a71025f71e..94d8a5d59e1f 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -331,9 +331,8 @@ def _find_module_non_stub_helper( # If this is not a directory then we can't traverse further into it if not self.fscache.isdir(dir_path): break - for i in range(len(components), 0, -1): - if approved_stub_package_exists(".".join(components[:i])): - return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED + if approved_stub_package_exists(".".join(components)): + return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED if plausible_match: return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS else: diff --git a/mypy/stubinfo.py b/mypy/stubinfo.py index 9d8dfbe43f37..0ec64b037fee 100644 --- a/mypy/stubinfo.py +++ b/mypy/stubinfo.py @@ -1,22 +1,53 @@ from __future__ import annotations -def is_legacy_bundled_package(prefix: str) -> bool: - return prefix in legacy_bundled_packages +def is_module_from_legacy_bundled_package(module: str) -> bool: + top_level = module.split(".", 1)[0] + return top_level in legacy_bundled_packages -def approved_stub_package_exists(prefix: str) -> bool: - return is_legacy_bundled_package(prefix) or prefix in non_bundled_packages +def approved_stub_package_exists(module: str) -> bool: + top_level = module.split(".", 1)[0] + if top_level in legacy_bundled_packages: + return True + if top_level in non_bundled_packages_flat: + return True + if top_level in non_bundled_packages_namespace: + namespace = non_bundled_packages_namespace[top_level] + components = module.split(".") + for i in range(len(components), 0, -1): + module = ".".join(components[:i]) + if module in namespace: + return True + return False -def stub_distribution_name(prefix: str) -> str: - return legacy_bundled_packages.get(prefix) or non_bundled_packages[prefix] +def stub_distribution_name(module: str) -> str | None: + top_level = module.split(".", 1)[0] + + dist = legacy_bundled_packages.get(top_level) + if dist: + return dist + dist = non_bundled_packages_flat.get(top_level) + if dist: + return dist + + if top_level in non_bundled_packages_namespace: + namespace = non_bundled_packages_namespace[top_level] + components = module.split(".") + for i in range(len(components), 0, -1): + module = ".".join(components[:i]) + dist = namespace.get(module) + if dist: + return dist + + return None # Stubs for these third-party packages used to be shipped with mypy. # # Map package name to PyPI stub distribution name. -legacy_bundled_packages = { +legacy_bundled_packages: dict[str, str] = { "aiofiles": "types-aiofiles", "bleach": "types-bleach", "boto": "types-boto", @@ -32,7 +63,6 @@ def stub_distribution_name(prefix: str) -> str: "docutils": "types-docutils", "first": "types-first", "gflags": "types-python-gflags", - "google.protobuf": "types-protobuf", "markdown": "types-Markdown", "mock": "types-mock", "OpenSSL": "types-pyOpenSSL", @@ -66,20 +96,17 @@ def stub_distribution_name(prefix: str) -> str: # include packages that have a release that includes PEP 561 type # information. # -# Package name can have one or two components ('a' or 'a.b'). -# # Note that these packages are omitted for now: # pika: typeshed's stubs are on PyPI as types-pika-ts. # types-pika already exists on PyPI, and is more complete in many ways, # but is a non-typeshed stubs package. -non_bundled_packages = { +non_bundled_packages_flat: dict[str, str] = { "MySQLdb": "types-mysqlclient", "PIL": "types-Pillow", "PyInstaller": "types-pyinstaller", "Xlib": "types-python-xlib", "aws_xray_sdk": "types-aws-xray-sdk", "babel": "types-babel", - "backports.ssl_match_hostname": "types-backports.ssl_match_hostname", "braintree": "types-braintree", "bs4": "types-beautifulsoup4", "bugbear": "types-flake8-bugbear", @@ -107,7 +134,6 @@ def stub_distribution_name(prefix: str) -> str: "flask_migrate": "types-Flask-Migrate", "fpdf": "types-fpdf2", "gdb": "types-gdb", - "google.cloud.ndb": "types-google-cloud-ndb", "hdbcli": "types-hdbcli", "html5lib": "types-html5lib", "httplib2": "types-httplib2", @@ -123,7 +149,6 @@ def stub_distribution_name(prefix: str) -> str: "oauthlib": "types-oauthlib", "openpyxl": "types-openpyxl", "opentracing": "types-opentracing", - "paho.mqtt": "types-paho-mqtt", "parsimonious": "types-parsimonious", "passlib": "types-passlib", "passpy": "types-passpy", @@ -171,3 +196,10 @@ def stub_distribution_name(prefix: str) -> str: "pandas": "pandas-stubs", # https://github.com/pandas-dev/pandas-stubs "lxml": "lxml-stubs", # https://github.com/lxml/lxml-stubs } + + +non_bundled_packages_namespace: dict[str, dict[str, str]] = { + "backports": {"backports.ssl_match_hostname": "types-backports.ssl_match_hostname"}, + "google": {"google.cloud.ndb": "types-google-cloud-ndb", "google.protobuf": "types-protobuf"}, + "paho": {"paho.mqtt": "types-paho-mqtt"}, +} diff --git a/mypy/test/teststubinfo.py b/mypy/test/teststubinfo.py index eccee90244f3..10ce408e7023 100644 --- a/mypy/test/teststubinfo.py +++ b/mypy/test/teststubinfo.py @@ -2,11 +2,45 @@ import unittest -from mypy.stubinfo import is_legacy_bundled_package +from mypy.stubinfo import ( + approved_stub_package_exists, + is_module_from_legacy_bundled_package, + legacy_bundled_packages, + non_bundled_packages_flat, + stub_distribution_name, +) class TestStubInfo(unittest.TestCase): def test_is_legacy_bundled_packages(self) -> None: - assert not is_legacy_bundled_package("foobar_asdf") - assert is_legacy_bundled_package("pycurl") - assert is_legacy_bundled_package("dataclasses") + assert not is_module_from_legacy_bundled_package("foobar_asdf") + assert not is_module_from_legacy_bundled_package("PIL") + assert is_module_from_legacy_bundled_package("pycurl") + assert is_module_from_legacy_bundled_package("dataclasses") + + def test_approved_stub_package_exists(self) -> None: + assert not approved_stub_package_exists("foobar_asdf") + assert approved_stub_package_exists("pycurl") + assert approved_stub_package_exists("babel") + assert approved_stub_package_exists("google.cloud.ndb") + assert approved_stub_package_exists("google.cloud.ndb.submodule") + assert not approved_stub_package_exists("google.cloud.unknown") + assert approved_stub_package_exists("google.protobuf") + assert approved_stub_package_exists("google.protobuf.submodule") + assert not approved_stub_package_exists("google") + + def test_stub_distribution_name(self) -> None: + assert stub_distribution_name("foobar_asdf") is None + assert stub_distribution_name("pycurl") == "types-pycurl" + assert stub_distribution_name("babel") == "types-babel" + assert stub_distribution_name("google.cloud.ndb") == "types-google-cloud-ndb" + assert stub_distribution_name("google.cloud.ndb.submodule") == "types-google-cloud-ndb" + assert stub_distribution_name("google.cloud.unknown") is None + assert stub_distribution_name("google.protobuf") == "types-protobuf" + assert stub_distribution_name("google.protobuf.submodule") == "types-protobuf" + assert stub_distribution_name("google") is None + + def test_period_in_top_level(self) -> None: + for packages in (non_bundled_packages_flat, legacy_bundled_packages): + for top_level_module in packages: + assert "." not in top_level_module From c201a187be998ec9e9c179004e0cd28dc2c86137 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 16 Oct 2024 18:52:20 -0700 Subject: [PATCH 224/247] Make is_sub_path faster (#17962) See #17948 - 1.01x faster on clean - 1.06x faster on long - 1.04x faster on openai - 1.26x faster on openai incremental --- mypy/build.py | 9 ++++----- mypy/modulefinder.py | 9 +++++++-- mypy/util.py | 24 ++++++++++++++++++++---- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 52c11e065b63..ac6471d2383f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -59,7 +59,7 @@ get_mypy_comments, hash_digest, is_stub_package_file, - is_sub_path, + is_sub_path_normabs, is_typeshed_file, module_prefix, read_py_file, @@ -3528,10 +3528,9 @@ def is_silent_import_module(manager: BuildManager, path: str) -> bool: if manager.options.no_silence_site_packages: return False # Silence errors in site-package dirs and typeshed - return any( - is_sub_path(path, dir) - for dir in manager.search_paths.package_path + manager.search_paths.typeshed_path - ) + if any(is_sub_path_normabs(path, dir) for dir in manager.search_paths.package_path): + return True + return any(is_sub_path_normabs(path, dir) for dir in manager.search_paths.typeshed_path) def write_undocumented_ref_info( diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 94d8a5d59e1f..49c39a9ce91c 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -668,10 +668,13 @@ def mypy_path() -> list[str]: def default_lib_path( data_dir: str, pyversion: tuple[int, int], custom_typeshed_dir: str | None ) -> list[str]: - """Return default standard library search paths.""" + """Return default standard library search paths. Guaranteed to be normalised.""" + + data_dir = os.path.abspath(data_dir) path: list[str] = [] if custom_typeshed_dir: + custom_typeshed_dir = os.path.abspath(custom_typeshed_dir) typeshed_dir = os.path.join(custom_typeshed_dir, "stdlib") mypy_extensions_dir = os.path.join(custom_typeshed_dir, "stubs", "mypy-extensions") versions_file = os.path.join(typeshed_dir, "VERSIONS") @@ -711,7 +714,7 @@ def default_lib_path( @functools.lru_cache(maxsize=None) def get_search_dirs(python_executable: str | None) -> tuple[list[str], list[str]]: - """Find package directories for given python. + """Find package directories for given python. Guaranteed to return absolute paths. This runs a subprocess call, which generates a list of the directories in sys.path. To avoid repeatedly calling a subprocess (which can be slow!) we @@ -773,6 +776,7 @@ def compute_search_paths( root_dir = os.getenv("MYPY_TEST_PREFIX", None) if not root_dir: root_dir = os.path.dirname(os.path.dirname(__file__)) + root_dir = os.path.abspath(root_dir) lib_path.appendleft(os.path.join(root_dir, "test-data", "unit", "lib-stub")) # alt_lib_path is used by some tests to bypass the normal lib_path mechanics. # If we don't have one, grab directories of source files. @@ -829,6 +833,7 @@ def compute_search_paths( return SearchPaths( python_path=tuple(reversed(python_path)), mypy_path=tuple(mypypath), + # package_path and typeshed_path must be normalised and absolute via os.path.abspath package_path=tuple(sys_path + site_packages), typeshed_path=tuple(lib_path), ) diff --git a/mypy/util.py b/mypy/util.py index 8ec979af27e1..2eac2a86dfd0 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -6,7 +6,6 @@ import io import json import os -import pathlib import re import shutil import sys @@ -418,9 +417,26 @@ def replace_object_state( pass -def is_sub_path(path1: str, path2: str) -> bool: - """Given two paths, return if path1 is a sub-path of path2.""" - return pathlib.Path(path2) in pathlib.Path(path1).parents +def is_sub_path_normabs(path: str, dir: str) -> bool: + """Given two paths, return if path is a sub-path of dir. + + Moral equivalent of: Path(dir) in Path(path).parents + + Similar to the pathlib version: + - Treats paths case-sensitively + - Does not fully handle unnormalised paths (e.g. paths with "..") + - Does not handle a mix of absolute and relative paths + Unlike the pathlib version: + - Fast + - On Windows, assumes input has been slash normalised + - Handles even fewer unnormalised paths (e.g. paths with "." and "//") + + As a result, callers should ensure that inputs have had os.path.abspath called on them + (note that os.path.abspath will normalise) + """ + if not dir.endswith(os.sep): + dir += os.sep + return path.startswith(dir) if sys.platform == "linux" or sys.platform == "darwin": From 946c1bf424f0102e36156efe5f8149cf5d37c4a4 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 17 Oct 2024 07:44:31 +0200 Subject: [PATCH 225/247] Sync typeshed (#17971) Source commit: https://github.com/python/typeshed/commit/2370b8b9d194d4a468907df8952ef0dd7daff187 --- ...e-of-LiteralString-in-builtins-13743.patch | 27 +- mypy/typeshed/stdlib/VERSIONS | 4 + mypy/typeshed/stdlib/_ast.pyi | 1807 ++------------- mypy/typeshed/stdlib/_asyncio.pyi | 120 + mypy/typeshed/stdlib/_csv.pyi | 17 +- mypy/typeshed/stdlib/_ctypes.pyi | 5 +- mypy/typeshed/stdlib/_curses.pyi | 31 +- mypy/typeshed/stdlib/_decimal.pyi | 265 +-- mypy/typeshed/stdlib/_io.pyi | 195 ++ mypy/typeshed/stdlib/_operator.pyi | 42 +- mypy/typeshed/stdlib/_socket.pyi | 20 +- mypy/typeshed/stdlib/_sqlite3.pyi | 312 +++ mypy/typeshed/stdlib/_ssl.pyi | 292 +++ mypy/typeshed/stdlib/_weakref.pyi | 31 +- mypy/typeshed/stdlib/ast.pyi | 2015 +++++++++++++++-- mypy/typeshed/stdlib/asyncio/events.pyi | 11 +- mypy/typeshed/stdlib/asyncio/futures.pyi | 47 +- mypy/typeshed/stdlib/asyncio/streams.pyi | 4 +- mypy/typeshed/stdlib/asyncio/tasks.pyi | 74 +- mypy/typeshed/stdlib/builtins.pyi | 7 +- mypy/typeshed/stdlib/bz2.pyi | 2 +- mypy/typeshed/stdlib/code.pyi | 6 +- mypy/typeshed/stdlib/collections/__init__.pyi | 15 +- .../stdlib/concurrent/futures/process.pyi | 5 +- mypy/typeshed/stdlib/contextlib.pyi | 14 +- mypy/typeshed/stdlib/csv.pyi | 11 +- mypy/typeshed/stdlib/curses/__init__.pyi | 9 +- mypy/typeshed/stdlib/curses/panel.pyi | 8 +- mypy/typeshed/stdlib/curses/textpad.pyi | 6 +- mypy/typeshed/stdlib/decimal.pyi | 258 ++- mypy/typeshed/stdlib/email/_policybase.pyi | 45 +- mypy/typeshed/stdlib/email/feedparser.pyi | 10 +- mypy/typeshed/stdlib/email/generator.pyi | 55 +- mypy/typeshed/stdlib/email/message.pyi | 14 +- mypy/typeshed/stdlib/email/parser.pyi | 10 +- mypy/typeshed/stdlib/email/policy.pyi | 38 +- mypy/typeshed/stdlib/enum.pyi | 15 +- mypy/typeshed/stdlib/fileinput.pyi | 6 +- mypy/typeshed/stdlib/hmac.pyi | 7 +- mypy/typeshed/stdlib/http/client.pyi | 5 +- mypy/typeshed/stdlib/http/cookiejar.pyi | 4 +- mypy/typeshed/stdlib/io.pyi | 233 +- mypy/typeshed/stdlib/ipaddress.pyi | 4 +- mypy/typeshed/stdlib/logging/config.pyi | 2 +- mypy/typeshed/stdlib/mmap.pyi | 4 +- .../stdlib/multiprocessing/connection.pyi | 46 +- .../stdlib/multiprocessing/context.pyi | 7 +- .../stdlib/multiprocessing/managers.pyi | 6 +- mypy/typeshed/stdlib/multiprocessing/pool.pyi | 4 +- .../stdlib/multiprocessing/reduction.pyi | 4 +- mypy/typeshed/stdlib/operator.pyi | 104 +- mypy/typeshed/stdlib/os/__init__.pyi | 27 +- mypy/typeshed/stdlib/pickle.pyi | 14 +- mypy/typeshed/stdlib/pyexpat/__init__.pyi | 6 +- mypy/typeshed/stdlib/re.pyi | 11 +- mypy/typeshed/stdlib/shlex.pyi | 2 +- mypy/typeshed/stdlib/socket.pyi | 16 +- mypy/typeshed/stdlib/sqlite3/__init__.pyi | 466 +++- mypy/typeshed/stdlib/sqlite3/dbapi2.pyi | 760 ++----- mypy/typeshed/stdlib/sre_constants.pyi | 9 +- mypy/typeshed/stdlib/ssl.pyi | 102 +- mypy/typeshed/stdlib/tarfile.pyi | 117 +- mypy/typeshed/stdlib/tokenize.pyi | 2 +- mypy/typeshed/stdlib/traceback.pyi | 4 +- mypy/typeshed/stdlib/types.pyi | 2 + mypy/typeshed/stdlib/unittest/mock.pyi | 4 +- mypy/typeshed/stdlib/unittest/suite.pyi | 2 +- mypy/typeshed/stdlib/weakref.pyi | 55 +- .../stdlib/xml/parsers/expat/__init__.pyi | 6 + mypy/typeshed/stdlib/xmlrpc/client.pyi | 2 +- mypy/typeshed/stdlib/xmlrpc/server.pyi | 5 +- mypy/typeshed/stdlib/zoneinfo/__init__.pyi | 61 +- mypy/typeshed/stdlib/zoneinfo/_common.pyi | 13 + mypy/typeshed/stdlib/zoneinfo/_tzpath.pyi | 13 + test-data/unit/pythoneval.test | 22 +- 75 files changed, 4655 insertions(+), 3349 deletions(-) create mode 100644 mypy/typeshed/stdlib/_asyncio.pyi create mode 100644 mypy/typeshed/stdlib/_io.pyi create mode 100644 mypy/typeshed/stdlib/_sqlite3.pyi create mode 100644 mypy/typeshed/stdlib/_ssl.pyi create mode 100644 mypy/typeshed/stdlib/zoneinfo/_common.pyi create mode 100644 mypy/typeshed/stdlib/zoneinfo/_tzpath.pyi diff --git a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch index 683b0c322b71..91e255242ee9 100644 --- a/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch +++ b/misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch @@ -1,17 +1,17 @@ -From 3ec9b878d6bbe3fae64a508a62372f10a886406f Mon Sep 17 00:00:00 2001 +From b4259edd94188f9e4cc77a22e768eea183a32053 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 26 Sep 2022 12:55:07 -0700 Subject: [PATCH] Remove use of LiteralString in builtins (#13743) --- - mypy/typeshed/stdlib/builtins.pyi | 95 ------------------------------- - 1 file changed, 95 deletions(-) + mypy/typeshed/stdlib/builtins.pyi | 100 +----------------------------- + 1 file changed, 1 insertion(+), 99 deletions(-) diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi -index 53e00ec6a..bad3250ef 100644 +index 63c53a5f6..d55042b56 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi -@@ -61,7 +61,6 @@ from typing import ( # noqa: Y022 +@@ -63,7 +63,6 @@ from typing import ( # noqa: Y022 from typing_extensions import ( # noqa: Y023 Concatenate, Literal, @@ -19,7 +19,7 @@ index 53e00ec6a..bad3250ef 100644 ParamSpec, Self, TypeAlias, -@@ -435,31 +434,16 @@ class str(Sequence[str]): +@@ -438,31 +437,16 @@ class str(Sequence[str]): def __new__(cls, object: object = ...) -> Self: ... @overload def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ... @@ -51,7 +51,7 @@ index 53e00ec6a..bad3250ef 100644 def format(self, *args: object, **kwargs: object) -> str: ... def format_map(self, mapping: _FormatMapMapping, /) -> str: ... def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ... -@@ -475,99 +459,35 @@ class str(Sequence[str]): +@@ -478,99 +462,35 @@ class str(Sequence[str]): def isspace(self) -> bool: ... def istitle(self) -> bool: ... def isupper(self) -> bool: ... @@ -151,7 +151,7 @@ index 53e00ec6a..bad3250ef 100644 def zfill(self, width: SupportsIndex, /) -> str: ... # type: ignore[misc] @staticmethod @overload -@@ -578,9 +498,6 @@ class str(Sequence[str]): +@@ -581,39 +501,21 @@ class str(Sequence[str]): @staticmethod @overload def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ... @@ -161,8 +161,13 @@ index 53e00ec6a..bad3250ef 100644 def __add__(self, value: str, /) -> str: ... # type: ignore[misc] # Incompatible with Sequence.__contains__ def __contains__(self, key: str, /) -> bool: ... # type: ignore[override] -@@ -589,25 +506,13 @@ class str(Sequence[str]): - def __getitem__(self, key: SupportsIndex | slice, /) -> str: ... + def __eq__(self, value: object, /) -> bool: ... + def __ge__(self, value: str, /) -> bool: ... +- @overload +- def __getitem__(self: LiteralString, key: SupportsIndex | slice, /) -> LiteralString: ... +- @overload +- def __getitem__(self, key: SupportsIndex | slice, /) -> str: ... # type: ignore[misc] ++ def __getitem__(self, key: SupportsIndex | slice, /) -> str: ... def __gt__(self, value: str, /) -> bool: ... def __hash__(self) -> int: ... - @overload @@ -188,5 +193,5 @@ index 53e00ec6a..bad3250ef 100644 def __getnewargs__(self) -> tuple[str]: ... -- -2.45.2 +2.47.0 diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index dfed62f694fc..ed23ee6ddcea 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -20,6 +20,7 @@ __future__: 3.0- __main__: 3.0- _ast: 3.0- +_asyncio: 3.0- _bisect: 3.0- _bootlocale: 3.4-3.9 _codecs: 3.0- @@ -37,6 +38,7 @@ _imp: 3.0- _interpchannels: 3.13- _interpqueues: 3.13- _interpreters: 3.13- +_io: 3.0- _json: 3.0- _locale: 3.0- _lsprof: 3.0- @@ -50,6 +52,8 @@ _pydecimal: 3.5- _random: 3.0- _sitebuiltins: 3.4- _socket: 3.0- # present in 3.0 at runtime, but not in typeshed +_sqlite3: 3.0- +_ssl: 3.0- _stat: 3.4- _thread: 3.0- _threading_local: 3.0- diff --git a/mypy/typeshed/stdlib/_ast.pyi b/mypy/typeshed/stdlib/_ast.pyi index 1dbceac428c1..8dc1bcbea32c 100644 --- a/mypy/typeshed/stdlib/_ast.pyi +++ b/mypy/typeshed/stdlib/_ast.pyi @@ -1,1676 +1,149 @@ import sys -import typing_extensions -from typing import Any, ClassVar, Generic, Literal, TypedDict, overload -from typing_extensions import Self, Unpack +from ast import ( + AST as AST, + Add as Add, + And as And, + AnnAssign as AnnAssign, + Assert as Assert, + Assign as Assign, + AsyncFor as AsyncFor, + AsyncFunctionDef as AsyncFunctionDef, + AsyncWith as AsyncWith, + Attribute as Attribute, + AugAssign as AugAssign, + Await as Await, + BinOp as BinOp, + BitAnd as BitAnd, + BitOr as BitOr, + BitXor as BitXor, + BoolOp as BoolOp, + Break as Break, + Call as Call, + ClassDef as ClassDef, + Compare as Compare, + Constant as Constant, + Continue as Continue, + Del as Del, + Delete as Delete, + Dict as Dict, + DictComp as DictComp, + Div as Div, + Eq as Eq, + ExceptHandler as ExceptHandler, + Expr as Expr, + Expression as Expression, + FloorDiv as FloorDiv, + For as For, + FormattedValue as FormattedValue, + FunctionDef as FunctionDef, + FunctionType as FunctionType, + GeneratorExp as GeneratorExp, + Global as Global, + Gt as Gt, + GtE as GtE, + If as If, + IfExp as IfExp, + Import as Import, + ImportFrom as ImportFrom, + In as In, + Interactive as Interactive, + Invert as Invert, + Is as Is, + IsNot as IsNot, + JoinedStr as JoinedStr, + Lambda as Lambda, + List as List, + ListComp as ListComp, + Load as Load, + LShift as LShift, + Lt as Lt, + LtE as LtE, + MatMult as MatMult, + Mod as Mod, + Module as Module, + Mult as Mult, + Name as Name, + NamedExpr as NamedExpr, + Nonlocal as Nonlocal, + Not as Not, + NotEq as NotEq, + NotIn as NotIn, + Or as Or, + Pass as Pass, + Pow as Pow, + Raise as Raise, + Return as Return, + RShift as RShift, + Set as Set, + SetComp as SetComp, + Slice as Slice, + Starred as Starred, + Store as Store, + Sub as Sub, + Subscript as Subscript, + Try as Try, + Tuple as Tuple, + TypeIgnore as TypeIgnore, + UAdd as UAdd, + UnaryOp as UnaryOp, + USub as USub, + While as While, + With as With, + Yield as Yield, + YieldFrom as YieldFrom, + alias as alias, + arg as arg, + arguments as arguments, + boolop as boolop, + cmpop as cmpop, + comprehension as comprehension, + excepthandler as excepthandler, + expr as expr, + expr_context as expr_context, + keyword as keyword, + mod as mod, + operator as operator, + stmt as stmt, + type_ignore as type_ignore, + unaryop as unaryop, + withitem as withitem, +) +from typing import Literal -PyCF_ONLY_AST: Literal[1024] -PyCF_TYPE_COMMENTS: Literal[4096] -PyCF_ALLOW_TOP_LEVEL_AWAIT: Literal[8192] - -if sys.version_info >= (3, 13): - PyCF_OPTIMIZED_AST: Literal[33792] - -# Used for node end positions in constructor keyword arguments -_EndPositionT = typing_extensions.TypeVar("_EndPositionT", int, int | None, default=int | None) - -# Alias used for fields that must always be valid identifiers -# A string `x` counts as a valid identifier if both the following are True -# (1) `x.isidentifier()` evaluates to `True` -# (2) `keyword.iskeyword(x)` evaluates to `False` -_Identifier: typing_extensions.TypeAlias = str - -# Corresponds to the names in the `_attributes` class variable which is non-empty in certain AST nodes -class _Attributes(TypedDict, Generic[_EndPositionT], total=False): - lineno: int - col_offset: int - end_lineno: _EndPositionT - end_col_offset: _EndPositionT - -class AST: - if sys.version_info >= (3, 10): - __match_args__ = () - _attributes: ClassVar[tuple[str, ...]] - _fields: ClassVar[tuple[str, ...]] - if sys.version_info >= (3, 13): - _field_types: ClassVar[dict[str, Any]] - - if sys.version_info >= (3, 14): - def __replace__(self) -> Self: ... - -class mod(AST): ... -class type_ignore(AST): ... - -class TypeIgnore(type_ignore): - if sys.version_info >= (3, 10): - __match_args__ = ("lineno", "tag") - lineno: int - tag: str - def __init__(self, lineno: int, tag: str) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, lineno: int = ..., tag: str = ...) -> Self: ... - -class FunctionType(mod): - if sys.version_info >= (3, 10): - __match_args__ = ("argtypes", "returns") - argtypes: list[expr] - returns: expr - if sys.version_info >= (3, 13): - @overload - def __init__(self, argtypes: list[expr], returns: expr) -> None: ... - @overload - def __init__(self, argtypes: list[expr] = ..., *, returns: expr) -> None: ... - else: - def __init__(self, argtypes: list[expr], returns: expr) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, argtypes: list[expr] = ..., returns: expr = ...) -> Self: ... - -class Module(mod): - if sys.version_info >= (3, 10): - __match_args__ = ("body", "type_ignores") - body: list[stmt] - type_ignores: list[TypeIgnore] - if sys.version_info >= (3, 13): - def __init__(self, body: list[stmt] = ..., type_ignores: list[TypeIgnore] = ...) -> None: ... - else: - def __init__(self, body: list[stmt], type_ignores: list[TypeIgnore]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, body: list[stmt] = ..., type_ignores: list[TypeIgnore] = ...) -> Self: ... - -class Interactive(mod): - if sys.version_info >= (3, 10): - __match_args__ = ("body",) - body: list[stmt] - if sys.version_info >= (3, 13): - def __init__(self, body: list[stmt] = ...) -> None: ... - else: - def __init__(self, body: list[stmt]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, body: list[stmt] = ...) -> Self: ... - -class Expression(mod): - if sys.version_info >= (3, 10): - __match_args__ = ("body",) - body: expr - def __init__(self, body: expr) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, body: expr = ...) -> Self: ... - -class stmt(AST): - lineno: int - col_offset: int - end_lineno: int | None - end_col_offset: int | None - def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, **kwargs: Unpack[_Attributes]) -> Self: ... - -class FunctionDef(stmt): - if sys.version_info >= (3, 12): - __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment", "type_params") - elif sys.version_info >= (3, 10): - __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment") - name: _Identifier - args: arguments - body: list[stmt] - decorator_list: list[expr] - returns: expr | None - type_comment: str | None - if sys.version_info >= (3, 12): - type_params: list[type_param] - if sys.version_info >= (3, 13): - def __init__( - self, - name: _Identifier, - args: arguments, - body: list[stmt] = ..., - decorator_list: list[expr] = ..., - returns: expr | None = None, - type_comment: str | None = None, - type_params: list[type_param] = ..., - **kwargs: Unpack[_Attributes], - ) -> None: ... - elif sys.version_info >= (3, 12): - @overload - def __init__( - self, - name: _Identifier, - args: arguments, - body: list[stmt], - decorator_list: list[expr], - returns: expr | None, - type_comment: str | None, - type_params: list[type_param], - **kwargs: Unpack[_Attributes], - ) -> None: ... - @overload - def __init__( - self, - name: _Identifier, - args: arguments, - body: list[stmt], - decorator_list: list[expr], - returns: expr | None = None, - type_comment: str | None = None, - *, - type_params: list[type_param], - **kwargs: Unpack[_Attributes], - ) -> None: ... - else: - def __init__( - self, - name: _Identifier, - args: arguments, - body: list[stmt], - decorator_list: list[expr], - returns: expr | None = None, - type_comment: str | None = None, - **kwargs: Unpack[_Attributes], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - name: _Identifier = ..., - args: arguments = ..., - body: list[stmt] = ..., - decorator_list: list[expr] = ..., - returns: expr | None = ..., - type_comment: str | None = ..., - type_params: list[type_param] = ..., - ) -> Self: ... - -class AsyncFunctionDef(stmt): - if sys.version_info >= (3, 12): - __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment", "type_params") - elif sys.version_info >= (3, 10): - __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment") - name: _Identifier - args: arguments - body: list[stmt] - decorator_list: list[expr] - returns: expr | None - type_comment: str | None - if sys.version_info >= (3, 12): - type_params: list[type_param] - if sys.version_info >= (3, 13): - def __init__( - self, - name: _Identifier, - args: arguments, - body: list[stmt] = ..., - decorator_list: list[expr] = ..., - returns: expr | None = None, - type_comment: str | None = None, - type_params: list[type_param] = ..., - **kwargs: Unpack[_Attributes], - ) -> None: ... - elif sys.version_info >= (3, 12): - @overload - def __init__( - self, - name: _Identifier, - args: arguments, - body: list[stmt], - decorator_list: list[expr], - returns: expr | None, - type_comment: str | None, - type_params: list[type_param], - **kwargs: Unpack[_Attributes], - ) -> None: ... - @overload - def __init__( - self, - name: _Identifier, - args: arguments, - body: list[stmt], - decorator_list: list[expr], - returns: expr | None = None, - type_comment: str | None = None, - *, - type_params: list[type_param], - **kwargs: Unpack[_Attributes], - ) -> None: ... - else: - def __init__( - self, - name: _Identifier, - args: arguments, - body: list[stmt], - decorator_list: list[expr], - returns: expr | None = None, - type_comment: str | None = None, - **kwargs: Unpack[_Attributes], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - name: _Identifier = ..., - args: arguments = ..., - body: list[stmt], - decorator_list: list[expr], - returns: expr | None, - type_comment: str | None, - type_params: list[type_param], - ) -> Self: ... - -class ClassDef(stmt): - if sys.version_info >= (3, 12): - __match_args__ = ("name", "bases", "keywords", "body", "decorator_list", "type_params") - elif sys.version_info >= (3, 10): - __match_args__ = ("name", "bases", "keywords", "body", "decorator_list") - name: _Identifier - bases: list[expr] - keywords: list[keyword] - body: list[stmt] - decorator_list: list[expr] - if sys.version_info >= (3, 12): - type_params: list[type_param] - if sys.version_info >= (3, 13): - def __init__( - self, - name: _Identifier, - bases: list[expr] = ..., - keywords: list[keyword] = ..., - body: list[stmt] = ..., - decorator_list: list[expr] = ..., - type_params: list[type_param] = ..., - **kwargs: Unpack[_Attributes], - ) -> None: ... - elif sys.version_info >= (3, 12): - def __init__( - self, - name: _Identifier, - bases: list[expr], - keywords: list[keyword], - body: list[stmt], - decorator_list: list[expr], - type_params: list[type_param], - **kwargs: Unpack[_Attributes], - ) -> None: ... - else: - def __init__( - self, - name: _Identifier, - bases: list[expr], - keywords: list[keyword], - body: list[stmt], - decorator_list: list[expr], - **kwargs: Unpack[_Attributes], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - name: _Identifier, - bases: list[expr], - keywords: list[keyword], - body: list[stmt], - decorator_list: list[expr], - type_params: list[type_param], - **kwargs: Unpack[_Attributes], - ) -> Self: ... - -class Return(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("value",) - value: expr | None - def __init__(self, value: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, value: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Delete(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("targets",) - targets: list[expr] - if sys.version_info >= (3, 13): - def __init__(self, targets: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, targets: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, targets: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Assign(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("targets", "value", "type_comment") - targets: list[expr] - value: expr - type_comment: str | None - if sys.version_info >= (3, 13): - @overload - def __init__( - self, targets: list[expr], value: expr, type_comment: str | None = None, **kwargs: Unpack[_Attributes] - ) -> None: ... - @overload - def __init__( - self, targets: list[expr] = ..., *, value: expr, type_comment: str | None = None, **kwargs: Unpack[_Attributes] - ) -> None: ... - else: - def __init__( - self, targets: list[expr], value: expr, type_comment: str | None = None, **kwargs: Unpack[_Attributes] - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, targets: list[expr] = ..., value: expr = ..., type_comment: str | None = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class AugAssign(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("target", "op", "value") - target: Name | Attribute | Subscript - op: operator - value: expr - def __init__( - self, target: Name | Attribute | Subscript, op: operator, value: expr, **kwargs: Unpack[_Attributes] - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - target: Name | Attribute | Subscript = ..., - op: operator = ..., - value: expr = ..., - **kwargs: Unpack[_Attributes], - ) -> Self: ... - -class AnnAssign(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("target", "annotation", "value", "simple") - target: Name | Attribute | Subscript - annotation: expr - value: expr | None - simple: int - @overload - def __init__( - self, - target: Name | Attribute | Subscript, - annotation: expr, - value: expr | None, - simple: int, - **kwargs: Unpack[_Attributes], - ) -> None: ... - @overload - def __init__( - self, - target: Name | Attribute | Subscript, - annotation: expr, - value: expr | None = None, - *, - simple: int, - **kwargs: Unpack[_Attributes], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - target: Name | Attribute | Subscript = ..., - annotation: expr = ..., - value: expr | None = ..., - simple: int = ..., - **kwargs: Unpack[_Attributes], - ) -> Self: ... - -class For(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("target", "iter", "body", "orelse", "type_comment") - target: expr - iter: expr - body: list[stmt] - orelse: list[stmt] - type_comment: str | None - if sys.version_info >= (3, 13): - def __init__( - self, - target: expr, - iter: expr, - body: list[stmt] = ..., - orelse: list[stmt] = ..., - type_comment: str | None = None, - **kwargs: Unpack[_Attributes], - ) -> None: ... - else: - def __init__( - self, - target: expr, - iter: expr, - body: list[stmt], - orelse: list[stmt], - type_comment: str | None = None, - **kwargs: Unpack[_Attributes], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - target: expr = ..., - iter: expr = ..., - body: list[stmt] = ..., - orelse: list[stmt] = ..., - type_comment: str | None = ..., - **kwargs: Unpack[_Attributes], - ) -> Self: ... - -class AsyncFor(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("target", "iter", "body", "orelse", "type_comment") - target: expr - iter: expr - body: list[stmt] - orelse: list[stmt] - type_comment: str | None - if sys.version_info >= (3, 13): - def __init__( - self, - target: expr, - iter: expr, - body: list[stmt] = ..., - orelse: list[stmt] = ..., - type_comment: str | None = None, - **kwargs: Unpack[_Attributes], - ) -> None: ... - else: - def __init__( - self, - target: expr, - iter: expr, - body: list[stmt], - orelse: list[stmt], - type_comment: str | None = None, - **kwargs: Unpack[_Attributes], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - target: expr = ..., - iter: expr = ..., - body: list[stmt] = ..., - orelse: list[stmt] = ..., - type_comment: str | None = ..., - **kwargs: Unpack[_Attributes], - ) -> Self: ... - -class While(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("test", "body", "orelse") - test: expr - body: list[stmt] - orelse: list[stmt] - if sys.version_info >= (3, 13): - def __init__( - self, test: expr, body: list[stmt] = ..., orelse: list[stmt] = ..., **kwargs: Unpack[_Attributes] - ) -> None: ... - else: - def __init__(self, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> Self: ... - -class If(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("test", "body", "orelse") - test: expr - body: list[stmt] - orelse: list[stmt] - if sys.version_info >= (3, 13): - def __init__( - self, test: expr, body: list[stmt] = ..., orelse: list[stmt] = ..., **kwargs: Unpack[_Attributes] - ) -> None: ... - else: - def __init__(self, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, test: expr = ..., body: list[stmt] = ..., orelse: list[stmt] = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class With(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("items", "body", "type_comment") - items: list[withitem] - body: list[stmt] - type_comment: str | None - if sys.version_info >= (3, 13): - def __init__( - self, - items: list[withitem] = ..., - body: list[stmt] = ..., - type_comment: str | None = None, - **kwargs: Unpack[_Attributes], - ) -> None: ... - else: - def __init__( - self, items: list[withitem], body: list[stmt], type_comment: str | None = None, **kwargs: Unpack[_Attributes] - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - items: list[withitem] = ..., - body: list[stmt] = ..., - type_comment: str | None = ..., - **kwargs: Unpack[_Attributes], - ) -> Self: ... - -class AsyncWith(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("items", "body", "type_comment") - items: list[withitem] - body: list[stmt] - type_comment: str | None - if sys.version_info >= (3, 13): - def __init__( - self, - items: list[withitem] = ..., - body: list[stmt] = ..., - type_comment: str | None = None, - **kwargs: Unpack[_Attributes], - ) -> None: ... - else: - def __init__( - self, items: list[withitem], body: list[stmt], type_comment: str | None = None, **kwargs: Unpack[_Attributes] - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - items: list[withitem] = ..., - body: list[stmt] = ..., - type_comment: str | None = ..., - **kwargs: Unpack[_Attributes], - ) -> Self: ... - -class Raise(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("exc", "cause") - exc: expr | None - cause: expr | None - def __init__(self, exc: expr | None = None, cause: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, exc: expr | None = ..., cause: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Try(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("body", "handlers", "orelse", "finalbody") - body: list[stmt] - handlers: list[ExceptHandler] - orelse: list[stmt] - finalbody: list[stmt] - if sys.version_info >= (3, 13): - def __init__( - self, - body: list[stmt] = ..., - handlers: list[ExceptHandler] = ..., - orelse: list[stmt] = ..., - finalbody: list[stmt] = ..., - **kwargs: Unpack[_Attributes], - ) -> None: ... - else: - def __init__( - self, - body: list[stmt], - handlers: list[ExceptHandler], - orelse: list[stmt], - finalbody: list[stmt], - **kwargs: Unpack[_Attributes], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - body: list[stmt] = ..., - handlers: list[ExceptHandler] = ..., - orelse: list[stmt] = ..., - finalbody: list[stmt] = ..., - **kwargs: Unpack[_Attributes], - ) -> Self: ... +if sys.version_info >= (3, 12): + from ast import ParamSpec as ParamSpec, TypeVar as TypeVar, TypeVarTuple as TypeVarTuple, type_param as type_param if sys.version_info >= (3, 11): - class TryStar(stmt): - __match_args__ = ("body", "handlers", "orelse", "finalbody") - body: list[stmt] - handlers: list[ExceptHandler] - orelse: list[stmt] - finalbody: list[stmt] - if sys.version_info >= (3, 13): - def __init__( - self, - body: list[stmt] = ..., - handlers: list[ExceptHandler] = ..., - orelse: list[stmt] = ..., - finalbody: list[stmt] = ..., - **kwargs: Unpack[_Attributes], - ) -> None: ... - else: - def __init__( - self, - body: list[stmt], - handlers: list[ExceptHandler], - orelse: list[stmt], - finalbody: list[stmt], - **kwargs: Unpack[_Attributes], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - body: list[stmt] = ..., - handlers: list[ExceptHandler] = ..., - orelse: list[stmt] = ..., - finalbody: list[stmt] = ..., - **kwargs: Unpack[_Attributes], - ) -> Self: ... - -class Assert(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("test", "msg") - test: expr - msg: expr | None - def __init__(self, test: expr, msg: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, test: expr, msg: expr | None, **kwargs: Unpack[_Attributes]) -> Self: ... - -class Import(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("names",) - names: list[alias] - if sys.version_info >= (3, 13): - def __init__(self, names: list[alias] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, names: list[alias], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, names: list[alias] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class ImportFrom(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("module", "names", "level") - module: str | None - names: list[alias] - level: int - if sys.version_info >= (3, 13): - @overload - def __init__(self, module: str | None, names: list[alias], level: int, **kwargs: Unpack[_Attributes]) -> None: ... - @overload - def __init__( - self, module: str | None = None, names: list[alias] = ..., *, level: int, **kwargs: Unpack[_Attributes] - ) -> None: ... - else: - @overload - def __init__(self, module: str | None, names: list[alias], level: int, **kwargs: Unpack[_Attributes]) -> None: ... - @overload - def __init__( - self, module: str | None = None, *, names: list[alias], level: int, **kwargs: Unpack[_Attributes] - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, module: str | None = ..., names: list[alias] = ..., level: int = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class Global(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("names",) - names: list[_Identifier] - if sys.version_info >= (3, 13): - def __init__(self, names: list[_Identifier] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> Self: ... - -class Nonlocal(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("names",) - names: list[_Identifier] - if sys.version_info >= (3, 13): - def __init__(self, names: list[_Identifier] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, names: list[_Identifier] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Expr(stmt): - if sys.version_info >= (3, 10): - __match_args__ = ("value",) - value: expr - def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Pass(stmt): ... -class Break(stmt): ... -class Continue(stmt): ... - -class expr(AST): - lineno: int - col_offset: int - end_lineno: int | None - end_col_offset: int | None - def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, **kwargs: Unpack[_Attributes]) -> Self: ... - -class BoolOp(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("op", "values") - op: boolop - values: list[expr] - if sys.version_info >= (3, 13): - def __init__(self, op: boolop, values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, op: boolop, values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, op: boolop = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class BinOp(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("left", "op", "right") - left: expr - op: operator - right: expr - def __init__(self, left: expr, op: operator, right: expr, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, left: expr = ..., op: operator = ..., right: expr = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class UnaryOp(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("op", "operand") - op: unaryop - operand: expr - def __init__(self, op: unaryop, operand: expr, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, op: unaryop = ..., operand: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Lambda(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("args", "body") - args: arguments - body: expr - def __init__(self, args: arguments, body: expr, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, args: arguments = ..., body: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class IfExp(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("test", "body", "orelse") - test: expr - body: expr - orelse: expr - def __init__(self, test: expr, body: expr, orelse: expr, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, test: expr = ..., body: expr = ..., orelse: expr = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class Dict(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("keys", "values") - keys: list[expr | None] - values: list[expr] - if sys.version_info >= (3, 13): - def __init__(self, keys: list[expr | None] = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, keys: list[expr | None], values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, keys: list[expr | None] = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class Set(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("elts",) - elts: list[expr] - if sys.version_info >= (3, 13): - def __init__(self, elts: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, elts: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, elts: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class ListComp(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("elt", "generators") - elt: expr - generators: list[comprehension] - if sys.version_info >= (3, 13): - def __init__(self, elt: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class SetComp(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("elt", "generators") - elt: expr - generators: list[comprehension] - if sys.version_info >= (3, 13): - def __init__(self, elt: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class DictComp(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("key", "value", "generators") - key: expr - value: expr - generators: list[comprehension] - if sys.version_info >= (3, 13): - def __init__( - self, key: expr, value: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] - ) -> None: ... - else: - def __init__(self, key: expr, value: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, key: expr = ..., value: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class GeneratorExp(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("elt", "generators") - elt: expr - generators: list[comprehension] - if sys.version_info >= (3, 13): - def __init__(self, elt: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class Await(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("value",) - value: expr - def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Yield(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("value",) - value: expr | None - def __init__(self, value: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, value: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class YieldFrom(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("value",) - value: expr - def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Compare(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("left", "ops", "comparators") - left: expr - ops: list[cmpop] - comparators: list[expr] - if sys.version_info >= (3, 13): - def __init__( - self, left: expr, ops: list[cmpop] = ..., comparators: list[expr] = ..., **kwargs: Unpack[_Attributes] - ) -> None: ... - else: - def __init__(self, left: expr, ops: list[cmpop], comparators: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, left: expr = ..., ops: list[cmpop] = ..., comparators: list[expr] = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class Call(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("func", "args", "keywords") - func: expr - args: list[expr] - keywords: list[keyword] - if sys.version_info >= (3, 13): - def __init__( - self, func: expr, args: list[expr] = ..., keywords: list[keyword] = ..., **kwargs: Unpack[_Attributes] - ) -> None: ... - else: - def __init__(self, func: expr, args: list[expr], keywords: list[keyword], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, func: expr = ..., args: list[expr] = ..., keywords: list[keyword] = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class FormattedValue(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("value", "conversion", "format_spec") - value: expr - conversion: int - format_spec: expr | None - def __init__(self, value: expr, conversion: int, format_spec: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, value: expr = ..., conversion: int = ..., format_spec: expr | None = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class JoinedStr(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("values",) - values: list[expr] - if sys.version_info >= (3, 13): - def __init__(self, values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Constant(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("value", "kind") - value: Any # None, str, bytes, bool, int, float, complex, Ellipsis - kind: str | None - if sys.version_info < (3, 14): - # Aliases for value, for backwards compatibility - s: Any - n: int | float | complex - - def __init__(self, value: Any, kind: str | None = None, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, value: Any = ..., kind: str | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class NamedExpr(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("target", "value") - target: Name - value: expr - def __init__(self, target: Name, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, target: Name = ..., value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Attribute(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("value", "attr", "ctx") - value: expr - attr: _Identifier - ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` - def __init__(self, value: expr, attr: _Identifier, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, value: expr = ..., attr: _Identifier = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -if sys.version_info >= (3, 9): - _Slice: typing_extensions.TypeAlias = expr - _SliceAttributes: typing_extensions.TypeAlias = _Attributes -else: - class slice(AST): ... - _Slice: typing_extensions.TypeAlias = slice - - class _SliceAttributes(TypedDict): ... - -class Slice(_Slice): - if sys.version_info >= (3, 10): - __match_args__ = ("lower", "upper", "step") - lower: expr | None - upper: expr | None - step: expr | None - def __init__( - self, lower: expr | None = None, upper: expr | None = None, step: expr | None = None, **kwargs: Unpack[_SliceAttributes] - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - lower: expr | None = ..., - upper: expr | None = ..., - step: expr | None = ..., - **kwargs: Unpack[_SliceAttributes], - ) -> Self: ... - -if sys.version_info < (3, 9): - class ExtSlice(slice): - dims: list[slice] - def __init__(self, dims: list[slice], **kwargs: Unpack[_SliceAttributes]) -> None: ... - - class Index(slice): - value: expr - def __init__(self, value: expr, **kwargs: Unpack[_SliceAttributes]) -> None: ... - -class Subscript(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("value", "slice", "ctx") - value: expr - slice: _Slice - ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` - def __init__(self, value: expr, slice: _Slice, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, value: expr = ..., slice: _Slice = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - -class Starred(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("value", "ctx") - value: expr - ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` - def __init__(self, value: expr, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, value: expr = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Name(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("id", "ctx") - id: _Identifier - ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` - def __init__(self, id: _Identifier, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, id: _Identifier = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class List(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("elts", "ctx") - elts: list[expr] - ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` - if sys.version_info >= (3, 13): - def __init__(self, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, elts: list[expr], ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class Tuple(expr): - if sys.version_info >= (3, 10): - __match_args__ = ("elts", "ctx") - elts: list[expr] - ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` - if sys.version_info >= (3, 9): - dims: list[expr] - if sys.version_info >= (3, 13): - def __init__(self, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, elts: list[expr], ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class expr_context(AST): ... - -if sys.version_info < (3, 9): - class AugLoad(expr_context): ... - class AugStore(expr_context): ... - class Param(expr_context): ... - - class Suite(mod): - body: list[stmt] - def __init__(self, body: list[stmt]) -> None: ... - -class Del(expr_context): ... -class Load(expr_context): ... -class Store(expr_context): ... -class boolop(AST): ... -class And(boolop): ... -class Or(boolop): ... -class operator(AST): ... -class Add(operator): ... -class BitAnd(operator): ... -class BitOr(operator): ... -class BitXor(operator): ... -class Div(operator): ... -class FloorDiv(operator): ... -class LShift(operator): ... -class Mod(operator): ... -class Mult(operator): ... -class MatMult(operator): ... -class Pow(operator): ... -class RShift(operator): ... -class Sub(operator): ... -class unaryop(AST): ... -class Invert(unaryop): ... -class Not(unaryop): ... -class UAdd(unaryop): ... -class USub(unaryop): ... -class cmpop(AST): ... -class Eq(cmpop): ... -class Gt(cmpop): ... -class GtE(cmpop): ... -class In(cmpop): ... -class Is(cmpop): ... -class IsNot(cmpop): ... -class Lt(cmpop): ... -class LtE(cmpop): ... -class NotEq(cmpop): ... -class NotIn(cmpop): ... - -class comprehension(AST): - if sys.version_info >= (3, 10): - __match_args__ = ("target", "iter", "ifs", "is_async") - target: expr - iter: expr - ifs: list[expr] - is_async: int - if sys.version_info >= (3, 13): - @overload - def __init__(self, target: expr, iter: expr, ifs: list[expr], is_async: int) -> None: ... - @overload - def __init__(self, target: expr, iter: expr, ifs: list[expr] = ..., *, is_async: int) -> None: ... - else: - def __init__(self, target: expr, iter: expr, ifs: list[expr], is_async: int) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, target: expr = ..., iter: expr = ..., ifs: list[expr] = ..., is_async: int = ...) -> Self: ... - -class excepthandler(AST): - lineno: int - col_offset: int - end_lineno: int | None - end_col_offset: int | None - def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, lineno: int = ..., col_offset: int = ..., end_lineno: int | None = ..., end_col_offset: int | None = ... - ) -> Self: ... - -class ExceptHandler(excepthandler): - if sys.version_info >= (3, 10): - __match_args__ = ("type", "name", "body") - type: expr | None - name: _Identifier | None - body: list[stmt] - if sys.version_info >= (3, 13): - def __init__( - self, type: expr | None = None, name: _Identifier | None = None, body: list[stmt] = ..., **kwargs: Unpack[_Attributes] - ) -> None: ... - else: - @overload - def __init__( - self, type: expr | None, name: _Identifier | None, body: list[stmt], **kwargs: Unpack[_Attributes] - ) -> None: ... - @overload - def __init__( - self, type: expr | None = None, name: _Identifier | None = None, *, body: list[stmt], **kwargs: Unpack[_Attributes] - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - type: expr | None = ..., - name: _Identifier | None = ..., - body: list[stmt] = ..., - **kwargs: Unpack[_Attributes], - ) -> Self: ... - -class arguments(AST): - if sys.version_info >= (3, 10): - __match_args__ = ("posonlyargs", "args", "vararg", "kwonlyargs", "kw_defaults", "kwarg", "defaults") - posonlyargs: list[arg] - args: list[arg] - vararg: arg | None - kwonlyargs: list[arg] - kw_defaults: list[expr | None] - kwarg: arg | None - defaults: list[expr] - if sys.version_info >= (3, 13): - def __init__( - self, - posonlyargs: list[arg] = ..., - args: list[arg] = ..., - vararg: arg | None = None, - kwonlyargs: list[arg] = ..., - kw_defaults: list[expr | None] = ..., - kwarg: arg | None = None, - defaults: list[expr] = ..., - ) -> None: ... - else: - @overload - def __init__( - self, - posonlyargs: list[arg], - args: list[arg], - vararg: arg | None, - kwonlyargs: list[arg], - kw_defaults: list[expr | None], - kwarg: arg | None, - defaults: list[expr], - ) -> None: ... - @overload - def __init__( - self, - posonlyargs: list[arg], - args: list[arg], - vararg: arg | None, - kwonlyargs: list[arg], - kw_defaults: list[expr | None], - kwarg: arg | None = None, - *, - defaults: list[expr], - ) -> None: ... - @overload - def __init__( - self, - posonlyargs: list[arg], - args: list[arg], - vararg: arg | None = None, - *, - kwonlyargs: list[arg], - kw_defaults: list[expr | None], - kwarg: arg | None = None, - defaults: list[expr], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - posonlyargs: list[arg] = ..., - args: list[arg] = ..., - vararg: arg | None = ..., - kwonlyargs: list[arg] = ..., - kw_defaults: list[expr | None] = ..., - kwarg: arg | None = ..., - defaults: list[expr] = ..., - ) -> Self: ... - -class arg(AST): - lineno: int - col_offset: int - end_lineno: int | None - end_col_offset: int | None - if sys.version_info >= (3, 10): - __match_args__ = ("arg", "annotation", "type_comment") - arg: _Identifier - annotation: expr | None - type_comment: str | None - def __init__( - self, arg: _Identifier, annotation: expr | None = None, type_comment: str | None = None, **kwargs: Unpack[_Attributes] - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - arg: _Identifier = ..., - annotation: expr | None = ..., - type_comment: str | None = ..., - **kwargs: Unpack[_Attributes], - ) -> Self: ... - -class keyword(AST): - lineno: int - col_offset: int - end_lineno: int | None - end_col_offset: int | None - if sys.version_info >= (3, 10): - __match_args__ = ("arg", "value") - arg: _Identifier | None - value: expr - @overload - def __init__(self, arg: _Identifier | None, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... - @overload - def __init__(self, arg: _Identifier | None = None, *, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, arg: _Identifier | None = ..., value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class alias(AST): - lineno: int - col_offset: int - end_lineno: int | None - end_col_offset: int | None - if sys.version_info >= (3, 10): - __match_args__ = ("name", "asname") - name: str - asname: _Identifier | None - def __init__(self, name: str, asname: _Identifier | None = None, **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, name: str = ..., asname: _Identifier | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - -class withitem(AST): - if sys.version_info >= (3, 10): - __match_args__ = ("context_expr", "optional_vars") - context_expr: expr - optional_vars: expr | None - def __init__(self, context_expr: expr, optional_vars: expr | None = None) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, context_expr: expr = ..., optional_vars: expr | None = ...) -> Self: ... + from ast import TryStar as TryStar if sys.version_info >= (3, 10): - class Match(stmt): - __match_args__ = ("subject", "cases") - subject: expr - cases: list[match_case] - if sys.version_info >= (3, 13): - def __init__(self, subject: expr, cases: list[match_case] = ..., **kwargs: Unpack[_Attributes]) -> None: ... - else: - def __init__(self, subject: expr, cases: list[match_case], **kwargs: Unpack[_Attributes]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, subject: expr = ..., cases: list[match_case] = ..., **kwargs: Unpack[_Attributes] - ) -> Self: ... - - class pattern(AST): - lineno: int - col_offset: int - end_lineno: int - end_col_offset: int - def __init__(self, **kwargs: Unpack[_Attributes[int]]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, lineno: int = ..., col_offset: int = ..., end_lineno: int = ..., end_col_offset: int = ... - ) -> Self: ... - - # Without the alias, Pyright complains variables named pattern are recursively defined - _Pattern: typing_extensions.TypeAlias = pattern - - class match_case(AST): - __match_args__ = ("pattern", "guard", "body") - pattern: _Pattern - guard: expr | None - body: list[stmt] - if sys.version_info >= (3, 13): - def __init__(self, pattern: _Pattern, guard: expr | None = None, body: list[stmt] = ...) -> None: ... - else: - @overload - def __init__(self, pattern: _Pattern, guard: expr | None, body: list[stmt]) -> None: ... - @overload - def __init__(self, pattern: _Pattern, guard: expr | None = None, *, body: list[stmt]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, pattern: _Pattern = ..., guard: expr | None = ..., body: list[stmt] = ...) -> Self: ... - - class MatchValue(pattern): - __match_args__ = ("value",) - value: expr - def __init__(self, value: expr, **kwargs: Unpack[_Attributes[int]]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... - - class MatchSingleton(pattern): - __match_args__ = ("value",) - value: Literal[True, False] | None - def __init__(self, value: Literal[True, False] | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, value: Literal[True, False] | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... - - class MatchSequence(pattern): - __match_args__ = ("patterns",) - patterns: list[pattern] - if sys.version_info >= (3, 13): - def __init__(self, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> None: ... - else: - def __init__(self, patterns: list[pattern], **kwargs: Unpack[_Attributes[int]]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... - - class MatchStar(pattern): - __match_args__ = ("name",) - name: _Identifier | None - def __init__(self, name: _Identifier | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + from ast import ( + MatchAs as MatchAs, + MatchClass as MatchClass, + MatchMapping as MatchMapping, + MatchOr as MatchOr, + MatchSequence as MatchSequence, + MatchSingleton as MatchSingleton, + MatchStar as MatchStar, + MatchValue as MatchValue, + match_case as match_case, + pattern as pattern, + ) - if sys.version_info >= (3, 14): - def __replace__(self, *, name: _Identifier | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... - - class MatchMapping(pattern): - __match_args__ = ("keys", "patterns", "rest") - keys: list[expr] - patterns: list[pattern] - rest: _Identifier | None - if sys.version_info >= (3, 13): - def __init__( - self, - keys: list[expr] = ..., - patterns: list[pattern] = ..., - rest: _Identifier | None = None, - **kwargs: Unpack[_Attributes[int]], - ) -> None: ... - else: - def __init__( - self, - keys: list[expr], - patterns: list[pattern], - rest: _Identifier | None = None, - **kwargs: Unpack[_Attributes[int]], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - keys: list[expr] = ..., - patterns: list[pattern] = ..., - rest: _Identifier | None = ..., - **kwargs: Unpack[_Attributes[int]], - ) -> Self: ... - - class MatchClass(pattern): - __match_args__ = ("cls", "patterns", "kwd_attrs", "kwd_patterns") - cls: expr - patterns: list[pattern] - kwd_attrs: list[_Identifier] - kwd_patterns: list[pattern] - if sys.version_info >= (3, 13): - def __init__( - self, - cls: expr, - patterns: list[pattern] = ..., - kwd_attrs: list[_Identifier] = ..., - kwd_patterns: list[pattern] = ..., - **kwargs: Unpack[_Attributes[int]], - ) -> None: ... - else: - def __init__( - self, - cls: expr, - patterns: list[pattern], - kwd_attrs: list[_Identifier], - kwd_patterns: list[pattern], - **kwargs: Unpack[_Attributes[int]], - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - cls: expr = ..., - patterns: list[pattern] = ..., - kwd_attrs: list[_Identifier] = ..., - kwd_patterns: list[pattern] = ..., - **kwargs: Unpack[_Attributes[int]], - ) -> Self: ... - - class MatchAs(pattern): - __match_args__ = ("pattern", "name") - pattern: _Pattern | None - name: _Identifier | None - def __init__( - self, pattern: _Pattern | None = None, name: _Identifier | None = None, **kwargs: Unpack[_Attributes[int]] - ) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, pattern: _Pattern | None = ..., name: _Identifier | None = ..., **kwargs: Unpack[_Attributes[int]] - ) -> Self: ... - - class MatchOr(pattern): - __match_args__ = ("patterns",) - patterns: list[pattern] - if sys.version_info >= (3, 13): - def __init__(self, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> None: ... - else: - def __init__(self, patterns: list[pattern], **kwargs: Unpack[_Attributes[int]]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, *, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... - -if sys.version_info >= (3, 12): - class type_param(AST): - lineno: int - col_offset: int - end_lineno: int - end_col_offset: int - def __init__(self, **kwargs: Unpack[_Attributes[int]]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__(self, **kwargs: Unpack[_Attributes[int]]) -> Self: ... - - class TypeVar(type_param): - if sys.version_info >= (3, 13): - __match_args__ = ("name", "bound", "default_value") - else: - __match_args__ = ("name", "bound") - name: _Identifier - bound: expr | None - if sys.version_info >= (3, 13): - default_value: expr | None - def __init__( - self, - name: _Identifier, - bound: expr | None = None, - default_value: expr | None = None, - **kwargs: Unpack[_Attributes[int]], - ) -> None: ... - else: - def __init__(self, name: _Identifier, bound: expr | None = None, **kwargs: Unpack[_Attributes[int]]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - name: _Identifier = ..., - bound: expr | None = ..., - default_value: expr | None = ..., - **kwargs: Unpack[_Attributes[int]], - ) -> Self: ... - - class ParamSpec(type_param): - if sys.version_info >= (3, 13): - __match_args__ = ("name", "default_value") - else: - __match_args__ = ("name",) - name: _Identifier - if sys.version_info >= (3, 13): - default_value: expr | None - def __init__( - self, name: _Identifier, default_value: expr | None = None, **kwargs: Unpack[_Attributes[int]] - ) -> None: ... - else: - def __init__(self, name: _Identifier, **kwargs: Unpack[_Attributes[int]]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, name: _Identifier = ..., default_value: expr | None = ..., **kwargs: Unpack[_Attributes[int]] - ) -> Self: ... - - class TypeVarTuple(type_param): - if sys.version_info >= (3, 13): - __match_args__ = ("name", "default_value") - else: - __match_args__ = ("name",) - name: _Identifier - if sys.version_info >= (3, 13): - default_value: expr | None - def __init__( - self, name: _Identifier, default_value: expr | None = None, **kwargs: Unpack[_Attributes[int]] - ) -> None: ... - else: - def __init__(self, name: _Identifier, **kwargs: Unpack[_Attributes[int]]) -> None: ... - - if sys.version_info >= (3, 14): - def __replace__( - self, *, name: _Identifier = ..., default_value: expr | None = ..., **kwargs: Unpack[_Attributes[int]] - ) -> Self: ... +if sys.version_info < (3, 9): + from ast import ( + AugLoad as AugLoad, + AugStore as AugStore, + ExtSlice as ExtSlice, + Index as Index, + Param as Param, + Suite as Suite, + slice as slice, + ) - class TypeAlias(stmt): - __match_args__ = ("name", "type_params", "value") - name: Name - type_params: list[type_param] - value: expr - if sys.version_info >= (3, 13): - @overload - def __init__( - self, name: Name, type_params: list[type_param], value: expr, **kwargs: Unpack[_Attributes[int]] - ) -> None: ... - @overload - def __init__( - self, name: Name, type_params: list[type_param] = ..., *, value: expr, **kwargs: Unpack[_Attributes[int]] - ) -> None: ... - else: - def __init__( - self, name: Name, type_params: list[type_param], value: expr, **kwargs: Unpack[_Attributes[int]] - ) -> None: ... +PyCF_ALLOW_TOP_LEVEL_AWAIT: Literal[8192] +PyCF_ONLY_AST: Literal[1024] +PyCF_TYPE_COMMENTS: Literal[4096] - if sys.version_info >= (3, 14): - def __replace__( - self, - *, - name: Name = ..., - type_params: list[type_param] = ..., - value: expr = ..., - **kwargs: Unpack[_Attributes[int]], - ) -> Self: ... +if sys.version_info >= (3, 13): + PyCF_OPTIMIZED_AST: Literal[33792] diff --git a/mypy/typeshed/stdlib/_asyncio.pyi b/mypy/typeshed/stdlib/_asyncio.pyi new file mode 100644 index 000000000000..18920cd8a8a4 --- /dev/null +++ b/mypy/typeshed/stdlib/_asyncio.pyi @@ -0,0 +1,120 @@ +import sys +from asyncio.events import AbstractEventLoop +from collections.abc import Awaitable, Callable, Coroutine, Generator, Iterable +from contextvars import Context +from types import FrameType +from typing import Any, Literal, TextIO, TypeVar +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 9): + from types import GenericAlias + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_TaskYieldType: TypeAlias = Future[object] | None + +class Future(Awaitable[_T], Iterable[_T]): + _state: str + @property + def _exception(self) -> BaseException | None: ... + _blocking: bool + @property + def _log_traceback(self) -> bool: ... + @_log_traceback.setter + def _log_traceback(self, val: Literal[False]) -> None: ... + _asyncio_future_blocking: bool # is a part of duck-typing contract for `Future` + def __init__(self, *, loop: AbstractEventLoop | None = ...) -> None: ... + def __del__(self) -> None: ... + def get_loop(self) -> AbstractEventLoop: ... + @property + def _callbacks(self) -> list[tuple[Callable[[Self], Any], Context]]: ... + def add_done_callback(self, fn: Callable[[Self], object], /, *, context: Context | None = None) -> None: ... + if sys.version_info >= (3, 9): + def cancel(self, msg: Any | None = None) -> bool: ... + else: + def cancel(self) -> bool: ... + + def cancelled(self) -> bool: ... + def done(self) -> bool: ... + def result(self) -> _T: ... + def exception(self) -> BaseException | None: ... + def remove_done_callback(self, fn: Callable[[Self], object], /) -> int: ... + def set_result(self, result: _T, /) -> None: ... + def set_exception(self, exception: type | BaseException, /) -> None: ... + def __iter__(self) -> Generator[Any, None, _T]: ... + def __await__(self) -> Generator[Any, None, _T]: ... + @property + def _loop(self) -> AbstractEventLoop: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +if sys.version_info >= (3, 12): + _TaskCompatibleCoro: TypeAlias = Coroutine[Any, Any, _T_co] +elif sys.version_info >= (3, 9): + _TaskCompatibleCoro: TypeAlias = Generator[_TaskYieldType, None, _T_co] | Coroutine[Any, Any, _T_co] +else: + _TaskCompatibleCoro: TypeAlias = Generator[_TaskYieldType, None, _T_co] | Awaitable[_T_co] + +# mypy and pyright complain that a subclass of an invariant class shouldn't be covariant. +# While this is true in general, here it's sort-of okay to have a covariant subclass, +# since the only reason why `asyncio.Future` is invariant is the `set_result()` method, +# and `asyncio.Task.set_result()` always raises. +class Task(Future[_T_co]): # type: ignore[type-var] # pyright: ignore[reportInvalidTypeArguments] + if sys.version_info >= (3, 12): + def __init__( + self, + coro: _TaskCompatibleCoro[_T_co], + *, + loop: AbstractEventLoop = ..., + name: str | None = ..., + context: Context | None = None, + eager_start: bool = False, + ) -> None: ... + elif sys.version_info >= (3, 11): + def __init__( + self, + coro: _TaskCompatibleCoro[_T_co], + *, + loop: AbstractEventLoop = ..., + name: str | None = ..., + context: Context | None = None, + ) -> None: ... + else: + def __init__( + self, coro: _TaskCompatibleCoro[_T_co], *, loop: AbstractEventLoop = ..., name: str | None = ... + ) -> None: ... + + if sys.version_info >= (3, 12): + def get_coro(self) -> _TaskCompatibleCoro[_T_co] | None: ... + else: + def get_coro(self) -> _TaskCompatibleCoro[_T_co]: ... + + def get_name(self) -> str: ... + def set_name(self, value: object, /) -> None: ... + if sys.version_info >= (3, 12): + def get_context(self) -> Context: ... + + def get_stack(self, *, limit: int | None = None) -> list[FrameType]: ... + def print_stack(self, *, limit: int | None = None, file: TextIO | None = None) -> None: ... + if sys.version_info >= (3, 11): + def cancelling(self) -> int: ... + def uncancel(self) -> int: ... + if sys.version_info < (3, 9): + @classmethod + def current_task(cls, loop: AbstractEventLoop | None = None) -> Task[Any] | None: ... + @classmethod + def all_tasks(cls, loop: AbstractEventLoop | None = None) -> set[Task[Any]]: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +def get_event_loop() -> AbstractEventLoop: ... +def get_running_loop() -> AbstractEventLoop: ... +def _set_running_loop(loop: AbstractEventLoop | None, /) -> None: ... +def _get_running_loop() -> AbstractEventLoop: ... +def _register_task(task: Task[Any]) -> None: ... +def _unregister_task(task: Task[Any]) -> None: ... +def _enter_task(loop: AbstractEventLoop, task: Task[Any]) -> None: ... +def _leave_task(loop: AbstractEventLoop, task: Task[Any]) -> None: ... + +if sys.version_info >= (3, 12): + def current_task(loop: AbstractEventLoop | None = None) -> Task[Any] | None: ... diff --git a/mypy/typeshed/stdlib/_csv.pyi b/mypy/typeshed/stdlib/_csv.pyi index 9bb5d27f6e35..6d1893cf8c16 100644 --- a/mypy/typeshed/stdlib/_csv.pyi +++ b/mypy/typeshed/stdlib/_csv.pyi @@ -20,6 +20,8 @@ _QuotingType: TypeAlias = int class Error(Exception): ... +_DialectLike: TypeAlias = str | Dialect | type[Dialect] + class Dialect: delimiter: str quotechar: str | None @@ -29,9 +31,18 @@ class Dialect: lineterminator: str quoting: _QuotingType strict: bool - def __init__(self) -> None: ... - -_DialectLike: TypeAlias = str | Dialect | type[Dialect] + def __init__( + self, + dialect: _DialectLike | None = ..., + delimiter: str = ",", + doublequote: bool = True, + escapechar: str | None = None, + lineterminator: str = "\r\n", + quotechar: str | None = '"', + quoting: _QuotingType = 0, + skipinitialspace: bool = False, + strict: bool = False, + ) -> None: ... class _reader(Iterator[list[str]]): @property diff --git a/mypy/typeshed/stdlib/_ctypes.pyi b/mypy/typeshed/stdlib/_ctypes.pyi index 0fe7521d7749..91d95a154f8e 100644 --- a/mypy/typeshed/stdlib/_ctypes.pyi +++ b/mypy/typeshed/stdlib/_ctypes.pyi @@ -2,7 +2,7 @@ import sys from _typeshed import ReadableBuffer, WriteableBuffer from abc import abstractmethod from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence -from ctypes import CDLL, ArgumentError as ArgumentError +from ctypes import CDLL, ArgumentError as ArgumentError, c_void_p from typing import Any, ClassVar, Generic, TypeVar, overload from typing_extensions import Self, TypeAlias @@ -99,6 +99,9 @@ class _Pointer(_PointerLike, _CData, Generic[_CT]): def __getitem__(self, key: slice, /) -> list[Any]: ... def __setitem__(self, key: int, value: Any, /) -> None: ... +@overload +def POINTER(type: None, /) -> type[c_void_p]: ... +@overload def POINTER(type: type[_CT], /) -> type[_Pointer[_CT]]: ... def pointer(obj: _CT, /) -> _Pointer[_CT]: ... diff --git a/mypy/typeshed/stdlib/_curses.pyi b/mypy/typeshed/stdlib/_curses.pyi index b68c8925a041..80075d77e8e6 100644 --- a/mypy/typeshed/stdlib/_curses.pyi +++ b/mypy/typeshed/stdlib/_curses.pyi @@ -298,7 +298,7 @@ if sys.version_info >= (3, 9): def getmouse() -> tuple[int, int, int, int, int]: ... def getsyx() -> tuple[int, int]: ... -def getwin(file: SupportsRead[bytes], /) -> _CursesWindow: ... +def getwin(file: SupportsRead[bytes], /) -> window: ... def halfdelay(tenths: int, /) -> None: ... def has_colors() -> bool: ... @@ -310,7 +310,7 @@ def has_il() -> bool: ... def has_key(key: int, /) -> bool: ... def init_color(color_number: int, r: int, g: int, b: int, /) -> None: ... def init_pair(pair_number: int, fg: int, bg: int, /) -> None: ... -def initscr() -> _CursesWindow: ... +def initscr() -> window: ... def intrflush(flag: bool, /) -> None: ... def is_term_resized(nlines: int, ncols: int, /) -> bool: ... def isendwin() -> bool: ... @@ -321,8 +321,8 @@ def meta(yes: bool, /) -> None: ... def mouseinterval(interval: int, /) -> None: ... def mousemask(newmask: int, /) -> tuple[int, int]: ... def napms(ms: int, /) -> int: ... -def newpad(nlines: int, ncols: int, /) -> _CursesWindow: ... -def newwin(nlines: int, ncols: int, begin_y: int = ..., begin_x: int = ..., /) -> _CursesWindow: ... +def newpad(nlines: int, ncols: int, /) -> window: ... +def newwin(nlines: int, ncols: int, begin_y: int = ..., begin_x: int = ..., /) -> window: ... def nl(flag: bool = True, /) -> None: ... def nocbreak() -> None: ... def noecho() -> None: ... @@ -378,7 +378,7 @@ def use_env(flag: bool, /) -> None: ... class error(Exception): ... @final -class _CursesWindow: +class window: # undocumented encoding: str @overload def addch(self, ch: _ChType, attr: int = ...) -> None: ... @@ -431,9 +431,9 @@ class _CursesWindow: def delch(self, y: int, x: int) -> None: ... def deleteln(self) -> None: ... @overload - def derwin(self, begin_y: int, begin_x: int) -> _CursesWindow: ... + def derwin(self, begin_y: int, begin_x: int) -> window: ... @overload - def derwin(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> _CursesWindow: ... + def derwin(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> window: ... def echochar(self, ch: _ChType, attr: int = ..., /) -> None: ... def enclose(self, y: int, x: int, /) -> bool: ... def erase(self) -> None: ... @@ -505,16 +505,16 @@ class _CursesWindow: @overload def noutrefresh(self, pminrow: int, pmincol: int, sminrow: int, smincol: int, smaxrow: int, smaxcol: int) -> None: ... @overload - def overlay(self, destwin: _CursesWindow) -> None: ... + def overlay(self, destwin: window) -> None: ... @overload def overlay( - self, destwin: _CursesWindow, sminrow: int, smincol: int, dminrow: int, dmincol: int, dmaxrow: int, dmaxcol: int + self, destwin: window, sminrow: int, smincol: int, dminrow: int, dmincol: int, dmaxrow: int, dmaxcol: int ) -> None: ... @overload - def overwrite(self, destwin: _CursesWindow) -> None: ... + def overwrite(self, destwin: window) -> None: ... @overload def overwrite( - self, destwin: _CursesWindow, sminrow: int, smincol: int, dminrow: int, dmincol: int, dmaxrow: int, dmaxcol: int + self, destwin: window, sminrow: int, smincol: int, dminrow: int, dmincol: int, dmaxrow: int, dmaxcol: int ) -> None: ... def putwin(self, file: IO[Any], /) -> None: ... def redrawln(self, beg: int, num: int, /) -> None: ... @@ -530,13 +530,13 @@ class _CursesWindow: def standend(self) -> None: ... def standout(self) -> None: ... @overload - def subpad(self, begin_y: int, begin_x: int) -> _CursesWindow: ... + def subpad(self, begin_y: int, begin_x: int) -> window: ... @overload - def subpad(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> _CursesWindow: ... + def subpad(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> window: ... @overload - def subwin(self, begin_y: int, begin_x: int) -> _CursesWindow: ... + def subwin(self, begin_y: int, begin_x: int) -> window: ... @overload - def subwin(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> _CursesWindow: ... + def subwin(self, nlines: int, ncols: int, begin_y: int, begin_x: int) -> window: ... def syncdown(self) -> None: ... def syncok(self, flag: bool) -> None: ... def syncup(self) -> None: ... @@ -555,4 +555,3 @@ class _ncurses_version(NamedTuple): patch: int ncurses_version: _ncurses_version -window = _CursesWindow # undocumented diff --git a/mypy/typeshed/stdlib/_decimal.pyi b/mypy/typeshed/stdlib/_decimal.pyi index 937a04ac3799..9fcc08dbb95d 100644 --- a/mypy/typeshed/stdlib/_decimal.pyi +++ b/mypy/typeshed/stdlib/_decimal.pyi @@ -1,22 +1,39 @@ -import numbers import sys -from collections.abc import Container, Sequence +from decimal import ( + Clamped as Clamped, + Context as Context, + ConversionSyntax as ConversionSyntax, + Decimal as Decimal, + DecimalException as DecimalException, + DecimalTuple as DecimalTuple, + DivisionByZero as DivisionByZero, + DivisionImpossible as DivisionImpossible, + DivisionUndefined as DivisionUndefined, + FloatOperation as FloatOperation, + Inexact as Inexact, + InvalidContext as InvalidContext, + InvalidOperation as InvalidOperation, + Overflow as Overflow, + Rounded as Rounded, + Subnormal as Subnormal, + Underflow as Underflow, +) from types import TracebackType -from typing import Any, ClassVar, Final, Literal, NamedTuple, overload -from typing_extensions import Self, TypeAlias +from typing import Final +from typing_extensions import TypeAlias -_Decimal: TypeAlias = Decimal | int -_DecimalNew: TypeAlias = Decimal | float | str | tuple[int, Sequence[int], int] -_ComparableNum: TypeAlias = Decimal | float | numbers.Rational +_TrapType: TypeAlias = type[DecimalException] + +class _ContextManager: + new_context: Context + saved_context: Context + def __init__(self, new_context: Context) -> None: ... + def __enter__(self) -> Context: ... + def __exit__(self, t: type[BaseException] | None, v: BaseException | None, tb: TracebackType | None) -> None: ... __version__: Final[str] __libmpdec_version__: Final[str] -class DecimalTuple(NamedTuple): - sign: int - digits: tuple[int, ...] - exponent: int | Literal["n", "N", "F"] - ROUND_DOWN: Final[str] ROUND_HALF_UP: Final[str] ROUND_HALF_EVEN: Final[str] @@ -32,21 +49,6 @@ MAX_PREC: Final[int] MIN_EMIN: Final[int] MIN_ETINY: Final[int] -class DecimalException(ArithmeticError): ... -class Clamped(DecimalException): ... -class InvalidOperation(DecimalException): ... -class ConversionSyntax(InvalidOperation): ... -class DivisionByZero(DecimalException, ZeroDivisionError): ... -class DivisionImpossible(InvalidOperation): ... -class DivisionUndefined(InvalidOperation, ZeroDivisionError): ... -class Inexact(DecimalException): ... -class InvalidContext(InvalidOperation): ... -class Rounded(DecimalException): ... -class Subnormal(DecimalException): ... -class Overflow(Inexact, Rounded): ... -class Underflow(Inexact, Rounded, Subnormal): ... -class FloatOperation(DecimalException, TypeError): ... - def setcontext(context: Context, /) -> None: ... def getcontext() -> Context: ... @@ -67,215 +69,6 @@ if sys.version_info >= (3, 11): else: def localcontext(ctx: Context | None = None) -> _ContextManager: ... -class Decimal: - def __new__(cls, value: _DecimalNew = ..., context: Context | None = ...) -> Self: ... - @classmethod - def from_float(cls, f: float, /) -> Self: ... - def __bool__(self) -> bool: ... - def compare(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def __hash__(self) -> int: ... - def as_tuple(self) -> DecimalTuple: ... - def as_integer_ratio(self) -> tuple[int, int]: ... - def to_eng_string(self, context: Context | None = None) -> str: ... - def __abs__(self) -> Decimal: ... - def __add__(self, value: _Decimal, /) -> Decimal: ... - def __divmod__(self, value: _Decimal, /) -> tuple[Decimal, Decimal]: ... - def __eq__(self, value: object, /) -> bool: ... - def __floordiv__(self, value: _Decimal, /) -> Decimal: ... - def __ge__(self, value: _ComparableNum, /) -> bool: ... - def __gt__(self, value: _ComparableNum, /) -> bool: ... - def __le__(self, value: _ComparableNum, /) -> bool: ... - def __lt__(self, value: _ComparableNum, /) -> bool: ... - def __mod__(self, value: _Decimal, /) -> Decimal: ... - def __mul__(self, value: _Decimal, /) -> Decimal: ... - def __neg__(self) -> Decimal: ... - def __pos__(self) -> Decimal: ... - def __pow__(self, value: _Decimal, mod: _Decimal | None = None, /) -> Decimal: ... - def __radd__(self, value: _Decimal, /) -> Decimal: ... - def __rdivmod__(self, value: _Decimal, /) -> tuple[Decimal, Decimal]: ... - def __rfloordiv__(self, value: _Decimal, /) -> Decimal: ... - def __rmod__(self, value: _Decimal, /) -> Decimal: ... - def __rmul__(self, value: _Decimal, /) -> Decimal: ... - def __rsub__(self, value: _Decimal, /) -> Decimal: ... - def __rtruediv__(self, value: _Decimal, /) -> Decimal: ... - def __sub__(self, value: _Decimal, /) -> Decimal: ... - def __truediv__(self, value: _Decimal, /) -> Decimal: ... - def remainder_near(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def __float__(self) -> float: ... - def __int__(self) -> int: ... - def __trunc__(self) -> int: ... - @property - def real(self) -> Decimal: ... - @property - def imag(self) -> Decimal: ... - def conjugate(self) -> Decimal: ... - def __complex__(self) -> complex: ... - @overload - def __round__(self) -> int: ... - @overload - def __round__(self, ndigits: int, /) -> Decimal: ... - def __floor__(self) -> int: ... - def __ceil__(self) -> int: ... - def fma(self, other: _Decimal, third: _Decimal, context: Context | None = None) -> Decimal: ... - def __rpow__(self, value: _Decimal, mod: Context | None = None, /) -> Decimal: ... - def normalize(self, context: Context | None = None) -> Decimal: ... - def quantize(self, exp: _Decimal, rounding: str | None = None, context: Context | None = None) -> Decimal: ... - def same_quantum(self, other: _Decimal, context: Context | None = None) -> bool: ... - def to_integral_exact(self, rounding: str | None = None, context: Context | None = None) -> Decimal: ... - def to_integral_value(self, rounding: str | None = None, context: Context | None = None) -> Decimal: ... - def to_integral(self, rounding: str | None = None, context: Context | None = None) -> Decimal: ... - def sqrt(self, context: Context | None = None) -> Decimal: ... - def max(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def min(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def adjusted(self) -> int: ... - def canonical(self) -> Decimal: ... - def compare_signal(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def compare_total(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def compare_total_mag(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def copy_abs(self) -> Decimal: ... - def copy_negate(self) -> Decimal: ... - def copy_sign(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def exp(self, context: Context | None = None) -> Decimal: ... - def is_canonical(self) -> bool: ... - def is_finite(self) -> bool: ... - def is_infinite(self) -> bool: ... - def is_nan(self) -> bool: ... - def is_normal(self, context: Context | None = None) -> bool: ... - def is_qnan(self) -> bool: ... - def is_signed(self) -> bool: ... - def is_snan(self) -> bool: ... - def is_subnormal(self, context: Context | None = None) -> bool: ... - def is_zero(self) -> bool: ... - def ln(self, context: Context | None = None) -> Decimal: ... - def log10(self, context: Context | None = None) -> Decimal: ... - def logb(self, context: Context | None = None) -> Decimal: ... - def logical_and(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def logical_invert(self, context: Context | None = None) -> Decimal: ... - def logical_or(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def logical_xor(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def max_mag(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def min_mag(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def next_minus(self, context: Context | None = None) -> Decimal: ... - def next_plus(self, context: Context | None = None) -> Decimal: ... - def next_toward(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def number_class(self, context: Context | None = None) -> str: ... - def radix(self) -> Decimal: ... - def rotate(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def scaleb(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def shift(self, other: _Decimal, context: Context | None = None) -> Decimal: ... - def __reduce__(self) -> tuple[type[Self], tuple[str]]: ... - def __copy__(self) -> Self: ... - def __deepcopy__(self, memo: Any, /) -> Self: ... - def __format__(self, specifier: str, context: Context | None = ..., /) -> str: ... - -class _ContextManager: - new_context: Context - saved_context: Context - def __init__(self, new_context: Context) -> None: ... - def __enter__(self) -> Context: ... - def __exit__(self, t: type[BaseException] | None, v: BaseException | None, tb: TracebackType | None) -> None: ... - -_TrapType: TypeAlias = type[DecimalException] - -class Context: - # TODO: Context doesn't allow you to delete *any* attributes from instances of the class at runtime, - # even settable attributes like `prec` and `rounding`, - # but that's inexpressable in the stub. - # Type checkers either ignore it or misinterpret it - # if you add a `def __delattr__(self, name: str, /) -> NoReturn` method to the stub - prec: int - rounding: str - Emin: int - Emax: int - capitals: int - clamp: int - traps: dict[_TrapType, bool] - flags: dict[_TrapType, bool] - def __init__( - self, - prec: int | None = ..., - rounding: str | None = ..., - Emin: int | None = ..., - Emax: int | None = ..., - capitals: int | None = ..., - clamp: int | None = ..., - flags: None | dict[_TrapType, bool] | Container[_TrapType] = ..., - traps: None | dict[_TrapType, bool] | Container[_TrapType] = ..., - _ignored_flags: list[_TrapType] | None = ..., - ) -> None: ... - def __reduce__(self) -> tuple[type[Self], tuple[Any, ...]]: ... - def clear_flags(self) -> None: ... - def clear_traps(self) -> None: ... - def copy(self) -> Context: ... - def __copy__(self) -> Context: ... - # see https://github.com/python/cpython/issues/94107 - __hash__: ClassVar[None] # type: ignore[assignment] - def Etiny(self) -> int: ... - def Etop(self) -> int: ... - def create_decimal(self, num: _DecimalNew = "0", /) -> Decimal: ... - def create_decimal_from_float(self, f: float, /) -> Decimal: ... - def abs(self, x: _Decimal, /) -> Decimal: ... - def add(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def canonical(self, x: Decimal, /) -> Decimal: ... - def compare(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def compare_signal(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def compare_total(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def compare_total_mag(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def copy_abs(self, x: _Decimal, /) -> Decimal: ... - def copy_decimal(self, x: _Decimal, /) -> Decimal: ... - def copy_negate(self, x: _Decimal, /) -> Decimal: ... - def copy_sign(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def divide(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def divide_int(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def divmod(self, x: _Decimal, y: _Decimal, /) -> tuple[Decimal, Decimal]: ... - def exp(self, x: _Decimal, /) -> Decimal: ... - def fma(self, x: _Decimal, y: _Decimal, z: _Decimal, /) -> Decimal: ... - def is_canonical(self, x: _Decimal, /) -> bool: ... - def is_finite(self, x: _Decimal, /) -> bool: ... - def is_infinite(self, x: _Decimal, /) -> bool: ... - def is_nan(self, x: _Decimal, /) -> bool: ... - def is_normal(self, x: _Decimal, /) -> bool: ... - def is_qnan(self, x: _Decimal, /) -> bool: ... - def is_signed(self, x: _Decimal, /) -> bool: ... - def is_snan(self, x: _Decimal, /) -> bool: ... - def is_subnormal(self, x: _Decimal, /) -> bool: ... - def is_zero(self, x: _Decimal, /) -> bool: ... - def ln(self, x: _Decimal, /) -> Decimal: ... - def log10(self, x: _Decimal, /) -> Decimal: ... - def logb(self, x: _Decimal, /) -> Decimal: ... - def logical_and(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def logical_invert(self, x: _Decimal, /) -> Decimal: ... - def logical_or(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def logical_xor(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def max(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def max_mag(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def min(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def min_mag(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def minus(self, x: _Decimal, /) -> Decimal: ... - def multiply(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def next_minus(self, x: _Decimal, /) -> Decimal: ... - def next_plus(self, x: _Decimal, /) -> Decimal: ... - def next_toward(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def normalize(self, x: _Decimal, /) -> Decimal: ... - def number_class(self, x: _Decimal, /) -> str: ... - def plus(self, x: _Decimal, /) -> Decimal: ... - def power(self, a: _Decimal, b: _Decimal, modulo: _Decimal | None = None) -> Decimal: ... - def quantize(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def radix(self) -> Decimal: ... - def remainder(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def remainder_near(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def rotate(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def same_quantum(self, x: _Decimal, y: _Decimal, /) -> bool: ... - def scaleb(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def shift(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def sqrt(self, x: _Decimal, /) -> Decimal: ... - def subtract(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... - def to_eng_string(self, x: _Decimal, /) -> str: ... - def to_sci_string(self, x: _Decimal, /) -> str: ... - def to_integral_exact(self, x: _Decimal, /) -> Decimal: ... - def to_integral_value(self, x: _Decimal, /) -> Decimal: ... - def to_integral(self, x: _Decimal, /) -> Decimal: ... - DefaultContext: Context BasicContext: Context ExtendedContext: Context diff --git a/mypy/typeshed/stdlib/_io.pyi b/mypy/typeshed/stdlib/_io.pyi new file mode 100644 index 000000000000..e8290daad106 --- /dev/null +++ b/mypy/typeshed/stdlib/_io.pyi @@ -0,0 +1,195 @@ +import builtins +import codecs +import sys +from _typeshed import FileDescriptorOrPath, MaybeNone, ReadableBuffer, WriteableBuffer +from collections.abc import Callable, Iterable, Iterator +from io import BufferedIOBase, RawIOBase, TextIOBase, UnsupportedOperation as UnsupportedOperation +from os import _Opener +from types import TracebackType +from typing import IO, Any, BinaryIO, Final, Generic, Literal, Protocol, TextIO, TypeVar, overload, type_check_only +from typing_extensions import Self + +_T = TypeVar("_T") + +DEFAULT_BUFFER_SIZE: Final = 8192 + +open = builtins.open + +def open_code(path: str) -> IO[bytes]: ... + +BlockingIOError = builtins.BlockingIOError + +class _IOBase: + def __iter__(self) -> Iterator[bytes]: ... + def __next__(self) -> bytes: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def close(self) -> None: ... + def fileno(self) -> int: ... + def flush(self) -> None: ... + def isatty(self) -> bool: ... + def readable(self) -> bool: ... + read: Callable[..., Any] + def readlines(self, hint: int = -1, /) -> list[bytes]: ... + def seek(self, offset: int, whence: int = ..., /) -> int: ... + def seekable(self) -> bool: ... + def tell(self) -> int: ... + def truncate(self, size: int | None = ..., /) -> int: ... + def writable(self) -> bool: ... + write: Callable[..., Any] + def writelines(self, lines: Iterable[ReadableBuffer], /) -> None: ... + def readline(self, size: int | None = -1, /) -> bytes: ... + def __del__(self) -> None: ... + @property + def closed(self) -> bool: ... + def _checkClosed(self) -> None: ... # undocumented + +class _RawIOBase(_IOBase): + def readall(self) -> bytes: ... + # The following methods can return None if the file is in non-blocking mode + # and no data is available. + def readinto(self, buffer: WriteableBuffer, /) -> int | MaybeNone: ... + def write(self, b: ReadableBuffer, /) -> int | MaybeNone: ... + def read(self, size: int = -1, /) -> bytes | MaybeNone: ... + +class _BufferedIOBase(_IOBase): + def detach(self) -> RawIOBase: ... + def readinto(self, buffer: WriteableBuffer, /) -> int: ... + def write(self, buffer: ReadableBuffer, /) -> int: ... + def readinto1(self, buffer: WriteableBuffer, /) -> int: ... + def read(self, size: int | None = ..., /) -> bytes: ... + def read1(self, size: int = ..., /) -> bytes: ... + +class FileIO(RawIOBase, _RawIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of writelines in the base classes + mode: str + # The type of "name" equals the argument passed in to the constructor, + # but that can make FileIO incompatible with other I/O types that assume + # "name" is a str. In the future, making FileIO generic might help. + name: Any + def __init__( + self, file: FileDescriptorOrPath, mode: str = ..., closefd: bool = ..., opener: _Opener | None = ... + ) -> None: ... + @property + def closefd(self) -> bool: ... + +class BytesIO(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of methods in the base classes + def __init__(self, initial_bytes: ReadableBuffer = ...) -> None: ... + # BytesIO does not contain a "name" field. This workaround is necessary + # to allow BytesIO sub-classes to add this field, as it is defined + # as a read-only property on IO[]. + name: Any + def getvalue(self) -> bytes: ... + def getbuffer(self) -> memoryview: ... + def read1(self, size: int | None = -1, /) -> bytes: ... + +class BufferedReader(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of methods in the base classes + raw: RawIOBase + def __init__(self, raw: RawIOBase, buffer_size: int = ...) -> None: ... + def peek(self, size: int = 0, /) -> bytes: ... + +class BufferedWriter(BufferedIOBase, _BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of writelines in the base classes + raw: RawIOBase + def __init__(self, raw: RawIOBase, buffer_size: int = ...) -> None: ... + def write(self, buffer: ReadableBuffer, /) -> int: ... + +class BufferedRandom(BufferedReader, BufferedWriter, BufferedIOBase, _BufferedIOBase): # type: ignore[misc] # incompatible definitions of methods in the base classes + def seek(self, target: int, whence: int = 0, /) -> int: ... # stubtest needs this + +class BufferedRWPair(BufferedIOBase, _BufferedIOBase): + def __init__(self, reader: RawIOBase, writer: RawIOBase, buffer_size: int = ...) -> None: ... + def peek(self, size: int = ..., /) -> bytes: ... + +class _TextIOBase(_IOBase): + encoding: str + errors: str | None + newlines: str | tuple[str, ...] | None + def __iter__(self) -> Iterator[str]: ... # type: ignore[override] + def __next__(self) -> str: ... # type: ignore[override] + def detach(self) -> BinaryIO: ... + def write(self, s: str, /) -> int: ... + def writelines(self, lines: Iterable[str], /) -> None: ... # type: ignore[override] + def readline(self, size: int = ..., /) -> str: ... # type: ignore[override] + def readlines(self, hint: int = -1, /) -> list[str]: ... # type: ignore[override] + def read(self, size: int | None = ..., /) -> str: ... + +@type_check_only +class _WrappedBuffer(Protocol): + # "name" is wrapped by TextIOWrapper. Its type is inconsistent between + # the various I/O types, see the comments on TextIOWrapper.name and + # TextIO.name. + @property + def name(self) -> Any: ... + @property + def closed(self) -> bool: ... + def read(self, size: int = ..., /) -> ReadableBuffer: ... + # Optional: def read1(self, size: int, /) -> ReadableBuffer: ... + def write(self, b: bytes, /) -> object: ... + def flush(self) -> object: ... + def close(self) -> object: ... + def seekable(self) -> bool: ... + def readable(self) -> bool: ... + def writable(self) -> bool: ... + def truncate(self, size: int, /) -> int: ... + def fileno(self) -> int: ... + def isatty(self) -> bool: ... + # Optional: Only needs to be present if seekable() returns True. + # def seek(self, offset: Literal[0], whence: Literal[2]) -> int: ... + # def tell(self) -> int: ... + +_BufferT_co = TypeVar("_BufferT_co", bound=_WrappedBuffer, default=_WrappedBuffer, covariant=True) + +class TextIOWrapper(TextIOBase, _TextIOBase, TextIO, Generic[_BufferT_co]): # type: ignore[misc] # incompatible definitions of write in the base classes + def __init__( + self, + buffer: _BufferT_co, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + line_buffering: bool = False, + write_through: bool = False, + ) -> None: ... + # Equals the "buffer" argument passed in to the constructor. + @property + def buffer(self) -> _BufferT_co: ... # type: ignore[override] + @property + def line_buffering(self) -> bool: ... + @property + def write_through(self) -> bool: ... + def reconfigure( + self, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + line_buffering: bool | None = None, + write_through: bool | None = None, + ) -> None: ... + def readline(self, size: int = -1, /) -> str: ... # type: ignore[override] + # Equals the "buffer" argument passed in to the constructor. + def detach(self) -> _BufferT_co: ... # type: ignore[override] + # TextIOWrapper's version of seek only supports a limited subset of + # operations. + def seek(self, cookie: int, whence: int = 0, /) -> int: ... + +class StringIO(TextIOWrapper, TextIOBase, _TextIOBase): # type: ignore[misc] # incompatible definitions of write in the base classes + def __init__(self, initial_value: str | None = ..., newline: str | None = ...) -> None: ... + # StringIO does not contain a "name" field. This workaround is necessary + # to allow StringIO sub-classes to add this field, as it is defined + # as a read-only property on IO[]. + name: Any + def getvalue(self) -> str: ... + +class IncrementalNewlineDecoder(codecs.IncrementalDecoder): + def __init__(self, decoder: codecs.IncrementalDecoder | None, translate: bool, errors: str = ...) -> None: ... + def decode(self, input: ReadableBuffer | str, final: bool = False) -> str: ... + @property + def newlines(self) -> str | tuple[str, ...] | None: ... + def setstate(self, state: tuple[bytes, int], /) -> None: ... + +if sys.version_info >= (3, 10): + @overload + def text_encoding(encoding: None, stacklevel: int = 2, /) -> Literal["locale", "utf-8"]: ... + @overload + def text_encoding(encoding: _T, stacklevel: int = 2, /) -> _T: ... diff --git a/mypy/typeshed/stdlib/_operator.pyi b/mypy/typeshed/stdlib/_operator.pyi index 1b0083f4e274..967215d8fa21 100644 --- a/mypy/typeshed/stdlib/_operator.pyi +++ b/mypy/typeshed/stdlib/_operator.pyi @@ -1,18 +1,16 @@ import sys from _typeshed import SupportsGetItem from collections.abc import Callable, Container, Iterable, MutableMapping, MutableSequence, Sequence -from typing import Any, AnyStr, Generic, Protocol, SupportsAbs, SupportsIndex, TypeVar, final, overload -from typing_extensions import ParamSpec, TypeAlias, TypeIs, TypeVarTuple, Unpack +from operator import attrgetter as attrgetter, itemgetter as itemgetter, methodcaller as methodcaller +from typing import Any, AnyStr, Protocol, SupportsAbs, SupportsIndex, TypeVar, overload +from typing_extensions import ParamSpec, TypeAlias, TypeIs _R = TypeVar("_R") _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) -_T1 = TypeVar("_T1") -_T2 = TypeVar("_T2") _K = TypeVar("_K") _V = TypeVar("_V") _P = ParamSpec("_P") -_Ts = TypeVarTuple("_Ts") # The following protocols return "Any" instead of bool, since the comparison # operators can be overloaded to return an arbitrary object. For example, @@ -92,40 +90,6 @@ def setitem(a: MutableSequence[_T], b: slice, c: Sequence[_T], /) -> None: ... @overload def setitem(a: MutableMapping[_K, _V], b: _K, c: _V, /) -> None: ... def length_hint(obj: object, default: int = 0, /) -> int: ... -@final -class attrgetter(Generic[_T_co]): - @overload - def __new__(cls, attr: str, /) -> attrgetter[Any]: ... - @overload - def __new__(cls, attr: str, attr2: str, /) -> attrgetter[tuple[Any, Any]]: ... - @overload - def __new__(cls, attr: str, attr2: str, attr3: str, /) -> attrgetter[tuple[Any, Any, Any]]: ... - @overload - def __new__(cls, attr: str, attr2: str, attr3: str, attr4: str, /) -> attrgetter[tuple[Any, Any, Any, Any]]: ... - @overload - def __new__(cls, attr: str, /, *attrs: str) -> attrgetter[tuple[Any, ...]]: ... - def __call__(self, obj: Any, /) -> _T_co: ... - -@final -class itemgetter(Generic[_T_co]): - @overload - def __new__(cls, item: _T, /) -> itemgetter[_T]: ... - @overload - def __new__(cls, item1: _T1, item2: _T2, /, *items: Unpack[_Ts]) -> itemgetter[tuple[_T1, _T2, Unpack[_Ts]]]: ... - # __key: _KT_contra in SupportsGetItem seems to be causing variance issues, ie: - # TypeVar "_KT_contra@SupportsGetItem" is contravariant - # "tuple[int, int]" is incompatible with protocol "SupportsIndex" - # preventing [_T_co, ...] instead of [Any, ...] - # - # A suspected mypy issue prevents using [..., _T] instead of [..., Any] here. - # https://github.com/python/mypy/issues/14032 - def __call__(self, obj: SupportsGetItem[Any, Any]) -> Any: ... - -@final -class methodcaller: - def __init__(self, name: str, /, *args: Any, **kwargs: Any) -> None: ... - def __call__(self, obj: Any) -> Any: ... - def iadd(a: Any, b: Any, /) -> Any: ... def iand(a: Any, b: Any, /) -> Any: ... def iconcat(a: Any, b: Any, /) -> Any: ... diff --git a/mypy/typeshed/stdlib/_socket.pyi b/mypy/typeshed/stdlib/_socket.pyi index affa8d63ecfa..7e4cf4e0364a 100644 --- a/mypy/typeshed/stdlib/_socket.pyi +++ b/mypy/typeshed/stdlib/_socket.pyi @@ -1,6 +1,7 @@ import sys from _typeshed import ReadableBuffer, WriteableBuffer from collections.abc import Iterable +from socket import error as error, gaierror as gaierror, herror as herror, timeout as timeout from typing import Any, SupportsIndex, overload from typing_extensions import TypeAlias @@ -666,18 +667,6 @@ if sys.platform != "win32": if sys.platform != "win32" and sys.platform != "darwin": IPX_TYPE: int -# ===== Exceptions ===== - -error = OSError - -class herror(error): ... -class gaierror(error): ... - -if sys.version_info >= (3, 10): - timeout = TimeoutError -else: - class timeout(error): ... - # ===== Classes ===== class socket: @@ -687,8 +676,9 @@ class socket: def type(self) -> int: ... @property def proto(self) -> int: ... + # F811: "Redefinition of unused `timeout`" @property - def timeout(self) -> float | None: ... + def timeout(self) -> float | None: ... # noqa: F811 if sys.platform == "win32": def __init__( self, family: int = ..., type: int = ..., proto: int = ..., fileno: SupportsIndex | bytes | None = ... @@ -788,7 +778,9 @@ def inet_ntoa(packed_ip: ReadableBuffer, /) -> str: ... def inet_pton(address_family: int, ip_string: str, /) -> bytes: ... def inet_ntop(address_family: int, packed_ip: ReadableBuffer, /) -> str: ... def getdefaulttimeout() -> float | None: ... -def setdefaulttimeout(timeout: float | None, /) -> None: ... + +# F811: "Redefinition of unused `timeout`" +def setdefaulttimeout(timeout: float | None, /) -> None: ... # noqa: F811 if sys.platform != "win32": def sethostname(name: str, /) -> None: ... diff --git a/mypy/typeshed/stdlib/_sqlite3.pyi b/mypy/typeshed/stdlib/_sqlite3.pyi new file mode 100644 index 000000000000..6f06542c1ba7 --- /dev/null +++ b/mypy/typeshed/stdlib/_sqlite3.pyi @@ -0,0 +1,312 @@ +import sys +from _typeshed import ReadableBuffer, StrOrBytesPath +from collections.abc import Callable +from sqlite3 import ( + Connection as Connection, + Cursor as Cursor, + DatabaseError as DatabaseError, + DataError as DataError, + Error as Error, + IntegrityError as IntegrityError, + InterfaceError as InterfaceError, + InternalError as InternalError, + NotSupportedError as NotSupportedError, + OperationalError as OperationalError, + PrepareProtocol as PrepareProtocol, + ProgrammingError as ProgrammingError, + Row as Row, + Warning as Warning, +) +from typing import Any, Final, Literal, TypeVar, overload +from typing_extensions import TypeAlias + +if sys.version_info >= (3, 11): + from sqlite3 import Blob as Blob + +_T = TypeVar("_T") +_ConnectionT = TypeVar("_ConnectionT", bound=Connection) +_SqliteData: TypeAlias = str | ReadableBuffer | int | float | None +_Adapter: TypeAlias = Callable[[_T], _SqliteData] +_Converter: TypeAlias = Callable[[bytes], Any] + +PARSE_COLNAMES: Final[int] +PARSE_DECLTYPES: Final[int] +SQLITE_ALTER_TABLE: Final[int] +SQLITE_ANALYZE: Final[int] +SQLITE_ATTACH: Final[int] +SQLITE_CREATE_INDEX: Final[int] +SQLITE_CREATE_TABLE: Final[int] +SQLITE_CREATE_TEMP_INDEX: Final[int] +SQLITE_CREATE_TEMP_TABLE: Final[int] +SQLITE_CREATE_TEMP_TRIGGER: Final[int] +SQLITE_CREATE_TEMP_VIEW: Final[int] +SQLITE_CREATE_TRIGGER: Final[int] +SQLITE_CREATE_VIEW: Final[int] +SQLITE_CREATE_VTABLE: Final[int] +SQLITE_DELETE: Final[int] +SQLITE_DENY: Final[int] +SQLITE_DETACH: Final[int] +SQLITE_DONE: Final[int] +SQLITE_DROP_INDEX: Final[int] +SQLITE_DROP_TABLE: Final[int] +SQLITE_DROP_TEMP_INDEX: Final[int] +SQLITE_DROP_TEMP_TABLE: Final[int] +SQLITE_DROP_TEMP_TRIGGER: Final[int] +SQLITE_DROP_TEMP_VIEW: Final[int] +SQLITE_DROP_TRIGGER: Final[int] +SQLITE_DROP_VIEW: Final[int] +SQLITE_DROP_VTABLE: Final[int] +SQLITE_FUNCTION: Final[int] +SQLITE_IGNORE: Final[int] +SQLITE_INSERT: Final[int] +SQLITE_OK: Final[int] +SQLITE_PRAGMA: Final[int] +SQLITE_READ: Final[int] +SQLITE_RECURSIVE: Final[int] +SQLITE_REINDEX: Final[int] +SQLITE_SAVEPOINT: Final[int] +SQLITE_SELECT: Final[int] +SQLITE_TRANSACTION: Final[int] +SQLITE_UPDATE: Final[int] +adapters: dict[tuple[type[Any], type[Any]], _Adapter[Any]] +converters: dict[str, _Converter] +sqlite_version: str + +if sys.version_info < (3, 12): + version: str + +if sys.version_info >= (3, 12): + LEGACY_TRANSACTION_CONTROL: Final[int] + SQLITE_DBCONFIG_DEFENSIVE: Final[int] + SQLITE_DBCONFIG_DQS_DDL: Final[int] + SQLITE_DBCONFIG_DQS_DML: Final[int] + SQLITE_DBCONFIG_ENABLE_FKEY: Final[int] + SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: Final[int] + SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: Final[int] + SQLITE_DBCONFIG_ENABLE_QPSG: Final[int] + SQLITE_DBCONFIG_ENABLE_TRIGGER: Final[int] + SQLITE_DBCONFIG_ENABLE_VIEW: Final[int] + SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: Final[int] + SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: Final[int] + SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: Final[int] + SQLITE_DBCONFIG_RESET_DATABASE: Final[int] + SQLITE_DBCONFIG_TRIGGER_EQP: Final[int] + SQLITE_DBCONFIG_TRUSTED_SCHEMA: Final[int] + SQLITE_DBCONFIG_WRITABLE_SCHEMA: Final[int] + +if sys.version_info >= (3, 11): + SQLITE_ABORT: Final[int] + SQLITE_ABORT_ROLLBACK: Final[int] + SQLITE_AUTH: Final[int] + SQLITE_AUTH_USER: Final[int] + SQLITE_BUSY: Final[int] + SQLITE_BUSY_RECOVERY: Final[int] + SQLITE_BUSY_SNAPSHOT: Final[int] + SQLITE_BUSY_TIMEOUT: Final[int] + SQLITE_CANTOPEN: Final[int] + SQLITE_CANTOPEN_CONVPATH: Final[int] + SQLITE_CANTOPEN_DIRTYWAL: Final[int] + SQLITE_CANTOPEN_FULLPATH: Final[int] + SQLITE_CANTOPEN_ISDIR: Final[int] + SQLITE_CANTOPEN_NOTEMPDIR: Final[int] + SQLITE_CANTOPEN_SYMLINK: Final[int] + SQLITE_CONSTRAINT: Final[int] + SQLITE_CONSTRAINT_CHECK: Final[int] + SQLITE_CONSTRAINT_COMMITHOOK: Final[int] + SQLITE_CONSTRAINT_FOREIGNKEY: Final[int] + SQLITE_CONSTRAINT_FUNCTION: Final[int] + SQLITE_CONSTRAINT_NOTNULL: Final[int] + SQLITE_CONSTRAINT_PINNED: Final[int] + SQLITE_CONSTRAINT_PRIMARYKEY: Final[int] + SQLITE_CONSTRAINT_ROWID: Final[int] + SQLITE_CONSTRAINT_TRIGGER: Final[int] + SQLITE_CONSTRAINT_UNIQUE: Final[int] + SQLITE_CONSTRAINT_VTAB: Final[int] + SQLITE_CORRUPT: Final[int] + SQLITE_CORRUPT_INDEX: Final[int] + SQLITE_CORRUPT_SEQUENCE: Final[int] + SQLITE_CORRUPT_VTAB: Final[int] + SQLITE_EMPTY: Final[int] + SQLITE_ERROR: Final[int] + SQLITE_ERROR_MISSING_COLLSEQ: Final[int] + SQLITE_ERROR_RETRY: Final[int] + SQLITE_ERROR_SNAPSHOT: Final[int] + SQLITE_FORMAT: Final[int] + SQLITE_FULL: Final[int] + SQLITE_INTERNAL: Final[int] + SQLITE_INTERRUPT: Final[int] + SQLITE_IOERR: Final[int] + SQLITE_IOERR_ACCESS: Final[int] + SQLITE_IOERR_AUTH: Final[int] + SQLITE_IOERR_BEGIN_ATOMIC: Final[int] + SQLITE_IOERR_BLOCKED: Final[int] + SQLITE_IOERR_CHECKRESERVEDLOCK: Final[int] + SQLITE_IOERR_CLOSE: Final[int] + SQLITE_IOERR_COMMIT_ATOMIC: Final[int] + SQLITE_IOERR_CONVPATH: Final[int] + SQLITE_IOERR_CORRUPTFS: Final[int] + SQLITE_IOERR_DATA: Final[int] + SQLITE_IOERR_DELETE: Final[int] + SQLITE_IOERR_DELETE_NOENT: Final[int] + SQLITE_IOERR_DIR_CLOSE: Final[int] + SQLITE_IOERR_DIR_FSYNC: Final[int] + SQLITE_IOERR_FSTAT: Final[int] + SQLITE_IOERR_FSYNC: Final[int] + SQLITE_IOERR_GETTEMPPATH: Final[int] + SQLITE_IOERR_LOCK: Final[int] + SQLITE_IOERR_MMAP: Final[int] + SQLITE_IOERR_NOMEM: Final[int] + SQLITE_IOERR_RDLOCK: Final[int] + SQLITE_IOERR_READ: Final[int] + SQLITE_IOERR_ROLLBACK_ATOMIC: Final[int] + SQLITE_IOERR_SEEK: Final[int] + SQLITE_IOERR_SHMLOCK: Final[int] + SQLITE_IOERR_SHMMAP: Final[int] + SQLITE_IOERR_SHMOPEN: Final[int] + SQLITE_IOERR_SHMSIZE: Final[int] + SQLITE_IOERR_SHORT_READ: Final[int] + SQLITE_IOERR_TRUNCATE: Final[int] + SQLITE_IOERR_UNLOCK: Final[int] + SQLITE_IOERR_VNODE: Final[int] + SQLITE_IOERR_WRITE: Final[int] + SQLITE_LIMIT_ATTACHED: Final[int] + SQLITE_LIMIT_COLUMN: Final[int] + SQLITE_LIMIT_COMPOUND_SELECT: Final[int] + SQLITE_LIMIT_EXPR_DEPTH: Final[int] + SQLITE_LIMIT_FUNCTION_ARG: Final[int] + SQLITE_LIMIT_LENGTH: Final[int] + SQLITE_LIMIT_LIKE_PATTERN_LENGTH: Final[int] + SQLITE_LIMIT_SQL_LENGTH: Final[int] + SQLITE_LIMIT_TRIGGER_DEPTH: Final[int] + SQLITE_LIMIT_VARIABLE_NUMBER: Final[int] + SQLITE_LIMIT_VDBE_OP: Final[int] + SQLITE_LIMIT_WORKER_THREADS: Final[int] + SQLITE_LOCKED: Final[int] + SQLITE_LOCKED_SHAREDCACHE: Final[int] + SQLITE_LOCKED_VTAB: Final[int] + SQLITE_MISMATCH: Final[int] + SQLITE_MISUSE: Final[int] + SQLITE_NOLFS: Final[int] + SQLITE_NOMEM: Final[int] + SQLITE_NOTADB: Final[int] + SQLITE_NOTFOUND: Final[int] + SQLITE_NOTICE: Final[int] + SQLITE_NOTICE_RECOVER_ROLLBACK: Final[int] + SQLITE_NOTICE_RECOVER_WAL: Final[int] + SQLITE_OK_LOAD_PERMANENTLY: Final[int] + SQLITE_OK_SYMLINK: Final[int] + SQLITE_PERM: Final[int] + SQLITE_PROTOCOL: Final[int] + SQLITE_RANGE: Final[int] + SQLITE_READONLY: Final[int] + SQLITE_READONLY_CANTINIT: Final[int] + SQLITE_READONLY_CANTLOCK: Final[int] + SQLITE_READONLY_DBMOVED: Final[int] + SQLITE_READONLY_DIRECTORY: Final[int] + SQLITE_READONLY_RECOVERY: Final[int] + SQLITE_READONLY_ROLLBACK: Final[int] + SQLITE_ROW: Final[int] + SQLITE_SCHEMA: Final[int] + SQLITE_TOOBIG: Final[int] + SQLITE_WARNING: Final[int] + SQLITE_WARNING_AUTOINDEX: Final[int] + threadsafety: Final[int] + +# Can take or return anything depending on what's in the registry. +@overload +def adapt(obj: Any, proto: Any, /) -> Any: ... +@overload +def adapt(obj: Any, proto: Any, alt: _T, /) -> Any | _T: ... +def complete_statement(statement: str) -> bool: ... + +if sys.version_info >= (3, 12): + @overload + def connect( + database: StrOrBytesPath, + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + check_same_thread: bool = True, + cached_statements: int = 128, + uri: bool = False, + *, + autocommit: bool = ..., + ) -> Connection: ... + @overload + def connect( + database: StrOrBytesPath, + timeout: float, + detect_types: int, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None, + check_same_thread: bool, + factory: type[_ConnectionT], + cached_statements: int = 128, + uri: bool = False, + *, + autocommit: bool = ..., + ) -> _ConnectionT: ... + @overload + def connect( + database: StrOrBytesPath, + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + check_same_thread: bool = True, + *, + factory: type[_ConnectionT], + cached_statements: int = 128, + uri: bool = False, + autocommit: bool = ..., + ) -> _ConnectionT: ... + +else: + @overload + def connect( + database: StrOrBytesPath, + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + check_same_thread: bool = True, + cached_statements: int = 128, + uri: bool = False, + ) -> Connection: ... + @overload + def connect( + database: StrOrBytesPath, + timeout: float, + detect_types: int, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None, + check_same_thread: bool, + factory: type[_ConnectionT], + cached_statements: int = 128, + uri: bool = False, + ) -> _ConnectionT: ... + @overload + def connect( + database: StrOrBytesPath, + timeout: float = 5.0, + detect_types: int = 0, + isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", + check_same_thread: bool = True, + *, + factory: type[_ConnectionT], + cached_statements: int = 128, + uri: bool = False, + ) -> _ConnectionT: ... + +def enable_callback_tracebacks(enable: bool, /) -> None: ... + +if sys.version_info < (3, 12): + # takes a pos-or-keyword argument because there is a C wrapper + def enable_shared_cache(do_enable: int) -> None: ... + +if sys.version_info >= (3, 10): + def register_adapter(type: type[_T], adapter: _Adapter[_T], /) -> None: ... + def register_converter(typename: str, converter: _Converter, /) -> None: ... + +else: + def register_adapter(type: type[_T], caster: _Adapter[_T], /) -> None: ... + def register_converter(name: str, converter: _Converter, /) -> None: ... + +if sys.version_info < (3, 10): + OptimizedUnicode = str diff --git a/mypy/typeshed/stdlib/_ssl.pyi b/mypy/typeshed/stdlib/_ssl.pyi new file mode 100644 index 000000000000..3e88874143df --- /dev/null +++ b/mypy/typeshed/stdlib/_ssl.pyi @@ -0,0 +1,292 @@ +import sys +from _typeshed import ReadableBuffer, StrOrBytesPath +from collections.abc import Callable +from ssl import ( + SSLCertVerificationError as SSLCertVerificationError, + SSLContext, + SSLEOFError as SSLEOFError, + SSLError as SSLError, + SSLObject, + SSLSyscallError as SSLSyscallError, + SSLWantReadError as SSLWantReadError, + SSLWantWriteError as SSLWantWriteError, + SSLZeroReturnError as SSLZeroReturnError, +) +from typing import Any, Literal, TypedDict, final, overload +from typing_extensions import NotRequired, Self, TypeAlias + +_PasswordType: TypeAlias = Callable[[], str | bytes | bytearray] | str | bytes | bytearray +_PCTRTT: TypeAlias = tuple[tuple[str, str], ...] +_PCTRTTT: TypeAlias = tuple[_PCTRTT, ...] +_PeerCertRetDictType: TypeAlias = dict[str, str | _PCTRTTT | _PCTRTT] + +class _Cipher(TypedDict): + aead: bool + alg_bits: int + auth: str + description: str + digest: str | None + id: int + kea: str + name: str + protocol: str + strength_bits: int + symmetric: str + +class _CertInfo(TypedDict): + subject: tuple[tuple[tuple[str, str], ...], ...] + issuer: tuple[tuple[tuple[str, str], ...], ...] + version: int + serialNumber: str + notBefore: str + notAfter: str + subjectAltName: NotRequired[tuple[tuple[str, str], ...] | None] + OCSP: NotRequired[tuple[str, ...] | None] + caIssuers: NotRequired[tuple[str, ...] | None] + crlDistributionPoints: NotRequired[tuple[str, ...] | None] + +def RAND_add(string: str | ReadableBuffer, entropy: float, /) -> None: ... +def RAND_bytes(n: int, /) -> bytes: ... + +if sys.version_info < (3, 12): + def RAND_pseudo_bytes(n: int, /) -> tuple[bytes, bool]: ... + +if sys.version_info < (3, 10): + def RAND_egd(path: str) -> None: ... + +def RAND_status() -> bool: ... +def get_default_verify_paths() -> tuple[str, str, str, str]: ... + +if sys.platform == "win32": + _EnumRetType: TypeAlias = list[tuple[bytes, str, set[str] | bool]] + def enum_certificates(store_name: str) -> _EnumRetType: ... + def enum_crls(store_name: str) -> _EnumRetType: ... + +def txt2obj(txt: str, name: bool = False) -> tuple[int, str, str, str]: ... +def nid2obj(nid: int, /) -> tuple[int, str, str, str]: ... + +class _SSLContext: + check_hostname: bool + keylog_filename: str | None + maximum_version: int + minimum_version: int + num_tickets: int + options: int + post_handshake_auth: bool + protocol: int + if sys.version_info >= (3, 10): + security_level: int + sni_callback: Callable[[SSLObject, str, SSLContext], None | int] | None + verify_flags: int + verify_mode: int + def __new__(cls, protocol: int, /) -> Self: ... + def cert_store_stats(self) -> dict[str, int]: ... + @overload + def get_ca_certs(self, binary_form: Literal[False] = False) -> list[_PeerCertRetDictType]: ... + @overload + def get_ca_certs(self, binary_form: Literal[True]) -> list[bytes]: ... + @overload + def get_ca_certs(self, binary_form: bool = False) -> Any: ... + def get_ciphers(self) -> list[_Cipher]: ... + def load_cert_chain( + self, certfile: StrOrBytesPath, keyfile: StrOrBytesPath | None = None, password: _PasswordType | None = None + ) -> None: ... + def load_dh_params(self, path: str, /) -> None: ... + def load_verify_locations( + self, + cafile: StrOrBytesPath | None = None, + capath: StrOrBytesPath | None = None, + cadata: str | ReadableBuffer | None = None, + ) -> None: ... + def session_stats(self) -> dict[str, int]: ... + def set_ciphers(self, cipherlist: str, /) -> None: ... + def set_default_verify_paths(self) -> None: ... + def set_ecdh_curve(self, name: str, /) -> None: ... + if sys.version_info >= (3, 13): + def set_psk_client_callback(self, callback: Callable[[str | None], tuple[str | None, bytes]] | None) -> None: ... + def set_psk_server_callback( + self, callback: Callable[[str | None], tuple[str | None, bytes]] | None, identity_hint: str | None = None + ) -> None: ... + +@final +class MemoryBIO: + eof: bool + pending: int + def __new__(self) -> Self: ... + def read(self, size: int = -1, /) -> bytes: ... + def write(self, b: ReadableBuffer, /) -> int: ... + def write_eof(self) -> None: ... + +@final +class SSLSession: + @property + def has_ticket(self) -> bool: ... + @property + def id(self) -> bytes: ... + @property + def ticket_lifetime_hint(self) -> int: ... + @property + def time(self) -> int: ... + @property + def timeout(self) -> int: ... + +# _ssl.Certificate is weird: it can't be instantiated or subclassed. +# Instances can only be created via methods of the private _ssl._SSLSocket class, +# for which the relevant method signatures are: +# +# class _SSLSocket: +# def get_unverified_chain(self) -> list[Certificate] | None: ... +# def get_verified_chain(self) -> list[Certificate] | None: ... +# +# You can find a _ssl._SSLSocket object as the _sslobj attribute of a ssl.SSLSocket object + +if sys.version_info >= (3, 10): + @final + class Certificate: + def get_info(self) -> _CertInfo: ... + @overload + def public_bytes(self) -> str: ... + @overload + def public_bytes(self, format: Literal[1] = 1, /) -> str: ... # ENCODING_PEM + @overload + def public_bytes(self, format: Literal[2], /) -> bytes: ... # ENCODING_DER + @overload + def public_bytes(self, format: int, /) -> str | bytes: ... + +if sys.version_info < (3, 12): + err_codes_to_names: dict[tuple[int, int], str] + err_names_to_codes: dict[str, tuple[int, int]] + lib_codes_to_names: dict[int, str] + +_DEFAULT_CIPHERS: str + +# SSL error numbers +SSL_ERROR_ZERO_RETURN: int +SSL_ERROR_WANT_READ: int +SSL_ERROR_WANT_WRITE: int +SSL_ERROR_WANT_X509_LOOKUP: int +SSL_ERROR_SYSCALL: int +SSL_ERROR_SSL: int +SSL_ERROR_WANT_CONNECT: int +SSL_ERROR_EOF: int +SSL_ERROR_INVALID_ERROR_CODE: int + +# verify modes +CERT_NONE: int +CERT_OPTIONAL: int +CERT_REQUIRED: int + +# verify flags +VERIFY_DEFAULT: int +VERIFY_CRL_CHECK_LEAF: int +VERIFY_CRL_CHECK_CHAIN: int +VERIFY_X509_STRICT: int +VERIFY_X509_TRUSTED_FIRST: int +if sys.version_info >= (3, 10): + VERIFY_ALLOW_PROXY_CERTS: int + VERIFY_X509_PARTIAL_CHAIN: int + +# alert descriptions +ALERT_DESCRIPTION_CLOSE_NOTIFY: int +ALERT_DESCRIPTION_UNEXPECTED_MESSAGE: int +ALERT_DESCRIPTION_BAD_RECORD_MAC: int +ALERT_DESCRIPTION_RECORD_OVERFLOW: int +ALERT_DESCRIPTION_DECOMPRESSION_FAILURE: int +ALERT_DESCRIPTION_HANDSHAKE_FAILURE: int +ALERT_DESCRIPTION_BAD_CERTIFICATE: int +ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE: int +ALERT_DESCRIPTION_CERTIFICATE_REVOKED: int +ALERT_DESCRIPTION_CERTIFICATE_EXPIRED: int +ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN: int +ALERT_DESCRIPTION_ILLEGAL_PARAMETER: int +ALERT_DESCRIPTION_UNKNOWN_CA: int +ALERT_DESCRIPTION_ACCESS_DENIED: int +ALERT_DESCRIPTION_DECODE_ERROR: int +ALERT_DESCRIPTION_DECRYPT_ERROR: int +ALERT_DESCRIPTION_PROTOCOL_VERSION: int +ALERT_DESCRIPTION_INSUFFICIENT_SECURITY: int +ALERT_DESCRIPTION_INTERNAL_ERROR: int +ALERT_DESCRIPTION_USER_CANCELLED: int +ALERT_DESCRIPTION_NO_RENEGOTIATION: int +ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION: int +ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE: int +ALERT_DESCRIPTION_UNRECOGNIZED_NAME: int +ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE: int +ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE: int +ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY: int + +# protocol versions +PROTOCOL_SSLv23: int +PROTOCOL_TLS: int +PROTOCOL_TLS_CLIENT: int +PROTOCOL_TLS_SERVER: int +PROTOCOL_TLSv1: int +PROTOCOL_TLSv1_1: int +PROTOCOL_TLSv1_2: int + +# protocol options +OP_ALL: int +OP_NO_SSLv2: int +OP_NO_SSLv3: int +OP_NO_TLSv1: int +OP_NO_TLSv1_1: int +OP_NO_TLSv1_2: int +OP_NO_TLSv1_3: int +OP_CIPHER_SERVER_PREFERENCE: int +OP_SINGLE_DH_USE: int +OP_NO_TICKET: int +OP_SINGLE_ECDH_USE: int +OP_NO_COMPRESSION: int +OP_ENABLE_MIDDLEBOX_COMPAT: int +OP_NO_RENEGOTIATION: int +if sys.version_info >= (3, 11): + OP_IGNORE_UNEXPECTED_EOF: int +elif sys.version_info >= (3, 8) and sys.platform == "linux": + OP_IGNORE_UNEXPECTED_EOF: int +if sys.version_info >= (3, 12): + OP_LEGACY_SERVER_CONNECT: int + OP_ENABLE_KTLS: int + +# host flags +HOSTFLAG_ALWAYS_CHECK_SUBJECT: int +HOSTFLAG_NEVER_CHECK_SUBJECT: int +HOSTFLAG_NO_WILDCARDS: int +HOSTFLAG_NO_PARTIAL_WILDCARDS: int +HOSTFLAG_MULTI_LABEL_WILDCARDS: int +HOSTFLAG_SINGLE_LABEL_SUBDOMAINS: int + +if sys.version_info >= (3, 10): + # certificate file types + # Typed as Literal so the overload on Certificate.public_bytes can work properly. + ENCODING_PEM: Literal[1] + ENCODING_DER: Literal[2] + +# protocol versions +PROTO_MINIMUM_SUPPORTED: int +PROTO_MAXIMUM_SUPPORTED: int +PROTO_SSLv3: int +PROTO_TLSv1: int +PROTO_TLSv1_1: int +PROTO_TLSv1_2: int +PROTO_TLSv1_3: int + +# feature support +HAS_SNI: bool +HAS_TLS_UNIQUE: bool +HAS_ECDH: bool +HAS_NPN: bool +if sys.version_info >= (3, 13): + HAS_PSK: bool +HAS_ALPN: bool +HAS_SSLv2: bool +HAS_SSLv3: bool +HAS_TLSv1: bool +HAS_TLSv1_1: bool +HAS_TLSv1_2: bool +HAS_TLSv1_3: bool + +# version info +OPENSSL_VERSION_NUMBER: int +OPENSSL_VERSION_INFO: tuple[int, int, int, int, int] +OPENSSL_VERSION: str +_OPENSSL_API_VERSION: tuple[int, int, int, int, int] diff --git a/mypy/typeshed/stdlib/_weakref.pyi b/mypy/typeshed/stdlib/_weakref.pyi index f142820c56c7..a744340afaab 100644 --- a/mypy/typeshed/stdlib/_weakref.pyi +++ b/mypy/typeshed/stdlib/_weakref.pyi @@ -1,37 +1,10 @@ -import sys from collections.abc import Callable -from typing import Any, Generic, TypeVar, final, overload -from typing_extensions import Self - -if sys.version_info >= (3, 9): - from types import GenericAlias +from typing import Any, TypeVar, overload +from weakref import CallableProxyType as CallableProxyType, ProxyType as ProxyType, ReferenceType as ReferenceType, ref as ref _C = TypeVar("_C", bound=Callable[..., Any]) _T = TypeVar("_T") -@final -class CallableProxyType(Generic[_C]): # "weakcallableproxy" - def __eq__(self, value: object, /) -> bool: ... - def __getattr__(self, attr: str) -> Any: ... - __call__: _C - -@final -class ProxyType(Generic[_T]): # "weakproxy" - def __eq__(self, value: object, /) -> bool: ... - def __getattr__(self, attr: str) -> Any: ... - -class ReferenceType(Generic[_T]): - __callback__: Callable[[Self], Any] - def __new__(cls, o: _T, callback: Callable[[Self], Any] | None = ..., /) -> Self: ... - def __init__(self, o: _T, callback: Callable[[Self], Any] | None = ..., /) -> None: ... - def __call__(self) -> _T | None: ... - def __eq__(self, value: object, /) -> bool: ... - def __hash__(self) -> int: ... - if sys.version_info >= (3, 9): - def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... - -ref = ReferenceType - def getweakrefcount(object: Any, /) -> int: ... def getweakrefs(object: Any, /) -> list[Any]: ... diff --git a/mypy/typeshed/stdlib/ast.pyi b/mypy/typeshed/stdlib/ast.pyi index 80049cff4ce0..3a43d62a3a60 100644 --- a/mypy/typeshed/stdlib/ast.pyi +++ b/mypy/typeshed/stdlib/ast.pyi @@ -1,184 +1,1717 @@ import os import sys -from _ast import * +import typing_extensions +from _ast import ( + PyCF_ALLOW_TOP_LEVEL_AWAIT as PyCF_ALLOW_TOP_LEVEL_AWAIT, + PyCF_ONLY_AST as PyCF_ONLY_AST, + PyCF_TYPE_COMMENTS as PyCF_TYPE_COMMENTS, +) from _typeshed import ReadableBuffer, Unused from collections.abc import Iterator -from typing import Any, Literal, TypeVar as _TypeVar, overload -from typing_extensions import deprecated +from typing import Any, ClassVar, Generic, Literal, TypedDict, TypeVar as _TypeVar, overload +from typing_extensions import Self, Unpack, deprecated -class _ABC(type): - if sys.version_info >= (3, 9): - def __init__(cls, *args: Unused) -> None: ... +if sys.version_info >= (3, 13): + from _ast import PyCF_OPTIMIZED_AST as PyCF_OPTIMIZED_AST -if sys.version_info < (3, 14): - @deprecated("Replaced by ast.Constant; removed in Python 3.14") - class Num(Constant, metaclass=_ABC): - value: int | float | complex +# Alias used for fields that must always be valid identifiers +# A string `x` counts as a valid identifier if both the following are True +# (1) `x.isidentifier()` evaluates to `True` +# (2) `keyword.iskeyword(x)` evaluates to `False` +_Identifier: typing_extensions.TypeAlias = str - @deprecated("Replaced by ast.Constant; removed in Python 3.14") - class Str(Constant, metaclass=_ABC): - value: str - # Aliases for value, for backwards compatibility - s: str +# Used for node end positions in constructor keyword arguments +_EndPositionT = typing_extensions.TypeVar("_EndPositionT", int, int | None, default=int | None) - @deprecated("Replaced by ast.Constant; removed in Python 3.14") - class Bytes(Constant, metaclass=_ABC): - value: bytes +# Corresponds to the names in the `_attributes` class variable which is non-empty in certain AST nodes +class _Attributes(TypedDict, Generic[_EndPositionT], total=False): + lineno: int + col_offset: int + end_lineno: _EndPositionT + end_col_offset: _EndPositionT + +# The various AST classes are implemented in C, and imported from _ast at runtime, +# but they consider themselves to live in the ast module, +# so we'll define the stubs in this file. +class AST: + if sys.version_info >= (3, 10): + __match_args__ = () + _attributes: ClassVar[tuple[str, ...]] + _fields: ClassVar[tuple[str, ...]] + if sys.version_info >= (3, 13): + _field_types: ClassVar[dict[str, Any]] + + if sys.version_info >= (3, 14): + def __replace__(self) -> Self: ... + +class mod(AST): ... + +class Module(mod): + if sys.version_info >= (3, 10): + __match_args__ = ("body", "type_ignores") + body: list[stmt] + type_ignores: list[TypeIgnore] + if sys.version_info >= (3, 13): + def __init__(self, body: list[stmt] = ..., type_ignores: list[TypeIgnore] = ...) -> None: ... + else: + def __init__(self, body: list[stmt], type_ignores: list[TypeIgnore]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, body: list[stmt] = ..., type_ignores: list[TypeIgnore] = ...) -> Self: ... + +class Interactive(mod): + if sys.version_info >= (3, 10): + __match_args__ = ("body",) + body: list[stmt] + if sys.version_info >= (3, 13): + def __init__(self, body: list[stmt] = ...) -> None: ... + else: + def __init__(self, body: list[stmt]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, body: list[stmt] = ...) -> Self: ... + +class Expression(mod): + if sys.version_info >= (3, 10): + __match_args__ = ("body",) + body: expr + def __init__(self, body: expr) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, body: expr = ...) -> Self: ... + +class FunctionType(mod): + if sys.version_info >= (3, 10): + __match_args__ = ("argtypes", "returns") + argtypes: list[expr] + returns: expr + if sys.version_info >= (3, 13): + @overload + def __init__(self, argtypes: list[expr], returns: expr) -> None: ... + @overload + def __init__(self, argtypes: list[expr] = ..., *, returns: expr) -> None: ... + else: + def __init__(self, argtypes: list[expr], returns: expr) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, argtypes: list[expr] = ..., returns: expr = ...) -> Self: ... + +class stmt(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, **kwargs: Unpack[_Attributes]) -> Self: ... + +class FunctionDef(stmt): + if sys.version_info >= (3, 12): + __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment", "type_params") + elif sys.version_info >= (3, 10): + __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment") + name: _Identifier + args: arguments + body: list[stmt] + decorator_list: list[expr] + returns: expr | None + type_comment: str | None + if sys.version_info >= (3, 12): + type_params: list[type_param] + if sys.version_info >= (3, 13): + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt] = ..., + decorator_list: list[expr] = ..., + returns: expr | None = None, + type_comment: str | None = None, + type_params: list[type_param] = ..., + **kwargs: Unpack[_Attributes], + ) -> None: ... + elif sys.version_info >= (3, 12): + @overload + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None, + type_comment: str | None, + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> None: ... + @overload + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None = None, + type_comment: str | None = None, + *, + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None = None, + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier = ..., + args: arguments = ..., + body: list[stmt] = ..., + decorator_list: list[expr] = ..., + returns: expr | None = ..., + type_comment: str | None = ..., + type_params: list[type_param] = ..., + ) -> Self: ... + +class AsyncFunctionDef(stmt): + if sys.version_info >= (3, 12): + __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment", "type_params") + elif sys.version_info >= (3, 10): + __match_args__ = ("name", "args", "body", "decorator_list", "returns", "type_comment") + name: _Identifier + args: arguments + body: list[stmt] + decorator_list: list[expr] + returns: expr | None + type_comment: str | None + if sys.version_info >= (3, 12): + type_params: list[type_param] + if sys.version_info >= (3, 13): + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt] = ..., + decorator_list: list[expr] = ..., + returns: expr | None = None, + type_comment: str | None = None, + type_params: list[type_param] = ..., + **kwargs: Unpack[_Attributes], + ) -> None: ... + elif sys.version_info >= (3, 12): + @overload + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None, + type_comment: str | None, + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> None: ... + @overload + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None = None, + type_comment: str | None = None, + *, + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + name: _Identifier, + args: arguments, + body: list[stmt], + decorator_list: list[expr], + returns: expr | None = None, + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier = ..., + args: arguments = ..., + body: list[stmt], + decorator_list: list[expr], + returns: expr | None, + type_comment: str | None, + type_params: list[type_param], + ) -> Self: ... + +class ClassDef(stmt): + if sys.version_info >= (3, 12): + __match_args__ = ("name", "bases", "keywords", "body", "decorator_list", "type_params") + elif sys.version_info >= (3, 10): + __match_args__ = ("name", "bases", "keywords", "body", "decorator_list") + name: _Identifier + bases: list[expr] + keywords: list[keyword] + body: list[stmt] + decorator_list: list[expr] + if sys.version_info >= (3, 12): + type_params: list[type_param] + if sys.version_info >= (3, 13): + def __init__( + self, + name: _Identifier, + bases: list[expr] = ..., + keywords: list[keyword] = ..., + body: list[stmt] = ..., + decorator_list: list[expr] = ..., + type_params: list[type_param] = ..., + **kwargs: Unpack[_Attributes], + ) -> None: ... + elif sys.version_info >= (3, 12): + def __init__( + self, + name: _Identifier, + bases: list[expr], + keywords: list[keyword], + body: list[stmt], + decorator_list: list[expr], + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + name: _Identifier, + bases: list[expr], + keywords: list[keyword], + body: list[stmt], + decorator_list: list[expr], + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier, + bases: list[expr], + keywords: list[keyword], + body: list[stmt], + decorator_list: list[expr], + type_params: list[type_param], + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class Return(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("value",) + value: expr | None + def __init__(self, value: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Delete(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("targets",) + targets: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, targets: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, targets: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, targets: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Assign(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("targets", "value", "type_comment") + targets: list[expr] + value: expr + type_comment: str | None + if sys.version_info >= (3, 13): + @overload + def __init__( + self, targets: list[expr], value: expr, type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + @overload + def __init__( + self, targets: list[expr] = ..., *, value: expr, type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__( + self, targets: list[expr], value: expr, type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, targets: list[expr] = ..., value: expr = ..., type_comment: str | None = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +if sys.version_info >= (3, 12): + class TypeAlias(stmt): + __match_args__ = ("name", "type_params", "value") + name: Name + type_params: list[type_param] + value: expr + if sys.version_info >= (3, 13): + @overload + def __init__( + self, name: Name, type_params: list[type_param], value: expr, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + @overload + def __init__( + self, name: Name, type_params: list[type_param] = ..., *, value: expr, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + else: + def __init__( + self, name: Name, type_params: list[type_param], value: expr, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: Name = ..., + type_params: list[type_param] = ..., + value: expr = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + +class AugAssign(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "op", "value") + target: Name | Attribute | Subscript + op: operator + value: expr + def __init__( + self, target: Name | Attribute | Subscript, op: operator, value: expr, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: Name | Attribute | Subscript = ..., + op: operator = ..., + value: expr = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class AnnAssign(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "annotation", "value", "simple") + target: Name | Attribute | Subscript + annotation: expr + value: expr | None + simple: int + @overload + def __init__( + self, + target: Name | Attribute | Subscript, + annotation: expr, + value: expr | None, + simple: int, + **kwargs: Unpack[_Attributes], + ) -> None: ... + @overload + def __init__( + self, + target: Name | Attribute | Subscript, + annotation: expr, + value: expr | None = None, + *, + simple: int, + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: Name | Attribute | Subscript = ..., + annotation: expr = ..., + value: expr | None = ..., + simple: int = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class For(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "iter", "body", "orelse", "type_comment") + target: expr + iter: expr + body: list[stmt] + orelse: list[stmt] + type_comment: str | None + if sys.version_info >= (3, 13): + def __init__( + self, + target: expr, + iter: expr, + body: list[stmt] = ..., + orelse: list[stmt] = ..., + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + target: expr, + iter: expr, + body: list[stmt], + orelse: list[stmt], + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: expr = ..., + iter: expr = ..., + body: list[stmt] = ..., + orelse: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class AsyncFor(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "iter", "body", "orelse", "type_comment") + target: expr + iter: expr + body: list[stmt] + orelse: list[stmt] + type_comment: str | None + if sys.version_info >= (3, 13): + def __init__( + self, + target: expr, + iter: expr, + body: list[stmt] = ..., + orelse: list[stmt] = ..., + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + target: expr, + iter: expr, + body: list[stmt], + orelse: list[stmt], + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + target: expr = ..., + iter: expr = ..., + body: list[stmt] = ..., + orelse: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class While(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("test", "body", "orelse") + test: expr + body: list[stmt] + orelse: list[stmt] + if sys.version_info >= (3, 13): + def __init__( + self, test: expr, body: list[stmt] = ..., orelse: list[stmt] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__(self, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> Self: ... + +class If(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("test", "body", "orelse") + test: expr + body: list[stmt] + orelse: list[stmt] + if sys.version_info >= (3, 13): + def __init__( + self, test: expr, body: list[stmt] = ..., orelse: list[stmt] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__(self, test: expr, body: list[stmt], orelse: list[stmt], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, test: expr = ..., body: list[stmt] = ..., orelse: list[stmt] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class With(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("items", "body", "type_comment") + items: list[withitem] + body: list[stmt] + type_comment: str | None + if sys.version_info >= (3, 13): + def __init__( + self, + items: list[withitem] = ..., + body: list[stmt] = ..., + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, items: list[withitem], body: list[stmt], type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + items: list[withitem] = ..., + body: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class AsyncWith(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("items", "body", "type_comment") + items: list[withitem] + body: list[stmt] + type_comment: str | None + if sys.version_info >= (3, 13): + def __init__( + self, + items: list[withitem] = ..., + body: list[stmt] = ..., + type_comment: str | None = None, + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, items: list[withitem], body: list[stmt], type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + items: list[withitem] = ..., + body: list[stmt] = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +if sys.version_info >= (3, 10): + class Match(stmt): + __match_args__ = ("subject", "cases") + subject: expr + cases: list[match_case] + if sys.version_info >= (3, 13): + def __init__(self, subject: expr, cases: list[match_case] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, subject: expr, cases: list[match_case], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, subject: expr = ..., cases: list[match_case] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Raise(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("exc", "cause") + exc: expr | None + cause: expr | None + def __init__(self, exc: expr | None = None, cause: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, exc: expr | None = ..., cause: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Try(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("body", "handlers", "orelse", "finalbody") + body: list[stmt] + handlers: list[ExceptHandler] + orelse: list[stmt] + finalbody: list[stmt] + if sys.version_info >= (3, 13): + def __init__( + self, + body: list[stmt] = ..., + handlers: list[ExceptHandler] = ..., + orelse: list[stmt] = ..., + finalbody: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + body: list[stmt], + handlers: list[ExceptHandler], + orelse: list[stmt], + finalbody: list[stmt], + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + body: list[stmt] = ..., + handlers: list[ExceptHandler] = ..., + orelse: list[stmt] = ..., + finalbody: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +if sys.version_info >= (3, 11): + class TryStar(stmt): + __match_args__ = ("body", "handlers", "orelse", "finalbody") + body: list[stmt] + handlers: list[ExceptHandler] + orelse: list[stmt] + finalbody: list[stmt] + if sys.version_info >= (3, 13): + def __init__( + self, + body: list[stmt] = ..., + handlers: list[ExceptHandler] = ..., + orelse: list[stmt] = ..., + finalbody: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> None: ... + else: + def __init__( + self, + body: list[stmt], + handlers: list[ExceptHandler], + orelse: list[stmt], + finalbody: list[stmt], + **kwargs: Unpack[_Attributes], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + body: list[stmt] = ..., + handlers: list[ExceptHandler] = ..., + orelse: list[stmt] = ..., + finalbody: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class Assert(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("test", "msg") + test: expr + msg: expr | None + def __init__(self, test: expr, msg: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, test: expr, msg: expr | None, **kwargs: Unpack[_Attributes]) -> Self: ... + +class Import(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("names",) + names: list[alias] + if sys.version_info >= (3, 13): + def __init__(self, names: list[alias] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, names: list[alias], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, names: list[alias] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class ImportFrom(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("module", "names", "level") + module: str | None + names: list[alias] + level: int + if sys.version_info >= (3, 13): + @overload + def __init__(self, module: str | None, names: list[alias], level: int, **kwargs: Unpack[_Attributes]) -> None: ... + @overload + def __init__( + self, module: str | None = None, names: list[alias] = ..., *, level: int, **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + @overload + def __init__(self, module: str | None, names: list[alias], level: int, **kwargs: Unpack[_Attributes]) -> None: ... + @overload + def __init__( + self, module: str | None = None, *, names: list[alias], level: int, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, module: str | None = ..., names: list[alias] = ..., level: int = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Global(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("names",) + names: list[_Identifier] + if sys.version_info >= (3, 13): + def __init__(self, names: list[_Identifier] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> Self: ... + +class Nonlocal(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("names",) + names: list[_Identifier] + if sys.version_info >= (3, 13): + def __init__(self, names: list[_Identifier] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, names: list[_Identifier], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, names: list[_Identifier] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Expr(stmt): + if sys.version_info >= (3, 10): + __match_args__ = ("value",) + value: expr + def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Pass(stmt): ... +class Break(stmt): ... +class Continue(stmt): ... + +class expr(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, **kwargs: Unpack[_Attributes]) -> Self: ... + +class BoolOp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("op", "values") + op: boolop + values: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, op: boolop, values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, op: boolop, values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, op: boolop = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class NamedExpr(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "value") + target: Name + value: expr + def __init__(self, target: Name, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, target: Name = ..., value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class BinOp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("left", "op", "right") + left: expr + op: operator + right: expr + def __init__(self, left: expr, op: operator, right: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, left: expr = ..., op: operator = ..., right: expr = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class UnaryOp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("op", "operand") + op: unaryop + operand: expr + def __init__(self, op: unaryop, operand: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, op: unaryop = ..., operand: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Lambda(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("args", "body") + args: arguments + body: expr + def __init__(self, args: arguments, body: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, args: arguments = ..., body: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class IfExp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("test", "body", "orelse") + test: expr + body: expr + orelse: expr + def __init__(self, test: expr, body: expr, orelse: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, test: expr = ..., body: expr = ..., orelse: expr = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Dict(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("keys", "values") + keys: list[expr | None] + values: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, keys: list[expr | None] = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, keys: list[expr | None], values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, keys: list[expr | None] = ..., values: list[expr] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Set(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elts",) + elts: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, elts: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elts: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, elts: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class ListComp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elt", "generators") + elt: expr + generators: list[comprehension] + if sys.version_info >= (3, 13): + def __init__(self, elt: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class SetComp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elt", "generators") + elt: expr + generators: list[comprehension] + if sys.version_info >= (3, 13): + def __init__(self, elt: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class DictComp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("key", "value", "generators") + key: expr + value: expr + generators: list[comprehension] + if sys.version_info >= (3, 13): + def __init__( + self, key: expr, value: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__(self, key: expr, value: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, key: expr = ..., value: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class GeneratorExp(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elt", "generators") + elt: expr + generators: list[comprehension] + if sys.version_info >= (3, 13): + def __init__(self, elt: expr, generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elt: expr, generators: list[comprehension], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, elt: expr = ..., generators: list[comprehension] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Await(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value",) + value: expr + def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Yield(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value",) + value: expr | None + def __init__(self, value: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class YieldFrom(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value",) + value: expr + def __init__(self, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Compare(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("left", "ops", "comparators") + left: expr + ops: list[cmpop] + comparators: list[expr] + if sys.version_info >= (3, 13): + def __init__( + self, left: expr, ops: list[cmpop] = ..., comparators: list[expr] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__(self, left: expr, ops: list[cmpop], comparators: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, left: expr = ..., ops: list[cmpop] = ..., comparators: list[expr] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Call(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("func", "args", "keywords") + func: expr + args: list[expr] + keywords: list[keyword] + if sys.version_info >= (3, 13): + def __init__( + self, func: expr, args: list[expr] = ..., keywords: list[keyword] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + def __init__(self, func: expr, args: list[expr], keywords: list[keyword], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, func: expr = ..., args: list[expr] = ..., keywords: list[keyword] = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class FormattedValue(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value", "conversion", "format_spec") + value: expr + conversion: int + format_spec: expr | None + def __init__(self, value: expr, conversion: int, format_spec: expr | None = None, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, value: expr = ..., conversion: int = ..., format_spec: expr | None = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class JoinedStr(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("values",) + values: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, values: list[expr], **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, values: list[expr] = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Constant(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value", "kind") + value: Any # None, str, bytes, bool, int, float, complex, Ellipsis + kind: str | None + if sys.version_info < (3, 14): # Aliases for value, for backwards compatibility - s: bytes + s: Any + n: int | float | complex - @deprecated("Replaced by ast.Constant; removed in Python 3.14") - class NameConstant(Constant, metaclass=_ABC): ... + def __init__(self, value: Any, kind: str | None = None, **kwargs: Unpack[_Attributes]) -> None: ... - @deprecated("Replaced by ast.Constant; removed in Python 3.14") - class Ellipsis(Constant, metaclass=_ABC): ... + if sys.version_info >= (3, 14): + def __replace__(self, *, value: Any = ..., kind: str | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Attribute(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value", "attr", "ctx") + value: expr + attr: _Identifier + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + def __init__(self, value: expr, attr: _Identifier, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, value: expr = ..., attr: _Identifier = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Subscript(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value", "slice", "ctx") + value: expr + slice: _Slice + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + def __init__(self, value: expr, slice: _Slice, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, value: expr = ..., slice: _Slice = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes] + ) -> Self: ... + +class Starred(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("value", "ctx") + value: expr + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + def __init__(self, value: expr, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Name(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("id", "ctx") + id: _Identifier + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + def __init__(self, id: _Identifier, ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, id: _Identifier = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class List(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elts", "ctx") + elts: list[expr] + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + if sys.version_info >= (3, 13): + def __init__(self, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elts: list[expr], ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class Tuple(expr): + if sys.version_info >= (3, 10): + __match_args__ = ("elts", "ctx") + elts: list[expr] + ctx: expr_context # Not present in Python < 3.13 if not passed to `__init__` + if sys.version_info >= (3, 9): + dims: list[expr] + if sys.version_info >= (3, 13): + def __init__(self, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + else: + def __init__(self, elts: list[expr], ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, elts: list[expr] = ..., ctx: expr_context = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class slice(AST): ... # deprecated and moved to ast.py for >= (3, 9) if sys.version_info >= (3, 9): - class slice(AST): ... - class ExtSlice(slice): ... - class Index(slice): ... - class Suite(mod): ... - class AugLoad(expr_context): ... - class AugStore(expr_context): ... - class Param(expr_context): ... + _Slice: typing_extensions.TypeAlias = expr + _SliceAttributes: typing_extensions.TypeAlias = _Attributes +else: + # alias for use with variables named slice + _Slice: typing_extensions.TypeAlias = slice -class NodeVisitor: - def visit(self, node: AST) -> Any: ... - def generic_visit(self, node: AST) -> Any: ... - def visit_Module(self, node: Module) -> Any: ... - def visit_Interactive(self, node: Interactive) -> Any: ... - def visit_Expression(self, node: Expression) -> Any: ... - def visit_FunctionDef(self, node: FunctionDef) -> Any: ... - def visit_AsyncFunctionDef(self, node: AsyncFunctionDef) -> Any: ... - def visit_ClassDef(self, node: ClassDef) -> Any: ... - def visit_Return(self, node: Return) -> Any: ... - def visit_Delete(self, node: Delete) -> Any: ... - def visit_Assign(self, node: Assign) -> Any: ... - def visit_AugAssign(self, node: AugAssign) -> Any: ... - def visit_AnnAssign(self, node: AnnAssign) -> Any: ... - def visit_For(self, node: For) -> Any: ... - def visit_AsyncFor(self, node: AsyncFor) -> Any: ... - def visit_While(self, node: While) -> Any: ... - def visit_If(self, node: If) -> Any: ... - def visit_With(self, node: With) -> Any: ... - def visit_AsyncWith(self, node: AsyncWith) -> Any: ... - def visit_Raise(self, node: Raise) -> Any: ... - def visit_Try(self, node: Try) -> Any: ... - def visit_Assert(self, node: Assert) -> Any: ... - def visit_Import(self, node: Import) -> Any: ... - def visit_ImportFrom(self, node: ImportFrom) -> Any: ... - def visit_Global(self, node: Global) -> Any: ... - def visit_Nonlocal(self, node: Nonlocal) -> Any: ... - def visit_Expr(self, node: Expr) -> Any: ... - def visit_Pass(self, node: Pass) -> Any: ... - def visit_Break(self, node: Break) -> Any: ... - def visit_Continue(self, node: Continue) -> Any: ... - def visit_Slice(self, node: Slice) -> Any: ... - def visit_BoolOp(self, node: BoolOp) -> Any: ... - def visit_BinOp(self, node: BinOp) -> Any: ... - def visit_UnaryOp(self, node: UnaryOp) -> Any: ... - def visit_Lambda(self, node: Lambda) -> Any: ... - def visit_IfExp(self, node: IfExp) -> Any: ... - def visit_Dict(self, node: Dict) -> Any: ... - def visit_Set(self, node: Set) -> Any: ... - def visit_ListComp(self, node: ListComp) -> Any: ... - def visit_SetComp(self, node: SetComp) -> Any: ... - def visit_DictComp(self, node: DictComp) -> Any: ... - def visit_GeneratorExp(self, node: GeneratorExp) -> Any: ... - def visit_Await(self, node: Await) -> Any: ... - def visit_Yield(self, node: Yield) -> Any: ... - def visit_YieldFrom(self, node: YieldFrom) -> Any: ... - def visit_Compare(self, node: Compare) -> Any: ... - def visit_Call(self, node: Call) -> Any: ... - def visit_FormattedValue(self, node: FormattedValue) -> Any: ... - def visit_JoinedStr(self, node: JoinedStr) -> Any: ... - def visit_Constant(self, node: Constant) -> Any: ... - def visit_NamedExpr(self, node: NamedExpr) -> Any: ... - def visit_TypeIgnore(self, node: TypeIgnore) -> Any: ... - def visit_Attribute(self, node: Attribute) -> Any: ... - def visit_Subscript(self, node: Subscript) -> Any: ... - def visit_Starred(self, node: Starred) -> Any: ... - def visit_Name(self, node: Name) -> Any: ... - def visit_List(self, node: List) -> Any: ... - def visit_Tuple(self, node: Tuple) -> Any: ... - def visit_Del(self, node: Del) -> Any: ... - def visit_Load(self, node: Load) -> Any: ... - def visit_Store(self, node: Store) -> Any: ... - def visit_And(self, node: And) -> Any: ... - def visit_Or(self, node: Or) -> Any: ... - def visit_Add(self, node: Add) -> Any: ... - def visit_BitAnd(self, node: BitAnd) -> Any: ... - def visit_BitOr(self, node: BitOr) -> Any: ... - def visit_BitXor(self, node: BitXor) -> Any: ... - def visit_Div(self, node: Div) -> Any: ... - def visit_FloorDiv(self, node: FloorDiv) -> Any: ... - def visit_LShift(self, node: LShift) -> Any: ... - def visit_Mod(self, node: Mod) -> Any: ... - def visit_Mult(self, node: Mult) -> Any: ... - def visit_MatMult(self, node: MatMult) -> Any: ... - def visit_Pow(self, node: Pow) -> Any: ... - def visit_RShift(self, node: RShift) -> Any: ... - def visit_Sub(self, node: Sub) -> Any: ... - def visit_Invert(self, node: Invert) -> Any: ... - def visit_Not(self, node: Not) -> Any: ... - def visit_UAdd(self, node: UAdd) -> Any: ... - def visit_USub(self, node: USub) -> Any: ... - def visit_Eq(self, node: Eq) -> Any: ... - def visit_Gt(self, node: Gt) -> Any: ... - def visit_GtE(self, node: GtE) -> Any: ... - def visit_In(self, node: In) -> Any: ... - def visit_Is(self, node: Is) -> Any: ... - def visit_IsNot(self, node: IsNot) -> Any: ... - def visit_Lt(self, node: Lt) -> Any: ... - def visit_LtE(self, node: LtE) -> Any: ... - def visit_NotEq(self, node: NotEq) -> Any: ... - def visit_NotIn(self, node: NotIn) -> Any: ... - def visit_comprehension(self, node: comprehension) -> Any: ... - def visit_ExceptHandler(self, node: ExceptHandler) -> Any: ... - def visit_arguments(self, node: arguments) -> Any: ... - def visit_arg(self, node: arg) -> Any: ... - def visit_keyword(self, node: keyword) -> Any: ... - def visit_alias(self, node: alias) -> Any: ... - def visit_withitem(self, node: withitem) -> Any: ... + class _SliceAttributes(TypedDict): ... + +class Slice(_Slice): + if sys.version_info >= (3, 10): + __match_args__ = ("lower", "upper", "step") + lower: expr | None + upper: expr | None + step: expr | None + def __init__( + self, lower: expr | None = None, upper: expr | None = None, step: expr | None = None, **kwargs: Unpack[_SliceAttributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + lower: expr | None = ..., + upper: expr | None = ..., + step: expr | None = ..., + **kwargs: Unpack[_SliceAttributes], + ) -> Self: ... + +class ExtSlice(slice): # deprecated and moved to ast.py if sys.version_info >= (3, 9) + dims: list[slice] + def __init__(self, dims: list[slice], **kwargs: Unpack[_SliceAttributes]) -> None: ... + +class Index(slice): # deprecated and moved to ast.py if sys.version_info >= (3, 9) + value: expr + def __init__(self, value: expr, **kwargs: Unpack[_SliceAttributes]) -> None: ... + +class expr_context(AST): ... +class AugLoad(expr_context): ... # deprecated and moved to ast.py if sys.version_info >= (3, 9) +class AugStore(expr_context): ... # deprecated and moved to ast.py if sys.version_info >= (3, 9) +class Param(expr_context): ... # deprecated and moved to ast.py if sys.version_info >= (3, 9) + +class Suite(mod): # deprecated and moved to ast.py if sys.version_info >= (3, 9) + body: list[stmt] + def __init__(self, body: list[stmt]) -> None: ... + +class Load(expr_context): ... +class Store(expr_context): ... +class Del(expr_context): ... +class boolop(AST): ... +class And(boolop): ... +class Or(boolop): ... +class operator(AST): ... +class Add(operator): ... +class Sub(operator): ... +class Mult(operator): ... +class MatMult(operator): ... +class Div(operator): ... +class Mod(operator): ... +class Pow(operator): ... +class LShift(operator): ... +class RShift(operator): ... +class BitOr(operator): ... +class BitXor(operator): ... +class BitAnd(operator): ... +class FloorDiv(operator): ... +class unaryop(AST): ... +class Invert(unaryop): ... +class Not(unaryop): ... +class UAdd(unaryop): ... +class USub(unaryop): ... +class cmpop(AST): ... +class Eq(cmpop): ... +class NotEq(cmpop): ... +class Lt(cmpop): ... +class LtE(cmpop): ... +class Gt(cmpop): ... +class GtE(cmpop): ... +class Is(cmpop): ... +class IsNot(cmpop): ... +class In(cmpop): ... +class NotIn(cmpop): ... + +class comprehension(AST): + if sys.version_info >= (3, 10): + __match_args__ = ("target", "iter", "ifs", "is_async") + target: expr + iter: expr + ifs: list[expr] + is_async: int + if sys.version_info >= (3, 13): + @overload + def __init__(self, target: expr, iter: expr, ifs: list[expr], is_async: int) -> None: ... + @overload + def __init__(self, target: expr, iter: expr, ifs: list[expr] = ..., *, is_async: int) -> None: ... + else: + def __init__(self, target: expr, iter: expr, ifs: list[expr], is_async: int) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, target: expr = ..., iter: expr = ..., ifs: list[expr] = ..., is_async: int = ...) -> Self: ... + +class excepthandler(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + def __init__(self, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, lineno: int = ..., col_offset: int = ..., end_lineno: int | None = ..., end_col_offset: int | None = ... + ) -> Self: ... + +class ExceptHandler(excepthandler): + if sys.version_info >= (3, 10): + __match_args__ = ("type", "name", "body") + type: expr | None + name: _Identifier | None + body: list[stmt] + if sys.version_info >= (3, 13): + def __init__( + self, type: expr | None = None, name: _Identifier | None = None, body: list[stmt] = ..., **kwargs: Unpack[_Attributes] + ) -> None: ... + else: + @overload + def __init__( + self, type: expr | None, name: _Identifier | None, body: list[stmt], **kwargs: Unpack[_Attributes] + ) -> None: ... + @overload + def __init__( + self, type: expr | None = None, name: _Identifier | None = None, *, body: list[stmt], **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + type: expr | None = ..., + name: _Identifier | None = ..., + body: list[stmt] = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class arguments(AST): + if sys.version_info >= (3, 10): + __match_args__ = ("posonlyargs", "args", "vararg", "kwonlyargs", "kw_defaults", "kwarg", "defaults") + posonlyargs: list[arg] + args: list[arg] + vararg: arg | None + kwonlyargs: list[arg] + kw_defaults: list[expr | None] + kwarg: arg | None + defaults: list[expr] + if sys.version_info >= (3, 13): + def __init__( + self, + posonlyargs: list[arg] = ..., + args: list[arg] = ..., + vararg: arg | None = None, + kwonlyargs: list[arg] = ..., + kw_defaults: list[expr | None] = ..., + kwarg: arg | None = None, + defaults: list[expr] = ..., + ) -> None: ... + else: + @overload + def __init__( + self, + posonlyargs: list[arg], + args: list[arg], + vararg: arg | None, + kwonlyargs: list[arg], + kw_defaults: list[expr | None], + kwarg: arg | None, + defaults: list[expr], + ) -> None: ... + @overload + def __init__( + self, + posonlyargs: list[arg], + args: list[arg], + vararg: arg | None, + kwonlyargs: list[arg], + kw_defaults: list[expr | None], + kwarg: arg | None = None, + *, + defaults: list[expr], + ) -> None: ... + @overload + def __init__( + self, + posonlyargs: list[arg], + args: list[arg], + vararg: arg | None = None, + *, + kwonlyargs: list[arg], + kw_defaults: list[expr | None], + kwarg: arg | None = None, + defaults: list[expr], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + posonlyargs: list[arg] = ..., + args: list[arg] = ..., + vararg: arg | None = ..., + kwonlyargs: list[arg] = ..., + kw_defaults: list[expr | None] = ..., + kwarg: arg | None = ..., + defaults: list[expr] = ..., + ) -> Self: ... + +class arg(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + if sys.version_info >= (3, 10): + __match_args__ = ("arg", "annotation", "type_comment") + arg: _Identifier + annotation: expr | None + type_comment: str | None + def __init__( + self, arg: _Identifier, annotation: expr | None = None, type_comment: str | None = None, **kwargs: Unpack[_Attributes] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + arg: _Identifier = ..., + annotation: expr | None = ..., + type_comment: str | None = ..., + **kwargs: Unpack[_Attributes], + ) -> Self: ... + +class keyword(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None + if sys.version_info >= (3, 10): + __match_args__ = ("arg", "value") + arg: _Identifier | None + value: expr + @overload + def __init__(self, arg: _Identifier | None, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + @overload + def __init__(self, arg: _Identifier | None = None, *, value: expr, **kwargs: Unpack[_Attributes]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, arg: _Identifier | None = ..., value: expr = ..., **kwargs: Unpack[_Attributes]) -> Self: ... + +class alias(AST): + lineno: int + col_offset: int + end_lineno: int | None + end_col_offset: int | None if sys.version_info >= (3, 10): - def visit_Match(self, node: Match) -> Any: ... - def visit_match_case(self, node: match_case) -> Any: ... - def visit_MatchValue(self, node: MatchValue) -> Any: ... - def visit_MatchSequence(self, node: MatchSequence) -> Any: ... - def visit_MatchSingleton(self, node: MatchSingleton) -> Any: ... - def visit_MatchStar(self, node: MatchStar) -> Any: ... - def visit_MatchMapping(self, node: MatchMapping) -> Any: ... - def visit_MatchClass(self, node: MatchClass) -> Any: ... - def visit_MatchAs(self, node: MatchAs) -> Any: ... - def visit_MatchOr(self, node: MatchOr) -> Any: ... + __match_args__ = ("name", "asname") + name: str + asname: _Identifier | None + def __init__(self, name: str, asname: _Identifier | None = None, **kwargs: Unpack[_Attributes]) -> None: ... - if sys.version_info >= (3, 11): - def visit_TryStar(self, node: TryStar) -> Any: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, name: str = ..., asname: _Identifier | None = ..., **kwargs: Unpack[_Attributes]) -> Self: ... - if sys.version_info >= (3, 12): - def visit_TypeVar(self, node: TypeVar) -> Any: ... - def visit_ParamSpec(self, node: ParamSpec) -> Any: ... - def visit_TypeVarTuple(self, node: TypeVarTuple) -> Any: ... - def visit_TypeAlias(self, node: TypeAlias) -> Any: ... +class withitem(AST): + if sys.version_info >= (3, 10): + __match_args__ = ("context_expr", "optional_vars") + context_expr: expr + optional_vars: expr | None + def __init__(self, context_expr: expr, optional_vars: expr | None = None) -> None: ... - # visit methods for deprecated nodes - def visit_ExtSlice(self, node: ExtSlice) -> Any: ... - def visit_Index(self, node: Index) -> Any: ... - def visit_Suite(self, node: Suite) -> Any: ... - def visit_AugLoad(self, node: AugLoad) -> Any: ... - def visit_AugStore(self, node: AugStore) -> Any: ... - def visit_Param(self, node: Param) -> Any: ... - def visit_Num(self, node: Num) -> Any: ... - def visit_Str(self, node: Str) -> Any: ... - def visit_Bytes(self, node: Bytes) -> Any: ... - def visit_NameConstant(self, node: NameConstant) -> Any: ... - def visit_Ellipsis(self, node: Ellipsis) -> Any: ... + if sys.version_info >= (3, 14): + def __replace__(self, *, context_expr: expr = ..., optional_vars: expr | None = ...) -> Self: ... -class NodeTransformer(NodeVisitor): - def generic_visit(self, node: AST) -> AST: ... - # TODO: Override the visit_* methods with better return types. - # The usual return type is AST | None, but Iterable[AST] - # is also allowed in some cases -- this needs to be mapped. +if sys.version_info >= (3, 10): + class match_case(AST): + __match_args__ = ("pattern", "guard", "body") + pattern: _Pattern + guard: expr | None + body: list[stmt] + if sys.version_info >= (3, 13): + def __init__(self, pattern: _Pattern, guard: expr | None = None, body: list[stmt] = ...) -> None: ... + else: + @overload + def __init__(self, pattern: _Pattern, guard: expr | None, body: list[stmt]) -> None: ... + @overload + def __init__(self, pattern: _Pattern, guard: expr | None = None, *, body: list[stmt]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, pattern: _Pattern = ..., guard: expr | None = ..., body: list[stmt] = ...) -> Self: ... + + class pattern(AST): + lineno: int + col_offset: int + end_lineno: int + end_col_offset: int + def __init__(self, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, lineno: int = ..., col_offset: int = ..., end_lineno: int = ..., end_col_offset: int = ... + ) -> Self: ... + + # Without the alias, Pyright complains variables named pattern are recursively defined + _Pattern: typing_extensions.TypeAlias = pattern + + class MatchValue(pattern): + __match_args__ = ("value",) + value: expr + def __init__(self, value: expr, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: expr = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + + class MatchSingleton(pattern): + __match_args__ = ("value",) + value: Literal[True, False] | None + def __init__(self, value: Literal[True, False] | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, value: Literal[True, False] | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + + class MatchSequence(pattern): + __match_args__ = ("patterns",) + patterns: list[pattern] + if sys.version_info >= (3, 13): + def __init__(self, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> None: ... + else: + def __init__(self, patterns: list[pattern], **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + + class MatchMapping(pattern): + __match_args__ = ("keys", "patterns", "rest") + keys: list[expr] + patterns: list[pattern] + rest: _Identifier | None + if sys.version_info >= (3, 13): + def __init__( + self, + keys: list[expr] = ..., + patterns: list[pattern] = ..., + rest: _Identifier | None = None, + **kwargs: Unpack[_Attributes[int]], + ) -> None: ... + else: + def __init__( + self, + keys: list[expr], + patterns: list[pattern], + rest: _Identifier | None = None, + **kwargs: Unpack[_Attributes[int]], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + keys: list[expr] = ..., + patterns: list[pattern] = ..., + rest: _Identifier | None = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + + class MatchClass(pattern): + __match_args__ = ("cls", "patterns", "kwd_attrs", "kwd_patterns") + cls: expr + patterns: list[pattern] + kwd_attrs: list[_Identifier] + kwd_patterns: list[pattern] + if sys.version_info >= (3, 13): + def __init__( + self, + cls: expr, + patterns: list[pattern] = ..., + kwd_attrs: list[_Identifier] = ..., + kwd_patterns: list[pattern] = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> None: ... + else: + def __init__( + self, + cls: expr, + patterns: list[pattern], + kwd_attrs: list[_Identifier], + kwd_patterns: list[pattern], + **kwargs: Unpack[_Attributes[int]], + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + cls: expr = ..., + patterns: list[pattern] = ..., + kwd_attrs: list[_Identifier] = ..., + kwd_patterns: list[pattern] = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + + class MatchStar(pattern): + __match_args__ = ("name",) + name: _Identifier | None + def __init__(self, name: _Identifier | None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, name: _Identifier | None = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + + class MatchAs(pattern): + __match_args__ = ("pattern", "name") + pattern: _Pattern | None + name: _Identifier | None + def __init__( + self, pattern: _Pattern | None = None, name: _Identifier | None = None, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, pattern: _Pattern | None = ..., name: _Identifier | None = ..., **kwargs: Unpack[_Attributes[int]] + ) -> Self: ... + + class MatchOr(pattern): + __match_args__ = ("patterns",) + patterns: list[pattern] + if sys.version_info >= (3, 13): + def __init__(self, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> None: ... + else: + def __init__(self, patterns: list[pattern], **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, patterns: list[pattern] = ..., **kwargs: Unpack[_Attributes[int]]) -> Self: ... + +class type_ignore(AST): ... + +class TypeIgnore(type_ignore): + if sys.version_info >= (3, 10): + __match_args__ = ("lineno", "tag") + lineno: int + tag: str + def __init__(self, lineno: int, tag: str) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, *, lineno: int = ..., tag: str = ...) -> Self: ... + +if sys.version_info >= (3, 12): + class type_param(AST): + lineno: int + col_offset: int + end_lineno: int + end_col_offset: int + def __init__(self, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__(self, **kwargs: Unpack[_Attributes[int]]) -> Self: ... + + class TypeVar(type_param): + if sys.version_info >= (3, 13): + __match_args__ = ("name", "bound", "default_value") + else: + __match_args__ = ("name", "bound") + name: _Identifier + bound: expr | None + if sys.version_info >= (3, 13): + default_value: expr | None + def __init__( + self, + name: _Identifier, + bound: expr | None = None, + default_value: expr | None = None, + **kwargs: Unpack[_Attributes[int]], + ) -> None: ... + else: + def __init__(self, name: _Identifier, bound: expr | None = None, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, + *, + name: _Identifier = ..., + bound: expr | None = ..., + default_value: expr | None = ..., + **kwargs: Unpack[_Attributes[int]], + ) -> Self: ... + + class ParamSpec(type_param): + if sys.version_info >= (3, 13): + __match_args__ = ("name", "default_value") + else: + __match_args__ = ("name",) + name: _Identifier + if sys.version_info >= (3, 13): + default_value: expr | None + def __init__( + self, name: _Identifier, default_value: expr | None = None, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + else: + def __init__(self, name: _Identifier, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, name: _Identifier = ..., default_value: expr | None = ..., **kwargs: Unpack[_Attributes[int]] + ) -> Self: ... + + class TypeVarTuple(type_param): + if sys.version_info >= (3, 13): + __match_args__ = ("name", "default_value") + else: + __match_args__ = ("name",) + name: _Identifier + if sys.version_info >= (3, 13): + default_value: expr | None + def __init__( + self, name: _Identifier, default_value: expr | None = None, **kwargs: Unpack[_Attributes[int]] + ) -> None: ... + else: + def __init__(self, name: _Identifier, **kwargs: Unpack[_Attributes[int]]) -> None: ... + + if sys.version_info >= (3, 14): + def __replace__( + self, *, name: _Identifier = ..., default_value: expr | None = ..., **kwargs: Unpack[_Attributes[int]] + ) -> Self: ... + +class _ABC(type): + if sys.version_info >= (3, 9): + def __init__(cls, *args: Unused) -> None: ... + +if sys.version_info < (3, 14): + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Num(Constant, metaclass=_ABC): + value: int | float | complex + + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Str(Constant, metaclass=_ABC): + value: str + # Aliases for value, for backwards compatibility + s: str + + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Bytes(Constant, metaclass=_ABC): + value: bytes + # Aliases for value, for backwards compatibility + s: bytes + + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class NameConstant(Constant, metaclass=_ABC): ... + + @deprecated("Replaced by ast.Constant; removed in Python 3.14") + class Ellipsis(Constant, metaclass=_ABC): ... + +# everything below here is defined in ast.py _T = _TypeVar("_T", bound=AST) @@ -332,10 +1865,7 @@ else: feature_version: None | int | tuple[int, int] = None, ) -> AST: ... -if sys.version_info >= (3, 9): - def unparse(ast_obj: AST) -> str: ... - -def copy_location(new_node: _T, old_node: AST) -> _T: ... +def literal_eval(node_or_string: str | AST) -> Any: ... if sys.version_info >= (3, 13): def dump( @@ -355,17 +1885,158 @@ elif sys.version_info >= (3, 9): else: def dump(node: AST, annotate_fields: bool = True, include_attributes: bool = False) -> str: ... +def copy_location(new_node: _T, old_node: AST) -> _T: ... def fix_missing_locations(node: _T) -> _T: ... -def get_docstring(node: AsyncFunctionDef | FunctionDef | ClassDef | Module, clean: bool = True) -> str | None: ... def increment_lineno(node: _T, n: int = 1) -> _T: ... -def iter_child_nodes(node: AST) -> Iterator[AST]: ... def iter_fields(node: AST) -> Iterator[tuple[str, Any]]: ... -def literal_eval(node_or_string: str | AST) -> Any: ... +def iter_child_nodes(node: AST) -> Iterator[AST]: ... +def get_docstring(node: AsyncFunctionDef | FunctionDef | ClassDef | Module, clean: bool = True) -> str | None: ... def get_source_segment(source: str, node: AST, *, padded: bool = False) -> str | None: ... def walk(node: AST) -> Iterator[AST]: ... -if sys.version_info >= (3, 9): - def main() -> None: ... - if sys.version_info >= (3, 14): def compare(left: AST, right: AST, /, *, compare_attributes: bool = False) -> bool: ... + +class NodeVisitor: + def visit(self, node: AST) -> Any: ... + def generic_visit(self, node: AST) -> Any: ... + def visit_Module(self, node: Module) -> Any: ... + def visit_Interactive(self, node: Interactive) -> Any: ... + def visit_Expression(self, node: Expression) -> Any: ... + def visit_FunctionDef(self, node: FunctionDef) -> Any: ... + def visit_AsyncFunctionDef(self, node: AsyncFunctionDef) -> Any: ... + def visit_ClassDef(self, node: ClassDef) -> Any: ... + def visit_Return(self, node: Return) -> Any: ... + def visit_Delete(self, node: Delete) -> Any: ... + def visit_Assign(self, node: Assign) -> Any: ... + def visit_AugAssign(self, node: AugAssign) -> Any: ... + def visit_AnnAssign(self, node: AnnAssign) -> Any: ... + def visit_For(self, node: For) -> Any: ... + def visit_AsyncFor(self, node: AsyncFor) -> Any: ... + def visit_While(self, node: While) -> Any: ... + def visit_If(self, node: If) -> Any: ... + def visit_With(self, node: With) -> Any: ... + def visit_AsyncWith(self, node: AsyncWith) -> Any: ... + def visit_Raise(self, node: Raise) -> Any: ... + def visit_Try(self, node: Try) -> Any: ... + def visit_Assert(self, node: Assert) -> Any: ... + def visit_Import(self, node: Import) -> Any: ... + def visit_ImportFrom(self, node: ImportFrom) -> Any: ... + def visit_Global(self, node: Global) -> Any: ... + def visit_Nonlocal(self, node: Nonlocal) -> Any: ... + def visit_Expr(self, node: Expr) -> Any: ... + def visit_Pass(self, node: Pass) -> Any: ... + def visit_Break(self, node: Break) -> Any: ... + def visit_Continue(self, node: Continue) -> Any: ... + def visit_Slice(self, node: Slice) -> Any: ... + def visit_BoolOp(self, node: BoolOp) -> Any: ... + def visit_BinOp(self, node: BinOp) -> Any: ... + def visit_UnaryOp(self, node: UnaryOp) -> Any: ... + def visit_Lambda(self, node: Lambda) -> Any: ... + def visit_IfExp(self, node: IfExp) -> Any: ... + def visit_Dict(self, node: Dict) -> Any: ... + def visit_Set(self, node: Set) -> Any: ... + def visit_ListComp(self, node: ListComp) -> Any: ... + def visit_SetComp(self, node: SetComp) -> Any: ... + def visit_DictComp(self, node: DictComp) -> Any: ... + def visit_GeneratorExp(self, node: GeneratorExp) -> Any: ... + def visit_Await(self, node: Await) -> Any: ... + def visit_Yield(self, node: Yield) -> Any: ... + def visit_YieldFrom(self, node: YieldFrom) -> Any: ... + def visit_Compare(self, node: Compare) -> Any: ... + def visit_Call(self, node: Call) -> Any: ... + def visit_FormattedValue(self, node: FormattedValue) -> Any: ... + def visit_JoinedStr(self, node: JoinedStr) -> Any: ... + def visit_Constant(self, node: Constant) -> Any: ... + def visit_NamedExpr(self, node: NamedExpr) -> Any: ... + def visit_TypeIgnore(self, node: TypeIgnore) -> Any: ... + def visit_Attribute(self, node: Attribute) -> Any: ... + def visit_Subscript(self, node: Subscript) -> Any: ... + def visit_Starred(self, node: Starred) -> Any: ... + def visit_Name(self, node: Name) -> Any: ... + def visit_List(self, node: List) -> Any: ... + def visit_Tuple(self, node: Tuple) -> Any: ... + def visit_Del(self, node: Del) -> Any: ... + def visit_Load(self, node: Load) -> Any: ... + def visit_Store(self, node: Store) -> Any: ... + def visit_And(self, node: And) -> Any: ... + def visit_Or(self, node: Or) -> Any: ... + def visit_Add(self, node: Add) -> Any: ... + def visit_BitAnd(self, node: BitAnd) -> Any: ... + def visit_BitOr(self, node: BitOr) -> Any: ... + def visit_BitXor(self, node: BitXor) -> Any: ... + def visit_Div(self, node: Div) -> Any: ... + def visit_FloorDiv(self, node: FloorDiv) -> Any: ... + def visit_LShift(self, node: LShift) -> Any: ... + def visit_Mod(self, node: Mod) -> Any: ... + def visit_Mult(self, node: Mult) -> Any: ... + def visit_MatMult(self, node: MatMult) -> Any: ... + def visit_Pow(self, node: Pow) -> Any: ... + def visit_RShift(self, node: RShift) -> Any: ... + def visit_Sub(self, node: Sub) -> Any: ... + def visit_Invert(self, node: Invert) -> Any: ... + def visit_Not(self, node: Not) -> Any: ... + def visit_UAdd(self, node: UAdd) -> Any: ... + def visit_USub(self, node: USub) -> Any: ... + def visit_Eq(self, node: Eq) -> Any: ... + def visit_Gt(self, node: Gt) -> Any: ... + def visit_GtE(self, node: GtE) -> Any: ... + def visit_In(self, node: In) -> Any: ... + def visit_Is(self, node: Is) -> Any: ... + def visit_IsNot(self, node: IsNot) -> Any: ... + def visit_Lt(self, node: Lt) -> Any: ... + def visit_LtE(self, node: LtE) -> Any: ... + def visit_NotEq(self, node: NotEq) -> Any: ... + def visit_NotIn(self, node: NotIn) -> Any: ... + def visit_comprehension(self, node: comprehension) -> Any: ... + def visit_ExceptHandler(self, node: ExceptHandler) -> Any: ... + def visit_arguments(self, node: arguments) -> Any: ... + def visit_arg(self, node: arg) -> Any: ... + def visit_keyword(self, node: keyword) -> Any: ... + def visit_alias(self, node: alias) -> Any: ... + def visit_withitem(self, node: withitem) -> Any: ... + if sys.version_info >= (3, 10): + def visit_Match(self, node: Match) -> Any: ... + def visit_match_case(self, node: match_case) -> Any: ... + def visit_MatchValue(self, node: MatchValue) -> Any: ... + def visit_MatchSequence(self, node: MatchSequence) -> Any: ... + def visit_MatchSingleton(self, node: MatchSingleton) -> Any: ... + def visit_MatchStar(self, node: MatchStar) -> Any: ... + def visit_MatchMapping(self, node: MatchMapping) -> Any: ... + def visit_MatchClass(self, node: MatchClass) -> Any: ... + def visit_MatchAs(self, node: MatchAs) -> Any: ... + def visit_MatchOr(self, node: MatchOr) -> Any: ... + + if sys.version_info >= (3, 11): + def visit_TryStar(self, node: TryStar) -> Any: ... + + if sys.version_info >= (3, 12): + def visit_TypeVar(self, node: TypeVar) -> Any: ... + def visit_ParamSpec(self, node: ParamSpec) -> Any: ... + def visit_TypeVarTuple(self, node: TypeVarTuple) -> Any: ... + def visit_TypeAlias(self, node: TypeAlias) -> Any: ... + + # visit methods for deprecated nodes + def visit_ExtSlice(self, node: ExtSlice) -> Any: ... + def visit_Index(self, node: Index) -> Any: ... + def visit_Suite(self, node: Suite) -> Any: ... + def visit_AugLoad(self, node: AugLoad) -> Any: ... + def visit_AugStore(self, node: AugStore) -> Any: ... + def visit_Param(self, node: Param) -> Any: ... + def visit_Num(self, node: Num) -> Any: ... + def visit_Str(self, node: Str) -> Any: ... + def visit_Bytes(self, node: Bytes) -> Any: ... + def visit_NameConstant(self, node: NameConstant) -> Any: ... + def visit_Ellipsis(self, node: Ellipsis) -> Any: ... + +class NodeTransformer(NodeVisitor): + def generic_visit(self, node: AST) -> AST: ... + # TODO: Override the visit_* methods with better return types. + # The usual return type is AST | None, but Iterable[AST] + # is also allowed in some cases -- this needs to be mapped. + +if sys.version_info >= (3, 9): + def unparse(ast_obj: AST) -> str: ... + +if sys.version_info >= (3, 9): + def main() -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/events.pyi b/mypy/typeshed/stdlib/asyncio/events.pyi index eed688fc792a..ead64070671f 100644 --- a/mypy/typeshed/stdlib/asyncio/events.pyi +++ b/mypy/typeshed/stdlib/asyncio/events.pyi @@ -1,5 +1,11 @@ import ssl import sys +from _asyncio import ( + _get_running_loop as _get_running_loop, + _set_running_loop as _set_running_loop, + get_event_loop as get_event_loop, + get_running_loop as get_running_loop, +) from _typeshed import FileDescriptorLike, ReadableBuffer, StrPath, Unused, WriteableBuffer from abc import ABCMeta, abstractmethod from collections.abc import Callable, Sequence @@ -632,7 +638,6 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy, metaclass=ABCMeta): def get_event_loop_policy() -> AbstractEventLoopPolicy: ... def set_event_loop_policy(policy: AbstractEventLoopPolicy | None) -> None: ... -def get_event_loop() -> AbstractEventLoop: ... def set_event_loop(loop: AbstractEventLoop | None) -> None: ... def new_event_loop() -> AbstractEventLoop: ... @@ -646,7 +651,3 @@ if sys.version_info < (3, 14): else: def get_child_watcher() -> AbstractChildWatcher: ... def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... - -def _set_running_loop(loop: AbstractEventLoop | None, /) -> None: ... -def _get_running_loop() -> AbstractEventLoop: ... -def get_running_loop() -> AbstractEventLoop: ... diff --git a/mypy/typeshed/stdlib/asyncio/futures.pyi b/mypy/typeshed/stdlib/asyncio/futures.pyi index e19fd53f3311..28e6ca8c86a3 100644 --- a/mypy/typeshed/stdlib/asyncio/futures.pyi +++ b/mypy/typeshed/stdlib/asyncio/futures.pyi @@ -1,15 +1,10 @@ -import sys -from collections.abc import Awaitable, Callable, Generator, Iterable +from _asyncio import Future as Future from concurrent.futures._base import Future as _ConcurrentFuture -from contextvars import Context -from typing import Any, Literal, TypeVar -from typing_extensions import Self, TypeIs +from typing import Any, TypeVar +from typing_extensions import TypeIs from .events import AbstractEventLoop -if sys.version_info >= (3, 9): - from types import GenericAlias - __all__ = ("Future", "wrap_future", "isfuture") _T = TypeVar("_T") @@ -18,40 +13,4 @@ _T = TypeVar("_T") # but it leads to circular import error in pytype tool. # That's why the import order is reversed. def isfuture(obj: object) -> TypeIs[Future[Any]]: ... - -class Future(Awaitable[_T], Iterable[_T]): - _state: str - @property - def _exception(self) -> BaseException | None: ... - _blocking: bool - @property - def _log_traceback(self) -> bool: ... - @_log_traceback.setter - def _log_traceback(self, val: Literal[False]) -> None: ... - _asyncio_future_blocking: bool # is a part of duck-typing contract for `Future` - def __init__(self, *, loop: AbstractEventLoop | None = ...) -> None: ... - def __del__(self) -> None: ... - def get_loop(self) -> AbstractEventLoop: ... - @property - def _callbacks(self) -> list[tuple[Callable[[Self], Any], Context]]: ... - def add_done_callback(self, fn: Callable[[Self], object], /, *, context: Context | None = None) -> None: ... - if sys.version_info >= (3, 9): - def cancel(self, msg: Any | None = None) -> bool: ... - else: - def cancel(self) -> bool: ... - - def cancelled(self) -> bool: ... - def done(self) -> bool: ... - def result(self) -> _T: ... - def exception(self) -> BaseException | None: ... - def remove_done_callback(self, fn: Callable[[Self], object], /) -> int: ... - def set_result(self, result: _T, /) -> None: ... - def set_exception(self, exception: type | BaseException, /) -> None: ... - def __iter__(self) -> Generator[Any, None, _T]: ... - def __await__(self) -> Generator[Any, None, _T]: ... - @property - def _loop(self) -> AbstractEventLoop: ... - if sys.version_info >= (3, 9): - def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... - def wrap_future(future: _ConcurrentFuture[_T] | Future[_T], *, loop: AbstractEventLoop | None = None) -> Future[_T]: ... diff --git a/mypy/typeshed/stdlib/asyncio/streams.pyi b/mypy/typeshed/stdlib/asyncio/streams.pyi index 0be5249e2169..ed95583c1847 100644 --- a/mypy/typeshed/stdlib/asyncio/streams.pyi +++ b/mypy/typeshed/stdlib/asyncio/streams.pyi @@ -1,7 +1,7 @@ import ssl import sys from _typeshed import ReadableBuffer, StrPath -from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Sequence, Sized +from collections.abc import Awaitable, Callable, Iterable, Sequence, Sized from types import ModuleType from typing import Any, Protocol, SupportsIndex from typing_extensions import Self, TypeAlias @@ -137,7 +137,7 @@ class StreamWriter: elif sys.version_info >= (3, 11): def __del__(self) -> None: ... -class StreamReader(AsyncIterator[bytes]): +class StreamReader: def __init__(self, limit: int = 65536, loop: events.AbstractEventLoop | None = None) -> None: ... def exception(self) -> Exception: ... def set_exception(self, exc: Exception) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncio/tasks.pyi b/mypy/typeshed/stdlib/asyncio/tasks.pyi index bb423e857399..47d1bb13e4a3 100644 --- a/mypy/typeshed/stdlib/asyncio/tasks.pyi +++ b/mypy/typeshed/stdlib/asyncio/tasks.pyi @@ -1,16 +1,20 @@ import concurrent.futures import sys +from _asyncio import ( + Task as Task, + _enter_task as _enter_task, + _leave_task as _leave_task, + _register_task as _register_task, + _unregister_task as _unregister_task, +) from collections.abc import Awaitable, Coroutine, Generator, Iterable, Iterator -from types import FrameType -from typing import Any, Literal, Protocol, TextIO, TypeVar, overload +from typing import Any, Literal, Protocol, TypeVar, overload from typing_extensions import TypeAlias from . import _CoroutineLike from .events import AbstractEventLoop from .futures import Future -if sys.version_info >= (3, 9): - from types import GenericAlias if sys.version_info >= (3, 11): from contextvars import Context @@ -400,58 +404,6 @@ elif sys.version_info >= (3, 9): else: _TaskCompatibleCoro: TypeAlias = Generator[_TaskYieldType, None, _T_co] | Awaitable[_T_co] -# mypy and pyright complain that a subclass of an invariant class shouldn't be covariant. -# While this is true in general, here it's sort-of okay to have a covariant subclass, -# since the only reason why `asyncio.Future` is invariant is the `set_result()` method, -# and `asyncio.Task.set_result()` always raises. -class Task(Future[_T_co]): # type: ignore[type-var] # pyright: ignore[reportInvalidTypeArguments] - if sys.version_info >= (3, 12): - def __init__( - self, - coro: _TaskCompatibleCoro[_T_co], - *, - loop: AbstractEventLoop = ..., - name: str | None = ..., - context: Context | None = None, - eager_start: bool = False, - ) -> None: ... - elif sys.version_info >= (3, 11): - def __init__( - self, - coro: _TaskCompatibleCoro[_T_co], - *, - loop: AbstractEventLoop = ..., - name: str | None = ..., - context: Context | None = None, - ) -> None: ... - else: - def __init__( - self, coro: _TaskCompatibleCoro[_T_co], *, loop: AbstractEventLoop = ..., name: str | None = ... - ) -> None: ... - - if sys.version_info >= (3, 12): - def get_coro(self) -> _TaskCompatibleCoro[_T_co] | None: ... - else: - def get_coro(self) -> _TaskCompatibleCoro[_T_co]: ... - - def get_name(self) -> str: ... - def set_name(self, value: object, /) -> None: ... - if sys.version_info >= (3, 12): - def get_context(self) -> Context: ... - - def get_stack(self, *, limit: int | None = None) -> list[FrameType]: ... - def print_stack(self, *, limit: int | None = None, file: TextIO | None = None) -> None: ... - if sys.version_info >= (3, 11): - def cancelling(self) -> int: ... - def uncancel(self) -> int: ... - if sys.version_info < (3, 9): - @classmethod - def current_task(cls, loop: AbstractEventLoop | None = None) -> Task[Any] | None: ... - @classmethod - def all_tasks(cls, loop: AbstractEventLoop | None = None) -> set[Task[Any]]: ... - if sys.version_info >= (3, 9): - def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... - def all_tasks(loop: AbstractEventLoop | None = None) -> set[Task[Any]]: ... if sys.version_info >= (3, 11): @@ -460,9 +412,10 @@ if sys.version_info >= (3, 11): else: def create_task(coro: _CoroutineLike[_T], *, name: str | None = None) -> Task[_T]: ... -def current_task(loop: AbstractEventLoop | None = None) -> Task[Any] | None: ... -def _enter_task(loop: AbstractEventLoop, task: Task[Any]) -> None: ... -def _leave_task(loop: AbstractEventLoop, task: Task[Any]) -> None: ... +if sys.version_info >= (3, 12): + from _asyncio import current_task as current_task +else: + def current_task(loop: AbstractEventLoop | None = None) -> Task[Any] | None: ... if sys.version_info >= (3, 12): _TaskT_co = TypeVar("_TaskT_co", bound=Task[Any], covariant=True) @@ -499,6 +452,3 @@ if sys.version_info >= (3, 12): name: str | None = None, context: Context | None = None, ) -> Task[_T_co]: ... - -def _register_task(task: Task[Any]) -> None: ... -def _unregister_task(task: Task[Any]) -> None: ... diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index 0999fb1d6c36..adfe1d83752b 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -799,6 +799,11 @@ class memoryview(Sequence[_I]): def __buffer__(self, flags: int, /) -> memoryview: ... def __release_buffer__(self, buffer: memoryview, /) -> None: ... + # These are inherited from the Sequence ABC, but don't actually exist on memoryview. + # See https://github.com/python/cpython/issues/125420 + index: ClassVar[None] # type: ignore[assignment] + count: ClassVar[None] # type: ignore[assignment] + @final class bool(int): def __new__(cls, o: object = ..., /) -> Self: ... @@ -1110,7 +1115,7 @@ class frozenset(AbstractSet[_T_co]): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... class enumerate(Iterator[tuple[int, _T]]): - def __new__(cls, iterable: Iterable[_T], start: int = ...) -> Self: ... + def __new__(cls, iterable: Iterable[_T], start: int = 0) -> Self: ... def __iter__(self) -> Self: ... def __next__(self) -> tuple[int, _T]: ... if sys.version_info >= (3, 9): diff --git a/mypy/typeshed/stdlib/bz2.pyi b/mypy/typeshed/stdlib/bz2.pyi index a7837e1b9ff8..cd11baa1a0c1 100644 --- a/mypy/typeshed/stdlib/bz2.pyi +++ b/mypy/typeshed/stdlib/bz2.pyi @@ -131,7 +131,7 @@ class BZ2File(BaseStream, IO[bytes]): @final class BZ2Compressor: - def __init__(self, compresslevel: int = ...) -> None: ... + def __init__(self, compresslevel: int = 9) -> None: ... def compress(self, data: ReadableBuffer, /) -> bytes: ... def flush(self) -> bytes: ... diff --git a/mypy/typeshed/stdlib/code.pyi b/mypy/typeshed/stdlib/code.pyi index 02689238a9a5..54971f3ae93c 100644 --- a/mypy/typeshed/stdlib/code.pyi +++ b/mypy/typeshed/stdlib/code.pyi @@ -12,7 +12,11 @@ class InteractiveInterpreter: def __init__(self, locals: Mapping[str, Any] | None = None) -> None: ... def runsource(self, source: str, filename: str = "", symbol: str = "single") -> bool: ... def runcode(self, code: CodeType) -> None: ... - def showsyntaxerror(self, filename: str | None = None) -> None: ... + if sys.version_info >= (3, 13): + def showsyntaxerror(self, filename: str | None = None, *, source: str = "") -> None: ... + else: + def showsyntaxerror(self, filename: str | None = None) -> None: ... + def showtraceback(self) -> None: ... def write(self, data: str) -> None: ... diff --git a/mypy/typeshed/stdlib/collections/__init__.pyi b/mypy/typeshed/stdlib/collections/__init__.pyi index b2ed53e4427e..2d136318813c 100644 --- a/mypy/typeshed/stdlib/collections/__init__.pyi +++ b/mypy/typeshed/stdlib/collections/__init__.pyi @@ -17,7 +17,6 @@ if sys.version_info >= (3, 10): Mapping, MutableMapping, MutableSequence, - Reversible, Sequence, ValuesView, ) @@ -331,13 +330,13 @@ class Counter(dict[_T, int], Generic[_T]): # The pure-Python implementations of the "views" classes # These are exposed at runtime in `collections/__init__.py` -class _OrderedDictKeysView(KeysView[_KT_co], Reversible[_KT_co]): +class _OrderedDictKeysView(KeysView[_KT_co]): def __reversed__(self) -> Iterator[_KT_co]: ... -class _OrderedDictItemsView(ItemsView[_KT_co, _VT_co], Reversible[tuple[_KT_co, _VT_co]]): +class _OrderedDictItemsView(ItemsView[_KT_co, _VT_co]): def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... -class _OrderedDictValuesView(ValuesView[_VT_co], Reversible[_VT_co]): +class _OrderedDictValuesView(ValuesView[_VT_co]): def __reversed__(self) -> Iterator[_VT_co]: ... # The C implementations of the "views" classes @@ -345,18 +344,18 @@ class _OrderedDictValuesView(ValuesView[_VT_co], Reversible[_VT_co]): # but they are not exposed anywhere) # pyright doesn't have a specific error code for subclassing error! @final -class _odict_keys(dict_keys[_KT_co, _VT_co], Reversible[_KT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] +class _odict_keys(dict_keys[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] def __reversed__(self) -> Iterator[_KT_co]: ... @final -class _odict_items(dict_items[_KT_co, _VT_co], Reversible[tuple[_KT_co, _VT_co]]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] +class _odict_items(dict_items[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] def __reversed__(self) -> Iterator[tuple[_KT_co, _VT_co]]: ... @final -class _odict_values(dict_values[_KT_co, _VT_co], Reversible[_VT_co], Generic[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] +class _odict_values(dict_values[_KT_co, _VT_co]): # type: ignore[misc] # pyright: ignore[reportGeneralTypeIssues] def __reversed__(self) -> Iterator[_VT_co]: ... -class OrderedDict(dict[_KT, _VT], Reversible[_KT], Generic[_KT, _VT]): +class OrderedDict(dict[_KT, _VT]): def popitem(self, last: bool = True) -> tuple[_KT, _VT]: ... def move_to_end(self, key: _KT, last: bool = True) -> None: ... def copy(self) -> Self: ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/process.pyi b/mypy/typeshed/stdlib/concurrent/futures/process.pyi index d3706a9c15a6..a1de3d679b23 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/process.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/process.pyi @@ -19,8 +19,9 @@ _global_shutdown: bool class _ThreadWakeup: _closed: bool - _reader: Connection - _writer: Connection + # Any: Unused send and recv methods + _reader: Connection[Any, Any] + _writer: Connection[Any, Any] def close(self) -> None: ... def wakeup(self) -> None: ... def clear(self) -> None: ... diff --git a/mypy/typeshed/stdlib/contextlib.pyi b/mypy/typeshed/stdlib/contextlib.pyi index daf218d5a138..fb34251693ad 100644 --- a/mypy/typeshed/stdlib/contextlib.pyi +++ b/mypy/typeshed/stdlib/contextlib.pyi @@ -143,13 +143,15 @@ class _RedirectStream(AbstractContextManager[_T_io, None]): class redirect_stdout(_RedirectStream[_T_io]): ... class redirect_stderr(_RedirectStream[_T_io]): ... -# In reality this is a subclass of `AbstractContextManager`; -# see #7961 for why we don't do that in the stub -class ExitStack(Generic[_ExitT_co], metaclass=abc.ABCMeta): +class _BaseExitStack(Generic[_ExitT_co]): def enter_context(self, cm: AbstractContextManager[_T, _ExitT_co]) -> _T: ... def push(self, exit: _CM_EF) -> _CM_EF: ... def callback(self, callback: Callable[_P, _T], /, *args: _P.args, **kwds: _P.kwargs) -> Callable[_P, _T]: ... def pop_all(self) -> Self: ... + +# In reality this is a subclass of `AbstractContextManager`; +# see #7961 for why we don't do that in the stub +class ExitStack(_BaseExitStack[_ExitT_co], metaclass=abc.ABCMeta): def close(self) -> None: ... def __enter__(self) -> Self: ... def __exit__( @@ -163,16 +165,12 @@ _ACM_EF = TypeVar("_ACM_EF", bound=AbstractAsyncContextManager[Any, Any] | _Exit # In reality this is a subclass of `AbstractAsyncContextManager`; # see #7961 for why we don't do that in the stub -class AsyncExitStack(Generic[_ExitT_co], metaclass=abc.ABCMeta): - def enter_context(self, cm: AbstractContextManager[_T, _ExitT_co]) -> _T: ... +class AsyncExitStack(_BaseExitStack[_ExitT_co], metaclass=abc.ABCMeta): async def enter_async_context(self, cm: AbstractAsyncContextManager[_T, _ExitT_co]) -> _T: ... - def push(self, exit: _CM_EF) -> _CM_EF: ... def push_async_exit(self, exit: _ACM_EF) -> _ACM_EF: ... - def callback(self, callback: Callable[_P, _T], /, *args: _P.args, **kwds: _P.kwargs) -> Callable[_P, _T]: ... def push_async_callback( self, callback: Callable[_P, Awaitable[_T]], /, *args: _P.args, **kwds: _P.kwargs ) -> Callable[_P, Awaitable[_T]]: ... - def pop_all(self) -> Self: ... async def aclose(self) -> None: ... async def __aenter__(self) -> Self: ... async def __aexit__( diff --git a/mypy/typeshed/stdlib/csv.pyi b/mypy/typeshed/stdlib/csv.pyi index 24f0db332165..89a019753f04 100644 --- a/mypy/typeshed/stdlib/csv.pyi +++ b/mypy/typeshed/stdlib/csv.pyi @@ -1,12 +1,10 @@ import sys - -# actually csv.Dialect is a different class to _csv.Dialect at runtime, but for typing purposes, they're identical from _csv import ( QUOTE_ALL as QUOTE_ALL, QUOTE_MINIMAL as QUOTE_MINIMAL, QUOTE_NONE as QUOTE_NONE, QUOTE_NONNUMERIC as QUOTE_NONNUMERIC, - Dialect as Dialect, + Dialect as _Dialect, Error as Error, __version__ as __version__, _DialectLike, @@ -26,7 +24,7 @@ if sys.version_info >= (3, 12): from _csv import QUOTE_NOTNULL as QUOTE_NOTNULL, QUOTE_STRINGS as QUOTE_STRINGS from _typeshed import SupportsWrite -from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence +from collections.abc import Collection, Iterable, Mapping, Sequence from typing import Any, Generic, Literal, TypeVar, overload from typing_extensions import Self @@ -61,11 +59,14 @@ if sys.version_info < (3, 13): _T = TypeVar("_T") +class Dialect(_Dialect): + def __init__(self) -> None: ... + class excel(Dialect): ... class excel_tab(excel): ... class unix_dialect(Dialect): ... -class DictReader(Iterator[dict[_T | Any, str | Any]], Generic[_T]): +class DictReader(Generic[_T]): fieldnames: Sequence[_T] | None restkey: _T | None restval: str | Any | None diff --git a/mypy/typeshed/stdlib/curses/__init__.pyi b/mypy/typeshed/stdlib/curses/__init__.pyi index 1df184dbaa60..939cec0cede9 100644 --- a/mypy/typeshed/stdlib/curses/__init__.pyi +++ b/mypy/typeshed/stdlib/curses/__init__.pyi @@ -1,5 +1,5 @@ from _curses import * -from _curses import _CursesWindow as _CursesWindow +from _curses import window as window from collections.abc import Callable from typing import TypeVar from typing_extensions import Concatenate, ParamSpec @@ -19,4 +19,9 @@ COLS: int COLORS: int COLOR_PAIRS: int -def wrapper(func: Callable[Concatenate[_CursesWindow, _P], _T], /, *arg: _P.args, **kwds: _P.kwargs) -> _T: ... +def wrapper(func: Callable[Concatenate[window, _P], _T], /, *arg: _P.args, **kwds: _P.kwargs) -> _T: ... + +# typeshed used the name _CursesWindow for the underlying C class before +# it was mapped to the name 'window' in 3.8. +# Kept here as a legacy alias in case any third-party code is relying on it. +_CursesWindow = window diff --git a/mypy/typeshed/stdlib/curses/panel.pyi b/mypy/typeshed/stdlib/curses/panel.pyi index 3d3448bd9584..d94f76635d8f 100644 --- a/mypy/typeshed/stdlib/curses/panel.pyi +++ b/mypy/typeshed/stdlib/curses/panel.pyi @@ -1,4 +1,4 @@ -from _curses import _CursesWindow +from _curses import window version: str @@ -9,14 +9,14 @@ class _Curses_Panel: # type is (note the s def hidden(self) -> bool: ... def hide(self) -> None: ... def move(self, y: int, x: int) -> None: ... - def replace(self, win: _CursesWindow) -> None: ... + def replace(self, win: window) -> None: ... def set_userptr(self, obj: object) -> None: ... def show(self) -> None: ... def top(self) -> None: ... def userptr(self) -> object: ... - def window(self) -> _CursesWindow: ... + def window(self) -> window: ... def bottom_panel() -> _Curses_Panel: ... -def new_panel(win: _CursesWindow, /) -> _Curses_Panel: ... +def new_panel(win: window, /) -> _Curses_Panel: ... def top_panel() -> _Curses_Panel: ... def update_panels() -> _Curses_Panel: ... diff --git a/mypy/typeshed/stdlib/curses/textpad.pyi b/mypy/typeshed/stdlib/curses/textpad.pyi index ce6eed09b289..48ef67c9d85f 100644 --- a/mypy/typeshed/stdlib/curses/textpad.pyi +++ b/mypy/typeshed/stdlib/curses/textpad.pyi @@ -1,11 +1,11 @@ -from _curses import _CursesWindow +from _curses import window from collections.abc import Callable -def rectangle(win: _CursesWindow, uly: int, ulx: int, lry: int, lrx: int) -> None: ... +def rectangle(win: window, uly: int, ulx: int, lry: int, lrx: int) -> None: ... class Textbox: stripspaces: bool - def __init__(self, win: _CursesWindow, insert_mode: bool = False) -> None: ... + def __init__(self, win: window, insert_mode: bool = False) -> None: ... def edit(self, validate: Callable[[int], int] | None = None) -> str: ... def do_command(self, ch: str | int) -> None: ... def gather(self) -> str: ... diff --git a/mypy/typeshed/stdlib/decimal.pyi b/mypy/typeshed/stdlib/decimal.pyi index 35fc4405f11b..3da75ec192ac 100644 --- a/mypy/typeshed/stdlib/decimal.pyi +++ b/mypy/typeshed/stdlib/decimal.pyi @@ -1,2 +1,256 @@ -from _decimal import * -from _decimal import __libmpdec_version__ as __libmpdec_version__, __version__ as __version__ +import numbers +from _decimal import ( + HAVE_CONTEXTVAR as HAVE_CONTEXTVAR, + HAVE_THREADS as HAVE_THREADS, + MAX_EMAX as MAX_EMAX, + MAX_PREC as MAX_PREC, + MIN_EMIN as MIN_EMIN, + MIN_ETINY as MIN_ETINY, + ROUND_05UP as ROUND_05UP, + ROUND_CEILING as ROUND_CEILING, + ROUND_DOWN as ROUND_DOWN, + ROUND_FLOOR as ROUND_FLOOR, + ROUND_HALF_DOWN as ROUND_HALF_DOWN, + ROUND_HALF_EVEN as ROUND_HALF_EVEN, + ROUND_HALF_UP as ROUND_HALF_UP, + ROUND_UP as ROUND_UP, + BasicContext as BasicContext, + DefaultContext as DefaultContext, + ExtendedContext as ExtendedContext, + __libmpdec_version__ as __libmpdec_version__, + __version__ as __version__, + getcontext as getcontext, + localcontext as localcontext, + setcontext as setcontext, +) +from collections.abc import Container, Sequence +from typing import Any, ClassVar, Literal, NamedTuple, overload +from typing_extensions import Self, TypeAlias + +_Decimal: TypeAlias = Decimal | int +_DecimalNew: TypeAlias = Decimal | float | str | tuple[int, Sequence[int], int] +_ComparableNum: TypeAlias = Decimal | float | numbers.Rational +_TrapType: TypeAlias = type[DecimalException] + +# At runtime, these classes are implemented in C as part of "_decimal". +# However, they consider themselves to live in "decimal", so we'll put them here. + +class DecimalTuple(NamedTuple): + sign: int + digits: tuple[int, ...] + exponent: int | Literal["n", "N", "F"] + +class DecimalException(ArithmeticError): ... +class Clamped(DecimalException): ... +class InvalidOperation(DecimalException): ... +class ConversionSyntax(InvalidOperation): ... +class DivisionByZero(DecimalException, ZeroDivisionError): ... +class DivisionImpossible(InvalidOperation): ... +class DivisionUndefined(InvalidOperation, ZeroDivisionError): ... +class Inexact(DecimalException): ... +class InvalidContext(InvalidOperation): ... +class Rounded(DecimalException): ... +class Subnormal(DecimalException): ... +class Overflow(Inexact, Rounded): ... +class Underflow(Inexact, Rounded, Subnormal): ... +class FloatOperation(DecimalException, TypeError): ... + +class Decimal: + def __new__(cls, value: _DecimalNew = ..., context: Context | None = ...) -> Self: ... + @classmethod + def from_float(cls, f: float, /) -> Self: ... + def __bool__(self) -> bool: ... + def compare(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def __hash__(self) -> int: ... + def as_tuple(self) -> DecimalTuple: ... + def as_integer_ratio(self) -> tuple[int, int]: ... + def to_eng_string(self, context: Context | None = None) -> str: ... + def __abs__(self) -> Decimal: ... + def __add__(self, value: _Decimal, /) -> Decimal: ... + def __divmod__(self, value: _Decimal, /) -> tuple[Decimal, Decimal]: ... + def __eq__(self, value: object, /) -> bool: ... + def __floordiv__(self, value: _Decimal, /) -> Decimal: ... + def __ge__(self, value: _ComparableNum, /) -> bool: ... + def __gt__(self, value: _ComparableNum, /) -> bool: ... + def __le__(self, value: _ComparableNum, /) -> bool: ... + def __lt__(self, value: _ComparableNum, /) -> bool: ... + def __mod__(self, value: _Decimal, /) -> Decimal: ... + def __mul__(self, value: _Decimal, /) -> Decimal: ... + def __neg__(self) -> Decimal: ... + def __pos__(self) -> Decimal: ... + def __pow__(self, value: _Decimal, mod: _Decimal | None = None, /) -> Decimal: ... + def __radd__(self, value: _Decimal, /) -> Decimal: ... + def __rdivmod__(self, value: _Decimal, /) -> tuple[Decimal, Decimal]: ... + def __rfloordiv__(self, value: _Decimal, /) -> Decimal: ... + def __rmod__(self, value: _Decimal, /) -> Decimal: ... + def __rmul__(self, value: _Decimal, /) -> Decimal: ... + def __rsub__(self, value: _Decimal, /) -> Decimal: ... + def __rtruediv__(self, value: _Decimal, /) -> Decimal: ... + def __sub__(self, value: _Decimal, /) -> Decimal: ... + def __truediv__(self, value: _Decimal, /) -> Decimal: ... + def remainder_near(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def __float__(self) -> float: ... + def __int__(self) -> int: ... + def __trunc__(self) -> int: ... + @property + def real(self) -> Decimal: ... + @property + def imag(self) -> Decimal: ... + def conjugate(self) -> Decimal: ... + def __complex__(self) -> complex: ... + @overload + def __round__(self) -> int: ... + @overload + def __round__(self, ndigits: int, /) -> Decimal: ... + def __floor__(self) -> int: ... + def __ceil__(self) -> int: ... + def fma(self, other: _Decimal, third: _Decimal, context: Context | None = None) -> Decimal: ... + def __rpow__(self, value: _Decimal, mod: Context | None = None, /) -> Decimal: ... + def normalize(self, context: Context | None = None) -> Decimal: ... + def quantize(self, exp: _Decimal, rounding: str | None = None, context: Context | None = None) -> Decimal: ... + def same_quantum(self, other: _Decimal, context: Context | None = None) -> bool: ... + def to_integral_exact(self, rounding: str | None = None, context: Context | None = None) -> Decimal: ... + def to_integral_value(self, rounding: str | None = None, context: Context | None = None) -> Decimal: ... + def to_integral(self, rounding: str | None = None, context: Context | None = None) -> Decimal: ... + def sqrt(self, context: Context | None = None) -> Decimal: ... + def max(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def min(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def adjusted(self) -> int: ... + def canonical(self) -> Decimal: ... + def compare_signal(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def compare_total(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def compare_total_mag(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def copy_abs(self) -> Decimal: ... + def copy_negate(self) -> Decimal: ... + def copy_sign(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def exp(self, context: Context | None = None) -> Decimal: ... + def is_canonical(self) -> bool: ... + def is_finite(self) -> bool: ... + def is_infinite(self) -> bool: ... + def is_nan(self) -> bool: ... + def is_normal(self, context: Context | None = None) -> bool: ... + def is_qnan(self) -> bool: ... + def is_signed(self) -> bool: ... + def is_snan(self) -> bool: ... + def is_subnormal(self, context: Context | None = None) -> bool: ... + def is_zero(self) -> bool: ... + def ln(self, context: Context | None = None) -> Decimal: ... + def log10(self, context: Context | None = None) -> Decimal: ... + def logb(self, context: Context | None = None) -> Decimal: ... + def logical_and(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def logical_invert(self, context: Context | None = None) -> Decimal: ... + def logical_or(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def logical_xor(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def max_mag(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def min_mag(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def next_minus(self, context: Context | None = None) -> Decimal: ... + def next_plus(self, context: Context | None = None) -> Decimal: ... + def next_toward(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def number_class(self, context: Context | None = None) -> str: ... + def radix(self) -> Decimal: ... + def rotate(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def scaleb(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def shift(self, other: _Decimal, context: Context | None = None) -> Decimal: ... + def __reduce__(self) -> tuple[type[Self], tuple[str]]: ... + def __copy__(self) -> Self: ... + def __deepcopy__(self, memo: Any, /) -> Self: ... + def __format__(self, specifier: str, context: Context | None = ..., /) -> str: ... + +class Context: + # TODO: Context doesn't allow you to delete *any* attributes from instances of the class at runtime, + # even settable attributes like `prec` and `rounding`, + # but that's inexpressable in the stub. + # Type checkers either ignore it or misinterpret it + # if you add a `def __delattr__(self, name: str, /) -> NoReturn` method to the stub + prec: int + rounding: str + Emin: int + Emax: int + capitals: int + clamp: int + traps: dict[_TrapType, bool] + flags: dict[_TrapType, bool] + def __init__( + self, + prec: int | None = ..., + rounding: str | None = ..., + Emin: int | None = ..., + Emax: int | None = ..., + capitals: int | None = ..., + clamp: int | None = ..., + flags: None | dict[_TrapType, bool] | Container[_TrapType] = ..., + traps: None | dict[_TrapType, bool] | Container[_TrapType] = ..., + _ignored_flags: list[_TrapType] | None = ..., + ) -> None: ... + def __reduce__(self) -> tuple[type[Self], tuple[Any, ...]]: ... + def clear_flags(self) -> None: ... + def clear_traps(self) -> None: ... + def copy(self) -> Context: ... + def __copy__(self) -> Context: ... + # see https://github.com/python/cpython/issues/94107 + __hash__: ClassVar[None] # type: ignore[assignment] + def Etiny(self) -> int: ... + def Etop(self) -> int: ... + def create_decimal(self, num: _DecimalNew = "0", /) -> Decimal: ... + def create_decimal_from_float(self, f: float, /) -> Decimal: ... + def abs(self, x: _Decimal, /) -> Decimal: ... + def add(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def canonical(self, x: Decimal, /) -> Decimal: ... + def compare(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def compare_signal(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def compare_total(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def compare_total_mag(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def copy_abs(self, x: _Decimal, /) -> Decimal: ... + def copy_decimal(self, x: _Decimal, /) -> Decimal: ... + def copy_negate(self, x: _Decimal, /) -> Decimal: ... + def copy_sign(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def divide(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def divide_int(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def divmod(self, x: _Decimal, y: _Decimal, /) -> tuple[Decimal, Decimal]: ... + def exp(self, x: _Decimal, /) -> Decimal: ... + def fma(self, x: _Decimal, y: _Decimal, z: _Decimal, /) -> Decimal: ... + def is_canonical(self, x: _Decimal, /) -> bool: ... + def is_finite(self, x: _Decimal, /) -> bool: ... + def is_infinite(self, x: _Decimal, /) -> bool: ... + def is_nan(self, x: _Decimal, /) -> bool: ... + def is_normal(self, x: _Decimal, /) -> bool: ... + def is_qnan(self, x: _Decimal, /) -> bool: ... + def is_signed(self, x: _Decimal, /) -> bool: ... + def is_snan(self, x: _Decimal, /) -> bool: ... + def is_subnormal(self, x: _Decimal, /) -> bool: ... + def is_zero(self, x: _Decimal, /) -> bool: ... + def ln(self, x: _Decimal, /) -> Decimal: ... + def log10(self, x: _Decimal, /) -> Decimal: ... + def logb(self, x: _Decimal, /) -> Decimal: ... + def logical_and(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def logical_invert(self, x: _Decimal, /) -> Decimal: ... + def logical_or(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def logical_xor(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def max(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def max_mag(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def min(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def min_mag(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def minus(self, x: _Decimal, /) -> Decimal: ... + def multiply(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def next_minus(self, x: _Decimal, /) -> Decimal: ... + def next_plus(self, x: _Decimal, /) -> Decimal: ... + def next_toward(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def normalize(self, x: _Decimal, /) -> Decimal: ... + def number_class(self, x: _Decimal, /) -> str: ... + def plus(self, x: _Decimal, /) -> Decimal: ... + def power(self, a: _Decimal, b: _Decimal, modulo: _Decimal | None = None) -> Decimal: ... + def quantize(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def radix(self) -> Decimal: ... + def remainder(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def remainder_near(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def rotate(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def same_quantum(self, x: _Decimal, y: _Decimal, /) -> bool: ... + def scaleb(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def shift(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def sqrt(self, x: _Decimal, /) -> Decimal: ... + def subtract(self, x: _Decimal, y: _Decimal, /) -> Decimal: ... + def to_eng_string(self, x: _Decimal, /) -> str: ... + def to_sci_string(self, x: _Decimal, /) -> str: ... + def to_integral_exact(self, x: _Decimal, /) -> Decimal: ... + def to_integral_value(self, x: _Decimal, /) -> Decimal: ... + def to_integral(self, x: _Decimal, /) -> Decimal: ... diff --git a/mypy/typeshed/stdlib/email/_policybase.pyi b/mypy/typeshed/stdlib/email/_policybase.pyi index 9e1f653c9d78..900224eabb3d 100644 --- a/mypy/typeshed/stdlib/email/_policybase.pyi +++ b/mypy/typeshed/stdlib/email/_policybase.pyi @@ -1,11 +1,29 @@ from abc import ABCMeta, abstractmethod -from collections.abc import Callable from email.errors import MessageDefect from email.header import Header from email.message import Message +from typing import Generic, Protocol, TypeVar, type_check_only from typing_extensions import Self -class _PolicyBase: +_MessageT = TypeVar("_MessageT", bound=Message, default=Message) + +@type_check_only +class _MessageFactory(Protocol[_MessageT]): + def __call__(self, policy: Policy[_MessageT]) -> _MessageT: ... + +# Policy below is the only known direct subclass of _PolicyBase. We therefore +# assume that the __init__ arguments and attributes of _PolicyBase are +# the same as those of Policy. +class _PolicyBase(Generic[_MessageT]): + max_line_length: int | None + linesep: str + cte_type: str + raise_on_defect: bool + mangle_from_: bool + message_factory: _MessageFactory[_MessageT] | None + # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 + verify_generated_headers: bool + def __init__( self, *, @@ -14,7 +32,7 @@ class _PolicyBase: cte_type: str = "8bit", raise_on_defect: bool = False, mangle_from_: bool = ..., # default depends on sub-class - message_factory: Callable[[Policy], Message] | None = None, + message_factory: _MessageFactory[_MessageT] | None = None, # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 verify_generated_headers: bool = True, ) -> None: ... @@ -26,24 +44,15 @@ class _PolicyBase: cte_type: str = ..., raise_on_defect: bool = ..., mangle_from_: bool = ..., - message_factory: Callable[[Policy], Message] | None = ..., + message_factory: _MessageFactory[_MessageT] | None = ..., # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 verify_generated_headers: bool = ..., ) -> Self: ... def __add__(self, other: Policy) -> Self: ... -class Policy(_PolicyBase, metaclass=ABCMeta): - max_line_length: int | None - linesep: str - cte_type: str - raise_on_defect: bool - mangle_from_: bool - message_factory: Callable[[Policy], Message] | None - # Added in Python 3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.5 - verify_generated_headers: bool - - def handle_defect(self, obj: Message, defect: MessageDefect) -> None: ... - def register_defect(self, obj: Message, defect: MessageDefect) -> None: ... +class Policy(_PolicyBase[_MessageT], metaclass=ABCMeta): + def handle_defect(self, obj: _MessageT, defect: MessageDefect) -> None: ... + def register_defect(self, obj: _MessageT, defect: MessageDefect) -> None: ... def header_max_count(self, name: str) -> int | None: ... @abstractmethod def header_source_parse(self, sourcelines: list[str]) -> tuple[str, str]: ... @@ -56,11 +65,11 @@ class Policy(_PolicyBase, metaclass=ABCMeta): @abstractmethod def fold_binary(self, name: str, value: str) -> bytes: ... -class Compat32(Policy): +class Compat32(Policy[_MessageT]): def header_source_parse(self, sourcelines: list[str]) -> tuple[str, str]: ... def header_store_parse(self, name: str, value: str) -> tuple[str, str]: ... def header_fetch_parse(self, name: str, value: str) -> str | Header: ... # type: ignore[override] def fold(self, name: str, value: str) -> str: ... def fold_binary(self, name: str, value: str) -> bytes: ... -compat32: Compat32 +compat32: Compat32[Message] diff --git a/mypy/typeshed/stdlib/email/feedparser.pyi b/mypy/typeshed/stdlib/email/feedparser.pyi index 22920fc99c69..8c268ca1ae18 100644 --- a/mypy/typeshed/stdlib/email/feedparser.pyi +++ b/mypy/typeshed/stdlib/email/feedparser.pyi @@ -5,19 +5,19 @@ from typing import Generic, TypeVar, overload __all__ = ["FeedParser", "BytesFeedParser"] -_MessageT = TypeVar("_MessageT", bound=Message) +_MessageT = TypeVar("_MessageT", bound=Message, default=Message) class FeedParser(Generic[_MessageT]): @overload - def __init__(self: FeedParser[Message], _factory: None = None, *, policy: Policy = ...) -> None: ... + def __init__(self: FeedParser[Message], _factory: None = None, *, policy: Policy[Message] = ...) -> None: ... @overload - def __init__(self, _factory: Callable[[], _MessageT], *, policy: Policy = ...) -> None: ... + def __init__(self, _factory: Callable[[], _MessageT], *, policy: Policy[_MessageT] = ...) -> None: ... def feed(self, data: str) -> None: ... def close(self) -> _MessageT: ... class BytesFeedParser(FeedParser[_MessageT]): @overload - def __init__(self: BytesFeedParser[Message], _factory: None = None, *, policy: Policy = ...) -> None: ... + def __init__(self: BytesFeedParser[Message], _factory: None = None, *, policy: Policy[Message] = ...) -> None: ... @overload - def __init__(self, _factory: Callable[[], _MessageT], *, policy: Policy = ...) -> None: ... + def __init__(self, _factory: Callable[[], _MessageT], *, policy: Policy[_MessageT] = ...) -> None: ... def feed(self, data: bytes | bytearray) -> None: ... # type: ignore[override] diff --git a/mypy/typeshed/stdlib/email/generator.pyi b/mypy/typeshed/stdlib/email/generator.pyi index faa6551fc925..dfa0604a20a9 100644 --- a/mypy/typeshed/stdlib/email/generator.pyi +++ b/mypy/typeshed/stdlib/email/generator.pyi @@ -1,34 +1,71 @@ from _typeshed import SupportsWrite from email.message import Message from email.policy import Policy +from typing import Any, Generic, TypeVar, overload from typing_extensions import Self __all__ = ["Generator", "DecodedGenerator", "BytesGenerator"] -class Generator: - def clone(self, fp: SupportsWrite[str]) -> Self: ... - def write(self, s: str) -> None: ... +# By default, generators do not have a message policy. +_MessageT = TypeVar("_MessageT", bound=Message, default=Any) + +class Generator(Generic[_MessageT]): + maxheaderlen: int | None + policy: Policy[_MessageT] | None + @overload + def __init__( + self: Generator[Any], # The Policy of the message is used. + outfp: SupportsWrite[str], + mangle_from_: bool | None = None, + maxheaderlen: int | None = None, + *, + policy: None = None, + ) -> None: ... + @overload def __init__( self, outfp: SupportsWrite[str], mangle_from_: bool | None = None, maxheaderlen: int | None = None, *, - policy: Policy | None = None, + policy: Policy[_MessageT], ) -> None: ... - def flatten(self, msg: Message, unixfrom: bool = False, linesep: str | None = None) -> None: ... + def write(self, s: str) -> None: ... + def flatten(self, msg: _MessageT, unixfrom: bool = False, linesep: str | None = None) -> None: ... + def clone(self, fp: SupportsWrite[str]) -> Self: ... -class BytesGenerator(Generator): +class BytesGenerator(Generator[_MessageT]): + @overload + def __init__( + self: BytesGenerator[Any], # The Policy of the message is used. + outfp: SupportsWrite[bytes], + mangle_from_: bool | None = None, + maxheaderlen: int | None = None, + *, + policy: None = None, + ) -> None: ... + @overload def __init__( self, outfp: SupportsWrite[bytes], mangle_from_: bool | None = None, maxheaderlen: int | None = None, *, - policy: Policy | None = None, + policy: Policy[_MessageT], ) -> None: ... -class DecodedGenerator(Generator): +class DecodedGenerator(Generator[_MessageT]): + @overload + def __init__( + self: DecodedGenerator[Any], # The Policy of the message is used. + outfp: SupportsWrite[str], + mangle_from_: bool | None = None, + maxheaderlen: int | None = None, + fmt: str | None = None, + *, + policy: None = None, + ) -> None: ... + @overload def __init__( self, outfp: SupportsWrite[str], @@ -36,5 +73,5 @@ class DecodedGenerator(Generator): maxheaderlen: int | None = None, fmt: str | None = None, *, - policy: Policy | None = None, + policy: Policy[_MessageT], ) -> None: ... diff --git a/mypy/typeshed/stdlib/email/message.pyi b/mypy/typeshed/stdlib/email/message.pyi index 7e80f13adb8f..f04b014431f9 100644 --- a/mypy/typeshed/stdlib/email/message.pyi +++ b/mypy/typeshed/stdlib/email/message.pyi @@ -30,10 +30,13 @@ class _SupportsDecodeToPayload(Protocol): def decode(self, encoding: str, errors: str, /) -> _PayloadType | _MultipartPayloadType: ... class Message(Generic[_HeaderT, _HeaderParamT]): - policy: Policy # undocumented + # The policy attributes and arguments in this class and its subclasses + # would ideally use Policy[Self], but this is not possible. + policy: Policy[Any] # undocumented preamble: str | None epilogue: str | None defects: list[MessageDefect] + def __init__(self, policy: Policy[Any] = ...) -> None: ... def is_multipart(self) -> bool: ... def set_unixfrom(self, unixfrom: str) -> None: ... def get_unixfrom(self) -> str | None: ... @@ -126,8 +129,8 @@ class Message(Generic[_HeaderT, _HeaderParamT]): def get_charsets(self, failobj: _T) -> list[str | _T]: ... def walk(self) -> Generator[Self, None, None]: ... def get_content_disposition(self) -> str | None: ... - def as_string(self, unixfrom: bool = False, maxheaderlen: int = 0, policy: Policy | None = None) -> str: ... - def as_bytes(self, unixfrom: bool = False, policy: Policy | None = None) -> bytes: ... + def as_string(self, unixfrom: bool = False, maxheaderlen: int = 0, policy: Policy[Any] | None = None) -> str: ... + def as_bytes(self, unixfrom: bool = False, policy: Policy[Any] | None = None) -> bytes: ... def __bytes__(self) -> bytes: ... def set_param( self, @@ -139,13 +142,12 @@ class Message(Generic[_HeaderT, _HeaderParamT]): language: str = "", replace: bool = False, ) -> None: ... - def __init__(self, policy: Policy = ...) -> None: ... # The following two methods are undocumented, but a source code comment states that they are public API def set_raw(self, name: str, value: _HeaderParamT) -> None: ... def raw_items(self) -> Iterator[tuple[str, _HeaderT]]: ... class MIMEPart(Message[_HeaderRegistryT, _HeaderRegistryParamT]): - def __init__(self, policy: Policy | None = None) -> None: ... + def __init__(self, policy: Policy[Any] | None = None) -> None: ... def get_body(self, preferencelist: Sequence[str] = ("related", "html", "plain")) -> MIMEPart[_HeaderRegistryT] | None: ... def attach(self, payload: Self) -> None: ... # type: ignore[override] # The attachments are created via type(self) in the attach method. It's theoretically @@ -163,7 +165,7 @@ class MIMEPart(Message[_HeaderRegistryT, _HeaderRegistryParamT]): def add_attachment(self, *args: Any, content_manager: ContentManager | None = ..., **kw: Any) -> None: ... def clear(self) -> None: ... def clear_content(self) -> None: ... - def as_string(self, unixfrom: bool = False, maxheaderlen: int | None = None, policy: Policy | None = None) -> str: ... + def as_string(self, unixfrom: bool = False, maxheaderlen: int | None = None, policy: Policy[Any] | None = None) -> str: ... def is_attachment(self) -> bool: ... class EmailMessage(MIMEPart): ... diff --git a/mypy/typeshed/stdlib/email/parser.pyi b/mypy/typeshed/stdlib/email/parser.pyi index fecb29d90b2e..a1a57b4eef4b 100644 --- a/mypy/typeshed/stdlib/email/parser.pyi +++ b/mypy/typeshed/stdlib/email/parser.pyi @@ -12,9 +12,9 @@ _MessageT = TypeVar("_MessageT", bound=Message, default=Message) class Parser(Generic[_MessageT]): @overload - def __init__(self: Parser[Message[str, str]], _class: None = None, *, policy: Policy = ...) -> None: ... + def __init__(self: Parser[Message[str, str]], _class: None = None, *, policy: Policy[Message[str, str]] = ...) -> None: ... @overload - def __init__(self, _class: Callable[[], _MessageT], *, policy: Policy = ...) -> None: ... + def __init__(self, _class: Callable[[], _MessageT], *, policy: Policy[_MessageT] = ...) -> None: ... def parse(self, fp: SupportsRead[str], headersonly: bool = False) -> _MessageT: ... def parsestr(self, text: str, headersonly: bool = False) -> _MessageT: ... @@ -25,9 +25,11 @@ class HeaderParser(Parser[_MessageT]): class BytesParser(Generic[_MessageT]): parser: Parser[_MessageT] @overload - def __init__(self: BytesParser[Message[str, str]], _class: None = None, *, policy: Policy = ...) -> None: ... + def __init__( + self: BytesParser[Message[str, str]], _class: None = None, *, policy: Policy[Message[str, str]] = ... + ) -> None: ... @overload - def __init__(self, _class: Callable[[], _MessageT], *, policy: Policy = ...) -> None: ... + def __init__(self, _class: Callable[[], _MessageT], *, policy: Policy[_MessageT] = ...) -> None: ... def parse(self, fp: _WrappedBuffer, headersonly: bool = False) -> _MessageT: ... def parsebytes(self, text: bytes | bytearray, headersonly: bool = False) -> _MessageT: ... diff --git a/mypy/typeshed/stdlib/email/policy.pyi b/mypy/typeshed/stdlib/email/policy.pyi index 5f1cf934eb3c..5adf4a8c2390 100644 --- a/mypy/typeshed/stdlib/email/policy.pyi +++ b/mypy/typeshed/stdlib/email/policy.pyi @@ -1,16 +1,34 @@ from collections.abc import Callable -from email._policybase import Compat32 as Compat32, Policy as Policy, compat32 as compat32 +from email._policybase import Compat32 as Compat32, Policy as Policy, _MessageFactory, compat32 as compat32 from email.contentmanager import ContentManager -from email.message import Message -from typing import Any +from email.message import EmailMessage, Message +from typing import Any, TypeVar, overload __all__ = ["Compat32", "compat32", "Policy", "EmailPolicy", "default", "strict", "SMTP", "HTTP"] -class EmailPolicy(Policy): +_MessageT = TypeVar("_MessageT", bound=Message, default=Message) + +class EmailPolicy(Policy[_MessageT]): utf8: bool refold_source: str header_factory: Callable[[str, Any], Any] content_manager: ContentManager + @overload + def __init__( + self: EmailPolicy[EmailMessage], + *, + max_line_length: int | None = ..., + linesep: str = ..., + cte_type: str = ..., + raise_on_defect: bool = ..., + mangle_from_: bool = ..., + message_factory: None = None, + utf8: bool = ..., + refold_source: str = ..., + header_factory: Callable[[str, str], str] = ..., + content_manager: ContentManager = ..., + ) -> None: ... + @overload def __init__( self, *, @@ -19,7 +37,7 @@ class EmailPolicy(Policy): cte_type: str = ..., raise_on_defect: bool = ..., mangle_from_: bool = ..., - message_factory: Callable[[Policy], Message] | None = ..., + message_factory: _MessageFactory[_MessageT] | None = ..., utf8: bool = ..., refold_source: str = ..., header_factory: Callable[[str, str], str] = ..., @@ -31,8 +49,8 @@ class EmailPolicy(Policy): def fold(self, name: str, value: str) -> Any: ... def fold_binary(self, name: str, value: str) -> bytes: ... -default: EmailPolicy -SMTP: EmailPolicy -SMTPUTF8: EmailPolicy -HTTP: EmailPolicy -strict: EmailPolicy +default: EmailPolicy[EmailMessage] +SMTP: EmailPolicy[EmailMessage] +SMTPUTF8: EmailPolicy[EmailMessage] +HTTP: EmailPolicy[EmailMessage] +strict: EmailPolicy[EmailMessage] diff --git a/mypy/typeshed/stdlib/enum.pyi b/mypy/typeshed/stdlib/enum.pyi index 5c82b07c4185..3b6c325522d7 100644 --- a/mypy/typeshed/stdlib/enum.pyi +++ b/mypy/typeshed/stdlib/enum.pyi @@ -316,13 +316,24 @@ else: __rand__ = __and__ __rxor__ = __xor__ -# subclassing IntFlag so it picks up all implemented base functions, best modeling behavior of enum.auto() -class auto(IntFlag): +class auto: _value_: Any @_magic_enum_attr def value(self) -> Any: ... def __new__(cls) -> Self: ... + # These don't exist, but auto is basically immediately replaced with + # either an int or a str depending on the type of the enum. StrEnum's auto + # shouldn't have these, but they're needed for int versions of auto (mostly the __or__). + # Ideally type checkers would special case auto enough to handle this, + # but until then this is a slightly inaccurate helping hand. + def __or__(self, other: int | Self) -> Self: ... + def __and__(self, other: int | Self) -> Self: ... + def __xor__(self, other: int | Self) -> Self: ... + __ror__ = __or__ + __rand__ = __and__ + __rxor__ = __xor__ + if sys.version_info >= (3, 11): def pickle_by_global_name(self: Enum, proto: int) -> str: ... def pickle_by_enum_name(self: _EnumMemberT, proto: int) -> tuple[Callable[..., Any], tuple[type[_EnumMemberT], str]]: ... diff --git a/mypy/typeshed/stdlib/fileinput.pyi b/mypy/typeshed/stdlib/fileinput.pyi index 1e6aa78e2607..bf6daad0aea7 100644 --- a/mypy/typeshed/stdlib/fileinput.pyi +++ b/mypy/typeshed/stdlib/fileinput.pyi @@ -1,8 +1,8 @@ import sys from _typeshed import AnyStr_co, StrOrBytesPath -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Callable, Iterable from types import TracebackType -from typing import IO, Any, AnyStr, Literal, Protocol, overload +from typing import IO, Any, AnyStr, Generic, Literal, Protocol, overload from typing_extensions import Self, TypeAlias if sys.version_info >= (3, 9): @@ -107,7 +107,7 @@ def fileno() -> int: ... def isfirstline() -> bool: ... def isstdin() -> bool: ... -class FileInput(Iterator[AnyStr]): +class FileInput(Generic[AnyStr]): if sys.version_info >= (3, 10): # encoding and errors are added @overload diff --git a/mypy/typeshed/stdlib/hmac.pyi b/mypy/typeshed/stdlib/hmac.pyi index ac1372dd1e9c..eccfbdc235f4 100644 --- a/mypy/typeshed/stdlib/hmac.pyi +++ b/mypy/typeshed/stdlib/hmac.pyi @@ -1,12 +1,11 @@ from _typeshed import ReadableBuffer, SizedBuffer from collections.abc import Callable +from hashlib import _Hash as _HashlibHash from types import ModuleType -from typing import Any, AnyStr, overload +from typing import AnyStr, overload from typing_extensions import TypeAlias -# TODO more precise type for object of hashlib -_Hash: TypeAlias = Any -_DigestMod: TypeAlias = str | Callable[[], _Hash] | ModuleType +_DigestMod: TypeAlias = str | Callable[[], _HashlibHash] | ModuleType trans_5C: bytes trans_36: bytes diff --git a/mypy/typeshed/stdlib/http/client.pyi b/mypy/typeshed/stdlib/http/client.pyi index f68d9d0ca7d7..f47c744a6c7d 100644 --- a/mypy/typeshed/stdlib/http/client.pyi +++ b/mypy/typeshed/stdlib/http/client.pyi @@ -34,6 +34,7 @@ __all__ = [ _DataType: TypeAlias = SupportsRead[bytes] | Iterable[ReadableBuffer] | ReadableBuffer _T = TypeVar("_T") _MessageT = TypeVar("_MessageT", bound=email.message.Message) +_HeaderValue: TypeAlias = ReadableBuffer | str | int HTTP_PORT: int HTTPS_PORT: int @@ -167,7 +168,7 @@ class HTTPConnection: method: str, url: str, body: _DataType | str | None = None, - headers: Mapping[str, str] = {}, + headers: Mapping[str, _HeaderValue] = {}, *, encode_chunked: bool = False, ) -> None: ... @@ -180,7 +181,7 @@ class HTTPConnection: def connect(self) -> None: ... def close(self) -> None: ... def putrequest(self, method: str, url: str, skip_host: bool = False, skip_accept_encoding: bool = False) -> None: ... - def putheader(self, header: str | bytes, *argument: str | bytes) -> None: ... + def putheader(self, header: str | bytes, *values: _HeaderValue) -> None: ... def endheaders(self, message_body: _DataType | None = None, *, encode_chunked: bool = False) -> None: ... def send(self, data: _DataType | str) -> None: ... diff --git a/mypy/typeshed/stdlib/http/cookiejar.pyi b/mypy/typeshed/stdlib/http/cookiejar.pyi index 56097f163afd..31e1d3fc8378 100644 --- a/mypy/typeshed/stdlib/http/cookiejar.pyi +++ b/mypy/typeshed/stdlib/http/cookiejar.pyi @@ -1,6 +1,6 @@ import sys from _typeshed import StrPath -from collections.abc import Iterable, Iterator, Sequence +from collections.abc import Iterator, Sequence from http.client import HTTPResponse from re import Pattern from typing import ClassVar, TypeVar, overload @@ -21,7 +21,7 @@ _T = TypeVar("_T") class LoadError(OSError): ... -class CookieJar(Iterable[Cookie]): +class CookieJar: non_word_re: ClassVar[Pattern[str]] # undocumented quote_re: ClassVar[Pattern[str]] # undocumented strict_domain_re: ClassVar[Pattern[str]] # undocumented diff --git a/mypy/typeshed/stdlib/io.pyi b/mypy/typeshed/stdlib/io.pyi index 7607608696dd..5c26cb245a2f 100644 --- a/mypy/typeshed/stdlib/io.pyi +++ b/mypy/typeshed/stdlib/io.pyi @@ -1,13 +1,26 @@ import abc -import builtins -import codecs import sys -from _typeshed import FileDescriptorOrPath, ReadableBuffer, WriteableBuffer -from collections.abc import Callable, Iterable, Iterator -from os import _Opener -from types import TracebackType -from typing import IO, Any, BinaryIO, Final, Generic, Literal, Protocol, TextIO, TypeVar, overload, type_check_only -from typing_extensions import Self +from _io import ( + DEFAULT_BUFFER_SIZE as DEFAULT_BUFFER_SIZE, + BlockingIOError as BlockingIOError, + BufferedRandom as BufferedRandom, + BufferedReader as BufferedReader, + BufferedRWPair as BufferedRWPair, + BufferedWriter as BufferedWriter, + BytesIO as BytesIO, + FileIO as FileIO, + IncrementalNewlineDecoder as IncrementalNewlineDecoder, + StringIO as StringIO, + TextIOWrapper as TextIOWrapper, + _BufferedIOBase, + _IOBase, + _RawIOBase, + _TextIOBase, + _WrappedBuffer as _WrappedBuffer, # used elsewhere in typeshed + open as open, + open_code as open_code, +) +from typing import Final __all__ = [ "BlockingIOError", @@ -32,208 +45,16 @@ __all__ = [ ] if sys.version_info >= (3, 11): - __all__ += ["DEFAULT_BUFFER_SIZE", "IncrementalNewlineDecoder", "text_encoding"] - -_T = TypeVar("_T") + from _io import text_encoding as text_encoding -DEFAULT_BUFFER_SIZE: Final = 8192 + __all__ += ["DEFAULT_BUFFER_SIZE", "IncrementalNewlineDecoder", "text_encoding"] SEEK_SET: Final = 0 SEEK_CUR: Final = 1 SEEK_END: Final = 2 -open = builtins.open - -def open_code(path: str) -> IO[bytes]: ... - -BlockingIOError = builtins.BlockingIOError - class UnsupportedOperation(OSError, ValueError): ... - -class IOBase(metaclass=abc.ABCMeta): - def __iter__(self) -> Iterator[bytes]: ... - def __next__(self) -> bytes: ... - def __enter__(self) -> Self: ... - def __exit__( - self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None - ) -> None: ... - def close(self) -> None: ... - def fileno(self) -> int: ... - def flush(self) -> None: ... - def isatty(self) -> bool: ... - def readable(self) -> bool: ... - read: Callable[..., Any] - def readlines(self, hint: int = -1, /) -> list[bytes]: ... - def seek(self, offset: int, whence: int = ..., /) -> int: ... - def seekable(self) -> bool: ... - def tell(self) -> int: ... - def truncate(self, size: int | None = ..., /) -> int: ... - def writable(self) -> bool: ... - write: Callable[..., Any] - def writelines(self, lines: Iterable[ReadableBuffer], /) -> None: ... - def readline(self, size: int | None = -1, /) -> bytes: ... - def __del__(self) -> None: ... - @property - def closed(self) -> bool: ... - def _checkClosed(self) -> None: ... # undocumented - -class RawIOBase(IOBase): - def readall(self) -> bytes: ... - def readinto(self, buffer: WriteableBuffer, /) -> int | None: ... - def write(self, b: ReadableBuffer, /) -> int | None: ... - def read(self, size: int = -1, /) -> bytes | None: ... - -class BufferedIOBase(IOBase): - def detach(self) -> RawIOBase: ... - def readinto(self, buffer: WriteableBuffer, /) -> int: ... - def write(self, buffer: ReadableBuffer, /) -> int: ... - def readinto1(self, buffer: WriteableBuffer, /) -> int: ... - def read(self, size: int | None = ..., /) -> bytes: ... - def read1(self, size: int = ..., /) -> bytes: ... - -class FileIO(RawIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of writelines in the base classes - mode: str - # The type of "name" equals the argument passed in to the constructor, - # but that can make FileIO incompatible with other I/O types that assume - # "name" is a str. In the future, making FileIO generic might help. - name: Any - def __init__( - self, file: FileDescriptorOrPath, mode: str = ..., closefd: bool = ..., opener: _Opener | None = ... - ) -> None: ... - @property - def closefd(self) -> bool: ... - def write(self, b: ReadableBuffer, /) -> int: ... - def read(self, size: int = -1, /) -> bytes: ... - def __enter__(self) -> Self: ... - -class BytesIO(BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of methods in the base classes - def __init__(self, initial_bytes: ReadableBuffer = ...) -> None: ... - # BytesIO does not contain a "name" field. This workaround is necessary - # to allow BytesIO sub-classes to add this field, as it is defined - # as a read-only property on IO[]. - name: Any - def __enter__(self) -> Self: ... - def getvalue(self) -> bytes: ... - def getbuffer(self) -> memoryview: ... - def read1(self, size: int | None = -1, /) -> bytes: ... - -class BufferedReader(BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of methods in the base classes - raw: RawIOBase - def __enter__(self) -> Self: ... - def __init__(self, raw: RawIOBase, buffer_size: int = ...) -> None: ... - def peek(self, size: int = 0, /) -> bytes: ... - -class BufferedWriter(BufferedIOBase, BinaryIO): # type: ignore[misc] # incompatible definitions of writelines in the base classes - raw: RawIOBase - def __enter__(self) -> Self: ... - def __init__(self, raw: RawIOBase, buffer_size: int = ...) -> None: ... - def write(self, buffer: ReadableBuffer, /) -> int: ... - -class BufferedRandom(BufferedReader, BufferedWriter): # type: ignore[misc] # incompatible definitions of methods in the base classes - def __enter__(self) -> Self: ... - def seek(self, target: int, whence: int = 0, /) -> int: ... # stubtest needs this - -class BufferedRWPair(BufferedIOBase): - def __init__(self, reader: RawIOBase, writer: RawIOBase, buffer_size: int = ...) -> None: ... - def peek(self, size: int = ..., /) -> bytes: ... - -class TextIOBase(IOBase): - encoding: str - errors: str | None - newlines: str | tuple[str, ...] | None - def __iter__(self) -> Iterator[str]: ... # type: ignore[override] - def __next__(self) -> str: ... # type: ignore[override] - def detach(self) -> BinaryIO: ... - def write(self, s: str, /) -> int: ... - def writelines(self, lines: Iterable[str], /) -> None: ... # type: ignore[override] - def readline(self, size: int = ..., /) -> str: ... # type: ignore[override] - def readlines(self, hint: int = -1, /) -> list[str]: ... # type: ignore[override] - def read(self, size: int | None = ..., /) -> str: ... - -@type_check_only -class _WrappedBuffer(Protocol): - # "name" is wrapped by TextIOWrapper. Its type is inconsistent between - # the various I/O types, see the comments on TextIOWrapper.name and - # TextIO.name. - @property - def name(self) -> Any: ... - @property - def closed(self) -> bool: ... - def read(self, size: int = ..., /) -> ReadableBuffer: ... - # Optional: def read1(self, size: int, /) -> ReadableBuffer: ... - def write(self, b: bytes, /) -> object: ... - def flush(self) -> object: ... - def close(self) -> object: ... - def seekable(self) -> bool: ... - def readable(self) -> bool: ... - def writable(self) -> bool: ... - def truncate(self, size: int, /) -> int: ... - def fileno(self) -> int: ... - def isatty(self) -> bool: ... - # Optional: Only needs to be present if seekable() returns True. - # def seek(self, offset: Literal[0], whence: Literal[2]) -> int: ... - # def tell(self) -> int: ... - -_BufferT_co = TypeVar("_BufferT_co", bound=_WrappedBuffer, default=_WrappedBuffer, covariant=True) - -class TextIOWrapper(TextIOBase, TextIO, Generic[_BufferT_co]): # type: ignore[misc] # incompatible definitions of write in the base classes - def __init__( - self, - buffer: _BufferT_co, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - line_buffering: bool = False, - write_through: bool = False, - ) -> None: ... - # Equals the "buffer" argument passed in to the constructor. - @property - def buffer(self) -> _BufferT_co: ... # type: ignore[override] - @property - def closed(self) -> bool: ... - @property - def line_buffering(self) -> bool: ... - @property - def write_through(self) -> bool: ... - def reconfigure( - self, - *, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - line_buffering: bool | None = None, - write_through: bool | None = None, - ) -> None: ... - # These are inherited from TextIOBase, but must exist in the stub to satisfy mypy. - def __enter__(self) -> Self: ... - def __iter__(self) -> Iterator[str]: ... # type: ignore[override] - def __next__(self) -> str: ... # type: ignore[override] - def writelines(self, lines: Iterable[str], /) -> None: ... # type: ignore[override] - def readline(self, size: int = -1, /) -> str: ... # type: ignore[override] - def readlines(self, hint: int = -1, /) -> list[str]: ... # type: ignore[override] - # Equals the "buffer" argument passed in to the constructor. - def detach(self) -> _BufferT_co: ... # type: ignore[override] - # TextIOWrapper's version of seek only supports a limited subset of - # operations. - def seek(self, cookie: int, whence: int = 0, /) -> int: ... - -class StringIO(TextIOWrapper): - def __init__(self, initial_value: str | None = ..., newline: str | None = ...) -> None: ... - # StringIO does not contain a "name" field. This workaround is necessary - # to allow StringIO sub-classes to add this field, as it is defined - # as a read-only property on IO[]. - name: Any - def getvalue(self) -> str: ... - -class IncrementalNewlineDecoder(codecs.IncrementalDecoder): - def __init__(self, decoder: codecs.IncrementalDecoder | None, translate: bool, errors: str = ...) -> None: ... - def decode(self, input: ReadableBuffer | str, final: bool = False) -> str: ... - @property - def newlines(self) -> str | tuple[str, ...] | None: ... - def setstate(self, state: tuple[bytes, int], /) -> None: ... - -if sys.version_info >= (3, 10): - @overload - def text_encoding(encoding: None, stacklevel: int = 2, /) -> Literal["locale", "utf-8"]: ... - @overload - def text_encoding(encoding: _T, stacklevel: int = 2, /) -> _T: ... +class IOBase(_IOBase, metaclass=abc.ABCMeta): ... +class RawIOBase(_RawIOBase, IOBase): ... +class BufferedIOBase(_BufferedIOBase, IOBase): ... +class TextIOBase(_TextIOBase, IOBase): ... diff --git a/mypy/typeshed/stdlib/ipaddress.pyi b/mypy/typeshed/stdlib/ipaddress.pyi index f51ea87dcfcf..fa2faf8e6c13 100644 --- a/mypy/typeshed/stdlib/ipaddress.pyi +++ b/mypy/typeshed/stdlib/ipaddress.pyi @@ -1,6 +1,6 @@ import sys from collections.abc import Iterable, Iterator -from typing import Any, Final, Generic, Literal, SupportsInt, TypeVar, overload +from typing import Any, Final, Generic, Literal, TypeVar, overload from typing_extensions import Self, TypeAlias # Undocumented length constants @@ -31,7 +31,7 @@ class _IPAddressBase: @property def version(self) -> int: ... -class _BaseAddress(_IPAddressBase, SupportsInt): +class _BaseAddress(_IPAddressBase): def __init__(self, address: object) -> None: ... def __add__(self, other: int) -> Self: ... def __hash__(self) -> int: ... diff --git a/mypy/typeshed/stdlib/logging/config.pyi b/mypy/typeshed/stdlib/logging/config.pyi index 83fe7461cb5c..af57f36715be 100644 --- a/mypy/typeshed/stdlib/logging/config.pyi +++ b/mypy/typeshed/stdlib/logging/config.pyi @@ -116,7 +116,7 @@ class BaseConfigurator: # undocumented def cfg_convert(self, value: str) -> Any: ... def convert(self, value: Any) -> Any: ... def configure_custom(self, config: dict[str, Any]) -> Any: ... - def as_tuple(self, value: list[Any] | tuple[Any]) -> tuple[Any]: ... + def as_tuple(self, value: list[Any] | tuple[Any, ...]) -> tuple[Any, ...]: ... class DictConfigurator(BaseConfigurator): def configure(self) -> None: ... # undocumented diff --git a/mypy/typeshed/stdlib/mmap.pyi b/mypy/typeshed/stdlib/mmap.pyi index a0c150d6e7e8..5be7f7b76ba1 100644 --- a/mypy/typeshed/stdlib/mmap.pyi +++ b/mypy/typeshed/stdlib/mmap.pyi @@ -1,6 +1,6 @@ import sys from _typeshed import ReadableBuffer, Unused -from collections.abc import Iterable, Iterator, Sized +from collections.abc import Iterator from typing import Final, Literal, NoReturn, overload from typing_extensions import Self @@ -30,7 +30,7 @@ if sys.platform != "win32": PAGESIZE: int -class mmap(Iterable[int], Sized): +class mmap: if sys.platform == "win32": def __init__(self, fileno: int, length: int, tagname: str | None = ..., access: int = ..., offset: int = ...) -> None: ... else: diff --git a/mypy/typeshed/stdlib/multiprocessing/connection.pyi b/mypy/typeshed/stdlib/multiprocessing/connection.pyi index 7045a81b85be..9998239d3119 100644 --- a/mypy/typeshed/stdlib/multiprocessing/connection.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/connection.pyi @@ -1,9 +1,9 @@ import socket import sys -import types -from _typeshed import ReadableBuffer +from _typeshed import Incomplete, ReadableBuffer from collections.abc import Iterable -from typing import Any, SupportsIndex +from types import TracebackType +from typing import Any, Generic, SupportsIndex, TypeVar from typing_extensions import Self, TypeAlias __all__ = ["Client", "Listener", "Pipe", "wait"] @@ -11,7 +11,11 @@ __all__ = ["Client", "Listener", "Pipe", "wait"] # https://docs.python.org/3/library/multiprocessing.html#address-formats _Address: TypeAlias = str | tuple[str, int] -class _ConnectionBase: +# Defaulting to Any to avoid forcing generics on a lot of pre-existing code +_SendT = TypeVar("_SendT", contravariant=True, default=Any) +_RecvT = TypeVar("_RecvT", covariant=True, default=Any) + +class _ConnectionBase(Generic[_SendT, _RecvT]): def __init__(self, handle: SupportsIndex, readable: bool = True, writable: bool = True) -> None: ... @property def closed(self) -> bool: ... # undocumented @@ -22,27 +26,27 @@ class _ConnectionBase: def fileno(self) -> int: ... def close(self) -> None: ... def send_bytes(self, buf: ReadableBuffer, offset: int = 0, size: int | None = None) -> None: ... - def send(self, obj: Any) -> None: ... + def send(self, obj: _SendT) -> None: ... def recv_bytes(self, maxlength: int | None = None) -> bytes: ... def recv_bytes_into(self, buf: Any, offset: int = 0) -> int: ... - def recv(self) -> Any: ... + def recv(self) -> _RecvT: ... def poll(self, timeout: float | None = 0.0) -> bool: ... def __enter__(self) -> Self: ... def __exit__( - self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_tb: types.TracebackType | None + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_tb: TracebackType | None ) -> None: ... def __del__(self) -> None: ... -class Connection(_ConnectionBase): ... +class Connection(_ConnectionBase[_SendT, _RecvT]): ... if sys.platform == "win32": - class PipeConnection(_ConnectionBase): ... + class PipeConnection(_ConnectionBase[_SendT, _RecvT]): ... class Listener: def __init__( self, address: _Address | None = None, family: str | None = None, backlog: int = 1, authkey: bytes | None = None ) -> None: ... - def accept(self) -> Connection: ... + def accept(self) -> Connection[Incomplete, Incomplete]: ... def close(self) -> None: ... @property def address(self) -> _Address: ... @@ -50,26 +54,30 @@ class Listener: def last_accepted(self) -> _Address | None: ... def __enter__(self) -> Self: ... def __exit__( - self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_tb: types.TracebackType | None + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_tb: TracebackType | None ) -> None: ... +# Any: send and recv methods unused if sys.version_info >= (3, 12): - def deliver_challenge(connection: Connection, authkey: bytes, digest_name: str = "sha256") -> None: ... + def deliver_challenge(connection: Connection[Any, Any], authkey: bytes, digest_name: str = "sha256") -> None: ... else: - def deliver_challenge(connection: Connection, authkey: bytes) -> None: ... + def deliver_challenge(connection: Connection[Any, Any], authkey: bytes) -> None: ... -def answer_challenge(connection: Connection, authkey: bytes) -> None: ... +def answer_challenge(connection: Connection[Any, Any], authkey: bytes) -> None: ... def wait( - object_list: Iterable[Connection | socket.socket | int], timeout: float | None = None -) -> list[Connection | socket.socket | int]: ... -def Client(address: _Address, family: str | None = None, authkey: bytes | None = None) -> Connection: ... + object_list: Iterable[Connection[_SendT, _RecvT] | socket.socket | int], timeout: float | None = None +) -> list[Connection[_SendT, _RecvT] | socket.socket | int]: ... +def Client(address: _Address, family: str | None = None, authkey: bytes | None = None) -> Connection[Any, Any]: ... # N.B. Keep this in sync with multiprocessing.context.BaseContext.Pipe. # _ConnectionBase is the common base class of Connection and PipeConnection # and can be used in cross-platform code. +# +# The two connections should have the same generic types but inverted (Connection[_T1, _T2], Connection[_T2, _T1]). +# However, TypeVars scoped entirely within a return annotation is unspecified in the spec. if sys.platform != "win32": - def Pipe(duplex: bool = True) -> tuple[Connection, Connection]: ... + def Pipe(duplex: bool = True) -> tuple[Connection[Any, Any], Connection[Any, Any]]: ... else: - def Pipe(duplex: bool = True) -> tuple[PipeConnection, PipeConnection]: ... + def Pipe(duplex: bool = True) -> tuple[PipeConnection[Any, Any], PipeConnection[Any, Any]]: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/context.pyi b/mypy/typeshed/stdlib/multiprocessing/context.pyi index 605be4686c1f..9900f009a398 100644 --- a/mypy/typeshed/stdlib/multiprocessing/context.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/context.pyi @@ -47,10 +47,13 @@ class BaseContext: # N.B. Keep this in sync with multiprocessing.connection.Pipe. # _ConnectionBase is the common base class of Connection and PipeConnection # and can be used in cross-platform code. + # + # The two connections should have the same generic types but inverted (Connection[_T1, _T2], Connection[_T2, _T1]). + # However, TypeVars scoped entirely within a return annotation is unspecified in the spec. if sys.platform != "win32": - def Pipe(self, duplex: bool = True) -> tuple[Connection, Connection]: ... + def Pipe(self, duplex: bool = True) -> tuple[Connection[Any, Any], Connection[Any, Any]]: ... else: - def Pipe(self, duplex: bool = True) -> tuple[PipeConnection, PipeConnection]: ... + def Pipe(self, duplex: bool = True) -> tuple[PipeConnection[Any, Any], PipeConnection[Any, Any]]: ... def Barrier( self, parties: int, action: Callable[..., object] | None = None, timeout: float | None = None diff --git a/mypy/typeshed/stdlib/multiprocessing/managers.pyi b/mypy/typeshed/stdlib/multiprocessing/managers.pyi index 5d5b9cdcb913..c5a1134377a1 100644 --- a/mypy/typeshed/stdlib/multiprocessing/managers.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/managers.pyi @@ -1,7 +1,7 @@ import queue import sys import threading -from _typeshed import SupportsKeysAndGetItem, SupportsRichComparison, SupportsRichComparisonT +from _typeshed import Incomplete, SupportsKeysAndGetItem, SupportsRichComparison, SupportsRichComparisonT from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, MutableSequence, Sequence from types import TracebackType from typing import Any, AnyStr, ClassVar, Generic, SupportsIndex, TypeVar, overload @@ -129,7 +129,9 @@ class Server: self, registry: dict[str, tuple[Callable[..., Any], Any, Any, Any]], address: Any, authkey: bytes, serializer: str ) -> None: ... def serve_forever(self) -> None: ... - def accept_connection(self, c: Connection, name: str) -> None: ... + def accept_connection( + self, c: Connection[tuple[str, str | None], tuple[str, str, Iterable[Incomplete], Mapping[str, Incomplete]]], name: str + ) -> None: ... class BaseManager: if sys.version_info >= (3, 11): diff --git a/mypy/typeshed/stdlib/multiprocessing/pool.pyi b/mypy/typeshed/stdlib/multiprocessing/pool.pyi index 950ed1d8c56b..61d6d0781213 100644 --- a/mypy/typeshed/stdlib/multiprocessing/pool.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/pool.pyi @@ -1,5 +1,5 @@ import sys -from collections.abc import Callable, Iterable, Iterator, Mapping +from collections.abc import Callable, Iterable, Mapping from types import TracebackType from typing import Any, Final, Generic, TypeVar from typing_extensions import Self @@ -36,7 +36,7 @@ class MapResult(ApplyResult[list[_T]]): error_callback: Callable[[BaseException], object] | None, ) -> None: ... -class IMapIterator(Iterator[_T]): +class IMapIterator(Generic[_T]): def __init__(self, pool: Pool) -> None: ... def __iter__(self) -> Self: ... def next(self, timeout: float | None = None) -> _T: ... diff --git a/mypy/typeshed/stdlib/multiprocessing/reduction.pyi b/mypy/typeshed/stdlib/multiprocessing/reduction.pyi index a31987bcc3cb..473e90936d71 100644 --- a/mypy/typeshed/stdlib/multiprocessing/reduction.pyi +++ b/mypy/typeshed/stdlib/multiprocessing/reduction.pyi @@ -35,8 +35,8 @@ if sys.platform == "win32": handle: int, target_process: int | None = None, inheritable: bool = False, *, source_process: int | None = None ) -> int: ... def steal_handle(source_pid: int, handle: int) -> int: ... - def send_handle(conn: connection.PipeConnection, handle: int, destination_pid: int) -> None: ... - def recv_handle(conn: connection.PipeConnection) -> int: ... + def send_handle(conn: connection.PipeConnection[DupHandle, Any], handle: int, destination_pid: int) -> None: ... + def recv_handle(conn: connection.PipeConnection[Any, DupHandle]) -> int: ... class DupHandle: def __init__(self, handle: int, access: int, pid: int | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/operator.pyi b/mypy/typeshed/stdlib/operator.pyi index 1a817f00f3c1..b73e037f3ed9 100644 --- a/mypy/typeshed/stdlib/operator.pyi +++ b/mypy/typeshed/stdlib/operator.pyi @@ -1,5 +1,66 @@ import sys -from _operator import * +from _operator import ( + abs as abs, + add as add, + and_ as and_, + concat as concat, + contains as contains, + countOf as countOf, + delitem as delitem, + eq as eq, + floordiv as floordiv, + ge as ge, + getitem as getitem, + gt as gt, + iadd as iadd, + iand as iand, + iconcat as iconcat, + ifloordiv as ifloordiv, + ilshift as ilshift, + imatmul as imatmul, + imod as imod, + imul as imul, + index as index, + indexOf as indexOf, + inv as inv, + invert as invert, + ior as ior, + ipow as ipow, + irshift as irshift, + is_ as is_, + is_not as is_not, + isub as isub, + itruediv as itruediv, + ixor as ixor, + le as le, + length_hint as length_hint, + lshift as lshift, + lt as lt, + matmul as matmul, + mod as mod, + mul as mul, + ne as ne, + neg as neg, + not_ as not_, + or_ as or_, + pos as pos, + pow as pow, + rshift as rshift, + setitem as setitem, + sub as sub, + truediv as truediv, + truth as truth, + xor as xor, +) +from _typeshed import SupportsGetItem +from typing import Any, Generic, TypeVar, final, overload +from typing_extensions import TypeVarTuple, Unpack + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_Ts = TypeVarTuple("_Ts") __all__ = [ "abs", @@ -59,9 +120,13 @@ __all__ = [ ] if sys.version_info >= (3, 11): + from _operator import call as call + __all__ += ["call"] if sys.version_info >= (3, 14): + from _operator import is_none as is_none, is_not_none as is_not_none + __all__ += ["is_none", "is_not_none"] __lt__ = lt @@ -111,3 +176,40 @@ __itruediv__ = itruediv __ixor__ = ixor if sys.version_info >= (3, 11): __call__ = call + +# At runtime, these classes are implemented in C as part of the _operator module +# However, they consider themselves to live in the operator module, so we'll put +# them here. +@final +class attrgetter(Generic[_T_co]): + @overload + def __new__(cls, attr: str, /) -> attrgetter[Any]: ... + @overload + def __new__(cls, attr: str, attr2: str, /) -> attrgetter[tuple[Any, Any]]: ... + @overload + def __new__(cls, attr: str, attr2: str, attr3: str, /) -> attrgetter[tuple[Any, Any, Any]]: ... + @overload + def __new__(cls, attr: str, attr2: str, attr3: str, attr4: str, /) -> attrgetter[tuple[Any, Any, Any, Any]]: ... + @overload + def __new__(cls, attr: str, /, *attrs: str) -> attrgetter[tuple[Any, ...]]: ... + def __call__(self, obj: Any, /) -> _T_co: ... + +@final +class itemgetter(Generic[_T_co]): + @overload + def __new__(cls, item: _T, /) -> itemgetter[_T]: ... + @overload + def __new__(cls, item1: _T1, item2: _T2, /, *items: Unpack[_Ts]) -> itemgetter[tuple[_T1, _T2, Unpack[_Ts]]]: ... + # __key: _KT_contra in SupportsGetItem seems to be causing variance issues, ie: + # TypeVar "_KT_contra@SupportsGetItem" is contravariant + # "tuple[int, int]" is incompatible with protocol "SupportsIndex" + # preventing [_T_co, ...] instead of [Any, ...] + # + # A suspected mypy issue prevents using [..., _T] instead of [..., Any] here. + # https://github.com/python/mypy/issues/14032 + def __call__(self, obj: SupportsGetItem[Any, Any]) -> Any: ... + +@final +class methodcaller: + def __init__(self, name: str, /, *args: Any, **kwargs: Any) -> None: ... + def __call__(self, obj: Any) -> Any: ... diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index d7bb4883a0f2..833f9b1e56cb 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -23,8 +23,9 @@ from abc import abstractmethod from builtins import OSError from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence from contextlib import AbstractContextManager -from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper as _TextIOWrapper +from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper from subprocess import Popen +from types import TracebackType from typing import ( IO, Any, @@ -578,7 +579,7 @@ def fdopen( newline: str | None = ..., closefd: bool = ..., opener: _Opener | None = ..., -) -> _TextIOWrapper: ... +) -> TextIOWrapper: ... @overload def fdopen( fd: int, @@ -917,9 +918,25 @@ if sys.platform != "win32": if sys.platform != "darwin" and sys.platform != "linux": def plock(op: int, /) -> None: ... -class _wrap_close(_TextIOWrapper): - def __init__(self, stream: _TextIOWrapper, proc: Popen[str]) -> None: ... - def close(self) -> int | None: ... # type: ignore[override] +class _wrap_close: + def __init__(self, stream: TextIOWrapper, proc: Popen[str]) -> None: ... + def close(self) -> int | None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: ... + def __iter__(self) -> Iterator[str]: ... + # Methods below here don't exist directly on the _wrap_close object, but + # are copied from the wrapped TextIOWrapper object via __getattr__. + # The full set of TextIOWrapper methods are technically available this way, + # but undocumented. Only a subset are currently included here. + def read(self, size: int | None = -1, /) -> str: ... + def readable(self) -> bool: ... + def readline(self, size: int = -1, /) -> str: ... + def readlines(self, hint: int = -1, /) -> list[str]: ... + def writable(self) -> bool: ... + def write(self, s: str, /) -> int: ... + def writelines(self, lines: Iterable[str], /) -> None: ... def popen(cmd: str, mode: str = "r", buffering: int = -1) -> _wrap_close: ... def spawnl(mode: int, file: StrOrBytesPath, arg0: StrOrBytesPath, *args: StrOrBytesPath) -> int: ... diff --git a/mypy/typeshed/stdlib/pickle.pyi b/mypy/typeshed/stdlib/pickle.pyi index 98ec80b0f14e..9bea92ef1c9e 100644 --- a/mypy/typeshed/stdlib/pickle.pyi +++ b/mypy/typeshed/stdlib/pickle.pyi @@ -157,10 +157,10 @@ class Pickler: def __init__( self, file: SupportsWrite[bytes], - protocol: int | None = ..., + protocol: int | None = None, *, - fix_imports: bool = ..., - buffer_callback: _BufferCallback = ..., + fix_imports: bool = True, + buffer_callback: _BufferCallback = None, ) -> None: ... def reducer_override(self, obj: Any) -> Any: ... def dump(self, obj: Any, /) -> None: ... @@ -174,10 +174,10 @@ class Unpickler: self, file: _ReadableFileobj, *, - fix_imports: bool = ..., - encoding: str = ..., - errors: str = ..., - buffers: Iterable[Any] | None = ..., + fix_imports: bool = True, + encoding: str = "ASCII", + errors: str = "strict", + buffers: Iterable[Any] | None = (), ) -> None: ... def load(self) -> Any: ... def find_class(self, module_name: str, global_name: str, /) -> Any: ... diff --git a/mypy/typeshed/stdlib/pyexpat/__init__.pyi b/mypy/typeshed/stdlib/pyexpat/__init__.pyi index dc0156ef13bd..d38259a20d72 100644 --- a/mypy/typeshed/stdlib/pyexpat/__init__.pyi +++ b/mypy/typeshed/stdlib/pyexpat/__init__.pyi @@ -3,17 +3,13 @@ from collections.abc import Callable from pyexpat import errors as errors, model as model from typing import Any, Final, final from typing_extensions import TypeAlias +from xml.parsers.expat import ExpatError as ExpatError EXPAT_VERSION: Final[str] # undocumented version_info: tuple[int, int, int] # undocumented native_encoding: str # undocumented features: list[tuple[str, int]] # undocumented -class ExpatError(Exception): - code: int - lineno: int - offset: int - error = ExpatError XML_PARAM_ENTITY_PARSING_NEVER: Final = 0 XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE: Final = 1 diff --git a/mypy/typeshed/stdlib/re.pyi b/mypy/typeshed/stdlib/re.pyi index 76f98dd9f2a2..ac297b7d4567 100644 --- a/mypy/typeshed/stdlib/re.pyi +++ b/mypy/typeshed/stdlib/re.pyi @@ -4,7 +4,6 @@ import sre_constants import sys from _typeshed import ReadableBuffer from collections.abc import Callable, Iterator, Mapping -from sre_constants import error as error from typing import Any, AnyStr, Generic, Literal, TypeVar, final, overload from typing_extensions import TypeAlias @@ -54,6 +53,16 @@ if sys.version_info >= (3, 13): _T = TypeVar("_T") +# The implementation defines this in re._constants (version_info >= 3, 11) or +# sre_constants. Typeshed has it here because its __module__ attribute is set to "re". +class error(Exception): + msg: str + pattern: str | bytes | None + pos: int | None + lineno: int + colno: int + def __init__(self, msg: str, pattern: str | bytes | None = None, pos: int | None = None) -> None: ... + @final class Match(Generic[AnyStr]): @property diff --git a/mypy/typeshed/stdlib/shlex.pyi b/mypy/typeshed/stdlib/shlex.pyi index daa8df439b26..1c27483782fb 100644 --- a/mypy/typeshed/stdlib/shlex.pyi +++ b/mypy/typeshed/stdlib/shlex.pyi @@ -27,7 +27,7 @@ def join(split_command: Iterable[str]) -> str: ... def quote(s: str) -> str: ... # TODO: Make generic over infile once PEP 696 is implemented. -class shlex(Iterable[str]): +class shlex: commenters: str wordchars: str whitespace: str diff --git a/mypy/typeshed/stdlib/socket.pyi b/mypy/typeshed/stdlib/socket.pyi index b626409d2dde..39a93ce4d0f3 100644 --- a/mypy/typeshed/stdlib/socket.pyi +++ b/mypy/typeshed/stdlib/socket.pyi @@ -109,8 +109,6 @@ from _socket import ( _RetAddress as _RetAddress, close as close, dup as dup, - error as error, - gaierror as gaierror, getdefaulttimeout as getdefaulttimeout, gethostbyaddr as gethostbyaddr, gethostbyname as gethostbyname, @@ -121,7 +119,6 @@ from _socket import ( getservbyname as getservbyname, getservbyport as getservbyport, has_ipv6 as has_ipv6, - herror as herror, htonl as htonl, htons as htons, if_indextoname as if_indextoname, @@ -134,7 +131,6 @@ from _socket import ( ntohl as ntohl, ntohs as ntohs, setdefaulttimeout as setdefaulttimeout, - timeout as timeout, ) from _typeshed import ReadableBuffer, Unused, WriteableBuffer from collections.abc import Iterable @@ -486,6 +482,18 @@ EBADF: int EAGAIN: int EWOULDBLOCK: int +# These errors are implemented in _socket at runtime +# but they consider themselves to live in socket so we'll put them here. +error = OSError + +class herror(error): ... +class gaierror(error): ... + +if sys.version_info >= (3, 10): + timeout = TimeoutError +else: + class timeout(error): ... + class AddressFamily(IntEnum): AF_INET = 2 AF_INET6 = 10 diff --git a/mypy/typeshed/stdlib/sqlite3/__init__.pyi b/mypy/typeshed/stdlib/sqlite3/__init__.pyi index d747be90fd0a..d6087aa755c1 100644 --- a/mypy/typeshed/stdlib/sqlite3/__init__.pyi +++ b/mypy/typeshed/stdlib/sqlite3/__init__.pyi @@ -1 +1,465 @@ -from sqlite3.dbapi2 import * +import sys +from _typeshed import ReadableBuffer, StrOrBytesPath, SupportsLenAndGetItem, Unused +from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence +from sqlite3.dbapi2 import ( + PARSE_COLNAMES as PARSE_COLNAMES, + PARSE_DECLTYPES as PARSE_DECLTYPES, + SQLITE_ALTER_TABLE as SQLITE_ALTER_TABLE, + SQLITE_ANALYZE as SQLITE_ANALYZE, + SQLITE_ATTACH as SQLITE_ATTACH, + SQLITE_CREATE_INDEX as SQLITE_CREATE_INDEX, + SQLITE_CREATE_TABLE as SQLITE_CREATE_TABLE, + SQLITE_CREATE_TEMP_INDEX as SQLITE_CREATE_TEMP_INDEX, + SQLITE_CREATE_TEMP_TABLE as SQLITE_CREATE_TEMP_TABLE, + SQLITE_CREATE_TEMP_TRIGGER as SQLITE_CREATE_TEMP_TRIGGER, + SQLITE_CREATE_TEMP_VIEW as SQLITE_CREATE_TEMP_VIEW, + SQLITE_CREATE_TRIGGER as SQLITE_CREATE_TRIGGER, + SQLITE_CREATE_VIEW as SQLITE_CREATE_VIEW, + SQLITE_CREATE_VTABLE as SQLITE_CREATE_VTABLE, + SQLITE_DELETE as SQLITE_DELETE, + SQLITE_DENY as SQLITE_DENY, + SQLITE_DETACH as SQLITE_DETACH, + SQLITE_DONE as SQLITE_DONE, + SQLITE_DROP_INDEX as SQLITE_DROP_INDEX, + SQLITE_DROP_TABLE as SQLITE_DROP_TABLE, + SQLITE_DROP_TEMP_INDEX as SQLITE_DROP_TEMP_INDEX, + SQLITE_DROP_TEMP_TABLE as SQLITE_DROP_TEMP_TABLE, + SQLITE_DROP_TEMP_TRIGGER as SQLITE_DROP_TEMP_TRIGGER, + SQLITE_DROP_TEMP_VIEW as SQLITE_DROP_TEMP_VIEW, + SQLITE_DROP_TRIGGER as SQLITE_DROP_TRIGGER, + SQLITE_DROP_VIEW as SQLITE_DROP_VIEW, + SQLITE_DROP_VTABLE as SQLITE_DROP_VTABLE, + SQLITE_FUNCTION as SQLITE_FUNCTION, + SQLITE_IGNORE as SQLITE_IGNORE, + SQLITE_INSERT as SQLITE_INSERT, + SQLITE_OK as SQLITE_OK, + SQLITE_PRAGMA as SQLITE_PRAGMA, + SQLITE_READ as SQLITE_READ, + SQLITE_RECURSIVE as SQLITE_RECURSIVE, + SQLITE_REINDEX as SQLITE_REINDEX, + SQLITE_SAVEPOINT as SQLITE_SAVEPOINT, + SQLITE_SELECT as SQLITE_SELECT, + SQLITE_TRANSACTION as SQLITE_TRANSACTION, + SQLITE_UPDATE as SQLITE_UPDATE, + Binary as Binary, + Date as Date, + DateFromTicks as DateFromTicks, + Time as Time, + TimeFromTicks as TimeFromTicks, + TimestampFromTicks as TimestampFromTicks, + adapt as adapt, + adapters as adapters, + apilevel as apilevel, + complete_statement as complete_statement, + connect as connect, + converters as converters, + enable_callback_tracebacks as enable_callback_tracebacks, + paramstyle as paramstyle, + register_adapter as register_adapter, + register_converter as register_converter, + sqlite_version as sqlite_version, + sqlite_version_info as sqlite_version_info, + threadsafety as threadsafety, + version_info as version_info, +) +from types import TracebackType +from typing import Any, Literal, Protocol, SupportsIndex, TypeVar, final, overload +from typing_extensions import Self, TypeAlias + +if sys.version_info >= (3, 12): + from sqlite3.dbapi2 import ( + LEGACY_TRANSACTION_CONTROL as LEGACY_TRANSACTION_CONTROL, + SQLITE_DBCONFIG_DEFENSIVE as SQLITE_DBCONFIG_DEFENSIVE, + SQLITE_DBCONFIG_DQS_DDL as SQLITE_DBCONFIG_DQS_DDL, + SQLITE_DBCONFIG_DQS_DML as SQLITE_DBCONFIG_DQS_DML, + SQLITE_DBCONFIG_ENABLE_FKEY as SQLITE_DBCONFIG_ENABLE_FKEY, + SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER as SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, + SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION as SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, + SQLITE_DBCONFIG_ENABLE_QPSG as SQLITE_DBCONFIG_ENABLE_QPSG, + SQLITE_DBCONFIG_ENABLE_TRIGGER as SQLITE_DBCONFIG_ENABLE_TRIGGER, + SQLITE_DBCONFIG_ENABLE_VIEW as SQLITE_DBCONFIG_ENABLE_VIEW, + SQLITE_DBCONFIG_LEGACY_ALTER_TABLE as SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, + SQLITE_DBCONFIG_LEGACY_FILE_FORMAT as SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, + SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE as SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, + SQLITE_DBCONFIG_RESET_DATABASE as SQLITE_DBCONFIG_RESET_DATABASE, + SQLITE_DBCONFIG_TRIGGER_EQP as SQLITE_DBCONFIG_TRIGGER_EQP, + SQLITE_DBCONFIG_TRUSTED_SCHEMA as SQLITE_DBCONFIG_TRUSTED_SCHEMA, + SQLITE_DBCONFIG_WRITABLE_SCHEMA as SQLITE_DBCONFIG_WRITABLE_SCHEMA, + ) + +if sys.version_info >= (3, 11): + from sqlite3.dbapi2 import ( + SQLITE_ABORT as SQLITE_ABORT, + SQLITE_ABORT_ROLLBACK as SQLITE_ABORT_ROLLBACK, + SQLITE_AUTH as SQLITE_AUTH, + SQLITE_AUTH_USER as SQLITE_AUTH_USER, + SQLITE_BUSY as SQLITE_BUSY, + SQLITE_BUSY_RECOVERY as SQLITE_BUSY_RECOVERY, + SQLITE_BUSY_SNAPSHOT as SQLITE_BUSY_SNAPSHOT, + SQLITE_BUSY_TIMEOUT as SQLITE_BUSY_TIMEOUT, + SQLITE_CANTOPEN as SQLITE_CANTOPEN, + SQLITE_CANTOPEN_CONVPATH as SQLITE_CANTOPEN_CONVPATH, + SQLITE_CANTOPEN_DIRTYWAL as SQLITE_CANTOPEN_DIRTYWAL, + SQLITE_CANTOPEN_FULLPATH as SQLITE_CANTOPEN_FULLPATH, + SQLITE_CANTOPEN_ISDIR as SQLITE_CANTOPEN_ISDIR, + SQLITE_CANTOPEN_NOTEMPDIR as SQLITE_CANTOPEN_NOTEMPDIR, + SQLITE_CANTOPEN_SYMLINK as SQLITE_CANTOPEN_SYMLINK, + SQLITE_CONSTRAINT as SQLITE_CONSTRAINT, + SQLITE_CONSTRAINT_CHECK as SQLITE_CONSTRAINT_CHECK, + SQLITE_CONSTRAINT_COMMITHOOK as SQLITE_CONSTRAINT_COMMITHOOK, + SQLITE_CONSTRAINT_FOREIGNKEY as SQLITE_CONSTRAINT_FOREIGNKEY, + SQLITE_CONSTRAINT_FUNCTION as SQLITE_CONSTRAINT_FUNCTION, + SQLITE_CONSTRAINT_NOTNULL as SQLITE_CONSTRAINT_NOTNULL, + SQLITE_CONSTRAINT_PINNED as SQLITE_CONSTRAINT_PINNED, + SQLITE_CONSTRAINT_PRIMARYKEY as SQLITE_CONSTRAINT_PRIMARYKEY, + SQLITE_CONSTRAINT_ROWID as SQLITE_CONSTRAINT_ROWID, + SQLITE_CONSTRAINT_TRIGGER as SQLITE_CONSTRAINT_TRIGGER, + SQLITE_CONSTRAINT_UNIQUE as SQLITE_CONSTRAINT_UNIQUE, + SQLITE_CONSTRAINT_VTAB as SQLITE_CONSTRAINT_VTAB, + SQLITE_CORRUPT as SQLITE_CORRUPT, + SQLITE_CORRUPT_INDEX as SQLITE_CORRUPT_INDEX, + SQLITE_CORRUPT_SEQUENCE as SQLITE_CORRUPT_SEQUENCE, + SQLITE_CORRUPT_VTAB as SQLITE_CORRUPT_VTAB, + SQLITE_EMPTY as SQLITE_EMPTY, + SQLITE_ERROR as SQLITE_ERROR, + SQLITE_ERROR_MISSING_COLLSEQ as SQLITE_ERROR_MISSING_COLLSEQ, + SQLITE_ERROR_RETRY as SQLITE_ERROR_RETRY, + SQLITE_ERROR_SNAPSHOT as SQLITE_ERROR_SNAPSHOT, + SQLITE_FORMAT as SQLITE_FORMAT, + SQLITE_FULL as SQLITE_FULL, + SQLITE_INTERNAL as SQLITE_INTERNAL, + SQLITE_INTERRUPT as SQLITE_INTERRUPT, + SQLITE_IOERR as SQLITE_IOERR, + SQLITE_IOERR_ACCESS as SQLITE_IOERR_ACCESS, + SQLITE_IOERR_AUTH as SQLITE_IOERR_AUTH, + SQLITE_IOERR_BEGIN_ATOMIC as SQLITE_IOERR_BEGIN_ATOMIC, + SQLITE_IOERR_BLOCKED as SQLITE_IOERR_BLOCKED, + SQLITE_IOERR_CHECKRESERVEDLOCK as SQLITE_IOERR_CHECKRESERVEDLOCK, + SQLITE_IOERR_CLOSE as SQLITE_IOERR_CLOSE, + SQLITE_IOERR_COMMIT_ATOMIC as SQLITE_IOERR_COMMIT_ATOMIC, + SQLITE_IOERR_CONVPATH as SQLITE_IOERR_CONVPATH, + SQLITE_IOERR_CORRUPTFS as SQLITE_IOERR_CORRUPTFS, + SQLITE_IOERR_DATA as SQLITE_IOERR_DATA, + SQLITE_IOERR_DELETE as SQLITE_IOERR_DELETE, + SQLITE_IOERR_DELETE_NOENT as SQLITE_IOERR_DELETE_NOENT, + SQLITE_IOERR_DIR_CLOSE as SQLITE_IOERR_DIR_CLOSE, + SQLITE_IOERR_DIR_FSYNC as SQLITE_IOERR_DIR_FSYNC, + SQLITE_IOERR_FSTAT as SQLITE_IOERR_FSTAT, + SQLITE_IOERR_FSYNC as SQLITE_IOERR_FSYNC, + SQLITE_IOERR_GETTEMPPATH as SQLITE_IOERR_GETTEMPPATH, + SQLITE_IOERR_LOCK as SQLITE_IOERR_LOCK, + SQLITE_IOERR_MMAP as SQLITE_IOERR_MMAP, + SQLITE_IOERR_NOMEM as SQLITE_IOERR_NOMEM, + SQLITE_IOERR_RDLOCK as SQLITE_IOERR_RDLOCK, + SQLITE_IOERR_READ as SQLITE_IOERR_READ, + SQLITE_IOERR_ROLLBACK_ATOMIC as SQLITE_IOERR_ROLLBACK_ATOMIC, + SQLITE_IOERR_SEEK as SQLITE_IOERR_SEEK, + SQLITE_IOERR_SHMLOCK as SQLITE_IOERR_SHMLOCK, + SQLITE_IOERR_SHMMAP as SQLITE_IOERR_SHMMAP, + SQLITE_IOERR_SHMOPEN as SQLITE_IOERR_SHMOPEN, + SQLITE_IOERR_SHMSIZE as SQLITE_IOERR_SHMSIZE, + SQLITE_IOERR_SHORT_READ as SQLITE_IOERR_SHORT_READ, + SQLITE_IOERR_TRUNCATE as SQLITE_IOERR_TRUNCATE, + SQLITE_IOERR_UNLOCK as SQLITE_IOERR_UNLOCK, + SQLITE_IOERR_VNODE as SQLITE_IOERR_VNODE, + SQLITE_IOERR_WRITE as SQLITE_IOERR_WRITE, + SQLITE_LIMIT_ATTACHED as SQLITE_LIMIT_ATTACHED, + SQLITE_LIMIT_COLUMN as SQLITE_LIMIT_COLUMN, + SQLITE_LIMIT_COMPOUND_SELECT as SQLITE_LIMIT_COMPOUND_SELECT, + SQLITE_LIMIT_EXPR_DEPTH as SQLITE_LIMIT_EXPR_DEPTH, + SQLITE_LIMIT_FUNCTION_ARG as SQLITE_LIMIT_FUNCTION_ARG, + SQLITE_LIMIT_LENGTH as SQLITE_LIMIT_LENGTH, + SQLITE_LIMIT_LIKE_PATTERN_LENGTH as SQLITE_LIMIT_LIKE_PATTERN_LENGTH, + SQLITE_LIMIT_SQL_LENGTH as SQLITE_LIMIT_SQL_LENGTH, + SQLITE_LIMIT_TRIGGER_DEPTH as SQLITE_LIMIT_TRIGGER_DEPTH, + SQLITE_LIMIT_VARIABLE_NUMBER as SQLITE_LIMIT_VARIABLE_NUMBER, + SQLITE_LIMIT_VDBE_OP as SQLITE_LIMIT_VDBE_OP, + SQLITE_LIMIT_WORKER_THREADS as SQLITE_LIMIT_WORKER_THREADS, + SQLITE_LOCKED as SQLITE_LOCKED, + SQLITE_LOCKED_SHAREDCACHE as SQLITE_LOCKED_SHAREDCACHE, + SQLITE_LOCKED_VTAB as SQLITE_LOCKED_VTAB, + SQLITE_MISMATCH as SQLITE_MISMATCH, + SQLITE_MISUSE as SQLITE_MISUSE, + SQLITE_NOLFS as SQLITE_NOLFS, + SQLITE_NOMEM as SQLITE_NOMEM, + SQLITE_NOTADB as SQLITE_NOTADB, + SQLITE_NOTFOUND as SQLITE_NOTFOUND, + SQLITE_NOTICE as SQLITE_NOTICE, + SQLITE_NOTICE_RECOVER_ROLLBACK as SQLITE_NOTICE_RECOVER_ROLLBACK, + SQLITE_NOTICE_RECOVER_WAL as SQLITE_NOTICE_RECOVER_WAL, + SQLITE_OK_LOAD_PERMANENTLY as SQLITE_OK_LOAD_PERMANENTLY, + SQLITE_OK_SYMLINK as SQLITE_OK_SYMLINK, + SQLITE_PERM as SQLITE_PERM, + SQLITE_PROTOCOL as SQLITE_PROTOCOL, + SQLITE_RANGE as SQLITE_RANGE, + SQLITE_READONLY as SQLITE_READONLY, + SQLITE_READONLY_CANTINIT as SQLITE_READONLY_CANTINIT, + SQLITE_READONLY_CANTLOCK as SQLITE_READONLY_CANTLOCK, + SQLITE_READONLY_DBMOVED as SQLITE_READONLY_DBMOVED, + SQLITE_READONLY_DIRECTORY as SQLITE_READONLY_DIRECTORY, + SQLITE_READONLY_RECOVERY as SQLITE_READONLY_RECOVERY, + SQLITE_READONLY_ROLLBACK as SQLITE_READONLY_ROLLBACK, + SQLITE_ROW as SQLITE_ROW, + SQLITE_SCHEMA as SQLITE_SCHEMA, + SQLITE_TOOBIG as SQLITE_TOOBIG, + SQLITE_WARNING as SQLITE_WARNING, + SQLITE_WARNING_AUTOINDEX as SQLITE_WARNING_AUTOINDEX, + ) + +if sys.version_info < (3, 12): + from sqlite3.dbapi2 import enable_shared_cache as enable_shared_cache, version as version + +if sys.version_info < (3, 10): + from sqlite3.dbapi2 import OptimizedUnicode as OptimizedUnicode + +_CursorT = TypeVar("_CursorT", bound=Cursor) +_SqliteData: TypeAlias = str | ReadableBuffer | int | float | None +# Data that is passed through adapters can be of any type accepted by an adapter. +_AdaptedInputData: TypeAlias = _SqliteData | Any +# The Mapping must really be a dict, but making it invariant is too annoying. +_Parameters: TypeAlias = SupportsLenAndGetItem[_AdaptedInputData] | Mapping[str, _AdaptedInputData] + +class _AnyParamWindowAggregateClass(Protocol): + def step(self, *args: Any) -> object: ... + def inverse(self, *args: Any) -> object: ... + def value(self) -> _SqliteData: ... + def finalize(self) -> _SqliteData: ... + +class _WindowAggregateClass(Protocol): + step: Callable[..., object] + inverse: Callable[..., object] + def value(self) -> _SqliteData: ... + def finalize(self) -> _SqliteData: ... + +class _AggregateProtocol(Protocol): + def step(self, value: int, /) -> object: ... + def finalize(self) -> int: ... + +class _SingleParamWindowAggregateClass(Protocol): + def step(self, param: Any, /) -> object: ... + def inverse(self, param: Any, /) -> object: ... + def value(self) -> _SqliteData: ... + def finalize(self) -> _SqliteData: ... + +# These classes are implemented in the C module _sqlite3. At runtime, they're imported +# from there into sqlite3.dbapi2 and from that module to here. However, they +# consider themselves to live in the sqlite3.* namespace, so we'll define them here. + +class Error(Exception): + if sys.version_info >= (3, 11): + sqlite_errorcode: int + sqlite_errorname: str + +class DatabaseError(Error): ... +class DataError(DatabaseError): ... +class IntegrityError(DatabaseError): ... +class InterfaceError(Error): ... +class InternalError(DatabaseError): ... +class NotSupportedError(DatabaseError): ... +class OperationalError(DatabaseError): ... +class ProgrammingError(DatabaseError): ... +class Warning(Exception): ... + +class Connection: + @property + def DataError(self) -> type[DataError]: ... + @property + def DatabaseError(self) -> type[DatabaseError]: ... + @property + def Error(self) -> type[Error]: ... + @property + def IntegrityError(self) -> type[IntegrityError]: ... + @property + def InterfaceError(self) -> type[InterfaceError]: ... + @property + def InternalError(self) -> type[InternalError]: ... + @property + def NotSupportedError(self) -> type[NotSupportedError]: ... + @property + def OperationalError(self) -> type[OperationalError]: ... + @property + def ProgrammingError(self) -> type[ProgrammingError]: ... + @property + def Warning(self) -> type[Warning]: ... + @property + def in_transaction(self) -> bool: ... + isolation_level: str | None # one of '', 'DEFERRED', 'IMMEDIATE' or 'EXCLUSIVE' + @property + def total_changes(self) -> int: ... + if sys.version_info >= (3, 12): + @property + def autocommit(self) -> int: ... + @autocommit.setter + def autocommit(self, val: int) -> None: ... + row_factory: Any + text_factory: Any + if sys.version_info >= (3, 12): + def __init__( + self, + database: StrOrBytesPath, + timeout: float = ..., + detect_types: int = ..., + isolation_level: str | None = ..., + check_same_thread: bool = ..., + factory: type[Connection] | None = ..., + cached_statements: int = ..., + uri: bool = ..., + autocommit: bool = ..., + ) -> None: ... + else: + def __init__( + self, + database: StrOrBytesPath, + timeout: float = ..., + detect_types: int = ..., + isolation_level: str | None = ..., + check_same_thread: bool = ..., + factory: type[Connection] | None = ..., + cached_statements: int = ..., + uri: bool = ..., + ) -> None: ... + + def close(self) -> None: ... + if sys.version_info >= (3, 11): + def blobopen(self, table: str, column: str, row: int, /, *, readonly: bool = False, name: str = "main") -> Blob: ... + + def commit(self) -> None: ... + def create_aggregate(self, name: str, n_arg: int, aggregate_class: Callable[[], _AggregateProtocol]) -> None: ... + if sys.version_info >= (3, 11): + # num_params determines how many params will be passed to the aggregate class. We provide an overload + # for the case where num_params = 1, which is expected to be the common case. + @overload + def create_window_function( + self, name: str, num_params: Literal[1], aggregate_class: Callable[[], _SingleParamWindowAggregateClass] | None, / + ) -> None: ... + # And for num_params = -1, which means the aggregate must accept any number of parameters. + @overload + def create_window_function( + self, name: str, num_params: Literal[-1], aggregate_class: Callable[[], _AnyParamWindowAggregateClass] | None, / + ) -> None: ... + @overload + def create_window_function( + self, name: str, num_params: int, aggregate_class: Callable[[], _WindowAggregateClass] | None, / + ) -> None: ... + + def create_collation(self, name: str, callback: Callable[[str, str], int | SupportsIndex] | None, /) -> None: ... + def create_function( + self, name: str, narg: int, func: Callable[..., _SqliteData] | None, *, deterministic: bool = False + ) -> None: ... + @overload + def cursor(self, factory: None = None) -> Cursor: ... + @overload + def cursor(self, factory: Callable[[Connection], _CursorT]) -> _CursorT: ... + def execute(self, sql: str, parameters: _Parameters = ..., /) -> Cursor: ... + def executemany(self, sql: str, parameters: Iterable[_Parameters], /) -> Cursor: ... + def executescript(self, sql_script: str, /) -> Cursor: ... + def interrupt(self) -> None: ... + if sys.version_info >= (3, 13): + def iterdump(self, *, filter: str | None = None) -> Generator[str, None, None]: ... + else: + def iterdump(self) -> Generator[str, None, None]: ... + + def rollback(self) -> None: ... + def set_authorizer( + self, authorizer_callback: Callable[[int, str | None, str | None, str | None, str | None], int] | None + ) -> None: ... + def set_progress_handler(self, progress_handler: Callable[[], int | None] | None, n: int) -> None: ... + def set_trace_callback(self, trace_callback: Callable[[str], object] | None) -> None: ... + # enable_load_extension and load_extension is not available on python distributions compiled + # without sqlite3 loadable extension support. see footnotes https://docs.python.org/3/library/sqlite3.html#f1 + def enable_load_extension(self, enable: bool, /) -> None: ... + if sys.version_info >= (3, 12): + def load_extension(self, name: str, /, *, entrypoint: str | None = None) -> None: ... + else: + def load_extension(self, name: str, /) -> None: ... + + def backup( + self, + target: Connection, + *, + pages: int = -1, + progress: Callable[[int, int, int], object] | None = None, + name: str = "main", + sleep: float = 0.25, + ) -> None: ... + if sys.version_info >= (3, 11): + def setlimit(self, category: int, limit: int, /) -> int: ... + def getlimit(self, category: int, /) -> int: ... + def serialize(self, *, name: str = "main") -> bytes: ... + def deserialize(self, data: ReadableBuffer, /, *, name: str = "main") -> None: ... + if sys.version_info >= (3, 12): + def getconfig(self, op: int, /) -> bool: ... + def setconfig(self, op: int, enable: bool = True, /) -> bool: ... + + def __call__(self, sql: str, /) -> _Statement: ... + def __enter__(self) -> Self: ... + def __exit__( + self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None, / + ) -> Literal[False]: ... + +class Cursor: + arraysize: int + @property + def connection(self) -> Connection: ... + # May be None, but using | Any instead to avoid slightly annoying false positives. + @property + def description(self) -> tuple[tuple[str, None, None, None, None, None, None], ...] | Any: ... + @property + def lastrowid(self) -> int | None: ... + row_factory: Callable[[Cursor, Row], object] | None + @property + def rowcount(self) -> int: ... + def __init__(self, cursor: Connection, /) -> None: ... + def close(self) -> None: ... + def execute(self, sql: str, parameters: _Parameters = (), /) -> Self: ... + def executemany(self, sql: str, seq_of_parameters: Iterable[_Parameters], /) -> Self: ... + def executescript(self, sql_script: str, /) -> Cursor: ... + def fetchall(self) -> list[Any]: ... + def fetchmany(self, size: int | None = 1) -> list[Any]: ... + # Returns either a row (as created by the row_factory) or None, but + # putting None in the return annotation causes annoying false positives. + def fetchone(self) -> Any: ... + def setinputsizes(self, sizes: Unused, /) -> None: ... # does nothing + def setoutputsize(self, size: Unused, column: Unused = None, /) -> None: ... # does nothing + def __iter__(self) -> Self: ... + def __next__(self) -> Any: ... + +@final +class PrepareProtocol: + def __init__(self, *args: object, **kwargs: object) -> None: ... + +class Row(Sequence[Any]): + def __init__(self, cursor: Cursor, data: tuple[Any, ...], /) -> None: ... + def keys(self) -> list[str]: ... + @overload + def __getitem__(self, key: int | str, /) -> Any: ... + @overload + def __getitem__(self, key: slice, /) -> tuple[Any, ...]: ... + def __hash__(self) -> int: ... + def __iter__(self) -> Iterator[Any]: ... + def __len__(self) -> int: ... + # These return NotImplemented for anything that is not a Row. + def __eq__(self, value: object, /) -> bool: ... + def __ge__(self, value: object, /) -> bool: ... + def __gt__(self, value: object, /) -> bool: ... + def __le__(self, value: object, /) -> bool: ... + def __lt__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + +@final +class _Statement: ... + +if sys.version_info >= (3, 11): + @final + class Blob: + def close(self) -> None: ... + def read(self, length: int = -1, /) -> bytes: ... + def write(self, data: ReadableBuffer, /) -> None: ... + def tell(self) -> int: ... + # whence must be one of os.SEEK_SET, os.SEEK_CUR, os.SEEK_END + def seek(self, offset: int, origin: int = 0, /) -> None: ... + def __len__(self) -> int: ... + def __enter__(self) -> Self: ... + def __exit__(self, type: object, val: object, tb: object, /) -> Literal[False]: ... + def __getitem__(self, key: SupportsIndex | slice, /) -> int: ... + def __setitem__(self, key: SupportsIndex | slice, value: int, /) -> None: ... diff --git a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi index 0ee511df4e37..d3ea3ef0e896 100644 --- a/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi +++ b/mypy/typeshed/stdlib/sqlite3/dbapi2.pyi @@ -1,22 +1,226 @@ -import sqlite3 import sys -from _typeshed import ReadableBuffer, StrOrBytesPath, SupportsLenAndGetItem, Unused -from collections.abc import Callable, Generator, Iterable, Iterator, Mapping +from _sqlite3 import ( + PARSE_COLNAMES as PARSE_COLNAMES, + PARSE_DECLTYPES as PARSE_DECLTYPES, + SQLITE_ALTER_TABLE as SQLITE_ALTER_TABLE, + SQLITE_ANALYZE as SQLITE_ANALYZE, + SQLITE_ATTACH as SQLITE_ATTACH, + SQLITE_CREATE_INDEX as SQLITE_CREATE_INDEX, + SQLITE_CREATE_TABLE as SQLITE_CREATE_TABLE, + SQLITE_CREATE_TEMP_INDEX as SQLITE_CREATE_TEMP_INDEX, + SQLITE_CREATE_TEMP_TABLE as SQLITE_CREATE_TEMP_TABLE, + SQLITE_CREATE_TEMP_TRIGGER as SQLITE_CREATE_TEMP_TRIGGER, + SQLITE_CREATE_TEMP_VIEW as SQLITE_CREATE_TEMP_VIEW, + SQLITE_CREATE_TRIGGER as SQLITE_CREATE_TRIGGER, + SQLITE_CREATE_VIEW as SQLITE_CREATE_VIEW, + SQLITE_CREATE_VTABLE as SQLITE_CREATE_VTABLE, + SQLITE_DELETE as SQLITE_DELETE, + SQLITE_DENY as SQLITE_DENY, + SQLITE_DETACH as SQLITE_DETACH, + SQLITE_DONE as SQLITE_DONE, + SQLITE_DROP_INDEX as SQLITE_DROP_INDEX, + SQLITE_DROP_TABLE as SQLITE_DROP_TABLE, + SQLITE_DROP_TEMP_INDEX as SQLITE_DROP_TEMP_INDEX, + SQLITE_DROP_TEMP_TABLE as SQLITE_DROP_TEMP_TABLE, + SQLITE_DROP_TEMP_TRIGGER as SQLITE_DROP_TEMP_TRIGGER, + SQLITE_DROP_TEMP_VIEW as SQLITE_DROP_TEMP_VIEW, + SQLITE_DROP_TRIGGER as SQLITE_DROP_TRIGGER, + SQLITE_DROP_VIEW as SQLITE_DROP_VIEW, + SQLITE_DROP_VTABLE as SQLITE_DROP_VTABLE, + SQLITE_FUNCTION as SQLITE_FUNCTION, + SQLITE_IGNORE as SQLITE_IGNORE, + SQLITE_INSERT as SQLITE_INSERT, + SQLITE_OK as SQLITE_OK, + SQLITE_PRAGMA as SQLITE_PRAGMA, + SQLITE_READ as SQLITE_READ, + SQLITE_RECURSIVE as SQLITE_RECURSIVE, + SQLITE_REINDEX as SQLITE_REINDEX, + SQLITE_SAVEPOINT as SQLITE_SAVEPOINT, + SQLITE_SELECT as SQLITE_SELECT, + SQLITE_TRANSACTION as SQLITE_TRANSACTION, + SQLITE_UPDATE as SQLITE_UPDATE, + adapt as adapt, + adapters as adapters, + complete_statement as complete_statement, + connect as connect, + converters as converters, + enable_callback_tracebacks as enable_callback_tracebacks, + register_adapter as register_adapter, + register_converter as register_converter, + sqlite_version as sqlite_version, +) from datetime import date, datetime, time -from types import TracebackType -from typing import Any, Final, Literal, Protocol, SupportsIndex, TypeVar, final, overload -from typing_extensions import Self, TypeAlias +from sqlite3 import ( + Connection as Connection, + Cursor as Cursor, + DatabaseError as DatabaseError, + DataError as DataError, + Error as Error, + IntegrityError as IntegrityError, + InterfaceError as InterfaceError, + InternalError as InternalError, + NotSupportedError as NotSupportedError, + OperationalError as OperationalError, + PrepareProtocol as PrepareProtocol, + ProgrammingError as ProgrammingError, + Row as Row, + Warning as Warning, +) -_T = TypeVar("_T") -_ConnectionT = TypeVar("_ConnectionT", bound=Connection) -_CursorT = TypeVar("_CursorT", bound=Cursor) -_SqliteData: TypeAlias = str | ReadableBuffer | int | float | None -# Data that is passed through adapters can be of any type accepted by an adapter. -_AdaptedInputData: TypeAlias = _SqliteData | Any -# The Mapping must really be a dict, but making it invariant is too annoying. -_Parameters: TypeAlias = SupportsLenAndGetItem[_AdaptedInputData] | Mapping[str, _AdaptedInputData] -_Adapter: TypeAlias = Callable[[_T], _SqliteData] -_Converter: TypeAlias = Callable[[bytes], Any] +if sys.version_info >= (3, 12): + from _sqlite3 import ( + LEGACY_TRANSACTION_CONTROL as LEGACY_TRANSACTION_CONTROL, + SQLITE_DBCONFIG_DEFENSIVE as SQLITE_DBCONFIG_DEFENSIVE, + SQLITE_DBCONFIG_DQS_DDL as SQLITE_DBCONFIG_DQS_DDL, + SQLITE_DBCONFIG_DQS_DML as SQLITE_DBCONFIG_DQS_DML, + SQLITE_DBCONFIG_ENABLE_FKEY as SQLITE_DBCONFIG_ENABLE_FKEY, + SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER as SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, + SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION as SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, + SQLITE_DBCONFIG_ENABLE_QPSG as SQLITE_DBCONFIG_ENABLE_QPSG, + SQLITE_DBCONFIG_ENABLE_TRIGGER as SQLITE_DBCONFIG_ENABLE_TRIGGER, + SQLITE_DBCONFIG_ENABLE_VIEW as SQLITE_DBCONFIG_ENABLE_VIEW, + SQLITE_DBCONFIG_LEGACY_ALTER_TABLE as SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, + SQLITE_DBCONFIG_LEGACY_FILE_FORMAT as SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, + SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE as SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, + SQLITE_DBCONFIG_RESET_DATABASE as SQLITE_DBCONFIG_RESET_DATABASE, + SQLITE_DBCONFIG_TRIGGER_EQP as SQLITE_DBCONFIG_TRIGGER_EQP, + SQLITE_DBCONFIG_TRUSTED_SCHEMA as SQLITE_DBCONFIG_TRUSTED_SCHEMA, + SQLITE_DBCONFIG_WRITABLE_SCHEMA as SQLITE_DBCONFIG_WRITABLE_SCHEMA, + ) + +if sys.version_info >= (3, 11): + from _sqlite3 import ( + SQLITE_ABORT as SQLITE_ABORT, + SQLITE_ABORT_ROLLBACK as SQLITE_ABORT_ROLLBACK, + SQLITE_AUTH as SQLITE_AUTH, + SQLITE_AUTH_USER as SQLITE_AUTH_USER, + SQLITE_BUSY as SQLITE_BUSY, + SQLITE_BUSY_RECOVERY as SQLITE_BUSY_RECOVERY, + SQLITE_BUSY_SNAPSHOT as SQLITE_BUSY_SNAPSHOT, + SQLITE_BUSY_TIMEOUT as SQLITE_BUSY_TIMEOUT, + SQLITE_CANTOPEN as SQLITE_CANTOPEN, + SQLITE_CANTOPEN_CONVPATH as SQLITE_CANTOPEN_CONVPATH, + SQLITE_CANTOPEN_DIRTYWAL as SQLITE_CANTOPEN_DIRTYWAL, + SQLITE_CANTOPEN_FULLPATH as SQLITE_CANTOPEN_FULLPATH, + SQLITE_CANTOPEN_ISDIR as SQLITE_CANTOPEN_ISDIR, + SQLITE_CANTOPEN_NOTEMPDIR as SQLITE_CANTOPEN_NOTEMPDIR, + SQLITE_CANTOPEN_SYMLINK as SQLITE_CANTOPEN_SYMLINK, + SQLITE_CONSTRAINT as SQLITE_CONSTRAINT, + SQLITE_CONSTRAINT_CHECK as SQLITE_CONSTRAINT_CHECK, + SQLITE_CONSTRAINT_COMMITHOOK as SQLITE_CONSTRAINT_COMMITHOOK, + SQLITE_CONSTRAINT_FOREIGNKEY as SQLITE_CONSTRAINT_FOREIGNKEY, + SQLITE_CONSTRAINT_FUNCTION as SQLITE_CONSTRAINT_FUNCTION, + SQLITE_CONSTRAINT_NOTNULL as SQLITE_CONSTRAINT_NOTNULL, + SQLITE_CONSTRAINT_PINNED as SQLITE_CONSTRAINT_PINNED, + SQLITE_CONSTRAINT_PRIMARYKEY as SQLITE_CONSTRAINT_PRIMARYKEY, + SQLITE_CONSTRAINT_ROWID as SQLITE_CONSTRAINT_ROWID, + SQLITE_CONSTRAINT_TRIGGER as SQLITE_CONSTRAINT_TRIGGER, + SQLITE_CONSTRAINT_UNIQUE as SQLITE_CONSTRAINT_UNIQUE, + SQLITE_CONSTRAINT_VTAB as SQLITE_CONSTRAINT_VTAB, + SQLITE_CORRUPT as SQLITE_CORRUPT, + SQLITE_CORRUPT_INDEX as SQLITE_CORRUPT_INDEX, + SQLITE_CORRUPT_SEQUENCE as SQLITE_CORRUPT_SEQUENCE, + SQLITE_CORRUPT_VTAB as SQLITE_CORRUPT_VTAB, + SQLITE_EMPTY as SQLITE_EMPTY, + SQLITE_ERROR as SQLITE_ERROR, + SQLITE_ERROR_MISSING_COLLSEQ as SQLITE_ERROR_MISSING_COLLSEQ, + SQLITE_ERROR_RETRY as SQLITE_ERROR_RETRY, + SQLITE_ERROR_SNAPSHOT as SQLITE_ERROR_SNAPSHOT, + SQLITE_FORMAT as SQLITE_FORMAT, + SQLITE_FULL as SQLITE_FULL, + SQLITE_INTERNAL as SQLITE_INTERNAL, + SQLITE_INTERRUPT as SQLITE_INTERRUPT, + SQLITE_IOERR as SQLITE_IOERR, + SQLITE_IOERR_ACCESS as SQLITE_IOERR_ACCESS, + SQLITE_IOERR_AUTH as SQLITE_IOERR_AUTH, + SQLITE_IOERR_BEGIN_ATOMIC as SQLITE_IOERR_BEGIN_ATOMIC, + SQLITE_IOERR_BLOCKED as SQLITE_IOERR_BLOCKED, + SQLITE_IOERR_CHECKRESERVEDLOCK as SQLITE_IOERR_CHECKRESERVEDLOCK, + SQLITE_IOERR_CLOSE as SQLITE_IOERR_CLOSE, + SQLITE_IOERR_COMMIT_ATOMIC as SQLITE_IOERR_COMMIT_ATOMIC, + SQLITE_IOERR_CONVPATH as SQLITE_IOERR_CONVPATH, + SQLITE_IOERR_CORRUPTFS as SQLITE_IOERR_CORRUPTFS, + SQLITE_IOERR_DATA as SQLITE_IOERR_DATA, + SQLITE_IOERR_DELETE as SQLITE_IOERR_DELETE, + SQLITE_IOERR_DELETE_NOENT as SQLITE_IOERR_DELETE_NOENT, + SQLITE_IOERR_DIR_CLOSE as SQLITE_IOERR_DIR_CLOSE, + SQLITE_IOERR_DIR_FSYNC as SQLITE_IOERR_DIR_FSYNC, + SQLITE_IOERR_FSTAT as SQLITE_IOERR_FSTAT, + SQLITE_IOERR_FSYNC as SQLITE_IOERR_FSYNC, + SQLITE_IOERR_GETTEMPPATH as SQLITE_IOERR_GETTEMPPATH, + SQLITE_IOERR_LOCK as SQLITE_IOERR_LOCK, + SQLITE_IOERR_MMAP as SQLITE_IOERR_MMAP, + SQLITE_IOERR_NOMEM as SQLITE_IOERR_NOMEM, + SQLITE_IOERR_RDLOCK as SQLITE_IOERR_RDLOCK, + SQLITE_IOERR_READ as SQLITE_IOERR_READ, + SQLITE_IOERR_ROLLBACK_ATOMIC as SQLITE_IOERR_ROLLBACK_ATOMIC, + SQLITE_IOERR_SEEK as SQLITE_IOERR_SEEK, + SQLITE_IOERR_SHMLOCK as SQLITE_IOERR_SHMLOCK, + SQLITE_IOERR_SHMMAP as SQLITE_IOERR_SHMMAP, + SQLITE_IOERR_SHMOPEN as SQLITE_IOERR_SHMOPEN, + SQLITE_IOERR_SHMSIZE as SQLITE_IOERR_SHMSIZE, + SQLITE_IOERR_SHORT_READ as SQLITE_IOERR_SHORT_READ, + SQLITE_IOERR_TRUNCATE as SQLITE_IOERR_TRUNCATE, + SQLITE_IOERR_UNLOCK as SQLITE_IOERR_UNLOCK, + SQLITE_IOERR_VNODE as SQLITE_IOERR_VNODE, + SQLITE_IOERR_WRITE as SQLITE_IOERR_WRITE, + SQLITE_LIMIT_ATTACHED as SQLITE_LIMIT_ATTACHED, + SQLITE_LIMIT_COLUMN as SQLITE_LIMIT_COLUMN, + SQLITE_LIMIT_COMPOUND_SELECT as SQLITE_LIMIT_COMPOUND_SELECT, + SQLITE_LIMIT_EXPR_DEPTH as SQLITE_LIMIT_EXPR_DEPTH, + SQLITE_LIMIT_FUNCTION_ARG as SQLITE_LIMIT_FUNCTION_ARG, + SQLITE_LIMIT_LENGTH as SQLITE_LIMIT_LENGTH, + SQLITE_LIMIT_LIKE_PATTERN_LENGTH as SQLITE_LIMIT_LIKE_PATTERN_LENGTH, + SQLITE_LIMIT_SQL_LENGTH as SQLITE_LIMIT_SQL_LENGTH, + SQLITE_LIMIT_TRIGGER_DEPTH as SQLITE_LIMIT_TRIGGER_DEPTH, + SQLITE_LIMIT_VARIABLE_NUMBER as SQLITE_LIMIT_VARIABLE_NUMBER, + SQLITE_LIMIT_VDBE_OP as SQLITE_LIMIT_VDBE_OP, + SQLITE_LIMIT_WORKER_THREADS as SQLITE_LIMIT_WORKER_THREADS, + SQLITE_LOCKED as SQLITE_LOCKED, + SQLITE_LOCKED_SHAREDCACHE as SQLITE_LOCKED_SHAREDCACHE, + SQLITE_LOCKED_VTAB as SQLITE_LOCKED_VTAB, + SQLITE_MISMATCH as SQLITE_MISMATCH, + SQLITE_MISUSE as SQLITE_MISUSE, + SQLITE_NOLFS as SQLITE_NOLFS, + SQLITE_NOMEM as SQLITE_NOMEM, + SQLITE_NOTADB as SQLITE_NOTADB, + SQLITE_NOTFOUND as SQLITE_NOTFOUND, + SQLITE_NOTICE as SQLITE_NOTICE, + SQLITE_NOTICE_RECOVER_ROLLBACK as SQLITE_NOTICE_RECOVER_ROLLBACK, + SQLITE_NOTICE_RECOVER_WAL as SQLITE_NOTICE_RECOVER_WAL, + SQLITE_OK_LOAD_PERMANENTLY as SQLITE_OK_LOAD_PERMANENTLY, + SQLITE_OK_SYMLINK as SQLITE_OK_SYMLINK, + SQLITE_PERM as SQLITE_PERM, + SQLITE_PROTOCOL as SQLITE_PROTOCOL, + SQLITE_RANGE as SQLITE_RANGE, + SQLITE_READONLY as SQLITE_READONLY, + SQLITE_READONLY_CANTINIT as SQLITE_READONLY_CANTINIT, + SQLITE_READONLY_CANTLOCK as SQLITE_READONLY_CANTLOCK, + SQLITE_READONLY_DBMOVED as SQLITE_READONLY_DBMOVED, + SQLITE_READONLY_DIRECTORY as SQLITE_READONLY_DIRECTORY, + SQLITE_READONLY_RECOVERY as SQLITE_READONLY_RECOVERY, + SQLITE_READONLY_ROLLBACK as SQLITE_READONLY_ROLLBACK, + SQLITE_ROW as SQLITE_ROW, + SQLITE_SCHEMA as SQLITE_SCHEMA, + SQLITE_TOOBIG as SQLITE_TOOBIG, + SQLITE_WARNING as SQLITE_WARNING, + SQLITE_WARNING_AUTOINDEX as SQLITE_WARNING_AUTOINDEX, + ) + from sqlite3 import Blob as Blob + +if sys.version_info < (3, 14): + # Deprecated and removed from _sqlite3 in 3.12, but removed from here in 3.14. + version: str + +if sys.version_info < (3, 12): + if sys.version_info >= (3, 10): + # deprecation wrapper that has a different name for the argument... + def enable_shared_cache(enable: int) -> None: ... + else: + from _sqlite3 import enable_shared_cache as enable_shared_cache + +if sys.version_info < (3, 10): + from _sqlite3 import OptimizedUnicode as OptimizedUnicode paramstyle: str threadsafety: int @@ -35,527 +239,3 @@ if sys.version_info < (3, 14): sqlite_version_info: tuple[int, int, int] Binary = memoryview - -# The remaining definitions are imported from _sqlite3. - -PARSE_COLNAMES: Final[int] -PARSE_DECLTYPES: Final[int] -SQLITE_ALTER_TABLE: Final[int] -SQLITE_ANALYZE: Final[int] -SQLITE_ATTACH: Final[int] -SQLITE_CREATE_INDEX: Final[int] -SQLITE_CREATE_TABLE: Final[int] -SQLITE_CREATE_TEMP_INDEX: Final[int] -SQLITE_CREATE_TEMP_TABLE: Final[int] -SQLITE_CREATE_TEMP_TRIGGER: Final[int] -SQLITE_CREATE_TEMP_VIEW: Final[int] -SQLITE_CREATE_TRIGGER: Final[int] -SQLITE_CREATE_VIEW: Final[int] -SQLITE_CREATE_VTABLE: Final[int] -SQLITE_DELETE: Final[int] -SQLITE_DENY: Final[int] -SQLITE_DETACH: Final[int] -SQLITE_DONE: Final[int] -SQLITE_DROP_INDEX: Final[int] -SQLITE_DROP_TABLE: Final[int] -SQLITE_DROP_TEMP_INDEX: Final[int] -SQLITE_DROP_TEMP_TABLE: Final[int] -SQLITE_DROP_TEMP_TRIGGER: Final[int] -SQLITE_DROP_TEMP_VIEW: Final[int] -SQLITE_DROP_TRIGGER: Final[int] -SQLITE_DROP_VIEW: Final[int] -SQLITE_DROP_VTABLE: Final[int] -SQLITE_FUNCTION: Final[int] -SQLITE_IGNORE: Final[int] -SQLITE_INSERT: Final[int] -SQLITE_OK: Final[int] -if sys.version_info >= (3, 11): - SQLITE_LIMIT_LENGTH: Final[int] - SQLITE_LIMIT_SQL_LENGTH: Final[int] - SQLITE_LIMIT_COLUMN: Final[int] - SQLITE_LIMIT_EXPR_DEPTH: Final[int] - SQLITE_LIMIT_COMPOUND_SELECT: Final[int] - SQLITE_LIMIT_VDBE_OP: Final[int] - SQLITE_LIMIT_FUNCTION_ARG: Final[int] - SQLITE_LIMIT_ATTACHED: Final[int] - SQLITE_LIMIT_LIKE_PATTERN_LENGTH: Final[int] - SQLITE_LIMIT_VARIABLE_NUMBER: Final[int] - SQLITE_LIMIT_TRIGGER_DEPTH: Final[int] - SQLITE_LIMIT_WORKER_THREADS: Final[int] -SQLITE_PRAGMA: Final[int] -SQLITE_READ: Final[int] -SQLITE_REINDEX: Final[int] -SQLITE_RECURSIVE: Final[int] -SQLITE_SAVEPOINT: Final[int] -SQLITE_SELECT: Final[int] -SQLITE_TRANSACTION: Final[int] -SQLITE_UPDATE: Final[int] -adapters: dict[tuple[type[Any], type[Any]], _Adapter[Any]] -converters: dict[str, _Converter] -sqlite_version: str - -if sys.version_info < (3, 14): - # Deprecated in 3.12, removed in 3.14. - version: str - -if sys.version_info >= (3, 11): - SQLITE_ABORT: Final[int] - SQLITE_ABORT_ROLLBACK: Final[int] - SQLITE_AUTH: Final[int] - SQLITE_AUTH_USER: Final[int] - SQLITE_BUSY: Final[int] - SQLITE_BUSY_RECOVERY: Final[int] - SQLITE_BUSY_SNAPSHOT: Final[int] - SQLITE_BUSY_TIMEOUT: Final[int] - SQLITE_CANTOPEN: Final[int] - SQLITE_CANTOPEN_CONVPATH: Final[int] - SQLITE_CANTOPEN_DIRTYWAL: Final[int] - SQLITE_CANTOPEN_FULLPATH: Final[int] - SQLITE_CANTOPEN_ISDIR: Final[int] - SQLITE_CANTOPEN_NOTEMPDIR: Final[int] - SQLITE_CANTOPEN_SYMLINK: Final[int] - SQLITE_CONSTRAINT: Final[int] - SQLITE_CONSTRAINT_CHECK: Final[int] - SQLITE_CONSTRAINT_COMMITHOOK: Final[int] - SQLITE_CONSTRAINT_FOREIGNKEY: Final[int] - SQLITE_CONSTRAINT_FUNCTION: Final[int] - SQLITE_CONSTRAINT_NOTNULL: Final[int] - SQLITE_CONSTRAINT_PINNED: Final[int] - SQLITE_CONSTRAINT_PRIMARYKEY: Final[int] - SQLITE_CONSTRAINT_ROWID: Final[int] - SQLITE_CONSTRAINT_TRIGGER: Final[int] - SQLITE_CONSTRAINT_UNIQUE: Final[int] - SQLITE_CONSTRAINT_VTAB: Final[int] - SQLITE_CORRUPT: Final[int] - SQLITE_CORRUPT_INDEX: Final[int] - SQLITE_CORRUPT_SEQUENCE: Final[int] - SQLITE_CORRUPT_VTAB: Final[int] - SQLITE_EMPTY: Final[int] - SQLITE_ERROR: Final[int] - SQLITE_ERROR_MISSING_COLLSEQ: Final[int] - SQLITE_ERROR_RETRY: Final[int] - SQLITE_ERROR_SNAPSHOT: Final[int] - SQLITE_FORMAT: Final[int] - SQLITE_FULL: Final[int] - SQLITE_INTERNAL: Final[int] - SQLITE_INTERRUPT: Final[int] - SQLITE_IOERR: Final[int] - SQLITE_IOERR_ACCESS: Final[int] - SQLITE_IOERR_AUTH: Final[int] - SQLITE_IOERR_BEGIN_ATOMIC: Final[int] - SQLITE_IOERR_BLOCKED: Final[int] - SQLITE_IOERR_CHECKRESERVEDLOCK: Final[int] - SQLITE_IOERR_CLOSE: Final[int] - SQLITE_IOERR_COMMIT_ATOMIC: Final[int] - SQLITE_IOERR_CONVPATH: Final[int] - SQLITE_IOERR_CORRUPTFS: Final[int] - SQLITE_IOERR_DATA: Final[int] - SQLITE_IOERR_DELETE: Final[int] - SQLITE_IOERR_DELETE_NOENT: Final[int] - SQLITE_IOERR_DIR_CLOSE: Final[int] - SQLITE_IOERR_DIR_FSYNC: Final[int] - SQLITE_IOERR_FSTAT: Final[int] - SQLITE_IOERR_FSYNC: Final[int] - SQLITE_IOERR_GETTEMPPATH: Final[int] - SQLITE_IOERR_LOCK: Final[int] - SQLITE_IOERR_MMAP: Final[int] - SQLITE_IOERR_NOMEM: Final[int] - SQLITE_IOERR_RDLOCK: Final[int] - SQLITE_IOERR_READ: Final[int] - SQLITE_IOERR_ROLLBACK_ATOMIC: Final[int] - SQLITE_IOERR_SEEK: Final[int] - SQLITE_IOERR_SHMLOCK: Final[int] - SQLITE_IOERR_SHMMAP: Final[int] - SQLITE_IOERR_SHMOPEN: Final[int] - SQLITE_IOERR_SHMSIZE: Final[int] - SQLITE_IOERR_SHORT_READ: Final[int] - SQLITE_IOERR_TRUNCATE: Final[int] - SQLITE_IOERR_UNLOCK: Final[int] - SQLITE_IOERR_VNODE: Final[int] - SQLITE_IOERR_WRITE: Final[int] - SQLITE_LOCKED: Final[int] - SQLITE_LOCKED_SHAREDCACHE: Final[int] - SQLITE_LOCKED_VTAB: Final[int] - SQLITE_MISMATCH: Final[int] - SQLITE_MISUSE: Final[int] - SQLITE_NOLFS: Final[int] - SQLITE_NOMEM: Final[int] - SQLITE_NOTADB: Final[int] - SQLITE_NOTFOUND: Final[int] - SQLITE_NOTICE: Final[int] - SQLITE_NOTICE_RECOVER_ROLLBACK: Final[int] - SQLITE_NOTICE_RECOVER_WAL: Final[int] - SQLITE_OK_LOAD_PERMANENTLY: Final[int] - SQLITE_OK_SYMLINK: Final[int] - SQLITE_PERM: Final[int] - SQLITE_PROTOCOL: Final[int] - SQLITE_RANGE: Final[int] - SQLITE_READONLY: Final[int] - SQLITE_READONLY_CANTINIT: Final[int] - SQLITE_READONLY_CANTLOCK: Final[int] - SQLITE_READONLY_DBMOVED: Final[int] - SQLITE_READONLY_DIRECTORY: Final[int] - SQLITE_READONLY_RECOVERY: Final[int] - SQLITE_READONLY_ROLLBACK: Final[int] - SQLITE_ROW: Final[int] - SQLITE_SCHEMA: Final[int] - SQLITE_TOOBIG: Final[int] - SQLITE_WARNING: Final[int] - SQLITE_WARNING_AUTOINDEX: Final[int] - -if sys.version_info >= (3, 12): - LEGACY_TRANSACTION_CONTROL: Final[int] - SQLITE_DBCONFIG_DEFENSIVE: Final[int] - SQLITE_DBCONFIG_DQS_DDL: Final[int] - SQLITE_DBCONFIG_DQS_DML: Final[int] - SQLITE_DBCONFIG_ENABLE_FKEY: Final[int] - SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: Final[int] - SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: Final[int] - SQLITE_DBCONFIG_ENABLE_QPSG: Final[int] - SQLITE_DBCONFIG_ENABLE_TRIGGER: Final[int] - SQLITE_DBCONFIG_ENABLE_VIEW: Final[int] - SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: Final[int] - SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: Final[int] - SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: Final[int] - SQLITE_DBCONFIG_RESET_DATABASE: Final[int] - SQLITE_DBCONFIG_TRIGGER_EQP: Final[int] - SQLITE_DBCONFIG_TRUSTED_SCHEMA: Final[int] - SQLITE_DBCONFIG_WRITABLE_SCHEMA: Final[int] - -# Can take or return anything depending on what's in the registry. -@overload -def adapt(obj: Any, proto: Any, /) -> Any: ... -@overload -def adapt(obj: Any, proto: Any, alt: _T, /) -> Any | _T: ... -def complete_statement(statement: str) -> bool: ... - -if sys.version_info >= (3, 12): - @overload - def connect( - database: StrOrBytesPath, - timeout: float = 5.0, - detect_types: int = 0, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", - check_same_thread: bool = True, - cached_statements: int = 128, - uri: bool = False, - *, - autocommit: bool = ..., - ) -> Connection: ... - @overload - def connect( - database: StrOrBytesPath, - timeout: float, - detect_types: int, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None, - check_same_thread: bool, - factory: type[_ConnectionT], - cached_statements: int = 128, - uri: bool = False, - *, - autocommit: bool = ..., - ) -> _ConnectionT: ... - @overload - def connect( - database: StrOrBytesPath, - timeout: float = 5.0, - detect_types: int = 0, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", - check_same_thread: bool = True, - *, - factory: type[_ConnectionT], - cached_statements: int = 128, - uri: bool = False, - autocommit: bool = ..., - ) -> _ConnectionT: ... - -else: - @overload - def connect( - database: StrOrBytesPath, - timeout: float = 5.0, - detect_types: int = 0, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", - check_same_thread: bool = True, - cached_statements: int = 128, - uri: bool = False, - ) -> Connection: ... - @overload - def connect( - database: StrOrBytesPath, - timeout: float, - detect_types: int, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None, - check_same_thread: bool, - factory: type[_ConnectionT], - cached_statements: int = 128, - uri: bool = False, - ) -> _ConnectionT: ... - @overload - def connect( - database: StrOrBytesPath, - timeout: float = 5.0, - detect_types: int = 0, - isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED", - check_same_thread: bool = True, - *, - factory: type[_ConnectionT], - cached_statements: int = 128, - uri: bool = False, - ) -> _ConnectionT: ... - -def enable_callback_tracebacks(enable: bool, /) -> None: ... - -if sys.version_info < (3, 12): - # takes a pos-or-keyword argument because there is a C wrapper - def enable_shared_cache(enable: int) -> None: ... - -if sys.version_info >= (3, 10): - def register_adapter(type: type[_T], adapter: _Adapter[_T], /) -> None: ... - def register_converter(typename: str, converter: _Converter, /) -> None: ... - -else: - def register_adapter(type: type[_T], caster: _Adapter[_T], /) -> None: ... - def register_converter(name: str, converter: _Converter, /) -> None: ... - -class _AggregateProtocol(Protocol): - def step(self, value: int, /) -> object: ... - def finalize(self) -> int: ... - -class _SingleParamWindowAggregateClass(Protocol): - def step(self, param: Any, /) -> object: ... - def inverse(self, param: Any, /) -> object: ... - def value(self) -> _SqliteData: ... - def finalize(self) -> _SqliteData: ... - -class _AnyParamWindowAggregateClass(Protocol): - def step(self, *args: Any) -> object: ... - def inverse(self, *args: Any) -> object: ... - def value(self) -> _SqliteData: ... - def finalize(self) -> _SqliteData: ... - -class _WindowAggregateClass(Protocol): - step: Callable[..., object] - inverse: Callable[..., object] - def value(self) -> _SqliteData: ... - def finalize(self) -> _SqliteData: ... - -class Connection: - @property - def DataError(self) -> type[sqlite3.DataError]: ... - @property - def DatabaseError(self) -> type[sqlite3.DatabaseError]: ... - @property - def Error(self) -> type[sqlite3.Error]: ... - @property - def IntegrityError(self) -> type[sqlite3.IntegrityError]: ... - @property - def InterfaceError(self) -> type[sqlite3.InterfaceError]: ... - @property - def InternalError(self) -> type[sqlite3.InternalError]: ... - @property - def NotSupportedError(self) -> type[sqlite3.NotSupportedError]: ... - @property - def OperationalError(self) -> type[sqlite3.OperationalError]: ... - @property - def ProgrammingError(self) -> type[sqlite3.ProgrammingError]: ... - @property - def Warning(self) -> type[sqlite3.Warning]: ... - @property - def in_transaction(self) -> bool: ... - isolation_level: str | None # one of '', 'DEFERRED', 'IMMEDIATE' or 'EXCLUSIVE' - @property - def total_changes(self) -> int: ... - if sys.version_info >= (3, 12): - @property - def autocommit(self) -> int: ... - @autocommit.setter - def autocommit(self, val: int) -> None: ... - row_factory: Any - text_factory: Any - if sys.version_info >= (3, 12): - def __init__( - self, - database: StrOrBytesPath, - timeout: float = ..., - detect_types: int = ..., - isolation_level: str | None = ..., - check_same_thread: bool = ..., - factory: type[Connection] | None = ..., - cached_statements: int = ..., - uri: bool = ..., - autocommit: bool = ..., - ) -> None: ... - else: - def __init__( - self, - database: StrOrBytesPath, - timeout: float = ..., - detect_types: int = ..., - isolation_level: str | None = ..., - check_same_thread: bool = ..., - factory: type[Connection] | None = ..., - cached_statements: int = ..., - uri: bool = ..., - ) -> None: ... - - def close(self) -> None: ... - if sys.version_info >= (3, 11): - def blobopen(self, table: str, column: str, row: int, /, *, readonly: bool = False, name: str = "main") -> Blob: ... - - def commit(self) -> None: ... - def create_aggregate(self, name: str, n_arg: int, aggregate_class: Callable[[], _AggregateProtocol]) -> None: ... - if sys.version_info >= (3, 11): - # num_params determines how many params will be passed to the aggregate class. We provide an overload - # for the case where num_params = 1, which is expected to be the common case. - @overload - def create_window_function( - self, name: str, num_params: Literal[1], aggregate_class: Callable[[], _SingleParamWindowAggregateClass] | None, / - ) -> None: ... - # And for num_params = -1, which means the aggregate must accept any number of parameters. - @overload - def create_window_function( - self, name: str, num_params: Literal[-1], aggregate_class: Callable[[], _AnyParamWindowAggregateClass] | None, / - ) -> None: ... - @overload - def create_window_function( - self, name: str, num_params: int, aggregate_class: Callable[[], _WindowAggregateClass] | None, / - ) -> None: ... - - def create_collation(self, name: str, callback: Callable[[str, str], int | SupportsIndex] | None, /) -> None: ... - def create_function( - self, name: str, narg: int, func: Callable[..., _SqliteData] | None, *, deterministic: bool = False - ) -> None: ... - @overload - def cursor(self, factory: None = None) -> Cursor: ... - @overload - def cursor(self, factory: Callable[[Connection], _CursorT]) -> _CursorT: ... - def execute(self, sql: str, parameters: _Parameters = ..., /) -> Cursor: ... - def executemany(self, sql: str, parameters: Iterable[_Parameters], /) -> Cursor: ... - def executescript(self, sql_script: str, /) -> Cursor: ... - def interrupt(self) -> None: ... - if sys.version_info >= (3, 13): - def iterdump(self, *, filter: str | None = None) -> Generator[str, None, None]: ... - else: - def iterdump(self) -> Generator[str, None, None]: ... - - def rollback(self) -> None: ... - def set_authorizer( - self, authorizer_callback: Callable[[int, str | None, str | None, str | None, str | None], int] | None - ) -> None: ... - def set_progress_handler(self, progress_handler: Callable[[], int | None] | None, n: int) -> None: ... - def set_trace_callback(self, trace_callback: Callable[[str], object] | None) -> None: ... - # enable_load_extension and load_extension is not available on python distributions compiled - # without sqlite3 loadable extension support. see footnotes https://docs.python.org/3/library/sqlite3.html#f1 - def enable_load_extension(self, enable: bool, /) -> None: ... - def load_extension(self, name: str, /) -> None: ... - def backup( - self, - target: Connection, - *, - pages: int = -1, - progress: Callable[[int, int, int], object] | None = None, - name: str = "main", - sleep: float = 0.25, - ) -> None: ... - if sys.version_info >= (3, 11): - def setlimit(self, category: int, limit: int, /) -> int: ... - def getlimit(self, category: int, /) -> int: ... - def serialize(self, *, name: str = "main") -> bytes: ... - def deserialize(self, data: ReadableBuffer, /, *, name: str = "main") -> None: ... - if sys.version_info >= (3, 12): - def getconfig(self, op: int, /) -> bool: ... - def setconfig(self, op: int, enable: bool = True, /) -> bool: ... - - def __call__(self, sql: str, /) -> _Statement: ... - def __enter__(self) -> Self: ... - def __exit__( - self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None, / - ) -> Literal[False]: ... - -class Cursor(Iterator[Any]): - arraysize: int - @property - def connection(self) -> Connection: ... - # May be None, but using | Any instead to avoid slightly annoying false positives. - @property - def description(self) -> tuple[tuple[str, None, None, None, None, None, None], ...] | Any: ... - @property - def lastrowid(self) -> int | None: ... - row_factory: Callable[[Cursor, Row], object] | None - @property - def rowcount(self) -> int: ... - def __init__(self, cursor: Connection, /) -> None: ... - def close(self) -> None: ... - def execute(self, sql: str, parameters: _Parameters = (), /) -> Self: ... - def executemany(self, sql: str, seq_of_parameters: Iterable[_Parameters], /) -> Self: ... - def executescript(self, sql_script: str, /) -> Cursor: ... - def fetchall(self) -> list[Any]: ... - def fetchmany(self, size: int | None = 1) -> list[Any]: ... - # Returns either a row (as created by the row_factory) or None, but - # putting None in the return annotation causes annoying false positives. - def fetchone(self) -> Any: ... - def setinputsizes(self, sizes: Unused, /) -> None: ... # does nothing - def setoutputsize(self, size: Unused, column: Unused = None, /) -> None: ... # does nothing - def __iter__(self) -> Self: ... - def __next__(self) -> Any: ... - -class Error(Exception): - if sys.version_info >= (3, 11): - sqlite_errorcode: int - sqlite_errorname: str - -class DatabaseError(Error): ... -class DataError(DatabaseError): ... -class IntegrityError(DatabaseError): ... -class InterfaceError(Error): ... -class InternalError(DatabaseError): ... -class NotSupportedError(DatabaseError): ... -class OperationalError(DatabaseError): ... - -if sys.version_info < (3, 10): - OptimizedUnicode = str - -@final -class PrepareProtocol: - def __init__(self, *args: object, **kwargs: object) -> None: ... - -class ProgrammingError(DatabaseError): ... - -class Row: - def __init__(self, cursor: Cursor, data: tuple[Any, ...], /) -> None: ... - def keys(self) -> list[str]: ... - @overload - def __getitem__(self, key: int | str, /) -> Any: ... - @overload - def __getitem__(self, key: slice, /) -> tuple[Any, ...]: ... - def __hash__(self) -> int: ... - def __iter__(self) -> Iterator[Any]: ... - def __len__(self) -> int: ... - # These return NotImplemented for anything that is not a Row. - def __eq__(self, value: object, /) -> bool: ... - def __ge__(self, value: object, /) -> bool: ... - def __gt__(self, value: object, /) -> bool: ... - def __le__(self, value: object, /) -> bool: ... - def __lt__(self, value: object, /) -> bool: ... - def __ne__(self, value: object, /) -> bool: ... - -@final -class _Statement: ... - -class Warning(Exception): ... - -if sys.version_info >= (3, 11): - @final - class Blob: - def close(self) -> None: ... - def read(self, length: int = -1, /) -> bytes: ... - def write(self, data: ReadableBuffer, /) -> None: ... - def tell(self) -> int: ... - # whence must be one of os.SEEK_SET, os.SEEK_CUR, os.SEEK_END - def seek(self, offset: int, origin: int = 0, /) -> None: ... - def __len__(self) -> int: ... - def __enter__(self) -> Self: ... - def __exit__(self, type: object, val: object, tb: object, /) -> Literal[False]: ... - def __getitem__(self, key: SupportsIndex | slice, /) -> int: ... - def __setitem__(self, key: SupportsIndex | slice, value: int, /) -> None: ... diff --git a/mypy/typeshed/stdlib/sre_constants.pyi b/mypy/typeshed/stdlib/sre_constants.pyi index 0c1e484bb07e..383f0f7eb8bd 100644 --- a/mypy/typeshed/stdlib/sre_constants.pyi +++ b/mypy/typeshed/stdlib/sre_constants.pyi @@ -1,4 +1,5 @@ import sys +from re import error as error from typing import Any from typing_extensions import Self @@ -6,14 +7,6 @@ MAXGROUPS: int MAGIC: int -class error(Exception): - msg: str - pattern: str | bytes | None - pos: int | None - lineno: int - colno: int - def __init__(self, msg: str, pattern: str | bytes | None = None, pos: int | None = None) -> None: ... - class _NamedIntConstant(int): name: Any def __new__(cls, value: int, name: str) -> Self: ... diff --git a/mypy/typeshed/stdlib/ssl.pyi b/mypy/typeshed/stdlib/ssl.pyi index 81c68c69ec4e..1d97c02acc5e 100644 --- a/mypy/typeshed/stdlib/ssl.pyi +++ b/mypy/typeshed/stdlib/ssl.pyi @@ -1,18 +1,51 @@ import enum import socket import sys +from _ssl import ( + _DEFAULT_CIPHERS as _DEFAULT_CIPHERS, + _OPENSSL_API_VERSION as _OPENSSL_API_VERSION, + HAS_ALPN as HAS_ALPN, + HAS_ECDH as HAS_ECDH, + HAS_NPN as HAS_NPN, + HAS_SNI as HAS_SNI, + OPENSSL_VERSION as OPENSSL_VERSION, + OPENSSL_VERSION_INFO as OPENSSL_VERSION_INFO, + OPENSSL_VERSION_NUMBER as OPENSSL_VERSION_NUMBER, + HAS_SSLv2 as HAS_SSLv2, + HAS_SSLv3 as HAS_SSLv3, + HAS_TLSv1 as HAS_TLSv1, + HAS_TLSv1_1 as HAS_TLSv1_1, + HAS_TLSv1_2 as HAS_TLSv1_2, + HAS_TLSv1_3 as HAS_TLSv1_3, + MemoryBIO as MemoryBIO, + RAND_add as RAND_add, + RAND_bytes as RAND_bytes, + RAND_status as RAND_status, + SSLSession as SSLSession, + _PasswordType as _PasswordType, # typeshed only, but re-export for other type stubs to use + _SSLContext, +) from _typeshed import ReadableBuffer, StrOrBytesPath, WriteableBuffer from collections.abc import Callable, Iterable -from typing import Any, Literal, NamedTuple, TypedDict, final, overload +from typing import Any, Literal, NamedTuple, TypedDict, overload from typing_extensions import Never, Self, TypeAlias +if sys.version_info >= (3, 13): + from _ssl import HAS_PSK as HAS_PSK + +if sys.version_info < (3, 12): + from _ssl import RAND_pseudo_bytes as RAND_pseudo_bytes + +if sys.version_info < (3, 10): + from _ssl import RAND_egd as RAND_egd + +if sys.platform == "win32": + from _ssl import enum_certificates as enum_certificates, enum_crls as enum_crls + _PCTRTT: TypeAlias = tuple[tuple[str, str], ...] _PCTRTTT: TypeAlias = tuple[_PCTRTT, ...] _PeerCertRetDictType: TypeAlias = dict[str, str | _PCTRTTT | _PCTRTT] _PeerCertRetType: TypeAlias = _PeerCertRetDictType | bytes | None -_EnumRetType: TypeAlias = list[tuple[bytes, str, set[str] | bool]] -_PasswordType: TypeAlias = Callable[[], str | bytes | bytearray] | str | bytes | bytearray - _SrvnmeCbType: TypeAlias = Callable[[SSLSocket | SSLObject, str | None, SSLSocket], int | None] socket_error = OSError @@ -98,15 +131,6 @@ else: _create_default_https_context: Callable[..., SSLContext] -def RAND_bytes(n: int, /) -> bytes: ... - -if sys.version_info < (3, 12): - def RAND_pseudo_bytes(n: int, /) -> tuple[bytes, bool]: ... - -def RAND_status() -> bool: ... -def RAND_egd(path: str) -> None: ... -def RAND_add(string: str | ReadableBuffer, entropy: float, /) -> None: ... - if sys.version_info < (3, 12): def match_hostname(cert: _PeerCertRetDictType, hostname: str) -> None: ... @@ -133,10 +157,6 @@ class DefaultVerifyPaths(NamedTuple): def get_default_verify_paths() -> DefaultVerifyPaths: ... -if sys.platform == "win32": - def enum_certificates(store_name: str) -> _EnumRetType: ... - def enum_crls(store_name: str) -> _EnumRetType: ... - class VerifyMode(enum.IntEnum): CERT_NONE = 0 CERT_OPTIONAL = 1 @@ -229,21 +249,8 @@ if sys.version_info >= (3, 11) or sys.platform == "linux": OP_IGNORE_UNEXPECTED_EOF: Options HAS_NEVER_CHECK_COMMON_NAME: bool -HAS_SSLv2: bool -HAS_SSLv3: bool -HAS_TLSv1: bool -HAS_TLSv1_1: bool -HAS_TLSv1_2: bool -HAS_TLSv1_3: bool -HAS_ALPN: bool -HAS_ECDH: bool -HAS_SNI: bool -HAS_NPN: bool -CHANNEL_BINDING_TYPES: list[str] -OPENSSL_VERSION: str -OPENSSL_VERSION_INFO: tuple[int, int, int, int, int] -OPENSSL_VERSION_NUMBER: int +CHANNEL_BINDING_TYPES: list[str] class AlertDescription(enum.IntEnum): ALERT_DESCRIPTION_ACCESS_DENIED = 49 @@ -379,17 +386,15 @@ class TLSVersion(enum.IntEnum): TLSv1_2 = 771 TLSv1_3 = 772 -class SSLContext: - check_hostname: bool +class SSLContext(_SSLContext): options: Options verify_flags: VerifyFlags verify_mode: VerifyMode @property - def protocol(self) -> _SSLMethod: ... + def protocol(self) -> _SSLMethod: ... # type: ignore[override] hostname_checks_common_name: bool maximum_version: TLSVersion minimum_version: TLSVersion - sni_callback: Callable[[SSLObject, str, SSLContext], None | int] | None # The following two attributes have class-level defaults. # However, the docs explicitly state that it's OK to override these attributes on instances, # so making these ClassVars wouldn't be appropriate @@ -406,10 +411,6 @@ class SSLContext: else: def __new__(cls, protocol: int = ..., *args: Any, **kwargs: Any) -> Self: ... - def cert_store_stats(self) -> dict[str, int]: ... - def load_cert_chain( - self, certfile: StrOrBytesPath, keyfile: StrOrBytesPath | None = None, password: _PasswordType | None = None - ) -> None: ... def load_default_certs(self, purpose: Purpose = ...) -> None: ... def load_verify_locations( self, @@ -448,7 +449,6 @@ class SSLContext: server_hostname: str | bytes | None = None, session: SSLSession | None = None, ) -> SSLObject: ... - def session_stats(self) -> dict[str, int]: ... class SSLObject: context: SSLContext @@ -483,28 +483,6 @@ class SSLObject: def get_verified_chain(self) -> list[bytes]: ... def get_unverified_chain(self) -> list[bytes]: ... -@final -class MemoryBIO: - pending: int - eof: bool - def read(self, size: int = -1, /) -> bytes: ... - def write(self, b: ReadableBuffer, /) -> int: ... - def write_eof(self) -> None: ... - -@final -class SSLSession: - @property - def has_ticket(self) -> bool: ... - @property - def id(self) -> bytes: ... - @property - def ticket_lifetime_hint(self) -> int: ... - @property - def time(self) -> int: ... - @property - def timeout(self) -> int: ... - def __eq__(self, value: object, /) -> bool: ... - class SSLErrorNumber(enum.IntEnum): SSL_ERROR_EOF = 8 SSL_ERROR_INVALID_ERROR_CODE = 10 diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi index e46903bf610f..4d9ede57ea99 100644 --- a/mypy/typeshed/stdlib/tarfile.pyi +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -103,12 +103,10 @@ PAX_NAME_FIELDS: set[str] ENCODING: str -_FileCreationModes: TypeAlias = Literal["a", "w", "x"] - @overload def open( name: StrOrBytesPath | None = None, - mode: str = "r", + mode: Literal["r", "r:*", "r:", "r:gz", "r:bz2", "r:xz"] = "r", fileobj: IO[bytes] | None = None, bufsize: int = 10240, *, @@ -121,16 +119,124 @@ def open( pax_headers: Mapping[str, str] | None = ..., debug: int | None = ..., errorlevel: int | None = ..., - compresslevel: int | None = ..., +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None, + mode: Literal["x", "x:", "a", "a:", "w", "w:"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None = None, + *, + mode: Literal["x", "x:", "a", "a:", "w", "w:"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None, + mode: Literal["x:gz", "x:bz2", "w:gz", "w:bz2"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + compresslevel: int = 9, +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None = None, + *, + mode: Literal["x:gz", "x:bz2", "w:gz", "w:bz2"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + compresslevel: int = 9, +) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None, + mode: Literal["x:xz", "w:xz"], + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., preset: Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] | None = ..., ) -> TarFile: ... @overload def open( name: StrOrBytesPath | None = None, - mode: _FileCreationModes = ..., + *, + mode: Literal["x:xz", "w:xz"], fileobj: _Fileobj | None = None, bufsize: int = 10240, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + preset: Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] | None = ..., +) -> TarFile: ... + +# TODO: Temporary fallback for modes containing pipe characters. These don't +# work with mypy 1.10, but this should be fixed with mypy 1.11. +# https://github.com/python/typeshed/issues/12182 +@overload +def open( + name: StrOrBytesPath | None = None, *, + mode: str, + fileobj: IO[bytes] | None = None, + bufsize: int = 10240, format: int | None = ..., tarinfo: type[TarInfo] | None = ..., dereference: bool | None = ..., @@ -140,7 +246,6 @@ def open( pax_headers: Mapping[str, str] | None = ..., debug: int | None = ..., errorlevel: int | None = ..., - compresslevel: int | None = ..., preset: int | None = ..., ) -> TarFile: ... diff --git a/mypy/typeshed/stdlib/tokenize.pyi b/mypy/typeshed/stdlib/tokenize.pyi index 3d2a93865df8..e1c8fedee55c 100644 --- a/mypy/typeshed/stdlib/tokenize.pyi +++ b/mypy/typeshed/stdlib/tokenize.pyi @@ -133,7 +133,7 @@ class Untokenizer: def untokenize(iterable: Iterable[_Token]) -> Any: ... def detect_encoding(readline: Callable[[], bytes | bytearray]) -> tuple[str, Sequence[bytes]]: ... def tokenize(readline: Callable[[], bytes | bytearray]) -> Generator[TokenInfo, None, None]: ... -def generate_tokens(readline: Callable[[], str]) -> Generator[TokenInfo, None, None]: ... # undocumented +def generate_tokens(readline: Callable[[], str]) -> Generator[TokenInfo, None, None]: ... def open(filename: FileDescriptorOrPath) -> TextIO: ... def group(*choices: str) -> str: ... # undocumented def any(*choices: str) -> str: ... # undocumented diff --git a/mypy/typeshed/stdlib/traceback.pyi b/mypy/typeshed/stdlib/traceback.pyi index 075c0f4b9de8..1c4a59de66aa 100644 --- a/mypy/typeshed/stdlib/traceback.pyi +++ b/mypy/typeshed/stdlib/traceback.pyi @@ -231,7 +231,7 @@ class TracebackException: if sys.version_info >= (3, 11): def print(self, *, file: SupportsWrite[str] | None = None, chain: bool = True) -> None: ... -class FrameSummary(Iterable[Any]): +class FrameSummary: if sys.version_info >= (3, 11): def __init__( self, @@ -276,6 +276,8 @@ class FrameSummary(Iterable[Any]): def __getitem__(self, pos: Literal[3]) -> str | None: ... @overload def __getitem__(self, pos: int) -> Any: ... + @overload + def __getitem__(self, pos: slice) -> tuple[Any, ...]: ... def __iter__(self) -> Iterator[Any]: ... def __eq__(self, other: object) -> bool: ... def __len__(self) -> Literal[4]: ... diff --git a/mypy/typeshed/stdlib/types.pyi b/mypy/typeshed/stdlib/types.pyi index 0f6592a9883e..0c5a43617040 100644 --- a/mypy/typeshed/stdlib/types.pyi +++ b/mypy/typeshed/stdlib/types.pyi @@ -424,6 +424,8 @@ class MethodType: @property def __closure__(self) -> tuple[CellType, ...] | None: ... # inherited from the added function @property + def __code__(self) -> CodeType: ... # inherited from the added function + @property def __defaults__(self) -> tuple[Any, ...] | None: ... # inherited from the added function @property def __func__(self) -> Callable[..., Any]: ... diff --git a/mypy/typeshed/stdlib/unittest/mock.pyi b/mypy/typeshed/stdlib/unittest/mock.pyi index 1cfd38f540a4..193a4123c395 100644 --- a/mypy/typeshed/stdlib/unittest/mock.pyi +++ b/mypy/typeshed/stdlib/unittest/mock.pyi @@ -363,7 +363,7 @@ class _patcher: patch: _patcher -class MagicMixin: +class MagicMixin(Base): def __init__(self, *args: Any, **kw: Any) -> None: ... class NonCallableMagicMock(MagicMixin, NonCallableMock): ... @@ -393,7 +393,7 @@ class AsyncMock(AsyncMockMixin, AsyncMagicMixin, Mock): # But, `NonCallableMock` super-class has the better version. def reset_mock(self, visited: Any = None, *, return_value: bool = False, side_effect: bool = False) -> None: ... -class MagicProxy: +class MagicProxy(Base): name: str parent: Any def __init__(self, name: str, parent: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/unittest/suite.pyi b/mypy/typeshed/stdlib/unittest/suite.pyi index c10cbc75d7fd..ff583d0766a0 100644 --- a/mypy/typeshed/stdlib/unittest/suite.pyi +++ b/mypy/typeshed/stdlib/unittest/suite.pyi @@ -5,7 +5,7 @@ from typing_extensions import TypeAlias _TestType: TypeAlias = unittest.case.TestCase | TestSuite -class BaseTestSuite(Iterable[_TestType]): +class BaseTestSuite: _tests: list[unittest.case.TestCase] _removed_tests: int def __init__(self, tests: Iterable[_TestType] = ()) -> None: ... diff --git a/mypy/typeshed/stdlib/weakref.pyi b/mypy/typeshed/stdlib/weakref.pyi index aaba7ffc98d9..853caf3e8abb 100644 --- a/mypy/typeshed/stdlib/weakref.pyi +++ b/mypy/typeshed/stdlib/weakref.pyi @@ -1,19 +1,14 @@ import sys from _typeshed import SupportsKeysAndGetItem -from _weakref import ( - CallableProxyType as CallableProxyType, - ProxyType as ProxyType, - ReferenceType as ReferenceType, - getweakrefcount as getweakrefcount, - getweakrefs as getweakrefs, - proxy as proxy, - ref as ref, -) +from _weakref import getweakrefcount as getweakrefcount, getweakrefs as getweakrefs, proxy as proxy from _weakrefset import WeakSet as WeakSet from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping -from typing import Any, Generic, TypeVar, overload +from typing import Any, Generic, TypeVar, final, overload from typing_extensions import ParamSpec, Self +if sys.version_info >= (3, 9): + from types import GenericAlias + __all__ = [ "ref", "proxy", @@ -40,11 +35,39 @@ _P = ParamSpec("_P") ProxyTypes: tuple[type[Any], ...] +# These classes are implemented in C and imported from _weakref at runtime. However, +# they consider themselves to live in the weakref module for sys.version_info >= (3, 11), +# so defining their stubs here means we match their __module__ value. +# Prior to 3.11 they did not declare a module for themselves and ended up looking like they +# came from the builtin module at runtime, which was just wrong, and we won't attempt to +# duplicate that. + +@final +class CallableProxyType(Generic[_CallableT]): # "weakcallableproxy" + def __eq__(self, value: object, /) -> bool: ... + def __getattr__(self, attr: str) -> Any: ... + __call__: _CallableT + +@final +class ProxyType(Generic[_T]): # "weakproxy" + def __eq__(self, value: object, /) -> bool: ... + def __getattr__(self, attr: str) -> Any: ... + +class ReferenceType(Generic[_T]): # "weakref" + __callback__: Callable[[Self], Any] + def __new__(cls, o: _T, callback: Callable[[Self], Any] | None = ..., /) -> Self: ... + def __call__(self) -> _T | None: ... + def __eq__(self, value: object, /) -> bool: ... + def __hash__(self) -> int: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... + +ref = ReferenceType + +# everything below here is implemented in weakref.py + class WeakMethod(ref[_CallableT]): - # `ref` is implemented in `C` so positional-only arguments are enforced, but not in `WeakMethod`. - def __new__( # pyright: ignore[reportInconsistentConstructor] - cls, meth: _CallableT, callback: Callable[[Self], Any] | None = None - ) -> Self: ... + def __new__(cls, meth: _CallableT, callback: Callable[[Self], Any] | None = None) -> Self: ... def __call__(self) -> _CallableT | None: ... def __eq__(self, other: object) -> bool: ... def __ne__(self, other: object) -> bool: ... @@ -103,8 +126,8 @@ class WeakValueDictionary(MutableMapping[_KT, _VT]): class KeyedRef(ref[_T], Generic[_KT, _T]): key: _KT - def __new__(type, ob: _T, callback: Callable[[_T], Any], key: _KT) -> Self: ... - def __init__(self, ob: _T, callback: Callable[[_T], Any], key: _KT) -> None: ... + def __new__(type, ob: _T, callback: Callable[[Self], Any], key: _KT) -> Self: ... + def __init__(self, ob: _T, callback: Callable[[Self], Any], key: _KT) -> None: ... class WeakKeyDictionary(MutableMapping[_KT, _VT]): @overload diff --git a/mypy/typeshed/stdlib/xml/parsers/expat/__init__.pyi b/mypy/typeshed/stdlib/xml/parsers/expat/__init__.pyi index 73f3758c61ec..d9b7ea536999 100644 --- a/mypy/typeshed/stdlib/xml/parsers/expat/__init__.pyi +++ b/mypy/typeshed/stdlib/xml/parsers/expat/__init__.pyi @@ -1 +1,7 @@ from pyexpat import * + +# This is actually implemented in the C module pyexpat, but considers itself to live here. +class ExpatError(Exception): + code: int + lineno: int + offset: int diff --git a/mypy/typeshed/stdlib/xmlrpc/client.pyi b/mypy/typeshed/stdlib/xmlrpc/client.pyi index d254102acc55..5899d1d72a38 100644 --- a/mypy/typeshed/stdlib/xmlrpc/client.pyi +++ b/mypy/typeshed/stdlib/xmlrpc/client.pyi @@ -200,7 +200,7 @@ def dumps( allow_none: bool = False, ) -> str: ... def loads( - data: str, use_datetime: bool = False, use_builtin_types: bool = False + data: str | ReadableBuffer, use_datetime: bool = False, use_builtin_types: bool = False ) -> tuple[tuple[_Marshallable, ...], str | None]: ... def gzip_encode(data: ReadableBuffer) -> bytes: ... # undocumented def gzip_decode(data: ReadableBuffer, max_decode: int = 20971520) -> bytes: ... # undocumented diff --git a/mypy/typeshed/stdlib/xmlrpc/server.pyi b/mypy/typeshed/stdlib/xmlrpc/server.pyi index 8ca3a4d1a33c..5f497aa7190e 100644 --- a/mypy/typeshed/stdlib/xmlrpc/server.pyi +++ b/mypy/typeshed/stdlib/xmlrpc/server.pyi @@ -1,6 +1,7 @@ import http.server import pydoc import socketserver +from _typeshed import ReadableBuffer from collections.abc import Callable, Iterable, Mapping from re import Pattern from typing import Any, ClassVar, Protocol @@ -48,8 +49,8 @@ class SimpleXMLRPCDispatcher: # undocumented def register_multicall_functions(self) -> None: ... def _marshaled_dispatch( self, - data: str, - dispatch_method: Callable[[str | None, tuple[_Marshallable, ...]], Fault | tuple[_Marshallable, ...]] | None = None, + data: str | ReadableBuffer, + dispatch_method: Callable[[str, tuple[_Marshallable, ...]], Fault | tuple[_Marshallable, ...]] | None = None, path: Any | None = None, ) -> str: ... # undocumented def system_listMethods(self) -> list[str]: ... # undocumented diff --git a/mypy/typeshed/stdlib/zoneinfo/__init__.pyi b/mypy/typeshed/stdlib/zoneinfo/__init__.pyi index 77930ac79dd5..cc483afad9ff 100644 --- a/mypy/typeshed/stdlib/zoneinfo/__init__.pyi +++ b/mypy/typeshed/stdlib/zoneinfo/__init__.pyi @@ -1,38 +1,35 @@ -from _typeshed import StrPath -from collections.abc import Iterable, Sequence +import sys +from collections.abc import Iterable from datetime import datetime, timedelta, tzinfo -from typing import Any, Protocol from typing_extensions import Self -__all__ = ["ZoneInfo", "reset_tzpath", "available_timezones", "TZPATH", "ZoneInfoNotFoundError", "InvalidTZPathWarning"] +# TODO: remove this version check +# In theory we shouldn't need this version check. Pyright complains about the imports +# from zoneinfo.* when run on 3.8 and 3.7 without this. Updates to typeshed's +# pyright test script are probably needed, see #11189 +if sys.version_info >= (3, 9): + from zoneinfo._common import ZoneInfoNotFoundError as ZoneInfoNotFoundError, _IOBytes + from zoneinfo._tzpath import ( + TZPATH as TZPATH, + InvalidTZPathWarning as InvalidTZPathWarning, + available_timezones as available_timezones, + reset_tzpath as reset_tzpath, + ) -class _IOBytes(Protocol): - def read(self, size: int, /) -> bytes: ... - def seek(self, size: int, whence: int = ..., /) -> Any: ... + __all__ = ["ZoneInfo", "reset_tzpath", "available_timezones", "TZPATH", "ZoneInfoNotFoundError", "InvalidTZPathWarning"] -class ZoneInfo(tzinfo): - @property - def key(self) -> str: ... - def __init__(self, key: str) -> None: ... - @classmethod - def no_cache(cls, key: str) -> Self: ... - @classmethod - def from_file(cls, fobj: _IOBytes, /, key: str | None = None) -> Self: ... - @classmethod - def clear_cache(cls, *, only_keys: Iterable[str] | None = None) -> None: ... - def tzname(self, dt: datetime | None, /) -> str | None: ... - def utcoffset(self, dt: datetime | None, /) -> timedelta | None: ... - def dst(self, dt: datetime | None, /) -> timedelta | None: ... + class ZoneInfo(tzinfo): + @property + def key(self) -> str: ... + def __init__(self, key: str) -> None: ... + @classmethod + def no_cache(cls, key: str) -> Self: ... + @classmethod + def from_file(cls, fobj: _IOBytes, /, key: str | None = None) -> Self: ... + @classmethod + def clear_cache(cls, *, only_keys: Iterable[str] | None = None) -> None: ... + def tzname(self, dt: datetime | None, /) -> str | None: ... + def utcoffset(self, dt: datetime | None, /) -> timedelta | None: ... + def dst(self, dt: datetime | None, /) -> timedelta | None: ... -# Note: Both here and in clear_cache, the types allow the use of `str` where -# a sequence of strings is required. This should be remedied if a solution -# to this typing bug is found: https://github.com/python/typing/issues/256 -def reset_tzpath(to: Sequence[StrPath] | None = None) -> None: ... -def available_timezones() -> set[str]: ... - -TZPATH: tuple[str, ...] - -class ZoneInfoNotFoundError(KeyError): ... -class InvalidTZPathWarning(RuntimeWarning): ... - -def __dir__() -> list[str]: ... + def __dir__() -> list[str]: ... diff --git a/mypy/typeshed/stdlib/zoneinfo/_common.pyi b/mypy/typeshed/stdlib/zoneinfo/_common.pyi new file mode 100644 index 000000000000..a2f29f2d14f0 --- /dev/null +++ b/mypy/typeshed/stdlib/zoneinfo/_common.pyi @@ -0,0 +1,13 @@ +import io +from typing import Any, Protocol + +class _IOBytes(Protocol): + def read(self, size: int, /) -> bytes: ... + def seek(self, size: int, whence: int = ..., /) -> Any: ... + +def load_tzdata(key: str) -> io.BufferedReader: ... +def load_data( + fobj: _IOBytes, +) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[str, ...], bytes | None]: ... + +class ZoneInfoNotFoundError(KeyError): ... diff --git a/mypy/typeshed/stdlib/zoneinfo/_tzpath.pyi b/mypy/typeshed/stdlib/zoneinfo/_tzpath.pyi new file mode 100644 index 000000000000..0ef78d03e5f4 --- /dev/null +++ b/mypy/typeshed/stdlib/zoneinfo/_tzpath.pyi @@ -0,0 +1,13 @@ +from _typeshed import StrPath +from collections.abc import Sequence + +# Note: Both here and in clear_cache, the types allow the use of `str` where +# a sequence of strings is required. This should be remedied if a solution +# to this typing bug is found: https://github.com/python/typing/issues/256 +def reset_tzpath(to: Sequence[StrPath] | None = None) -> None: ... +def find_tzfile(key: str) -> str | None: ... +def available_timezones() -> set[str]: ... + +TZPATH: tuple[str, ...] + +class InvalidTZPathWarning(RuntimeWarning): ... diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 174ba8e98c3f..0dbd18ef188f 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -283,7 +283,7 @@ f.write('x') f.write(b'x') f.foobar() [out] -_program.py:3: error: Argument 1 to "write" of "TextIOBase" has incompatible type "bytes"; expected "str" +_program.py:3: error: Argument 1 to "write" of "_TextIOBase" has incompatible type "bytes"; expected "str" _program.py:4: error: "TextIOWrapper[_WrappedBuffer]" has no attribute "foobar" [case testOpenReturnTypeInference] @@ -293,9 +293,9 @@ reveal_type(open('x', 'rb')) mode = 'rb' reveal_type(open('x', mode)) [out] -_program.py:1: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" -_program.py:2: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" -_program.py:3: note: Revealed type is "io.BufferedReader" +_program.py:1: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:2: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:3: note: Revealed type is "_io.BufferedReader" _program.py:5: note: Revealed type is "typing.IO[Any]" [case testOpenReturnTypeInferenceSpecialCases] @@ -304,8 +304,8 @@ reveal_type(open(file='x', mode='rb')) mode = 'rb' reveal_type(open(mode=mode, file='r')) [out] -_testOpenReturnTypeInferenceSpecialCases.py:1: note: Revealed type is "io.BufferedReader" -_testOpenReturnTypeInferenceSpecialCases.py:2: note: Revealed type is "io.BufferedReader" +_testOpenReturnTypeInferenceSpecialCases.py:1: note: Revealed type is "_io.BufferedReader" +_testOpenReturnTypeInferenceSpecialCases.py:2: note: Revealed type is "_io.BufferedReader" _testOpenReturnTypeInferenceSpecialCases.py:4: note: Revealed type is "typing.IO[Any]" [case testPathOpenReturnTypeInference] @@ -317,9 +317,9 @@ reveal_type(p.open('rb')) mode = 'rb' reveal_type(p.open(mode)) [out] -_program.py:3: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" -_program.py:4: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" -_program.py:5: note: Revealed type is "io.BufferedReader" +_program.py:3: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:4: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:5: note: Revealed type is "_io.BufferedReader" _program.py:7: note: Revealed type is "typing.IO[Any]" [case testPathOpenReturnTypeInferenceSpecialCases] @@ -330,8 +330,8 @@ reveal_type(p.open(errors='replace', mode='r')) mode = 'r' reveal_type(p.open(mode=mode, errors='replace')) [out] -_program.py:3: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" -_program.py:4: note: Revealed type is "io.TextIOWrapper[io._WrappedBuffer]" +_program.py:3: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" +_program.py:4: note: Revealed type is "_io.TextIOWrapper[_io._WrappedBuffer]" _program.py:6: note: Revealed type is "typing.IO[Any]" [case testGenericPatterns] From e1d09d3a1a75a2af6bd6406b7e06288b281c3364 Mon Sep 17 00:00:00 2001 From: Ihor <31508183+nautics889@users.noreply.github.com> Date: Thu, 17 Oct 2024 09:05:34 +0300 Subject: [PATCH 226/247] Minor refactoring of dict literal usage (#16837) --- mypy/dmypy_util.py | 3 +-- mypy/gclogger.py | 13 +++++++------ mypy/server/update.py | 3 +-- mypy/types.py | 8 +++++--- mypyc/codegen/emitclass.py | 3 +-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/mypy/dmypy_util.py b/mypy/dmypy_util.py index 0baff863b3c3..9b21d78ce599 100644 --- a/mypy/dmypy_util.py +++ b/mypy/dmypy_util.py @@ -104,8 +104,7 @@ def truncate(self, size: int | None = 0) -> int: raise io.UnsupportedOperation def write(self, output: str) -> int: - resp: dict[str, Any] = {} - resp[self.output_key] = output + resp: dict[str, Any] = {self.output_key: output} send(self.server, resp) return len(output) diff --git a/mypy/gclogger.py b/mypy/gclogger.py index 75f754ddf4d5..d111e609223c 100644 --- a/mypy/gclogger.py +++ b/mypy/gclogger.py @@ -38,10 +38,11 @@ def __exit__(self, *args: object) -> None: def get_stats(self) -> Mapping[str, float]: end_time = time.time() - result = {} - result["gc_time"] = self.gc_time - result["gc_calls"] = self.gc_calls - result["gc_collected"] = self.gc_collected - result["gc_uncollectable"] = self.gc_uncollectable - result["build_time"] = end_time - self.start_time + result = { + "gc_time": self.gc_time, + "gc_calls": self.gc_calls, + "gc_collected": self.gc_collected, + "gc_uncollectable": self.gc_uncollectable, + "build_time": end_time - self.start_time, + } return result diff --git a/mypy/server/update.py b/mypy/server/update.py index 0cc7a2229514..6bf8c8d07c2d 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -1059,8 +1059,7 @@ def find_symbol_tables_recursive(prefix: str, symbols: SymbolTable) -> dict[str, Returns a dictionary from full name to corresponding symbol table. """ - result = {} - result[prefix] = symbols + result = {prefix: symbols} for name, node in symbols.items(): if isinstance(node.node, TypeInfo) and node.node.fullname.startswith(prefix + "."): more = find_symbol_tables_recursive(prefix + "." + name, node.node.names) diff --git a/mypy/types.py b/mypy/types.py index dff7e2c0c829..897e19d6ee19 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1493,9 +1493,11 @@ def serialize(self) -> JsonDict | str: type_ref = self.type.fullname if not self.args and not self.last_known_value: return type_ref - data: JsonDict = {".class": "Instance"} - data["type_ref"] = type_ref - data["args"] = [arg.serialize() for arg in self.args] + data: JsonDict = { + ".class": "Instance", + "type_ref": type_ref, + "args": [arg.serialize() for arg in self.args], + } if self.last_known_value is not None: data["last_known_value"] = self.last_known_value.serialize() data["extra_attrs"] = self.extra_attrs.serialize() if self.extra_attrs else None diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 3ab6932546a6..d1a9ad3bace1 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -213,8 +213,7 @@ def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None: methods_name = f"{name_prefix}_methods" vtable_setup_name = f"{name_prefix}_trait_vtable_setup" - fields: dict[str, str] = {} - fields["tp_name"] = f'"{name}"' + fields: dict[str, str] = {"tp_name": f'"{name}"'} generate_full = not cl.is_trait and not cl.builtin_base needs_getseters = cl.needs_getseters or not cl.is_generated or cl.has_dict From dc352e90852bec0a9cbea92d00589c547f53f779 Mon Sep 17 00:00:00 2001 From: jianghuyiyuan Date: Thu, 17 Oct 2024 15:21:42 +0800 Subject: [PATCH 227/247] chore: fix some comments (#17615) --- mypy/checker.py | 2 +- mypy/main.py | 2 +- mypy/server/deps.py | 2 +- mypy/util.py | 2 +- mypyc/external/googletest/src/gtest.cc | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index c646ebf3736b..74890aa86cda 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5575,7 +5575,7 @@ def partition_by_callable( if isinstance(typ, TypeVarType): # We could do better probably? - # Refine the the type variable's bound as our type in the case that + # Refine the type variable's bound as our type in the case that # callable() is true. This unfortunately loses the information that # the type is a type variable in that branch. # This matches what is done for isinstance, but it may be possible to diff --git a/mypy/main.py b/mypy/main.py index 3f25ced16106..5991ddd9e62f 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1479,7 +1479,7 @@ def process_package_roots( root = "" package_root.append(root) options.package_root = package_root - # Pass the package root on the the filesystem cache. + # Pass the package root on the filesystem cache. fscache.set_package_root(package_root) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index 9ed2d4549629..6376600ffc0c 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -56,7 +56,7 @@ class 'mod.Cls'. This can also refer to an attribute inherited from a * 'mod.Cls' represents each method in class 'mod.Cls' + the top-level of the module 'mod'. (To simplify the implementation, there is no location that only includes the body of a class without the entire surrounding module top level.) -* Trigger '<...>' as a location is an indirect way of referring to to all +* Trigger '<...>' as a location is an indirect way of referring to all locations triggered by the trigger. These indirect locations keep the dependency map smaller and easier to manage. diff --git a/mypy/util.py b/mypy/util.py index 2eac2a86dfd0..3c550958a659 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -201,7 +201,7 @@ def trim_source_line(line: str, max_len: int, col: int, min_width: int) -> tuple A typical result looks like this: ...some_variable = function_to_call(one_arg, other_arg) or... - Return the trimmed string and the column offset to to adjust error location. + Return the trimmed string and the column offset to adjust error location. """ if max_len < 2 * min_width + 1: # In case the window is too tiny it is better to still show something. diff --git a/mypyc/external/googletest/src/gtest.cc b/mypyc/external/googletest/src/gtest.cc index d882ab2e36a1..4df3bd6b418a 100644 --- a/mypyc/external/googletest/src/gtest.cc +++ b/mypyc/external/googletest/src/gtest.cc @@ -1784,7 +1784,7 @@ std::string CodePointToUtf8(UInt32 code_point) { return str; } -// The following two functions only make sense if the the system +// The following two functions only make sense if the system // uses UTF-16 for wide string encoding. All supported systems // with 16 bit wchar_t (Windows, Cygwin, Symbian OS) do use UTF-16. From d1a7d09e1a30c74e3a197295f8a4e134e34316a1 Mon Sep 17 00:00:00 2001 From: Evgeniy Slobodkin Date: Thu, 17 Oct 2024 10:28:09 +0300 Subject: [PATCH 228/247] Move some tests from cmdline.test (#5966 ) (#17565) Relates #5966. Below is the info what happened with the concrete test from `cmdline.test`: 1. `testErrorContextConfig` => check-flags.test (`testShowErrorContextFunction`) [duplicate] 2. `testNoConfigFile` => check-flags.test (`testNoConfigFile`) [move] 3. `testPerFileConfigSection` => check-flags.test (`testPerFileUntypedDefs`) [move] 4. `testIgnoreErrorsConfig` => check-flags.test (`testPerFileIgnoreErrors`) [move] 5. `testConfigFollowImportsNormal` => check-flags.test (`testFollowImportsNormal`) [move + modified] 6. `testConfigFollowImportsSilent` => check-flags (`testFollowImportsSilent`) [move + modified] 7. `testConfigFollowImportsSkip` => check-flags (`testFollowImportsSkip`) [move + modified] 8. `testConfigFollowImportsError` => check-flags.test (`testFollowImportsError`) [move + modified] 9. `testConfigFollowImportsSelective` => check-flags.test (`testFollowImportsSelective`) [move] 10. `testConfigSilentMissingImportsOff` => check-flags.test (`testSilentMissingImportsOff`) [move] 11. `testConfigSilentMissingImportsOn` => check-flags.test (`testSilentMissingImportsOn`) [move] 12. `testDisallowAnyGenericsBuiltinCollectionsPre39` => check-flags.test (`testDisallowAnyGenericsBuiltinTuplePre39`, `testDisallowAnyGenericsBuiltinListPre39`, `testDisallowAnyGenericsBuiltinSetPre39`, `testDisallowAnyGenericsBuiltinDictPre39`) [split] 13. `testDisallowAnyGenericsTypingCollections` => check-flags.test (`testDisallowAnyGenericsTupleWithNoTypeParamsGeneric`, `testDisallowAnyGenericsPlainList`, `testDisallowAnyGenericsPlainDict`, `testDisallowAnyGenericsPlainSet`) [split] 14. `testDisallowUntypedDefsAndGeneric` => check-flags.test (`testDisallowUntypedDefsAndGeneric`) [move] 15. `testParseError` => parse-errors.test (`testMissingBracket`) [move] 16. `testParseErrorAnnots` => check-fastparse.test (`testFasterParseTooManyArgumentsAnnotation`) [duplicate] 17. `testNotesOnlyResultInExitSuccess` => check-flags.test (`testNotesOnlyResultInExitSuccess`) [move] Let's compare test execution time. I've run `pytest -n 4 mypy/test/testcmdline.py` 3 times on my machine and calculated the average time. - Before: 130 tests, 1m 02s - After: 115 tests, 0m 55s Also, if it's possible to use fixture `FrozenSet` in `check-flags.test`, we'd be able to totally split items 12 and 13 from the above list. And `testMissingBracket` is skipped by pytest in the `parse-errors.test`-file, but, probably, this file is the best variant for it (not sure about it). --- test-data/unit/check-flags.test | 179 ++++++++++++++++++-- test-data/unit/cmdline.test | 275 +------------------------------ test-data/unit/parse-errors.test | 7 + 3 files changed, 176 insertions(+), 285 deletions(-) diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index 4f327a2f0edc..dd7bee3f7aec 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -79,6 +79,13 @@ async def g(x: int) -> Any: [builtins fixtures/async_await.pyi] [typing fixtures/typing-async.pyi] +[case testDisallowUntypedDefsAndGeneric] +# flags: --disallow-untyped-defs --disallow-any-generics +def get_tasks(self): + return 'whatever' +[out] +main:2: error: Function is missing a return type annotation + [case testDisallowUntypedDefsUntypedDecorator] # flags: --disallow-untyped-decorators def d(p): @@ -540,21 +547,30 @@ tmp/b.py:1: error: Unsupported operand types for + ("int" and "str") [case testFollowImportsNormal] # flags: --follow-imports=normal from mod import x -x + "" +x + 0 +x + "" # E: Unsupported operand types for + ("int" and "str") +import mod +mod.x + 0 +mod.x + "" # E: Unsupported operand types for + ("int" and "str") +mod.y # E: "object" has no attribute "y" +mod + 0 # E: Unsupported left operand type for + ("object") [file mod.py] -1 + "" +1 + "" # E: Unsupported operand types for + ("int" and "str") x = 0 -[out] -tmp/mod.py:1: error: Unsupported operand types for + ("int" and "str") -main:3: error: Unsupported operand types for + ("int" and "str") +x += "" # E: Unsupported operand types for + ("int" and "str") [case testFollowImportsSilent] # flags: --follow-imports=silent from mod import x x + "" # E: Unsupported operand types for + ("int" and "str") +import mod +mod.x + "" # E: Unsupported operand types for + ("int" and "str") +mod.y # E: "object" has no attribute "y" +mod + 0 # E: Unsupported left operand type for + ("object") [file mod.py] 1 + "" x = 0 +x += "" [case testFollowImportsSilentTypeIgnore] # flags: --warn-unused-ignores --follow-imports=silent @@ -565,20 +581,55 @@ x = 3 # type: ignore [case testFollowImportsSkip] # flags: --follow-imports=skip from mod import x +reveal_type(x) # N: Revealed type is "Any" x + "" +import mod +reveal_type(mod.x) # N: Revealed type is "Any" [file mod.py] this deliberate syntax error will not be reported -[out] [case testFollowImportsError] # flags: --follow-imports=error -from mod import x +from mod import x # E: Import of "mod" ignored \ + # N: (Using --follow-imports=error, module not passed on command line) x + "" +reveal_type(x) # N: Revealed type is "Any" +import mod +reveal_type(mod.x) # N: Revealed type is "Any" [file mod.py] deliberate syntax error -[out] -main:2: error: Import of "mod" ignored -main:2: note: (Using --follow-imports=error, module not passed on command line) + +[case testFollowImportsSelective] +# flags: --config-file tmp/mypy.ini +import normal +import silent +import skip +import error # E: Import of "error" ignored \ + # N: (Using --follow-imports=error, module not passed on command line) +reveal_type(normal.x) # N: Revealed type is "builtins.int" +reveal_type(silent.x) # N: Revealed type is "builtins.int" +reveal_type(skip) # N: Revealed type is "Any" +reveal_type(error) # N: Revealed type is "Any" +[file mypy.ini] +\[mypy] +\[mypy-normal] +follow_imports = normal +\[mypy-silent] +follow_imports = silent +\[mypy-skip] +follow_imports = skip +\[mypy-error] +follow_imports = error +[file normal.py] +x = 0 +x += '' # E: Unsupported operand types for + ("int" and "str") +[file silent.py] +x = 0 +x += '' +[file skip.py] +bla bla +[file error.py] +bla bla [case testIgnoreMissingImportsFalse] from mod import x @@ -591,6 +642,15 @@ main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missin from mod import x [out] +[case testNoConfigFile] +# flags: --config-file= +# type: ignore + +[file mypy.ini] +\[mypy] +warn_unused_ignores = True +[out] + [case testPerFileIncompleteDefsBasic] # flags: --config-file tmp/mypy.ini import standard, incomplete @@ -868,6 +928,16 @@ implicit_optional = true module = 'optional' strict_optional = true +[case testSilentMissingImportsOff] +-- ignore_missing_imports is False by default. +import missing # E: Cannot find implementation or library stub for module named "missing" \ + # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +reveal_type(missing.x) # N: Revealed type is "Any" + +[case testSilentMissingImportsOn] +# flags: --ignore-missing-imports +import missing +reveal_type(missing.x) # N: Revealed type is "Any" [case testDisallowImplicitTypesIgnoreMissingTypes] # flags: --ignore-missing-imports --disallow-any-unimported @@ -1447,6 +1517,29 @@ class Queue(Generic[_T]): ... [builtins fixtures/async_await.pyi] [typing fixtures/typing-full.pyi] +[case testDisallowAnyGenericsBuiltinTuplePre39] +# flags: --disallow-any-generics --python-version 3.8 +s = tuple([1, 2, 3]) +def f(t: tuple) -> None: pass # E: Implicit generic "Any". Use "typing.Tuple" and specify generic parameters +[builtins fixtures/tuple.pyi] + +[case testDisallowAnyGenericsBuiltinListPre39] +# flags: --disallow-any-generics --python-version 3.8 +l = list([1, 2, 3]) +def f(t: list) -> None: pass # E: Implicit generic "Any". Use "typing.List" and specify generic parameters +[builtins fixtures/list.pyi] + +[case testDisallowAnyGenericsBuiltinSetPre39] +# flags: --disallow-any-generics --python-version 3.8 +l = set({1, 2, 3}) +def f(s: set) -> None: pass # E: Implicit generic "Any". Use "typing.Set" and specify generic parameters +[builtins fixtures/set.pyi] + +[case testDisallowAnyGenericsBuiltinDictPre39] +# flags: --disallow-any-generics --python-version 3.8 +l = dict([('a', 1)]) +def f(d: dict) -> None: pass # E: Implicit generic "Any". Use "typing.Dict" and specify generic parameters +[builtins fixtures/dict.pyi] [case testCheckDefaultAllowAnyGeneric] from typing import TypeVar, Callable @@ -1863,8 +1956,9 @@ x: Tuple = () # E: Missing type parameters for generic type "Tuple" # flags: --disallow-any-generics from typing import Tuple, List -def f(s: List[Tuple]) -> None: pass # E: Missing type parameters for generic type "Tuple" -def g(s: List[Tuple[str, str]]) -> None: pass # no error +def f(s: Tuple) -> None: pass # E: Missing type parameters for generic type "Tuple" +def g(s: List[Tuple]) -> None: pass # E: Missing type parameters for generic type "Tuple" +def h(s: List[Tuple[str, str]]) -> None: pass # no error [builtins fixtures/list.pyi] [case testDisallowAnyGenericsTypeType] @@ -1908,14 +2002,36 @@ x: A = ('a', 'b', 1) # E: Missing type parameters for generic type "A" from typing import List def f(l: List) -> None: pass # E: Missing type parameters for generic type "List" -def g(l: List[str]) -> None: pass # no error +def g(l: List[str]) -> None: pass def h(l: List[List]) -> None: pass # E: Missing type parameters for generic type "List" def i(l: List[List[List[List]]]) -> None: pass # E: Missing type parameters for generic type "List" +def j() -> List: pass # E: Missing type parameters for generic type "List" x = [] # E: Need type annotation for "x" (hint: "x: List[] = ...") y: List = [] # E: Missing type parameters for generic type "List" [builtins fixtures/list.pyi] +[case testDisallowAnyGenericsPlainDict] +# flags: --disallow-any-generics +from typing import List, Dict + +def f(d: Dict) -> None: pass # E: Missing type parameters for generic type "Dict" +def g(d: Dict[str, Dict]) -> None: pass # E: Missing type parameters for generic type "Dict" +def h(d: List[Dict]) -> None: pass # E: Missing type parameters for generic type "Dict" + +d: Dict = {} # E: Missing type parameters for generic type "Dict" +[builtins fixtures/dict.pyi] + +[case testDisallowAnyGenericsPlainSet] +# flags: --disallow-any-generics +from typing import Set + +def f(s: Set) -> None: pass # E: Missing type parameters for generic type "Set" +def g(s: Set[Set]) -> None: pass # E: Missing type parameters for generic type "Set" + +s: Set = set() # E: Missing type parameters for generic type "Set" +[builtins fixtures/set.pyi] + [case testDisallowAnyGenericsCustomGenericClass] # flags: --disallow-any-generics from typing import Generic, TypeVar, Any @@ -2162,6 +2278,38 @@ allow_untyped_defs = True allow_untyped_calls = True disable_error_code = var-annotated +[case testPerFileIgnoreErrors] +# flags: --config-file tmp/mypy.ini +import foo, bar +[file foo.py] +x: str = 5 +[file bar.py] +x: str = 5 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +[file mypy.ini] +\[mypy] +\[mypy-foo] +ignore_errors = True + +[case testPerFileUntypedDefs] +# flags: --config-file tmp/mypy.ini +import x, y, z +[file x.py] +def f(a): ... # E: Function is missing a type annotation +def g(a: int) -> int: return f(a) +[file y.py] +def f(a): pass +def g(a: int) -> int: return f(a) +[file z.py] +def f(a): pass # E: Function is missing a type annotation +def g(a: int) -> int: return f(a) # E: Call to untyped function "f" in typed context +[file mypy.ini] +\[mypy] +disallow_untyped_defs = True +\[mypy-y] +disallow_untyped_defs = False +\[mypy-z] +disallow_untyped_calls = True + [case testPerModuleErrorCodesOverride] # flags: --config-file tmp/mypy.ini import tests.foo @@ -2284,3 +2432,8 @@ class C(Generic[T]): ... A = Union[C, List] # OK [builtins fixtures/list.pyi] + +[case testNotesOnlyResultInExitSuccess] +-- check_untyped_defs is False by default. +def f(): + x: int = "no" # N: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 38ea83cdbcf4..2bab19e0d42f 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -153,18 +153,6 @@ FLAG = False if not FLAG: x = "unreachable" -[case testErrorContextConfig] -# cmd: mypy main.py -[file mypy.ini] -\[mypy] -show_error_context=True -[file main.py] -def f() -> None: - 0 + "" -[out] -main.py: note: In function "f": -main.py:2: error: Unsupported operand types for + ("int" and "str") - [case testAltConfigFile] # cmd: mypy --config-file config.ini main.py [file config.ini] @@ -176,43 +164,6 @@ FLAG = False if not FLAG: x = "unreachable" -[case testNoConfigFile] -# cmd: mypy main.py --config-file= -[file mypy.ini] -\[mypy] -warn_unused_ignores = True -[file main.py] -# type: ignore - -[case testPerFileConfigSection] -# cmd: mypy x.py y.py z.py -[file mypy.ini] -\[mypy] -disallow_untyped_defs = True -\[mypy-y] -disallow_untyped_defs = False -\[mypy-z] -disallow_untyped_calls = True -[file x.py] -def f(a): - pass -def g(a: int) -> int: - return f(a) -[file y.py] -def f(a): - pass -def g(a: int) -> int: - return f(a) -[file z.py] -def f(a): - pass -def g(a: int) -> int: - return f(a) -[out] -z.py:1: error: Function is missing a type annotation -z.py:4: error: Call to untyped function "f" in typed context -x.py:1: error: Function is missing a type annotation - [case testPerFileConfigSectionMultipleMatchesDisallowed] # cmd: mypy xx.py xy.py yx.py yy.py [file mypy.ini] @@ -326,43 +277,6 @@ file.py:1: error: Cannot find implementation or library stub for module named "n file.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports file.py:6: error: Argument 1 to "foo" has incompatible type "str"; expected "int" -[case testIgnoreErrorsConfig] -# cmd: mypy x.py y.py -[file mypy.ini] -\[mypy] -\[mypy-x] -ignore_errors = True -[file x.py] -x: str = 5 -[file y.py] -x: str = 5 -[out] -y.py:1: error: Incompatible types in assignment (expression has type "int", variable has type "str") - -[case testConfigFollowImportsNormal] -# cmd: mypy main.py -[file main.py] -from a import x -x + 0 -x + '' # E -import a -a.x + 0 -a.x + '' # E -a.y # E -a + 0 # E -[file mypy.ini] -\[mypy] -follow_imports = normal -[file a.py] -x = 0 -x += '' # Error reported here -[out] -a.py:2: error: Unsupported operand types for + ("int" and "str") -main.py:3: error: Unsupported operand types for + ("int" and "str") -main.py:6: error: Unsupported operand types for + ("int" and "str") -main.py:7: error: Module has no attribute "y" -main.py:8: error: Unsupported operand types for + (Module and "int") - [case testConfigFollowImportsSysPath] # cmd: mypy main.py [file main.py] @@ -389,102 +303,6 @@ main.py:6: error: Unsupported operand types for + ("int" and "str") main.py:7: error: Module has no attribute "y" main.py:8: error: Unsupported operand types for + (Module and "int") -[case testConfigFollowImportsSilent] -# cmd: mypy main.py -[file main.py] -from a import x -x + '' -import a -a.x + '' -a.y -a + 0 -[file mypy.ini] -\[mypy] -follow_imports = silent -[file a.py] -x = 0 -x += '' # No error reported -[out] -main.py:2: error: Unsupported operand types for + ("int" and "str") -main.py:4: error: Unsupported operand types for + ("int" and "str") -main.py:5: error: Module has no attribute "y" -main.py:6: error: Unsupported operand types for + (Module and "int") - -[case testConfigFollowImportsSkip] -# cmd: mypy main.py -[file main.py] -from a import x -reveal_type(x) # Expect Any -import a -reveal_type(a.x) # Expect Any -[file mypy.ini] -\[mypy] -follow_imports = skip -[file a.py] -/ # No error reported -[out] -main.py:2: note: Revealed type is "Any" -main.py:4: note: Revealed type is "Any" -== Return code: 0 - -[case testConfigFollowImportsError] -# cmd: mypy main.py -[file main.py] -from a import x # Error reported here -reveal_type(x) # Expect Any -import a -reveal_type(a.x) # Expect Any -[file mypy.ini] -\[mypy] -follow_imports = error -[file a.py] -/ # No error reported -[out] -main.py:1: error: Import of "a" ignored -main.py:1: note: (Using --follow-imports=error, module not passed on command line) -main.py:2: note: Revealed type is "Any" -main.py:4: note: Revealed type is "Any" - -[case testConfigFollowImportsSelective] -# cmd: mypy main.py -[file mypy.ini] -\[mypy] -\[mypy-normal] -follow_imports = normal -\[mypy-silent] -follow_imports = silent -\[mypy-skip] -follow_imports = skip -\[mypy-error] -follow_imports = error -[file main.py] -import normal -import silent -import skip -import error -reveal_type(normal.x) -reveal_type(silent.x) -reveal_type(skip) -reveal_type(error) -[file normal.py] -x = 0 -x += '' -[file silent.py] -x = 0 -x += '' -[file skip.py] -bla bla -[file error.py] -bla bla -[out] -normal.py:2: error: Unsupported operand types for + ("int" and "str") -main.py:4: error: Import of "error" ignored -main.py:4: note: (Using --follow-imports=error, module not passed on command line) -main.py:5: note: Revealed type is "builtins.int" -main.py:6: note: Revealed type is "builtins.int" -main.py:7: note: Revealed type is "Any" -main.py:8: note: Revealed type is "Any" - [case testConfigFollowImportsInvalid] # cmd: mypy main.py [file mypy.ini] @@ -495,31 +313,6 @@ follow_imports =True mypy.ini: [mypy]: follow_imports: invalid choice 'True' (choose from 'normal', 'silent', 'skip', 'error') == Return code: 0 -[case testConfigSilentMissingImportsOff] -# cmd: mypy main.py -[file main.py] -import missing # Expect error here -reveal_type(missing.x) # Expect Any -[file mypy.ini] -\[mypy] -ignore_missing_imports = False -[out] -main.py:1: error: Cannot find implementation or library stub for module named "missing" -main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports -main.py:2: note: Revealed type is "Any" - -[case testConfigSilentMissingImportsOn] -# cmd: mypy main.py -[file main.py] -import missing # No error here -reveal_type(missing.x) # Expect Any -[file mypy.ini] -\[mypy] -ignore_missing_imports = True -[out] -main.py:2: note: Revealed type is "Any" -== Return code: 0 - [case testFailedImportOnWrongCWD] # cmd: mypy main.py # cwd: main/subdir1/subdir2 @@ -683,21 +476,10 @@ int_pow.py:11: note: Revealed type is "Any" python_version = 3.8 \[mypy-m] disallow_any_generics = True - [file m.py] -s = tuple([1, 2, 3]) # no error - -def f(t: tuple) -> None: pass -def g() -> list: pass -def h(s: dict) -> None: pass -def i(s: set) -> None: pass def j(s: frozenset) -> None: pass [out] -m.py:3: error: Implicit generic "Any". Use "typing.Tuple" and specify generic parameters -m.py:4: error: Implicit generic "Any". Use "typing.List" and specify generic parameters -m.py:5: error: Implicit generic "Any". Use "typing.Dict" and specify generic parameters -m.py:6: error: Implicit generic "Any". Use "typing.Set" and specify generic parameters -m.py:7: error: Implicit generic "Any". Use "typing.FrozenSet" and specify generic parameters +m.py:1: error: Implicit generic "Any". Use "typing.FrozenSet" and specify generic parameters [case testDisallowAnyGenericsTypingCollections] # cmd: mypy m.py @@ -705,21 +487,11 @@ m.py:7: error: Implicit generic "Any". Use "typing.FrozenSet" and specify generi \[mypy] \[mypy-m] disallow_any_generics = True - [file m.py] -from typing import Tuple, List, Dict, Set, FrozenSet - -def f(t: Tuple) -> None: pass -def g() -> List: pass -def h(s: Dict) -> None: pass -def i(s: Set) -> None: pass +from typing import FrozenSet def j(s: FrozenSet) -> None: pass [out] -m.py:3: error: Missing type parameters for generic type "Tuple" -m.py:4: error: Missing type parameters for generic type "List" -m.py:5: error: Missing type parameters for generic type "Dict" -m.py:6: error: Missing type parameters for generic type "Set" -m.py:7: error: Missing type parameters for generic type "FrozenSet" +m.py:2: error: Missing type parameters for generic type "FrozenSet" [case testSectionInheritance] # cmd: mypy a @@ -756,18 +528,6 @@ ignore_errors = False a/b/c/d/e/__init__.py:2: error: Missing type parameters for generic type "List" a/b/c/d/e/__init__.py:3: error: Argument 1 to "g" has incompatible type "None"; expected "List[Any]" -[case testDisallowUntypedDefsAndGenerics] -# cmd: mypy a.py -[file mypy.ini] -\[mypy] -disallow_untyped_defs = True -disallow_any_generics = True -[file a.py] -def get_tasks(self): - return 'whatever' -[out] -a.py:1: error: Function is missing a return type annotation - [case testMissingFile] # cmd: mypy nope.py [out] @@ -775,26 +535,6 @@ mypy: can't read file 'nope.py': No such file or directory == Return code: 2 --' -[case testParseError] -# cmd: mypy a.py -[file a.py] -def foo( -[out] -a.py:1: error: unexpected EOF while parsing -== Return code: 2 -[out version>=3.10] -a.py:1: error: '(' was never closed -== Return code: 2 - -[case testParseErrorAnnots] -# cmd: mypy a.py -[file a.py] -def foo(x): - # type: (str, int) -> None - return -[out] -a.py:1: error: Type signature has too many arguments - [case testModulesAndPackages] # cmd: mypy --package p.a --package p.b --module c [file p/__init__.py] @@ -1489,15 +1229,6 @@ pass Warning: --new-type-inference flag is deprecated; new type inference algorithm is already enabled by default == Return code: 0 -[case testNotesOnlyResultInExitSuccess] -# cmd: mypy a.py -[file a.py] -def f(): - x: int = "no" -[out] -a.py:2: note: By default the bodies of untyped functions are not checked, consider using --check-untyped-defs -== Return code: 0 - [case testCustomTypeshedDirFilePassedExplicitly] # cmd: mypy --custom-typeshed-dir dir m.py dir/stdlib/foo.pyi [file m.py] diff --git a/test-data/unit/parse-errors.test b/test-data/unit/parse-errors.test index c6b1c00a6169..7b1078d3fa2f 100644 --- a/test-data/unit/parse-errors.test +++ b/test-data/unit/parse-errors.test @@ -151,6 +151,13 @@ x = 0 # type: A B #comment #7 [out] file:2: error: Syntax error in type comment "A B" +[case testMissingBracket] +def foo( +[out] +file:1: error: unexpected EOF while parsing +[out version>=3.10] +file:1: error: '(' was never closed + [case testInvalidSignatureInComment1] def f(): # type: x pass From 61ad5a47c0185549cb40b287aca8df1e5e862f0c Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 17 Oct 2024 02:17:33 -0700 Subject: [PATCH 229/247] Add faster-cache extra, test in CI (#17978) Follow up to https://github.com/python/mypy/pull/17955 --- setup.py | 1 + tox.ini | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index a50afde4ce6b..71124916a3e4 100644 --- a/setup.py +++ b/setup.py @@ -232,6 +232,7 @@ def run(self): "python2": "", "reports": "lxml", "install-types": "pip", + "faster-cache": "orjson", }, python_requires=">=3.8", include_package_data=True, diff --git a/tox.ini b/tox.ini index 7f89b7d4f7ef..a505950521fa 100644 --- a/tox.ini +++ b/tox.ini @@ -20,15 +20,18 @@ passenv = PROGRAMDATA PROGRAMFILES(X86) PYTEST_ADDOPTS -deps = -rtest-requirements.txt +deps = + -r test-requirements.txt + # This is a bit of a hack, but ensures the faster-cache path is tested in CI + orjson;python_version=='3.12' commands = python -m pytest {posargs} [testenv:dev] description = generate a DEV environment, that has all project libraries usedevelop = True deps = - -rtest-requirements.txt - -rdocs/requirements-docs.txt + -r test-requirements.txt + -r docs/requirements-docs.txt commands = python -m pip list --format=columns python -c 'import sys; print(sys.executable)' @@ -38,7 +41,7 @@ commands = description = invoke sphinx-build to build the HTML docs passenv = VERIFY_MYPY_ERROR_CODES -deps = -rdocs/requirements-docs.txt +deps = -r docs/requirements-docs.txt commands = sphinx-build -n -d "{toxworkdir}/docs_doctree" docs/source "{toxworkdir}/docs_out" --color -W -bhtml {posargs} python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' From 539513a5ff41954b0d0d5b0cd25b1cb03a10122b Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 17 Oct 2024 02:18:47 -0700 Subject: [PATCH 230/247] Fix iteration over union (when self type is involved) (#17976) Fixes https://github.com/python/mypy/issues/17945 --- mypy/checkexpr.py | 1 + test-data/unit/check-selftype.test | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 8d1833a772e2..b06aaa8f89f5 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -3814,6 +3814,7 @@ def check_method_call_by_name( is_operator=True, msg=self.msg, original_type=original_type, + self_type=base_type, chk=self.chk, in_literal_context=self.is_literal_context(), ) diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 9601852ef823..1a088fe05092 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2160,3 +2160,19 @@ class MyProtocol(Protocol): def test() -> None: ... value: MyProtocol = test + +[case testSelfTypeUnionIter] +from typing import Self, Iterator, Generic, TypeVar, Union + +T = TypeVar("T") + +class range(Generic[T]): + def __iter__(self) -> Self: ... + def __next__(self) -> T: ... + +class count: + def __iter__(self) -> Iterator[int]: ... + +def foo(x: Union[range[int], count]) -> None: + for item in x: + reveal_type(item) # N: Revealed type is "builtins.int" From 123eb919a806212438cfbf9a18284fb830bf3dba Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Fri, 18 Oct 2024 03:16:58 +0200 Subject: [PATCH 231/247] Update mypy_primer workflow (#17987) Update the mypy_primer workflow to use actions/upload-artifact@v4. https://github.com/actions/upload-artifact/tree/v4 --- .github/workflows/mypy_primer.yml | 37 +++++++++++++++++------ .github/workflows/mypy_primer_comment.yml | 2 +- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index 0c77d3a255d8..fee7e6d079f1 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -69,18 +69,35 @@ jobs: --output concise \ | tee diff_${{ matrix.shard-index }}.txt ) || [ $? -eq 1 ] - - name: Upload mypy_primer diff - uses: actions/upload-artifact@v3 - with: - name: mypy_primer_diffs - path: diff_${{ matrix.shard-index }}.txt - - if: ${{ matrix.shard-index }} == 0 + - if: ${{ matrix.shard-index == 0 }} name: Save PR number run: | echo ${{ github.event.pull_request.number }} | tee pr_number.txt - - if: ${{ matrix.shard-index }} == 0 - name: Upload PR number - uses: actions/upload-artifact@v3 + - if: ${{ matrix.shard-index == 0 }} + name: Upload mypy_primer diff + PR number + uses: actions/upload-artifact@v4 + with: + name: mypy_primer_diffs-${{ matrix.shard-index }} + path: | + diff_${{ matrix.shard-index }}.txt + pr_number.txt + - name: Upload mypy_primer diff + uses: actions/upload-artifact@v4 + if: ${{ matrix.shard-index != 0 }} + with: + name: mypy_primer_diffs-${{ matrix.shard-index }} + path: diff_${{ matrix.shard-index }}.txt + + join_artifacts: + name: Join artifacts + runs-on: ubuntu-latest + needs: [mypy_primer] + permissions: + contents: read + steps: + - name: Merge artifacts + uses: actions/upload-artifact/merge@v4 with: name: mypy_primer_diffs - path: pr_number.txt + pattern: mypy_primer_diffs-* + delete-merged: true diff --git a/.github/workflows/mypy_primer_comment.yml b/.github/workflows/mypy_primer_comment.yml index 492e03aff16e..6e62d8c51713 100644 --- a/.github/workflows/mypy_primer_comment.yml +++ b/.github/workflows/mypy_primer_comment.yml @@ -18,7 +18,7 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: Download diffs - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: script: | const fs = require('fs'); From 5d9b1f92ec3078fe93a0ca46e0741c8ce19fe73e Mon Sep 17 00:00:00 2001 From: Evgeniy Slobodkin Date: Fri, 18 Oct 2024 04:18:41 +0300 Subject: [PATCH 232/247] TypeGuard should be bool not Any when matching TypeVar (#17145) Fixes #17117 --- mypy/constraints.py | 8 -------- test-data/unit/check-typeguard.test | 9 +++++++++ test-data/unit/check-typeis.test | 9 +++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 49a2aea8fa05..5c815bf2af65 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1029,18 +1029,10 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: if template.type_guard is not None and cactual.type_guard is not None: template_ret_type = template.type_guard cactual_ret_type = cactual.type_guard - elif template.type_guard is not None: - template_ret_type = AnyType(TypeOfAny.special_form) - elif cactual.type_guard is not None: - cactual_ret_type = AnyType(TypeOfAny.special_form) if template.type_is is not None and cactual.type_is is not None: template_ret_type = template.type_is cactual_ret_type = cactual.type_is - elif template.type_is is not None: - template_ret_type = AnyType(TypeOfAny.special_form) - elif cactual.type_is is not None: - cactual_ret_type = AnyType(TypeOfAny.special_form) res.extend(infer_constraints(template_ret_type, cactual_ret_type, self.direction)) diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 27b88553fb43..e7a8eac4f043 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -87,6 +87,15 @@ def main(a: Tuple[T, ...]): reveal_type(a) # N: Revealed type is "Tuple[T`-1, T`-1]" [builtins fixtures/tuple.pyi] +[case testTypeGuardPassedAsTypeVarIsBool] +from typing import Callable, TypeVar +from typing_extensions import TypeGuard +T = TypeVar('T') +def is_str(x: object) -> TypeGuard[str]: ... +def main(f: Callable[[object], T]) -> T: ... +reveal_type(main(is_str)) # N: Revealed type is "builtins.bool" +[builtins fixtures/tuple.pyi] + [case testTypeGuardNonOverlapping] from typing import List from typing_extensions import TypeGuard diff --git a/test-data/unit/check-typeis.test b/test-data/unit/check-typeis.test index 6b96845504ab..2372f990fda1 100644 --- a/test-data/unit/check-typeis.test +++ b/test-data/unit/check-typeis.test @@ -104,6 +104,15 @@ def main(x: object, type_check_func: Callable[[object], TypeIs[T]]) -> T: reveal_type(main("a", is_str)) # N: Revealed type is "builtins.str" [builtins fixtures/exception.pyi] +[case testTypeIsPassedAsTypeVarIsBool] +from typing import Callable, TypeVar +from typing_extensions import TypeIs +T = TypeVar('T') +def is_str(x: object) -> TypeIs[str]: pass +def main(f: Callable[[object], T]) -> T: pass +reveal_type(main(is_str)) # N: Revealed type is "builtins.bool" +[builtins fixtures/tuple.pyi] + [case testTypeIsUnionIn] from typing import Union from typing_extensions import TypeIs From 8150b513f2b298ee45a8ea02644aa7bc44786bd1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 19:53:47 -0700 Subject: [PATCH 233/247] Sync typeshed (#17988) Source commit: https://github.com/python/typeshed/commit/559ae9730ba3dab1305cdbaf2c29786ff38d740d --- mypy/typeshed/stdlib/_asyncio.pyi | 4 ++-- mypy/typeshed/stdlib/_csv.pyi | 3 ++- mypy/typeshed/stdlib/builtins.pyi | 5 ++++- mypy/typeshed/stdlib/csv.pyi | 11 +++++++++-- mypy/typeshed/stdlib/posixpath.pyi | 16 +++++++++++++--- mypy/typeshed/stdlib/unittest/runner.pyi | 3 ++- 6 files changed, 32 insertions(+), 10 deletions(-) diff --git a/mypy/typeshed/stdlib/_asyncio.pyi b/mypy/typeshed/stdlib/_asyncio.pyi index 18920cd8a8a4..a259026615aa 100644 --- a/mypy/typeshed/stdlib/_asyncio.pyi +++ b/mypy/typeshed/stdlib/_asyncio.pyi @@ -1,6 +1,6 @@ import sys from asyncio.events import AbstractEventLoop -from collections.abc import Awaitable, Callable, Coroutine, Generator, Iterable +from collections.abc import Awaitable, Callable, Coroutine, Generator from contextvars import Context from types import FrameType from typing import Any, Literal, TextIO, TypeVar @@ -13,7 +13,7 @@ _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) _TaskYieldType: TypeAlias = Future[object] | None -class Future(Awaitable[_T], Iterable[_T]): +class Future(Awaitable[_T]): _state: str @property def _exception(self) -> BaseException | None: ... diff --git a/mypy/typeshed/stdlib/_csv.pyi b/mypy/typeshed/stdlib/_csv.pyi index 6d1893cf8c16..0e206a63b260 100644 --- a/mypy/typeshed/stdlib/_csv.pyi +++ b/mypy/typeshed/stdlib/_csv.pyi @@ -1,3 +1,4 @@ +import csv import sys from _typeshed import SupportsWrite from collections.abc import Iterable, Iterator @@ -20,7 +21,7 @@ _QuotingType: TypeAlias = int class Error(Exception): ... -_DialectLike: TypeAlias = str | Dialect | type[Dialect] +_DialectLike: TypeAlias = str | Dialect | csv.Dialect | type[Dialect | csv.Dialect] class Dialect: delimiter: str diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index adfe1d83752b..eaa6695f7045 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -850,7 +850,10 @@ class slice: @overload def __new__(cls, start: Any, stop: Any, step: Any = ..., /) -> Self: ... def __eq__(self, value: object, /) -> bool: ... - __hash__: ClassVar[None] # type: ignore[assignment] + if sys.version_info >= (3, 12): + def __hash__(self) -> int: ... + else: + __hash__: ClassVar[None] # type: ignore[assignment] def indices(self, len: SupportsIndex, /) -> tuple[int, int, int]: ... class tuple(Sequence[_T_co]): diff --git a/mypy/typeshed/stdlib/csv.pyi b/mypy/typeshed/stdlib/csv.pyi index 89a019753f04..c5c7fea8fa25 100644 --- a/mypy/typeshed/stdlib/csv.pyi +++ b/mypy/typeshed/stdlib/csv.pyi @@ -4,7 +4,6 @@ from _csv import ( QUOTE_MINIMAL as QUOTE_MINIMAL, QUOTE_NONE as QUOTE_NONE, QUOTE_NONNUMERIC as QUOTE_NONNUMERIC, - Dialect as _Dialect, Error as Error, __version__ as __version__, _DialectLike, @@ -59,7 +58,15 @@ if sys.version_info < (3, 13): _T = TypeVar("_T") -class Dialect(_Dialect): +class Dialect: + delimiter: str + quotechar: str | None + escapechar: str | None + doublequote: bool + skipinitialspace: bool + lineterminator: str + quoting: _QuotingType + strict: bool def __init__(self) -> None: ... class excel(Dialect): ... diff --git a/mypy/typeshed/stdlib/posixpath.pyi b/mypy/typeshed/stdlib/posixpath.pyi index 31406f8df950..3313667f1781 100644 --- a/mypy/typeshed/stdlib/posixpath.pyi +++ b/mypy/typeshed/stdlib/posixpath.pyi @@ -77,7 +77,11 @@ pathsep: LiteralString defpath: LiteralString devnull: LiteralString -def abspath(path: PathLike[AnyStr] | AnyStr) -> AnyStr: ... +# Overloads are necessary to work around python/mypy#17952 & python/mypy#11880 +@overload +def abspath(path: PathLike[AnyStr]) -> AnyStr: ... +@overload +def abspath(path: AnyStr) -> AnyStr: ... @overload def basename(p: PathLike[AnyStr]) -> AnyStr: ... @overload @@ -86,8 +90,14 @@ def basename(p: AnyOrLiteralStr) -> AnyOrLiteralStr: ... def dirname(p: PathLike[AnyStr]) -> AnyStr: ... @overload def dirname(p: AnyOrLiteralStr) -> AnyOrLiteralStr: ... -def expanduser(path: PathLike[AnyStr] | AnyStr) -> AnyStr: ... -def expandvars(path: PathLike[AnyStr] | AnyStr) -> AnyStr: ... +@overload +def expanduser(path: PathLike[AnyStr]) -> AnyStr: ... +@overload +def expanduser(path: AnyStr) -> AnyStr: ... +@overload +def expandvars(path: PathLike[AnyStr]) -> AnyStr: ... +@overload +def expandvars(path: AnyStr) -> AnyStr: ... @overload def normcase(s: PathLike[AnyStr]) -> AnyStr: ... @overload diff --git a/mypy/typeshed/stdlib/unittest/runner.pyi b/mypy/typeshed/stdlib/unittest/runner.pyi index 46da85619d30..be546f42315e 100644 --- a/mypy/typeshed/stdlib/unittest/runner.pyi +++ b/mypy/typeshed/stdlib/unittest/runner.pyi @@ -13,13 +13,14 @@ class _SupportsWriteAndFlush(SupportsWrite[str], SupportsFlush, Protocol): ... # All methods used by unittest.runner.TextTestResult's stream class _TextTestStream(_SupportsWriteAndFlush, Protocol): - def writeln(self, arg: str | None = None) -> str: ... + def writeln(self, arg: str | None = None, /) -> str: ... # _WritelnDecorator should have all the same attrs as its stream param. # But that's not feasible to do Generically # We can expand the attributes if requested class _WritelnDecorator(_TextTestStream): def __init__(self, stream: _TextTestStream) -> None: ... + def writeln(self, arg: str | None = None) -> str: ... def __getattr__(self, attr: str) -> Any: ... # Any attribute from the stream type passed to __init__ # These attributes are prevented by __getattr__ stream: Never From df7cd88eded0702fc1b8bb225b286a514a442e37 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Oct 2024 11:01:16 +0100 Subject: [PATCH 234/247] Merge some pythoneval tests to speed them up (#17990) Each test has some non-trivial constant overhead, so merging tests will speed them up. Merge a few tests. The merged tests are old and unlikely to fail (but still marginally useful), so it isn't important if test failures are now a bit more difficult to investigate. --- test-data/unit/pythoneval.test | 177 +++++++++++++-------------------- 1 file changed, 68 insertions(+), 109 deletions(-) diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 0dbd18ef188f..997cdc0d0ff0 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -11,138 +11,97 @@ print('hello, world') [out] hello, world -[case testReversed] +[case testMiscStdlibFeatures] +# Various legacy tests merged together to speed up test runtimes. + +def f(x: object) -> None: pass + +# testReversed from typing import Reversible -class A(Reversible): +class R(Reversible): def __iter__(self): return iter('oof') def __reversed__(self): return iter('foo') -print(list(reversed(range(5)))) -print(list(reversed([1,2,3]))) -print(list(reversed('abc'))) -print(list(reversed(A()))) -[out] --- Escape bracket at line beginning -\[4, 3, 2, 1, 0] -\[3, 2, 1] -\['c', 'b', 'a'] -\['f', 'o', 'o'] - -[case testIntAndFloatConversion] +f(list(reversed(range(5)))) +f(list(reversed([1,2,3]))) +f(list(reversed('abc'))) +f(list(reversed(R()))) + +# testIntAndFloatConversion from typing import SupportsInt, SupportsFloat class A(SupportsInt): def __int__(self): return 5 class B(SupportsFloat): def __float__(self): return 1.2 -print(int(1)) -print(int(6.2)) -print(int('3')) -print(int(b'4')) -print(int(A())) -print(float(-9)) -print(float(B())) -[out] -1 -6 -3 -4 -5 --9.0 -1.2 - -[case testAbs] +f(int(1)) +f(int(6.2)) +f(int('3')) +f(int(b'4')) +f(int(A())) +f(float(-9)) +f(float(B())) + +# testAbs from typing import SupportsAbs -class A(SupportsAbs[float]): +class Ab(SupportsAbs[float]): def __abs__(self) -> float: return 5.5 -print(abs(-1)) -print(abs(-1.2)) -print(abs(A())) -[out] -1 -1.2 -5.5 - -[case testAbs2] -n: int -f: float -n = abs(1) -abs(1) + 'x' # Error -f = abs(1.1) -abs(1.1) + 'x' # Error -[out] -_program.py:4: error: Unsupported operand types for + ("int" and "str") -_program.py:6: error: Unsupported operand types for + ("float" and "str") +f(abs(-1)) +f(abs(-1.2)) +f(abs(Ab())) -[case testRound] +# testRound from typing import SupportsRound -class A(SupportsRound): +class Ro(SupportsRound): def __round__(self, ndigits=0): return 'x%d' % ndigits -print(round(1.6)) -print(round(A())) -print(round(A(), 2)) -[out] -2 -x0 -x2 +f(round(1.6)) +f(round(Ro())) +f(round(Ro(), 2)) -[case testCallMethodViaTypeObject] -import typing -print(list.__add__([1, 2], [3, 4])) -[out] -\[1, 2, 3, 4] +# testCallMethodViaTypeObject +list.__add__([1, 2], [3, 4]) -[case testInheritedClassAttribute] +# testInheritedClassAttribute import typing -class A: +class AA: x = 1 - def f(self: typing.Optional["A"]) -> None: print('f') -class B(A): + def f(self: typing.Optional["AA"]) -> None: pass +class BB(AA): pass -B.f(None) -print(B.x) -[out] -f -1 - -[case testModuleAttributes] -import math -import typing -print(type(__spec__)) -print(math.__name__) -print(math.__spec__.name) -print(type(math.__dict__)) -print(type(math.__doc__ or '')) -print(type(math.__spec__).__name__) -print(math.__class__) -[out] - -math -math - - -ModuleSpec - +BB.f(None) +f(BB.x) -[case testSpecialAttributes] -import typing -class A: +# testSpecialAttributes +class Doc: """A docstring!""" -print(A().__doc__) -print(A().__class__) -[out] -A docstring! - +f(Doc().__doc__) +f(Doc().__class__) -[case testFunctionAttributes] -import typing -ord.__class__ -print(type(ord.__doc__ or '' + '')) -print(ord.__name__) -print(ord.__module__) +# testFunctionAttributes +f(ord.__class__) +f(type(ord.__doc__ or '' + '')) +f(ord.__name__) +f(ord.__module__) + +# testModuleAttributes +import math +f(type(__spec__)) +f(math.__name__) +f(math.__spec__.name) +f(type(math.__dict__)) +f(type(math.__doc__ or '')) +f(type(math.__spec__).__name__) +f(math.__class__) + +[case testAbs2] +n: int +f: float +n = abs(1) +abs(1) + 'x' # Error +f = abs(1.1) +abs(1.1) + 'x' # Error [out] - -ord -builtins +_program.py:4: error: Unsupported operand types for + ("int" and "str") +_program.py:6: error: Unsupported operand types for + ("float" and "str") [case testTypeAttributes] import typing From bd2aafc13686711d0db11d30efbf1d4f68718136 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 18 Oct 2024 11:44:58 +0100 Subject: [PATCH 235/247] Fix type object with type var default in union context (#17991) Union type context wasn't handled previously, and it triggered false positives, but apparently only if a type object had type var defaults. Fixes #17942. --- mypy/checkexpr.py | 13 +++++++++++-- test-data/unit/check-typevar-defaults.test | 12 ++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index b06aaa8f89f5..08d7345452fb 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -399,8 +399,8 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: # TODO: always do this in type_object_type by passing the original context result.ret_type.line = e.line result.ret_type.column = e.column - if isinstance(get_proper_type(self.type_context[-1]), TypeType): - # This is the type in a Type[] expression, so substitute type + if is_type_type_context(self.type_context[-1]): + # This is the type in a type[] expression, so substitute type # variables with Any. result = erasetype.erase_typevars(result) elif isinstance(node, MypyFile): @@ -6617,3 +6617,12 @@ def get_partial_instance_type(t: Type | None) -> PartialType | None: if t is None or not isinstance(t, PartialType) or t.type is None: return None return t + + +def is_type_type_context(context: Type | None) -> bool: + context = get_proper_type(context) + if isinstance(context, TypeType): + return True + if isinstance(context, UnionType): + return any(is_type_type_context(item) for item in context.items) + return False diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test index 9ca67376da26..22e2594eb38b 100644 --- a/test-data/unit/check-typevar-defaults.test +++ b/test-data/unit/check-typevar-defaults.test @@ -717,3 +717,15 @@ def func_d3( reveal_type(c) # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.int, builtins.float]]]" reveal_type(d) # N: Revealed type is "__main__.B[builtins.int]" [builtins fixtures/dict.pyi] + +[case testTypeVarDefaultsAndTypeObjectTypeInUnion] +from __future__ import annotations +from typing import Generic +from typing_extensions import TypeVar + +_I = TypeVar("_I", default=int) + +class C(Generic[_I]): pass + +t: type[C] | int = C +[builtins fixtures/tuple.pyi] From c9d4c61d9c80b02279d1fcc9ca8a1974717b5e1c Mon Sep 17 00:00:00 2001 From: jairov4 <1904410+jairov4@users.noreply.github.com> Date: Fri, 18 Oct 2024 09:37:40 -0500 Subject: [PATCH 236/247] [mypyc] Optimize dunder methods (#17934) This change gives mypyc the ability to optionally optimize dunder methods that can guarantee strict adherence to its signature typing. The optimization allows to bypass vtable for dunder methods in certain cases that are applicable. Currently, mypy has adopted the convention of accept dunder methods that return `NotImplemented` value even when its signature do not reflect this possibility. With this change and by enabling an special flag, mypyc will expect strict typing be honored and will unleash more optimizations like native call without vtable lookup for some cases on dunder method calls. For example it could avoid calls to RichCompare Python API making the code can be fully optimized in by the C compiler when some comparison with dunders are required. Example: ```python @final class A: def __init__(self, x: i32) -> None: self.x: Final = x def __lt__(self, other: "A") -> bool: return self.x < other.x A(1) < A(2) ``` would produce: ```c char CPyDef_A_____lt__(PyObject *cpy_r_self, PyObject *cpy_r_other) { int32_t cpy_r_r0; int32_t cpy_r_r1; char cpy_r_r2; cpy_r_r0 = ((AObject *)cpy_r_self)->_x; cpy_r_r1 = ((AObject *)cpy_r_other)->_x; cpy_r_r2 = cpy_r_r0 < cpy_r_r1; return cpy_r_r2; } ... cpy_r_r29 = CPyDef_A_____lt__(cpy_r_r27, cpy_r_r28); ... ``` Instead of: ```c PyObject *CPyDef_A_____lt__(PyObject *cpy_r_self, PyObject *cpy_r_other) { int32_t cpy_r_r0; int32_t cpy_r_r1; char cpy_r_r2; PyObject *cpy_r_r3; cpy_r_r0 = ((AObject *)cpy_r_self)->_x; cpy_r_r1 = ((AObject *)cpy_r_other)->_x; cpy_r_r2 = cpy_r_r0 < cpy_r_r1; cpy_r_r3 = cpy_r_r2 ? Py_True : Py_False; CPy_INCREF(cpy_r_r3); return cpy_r_r3; } ... cpy_r_r29 = PyObject_RichCompare(cpy_r_r27, cpy_r_r28, 0); ... ``` Default behavior is kept. Tests run with both of strict typing enabled and disabled. --- mypyc/__main__.py | 5 +- mypyc/build.py | 5 ++ mypyc/irbuild/classdef.py | 55 +++++++++++-------- mypyc/irbuild/function.py | 14 ++--- mypyc/irbuild/ll_builder.py | 53 ++++++++++++++---- mypyc/irbuild/mapper.py | 15 +++--- mypyc/irbuild/prepare.py | 68 +++++++++++++++++------- mypyc/options.py | 8 +++ mypyc/test-data/run-dunders-special.test | 9 ++++ mypyc/test-data/run-dunders.test | 34 +++++++++--- mypyc/test/test_run.py | 16 +++++- 11 files changed, 210 insertions(+), 72 deletions(-) create mode 100644 mypyc/test-data/run-dunders-special.test diff --git a/mypyc/__main__.py b/mypyc/__main__.py index a3b9d21bc65a..653199e0fb55 100644 --- a/mypyc/__main__.py +++ b/mypyc/__main__.py @@ -24,7 +24,7 @@ from mypyc.build import mypycify setup(name='mypyc_output', - ext_modules=mypycify({}, opt_level="{}", debug_level="{}"), + ext_modules=mypycify({}, opt_level="{}", debug_level="{}", strict_dunder_typing={}), ) """ @@ -38,10 +38,11 @@ def main() -> None: opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1") + strict_dunder_typing = bool(int(os.getenv("MYPYC_STRICT_DUNDER_TYPING", "0"))) setup_file = os.path.join(build_dir, "setup.py") with open(setup_file, "w") as f: - f.write(setup_format.format(sys.argv[1:], opt_level, debug_level)) + f.write(setup_format.format(sys.argv[1:], opt_level, debug_level, strict_dunder_typing)) # We don't use run_setup (like we do in the test suite) because it throws # away the error code from distutils, and we don't care about the slight diff --git a/mypyc/build.py b/mypyc/build.py index 485803acba46..6d59113ef872 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -470,6 +470,7 @@ def mypycify( skip_cgen_input: Any | None = None, target_dir: str | None = None, include_runtime_files: bool | None = None, + strict_dunder_typing: bool = False, ) -> list[Extension]: """Main entry point to building using mypyc. @@ -509,6 +510,9 @@ def mypycify( should be directly #include'd instead of linked separately in order to reduce compiler invocations. Defaults to False in multi_file mode, True otherwise. + strict_dunder_typing: If True, force dunder methods to have the return type + of the method strictly, which can lead to more + optimization opportunities. Defaults to False. """ # Figure out our configuration @@ -519,6 +523,7 @@ def mypycify( separate=separate is not False, target_dir=target_dir, include_runtime_files=include_runtime_files, + strict_dunder_typing=strict_dunder_typing, ) # Generate all the actual important C code diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index bcc9594adcb9..6ff6308b81d8 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -81,6 +81,7 @@ pytype_from_template_op, type_object_op, ) +from mypyc.subtype import is_subtype def transform_class_def(builder: IRBuilder, cdef: ClassDef) -> None: @@ -801,30 +802,42 @@ def create_ne_from_eq(builder: IRBuilder, cdef: ClassDef) -> None: def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> None: """Generate a "__ne__" method from a "__eq__" method.""" - with builder.enter_method(cls, "__ne__", object_rprimitive): - rhs_arg = builder.add_argument("rhs", object_rprimitive) - - # If __eq__ returns NotImplemented, then __ne__ should also - not_implemented_block, regular_block = BasicBlock(), BasicBlock() + func_ir = cls.get_method("__eq__") + assert func_ir + eq_sig = func_ir.decl.sig + strict_typing = builder.options.strict_dunders_typing + with builder.enter_method(cls, "__ne__", eq_sig.ret_type): + rhs_type = eq_sig.args[0].type if strict_typing else object_rprimitive + rhs_arg = builder.add_argument("rhs", rhs_type) eqval = builder.add(MethodCall(builder.self(), "__eq__", [rhs_arg], line)) - not_implemented = builder.add( - LoadAddress(not_implemented_op.type, not_implemented_op.src, line) - ) - builder.add( - Branch( - builder.translate_is_op(eqval, not_implemented, "is", line), - not_implemented_block, - regular_block, - Branch.BOOL, - ) - ) - builder.activate_block(regular_block) - retval = builder.coerce(builder.unary_op(eqval, "not", line), object_rprimitive, line) - builder.add(Return(retval)) + can_return_not_implemented = is_subtype(not_implemented_op.type, eq_sig.ret_type) + return_bool = is_subtype(eq_sig.ret_type, bool_rprimitive) - builder.activate_block(not_implemented_block) - builder.add(Return(not_implemented)) + if not strict_typing or can_return_not_implemented: + # If __eq__ returns NotImplemented, then __ne__ should also + not_implemented_block, regular_block = BasicBlock(), BasicBlock() + not_implemented = builder.add( + LoadAddress(not_implemented_op.type, not_implemented_op.src, line) + ) + builder.add( + Branch( + builder.translate_is_op(eqval, not_implemented, "is", line), + not_implemented_block, + regular_block, + Branch.BOOL, + ) + ) + builder.activate_block(regular_block) + rettype = bool_rprimitive if return_bool and strict_typing else object_rprimitive + retval = builder.coerce(builder.unary_op(eqval, "not", line), rettype, line) + builder.add(Return(retval)) + builder.activate_block(not_implemented_block) + builder.add(Return(not_implemented)) + else: + rettype = bool_rprimitive if return_bool and strict_typing else object_rprimitive + retval = builder.coerce(builder.unary_op(eqval, "not", line), rettype, line) + builder.add(Return(retval)) def load_non_ext_class( diff --git a/mypyc/irbuild/function.py b/mypyc/irbuild/function.py index c985e88b0e0c..f9d55db50f27 100644 --- a/mypyc/irbuild/function.py +++ b/mypyc/irbuild/function.py @@ -96,7 +96,8 @@ def transform_func_def(builder: IRBuilder, fdef: FuncDef) -> None: - func_ir, func_reg = gen_func_item(builder, fdef, fdef.name, builder.mapper.fdef_to_sig(fdef)) + sig = builder.mapper.fdef_to_sig(fdef, builder.options.strict_dunders_typing) + func_ir, func_reg = gen_func_item(builder, fdef, fdef.name, sig) # If the function that was visited was a nested function, then either look it up in our # current environment or define it if it was not already defined. @@ -113,9 +114,8 @@ def transform_overloaded_func_def(builder: IRBuilder, o: OverloadedFuncDef) -> N def transform_decorator(builder: IRBuilder, dec: Decorator) -> None: - func_ir, func_reg = gen_func_item( - builder, dec.func, dec.func.name, builder.mapper.fdef_to_sig(dec.func) - ) + sig = builder.mapper.fdef_to_sig(dec.func, builder.options.strict_dunders_typing) + func_ir, func_reg = gen_func_item(builder, dec.func, dec.func.name, sig) decorated_func: Value | None = None if func_reg: decorated_func = load_decorated_func(builder, dec.func, func_reg) @@ -416,7 +416,8 @@ def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None # Perform the function of visit_method for methods inside extension classes. name = fdef.name class_ir = builder.mapper.type_to_ir[cdef.info] - func_ir, func_reg = gen_func_item(builder, fdef, name, builder.mapper.fdef_to_sig(fdef), cdef) + sig = builder.mapper.fdef_to_sig(fdef, builder.options.strict_dunders_typing) + func_ir, func_reg = gen_func_item(builder, fdef, name, sig, cdef) builder.functions.append(func_ir) if is_decorated(builder, fdef): @@ -481,7 +482,8 @@ def handle_non_ext_method( ) -> None: # Perform the function of visit_method for methods inside non-extension classes. name = fdef.name - func_ir, func_reg = gen_func_item(builder, fdef, name, builder.mapper.fdef_to_sig(fdef), cdef) + sig = builder.mapper.fdef_to_sig(fdef, builder.options.strict_dunders_typing) + func_ir, func_reg = gen_func_item(builder, fdef, name, sig, cdef) assert func_reg is not None builder.functions.append(func_ir) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index c98136ce06d2..cb23a74c69c6 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -14,7 +14,7 @@ from mypy.argmap import map_actuals_to_formals from mypy.nodes import ARG_POS, ARG_STAR, ARG_STAR2, ArgKind -from mypy.operators import op_methods +from mypy.operators import op_methods, unary_op_methods from mypy.types import AnyType, TypeOfAny from mypyc.common import ( BITMAP_BITS, @@ -167,6 +167,7 @@ buf_init_item, fast_isinstance_op, none_object_op, + not_implemented_op, var_object_size, ) from mypyc.primitives.registry import ( @@ -1398,11 +1399,48 @@ def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: if base_op in float_op_to_id: return self.float_op(lreg, rreg, base_op, line) + dunder_op = self.dunder_op(lreg, rreg, op, line) + if dunder_op: + return dunder_op + primitive_ops_candidates = binary_ops.get(op, []) target = self.matching_primitive_op(primitive_ops_candidates, [lreg, rreg], line) assert target, "Unsupported binary operation: %s" % op return target + def dunder_op(self, lreg: Value, rreg: Value | None, op: str, line: int) -> Value | None: + """ + Dispatch a dunder method if applicable. + For example for `a + b` it will use `a.__add__(b)` which can lead to higher performance + due to the fact that the method could be already compiled and optimized instead of going + all the way through `PyNumber_Add(a, b)` python api (making a jump into the python DL). + """ + ltype = lreg.type + if not isinstance(ltype, RInstance): + return None + + method_name = op_methods.get(op) if rreg else unary_op_methods.get(op) + if method_name is None: + return None + + if not ltype.class_ir.has_method(method_name): + return None + + decl = ltype.class_ir.method_decl(method_name) + if not rreg and len(decl.sig.args) != 1: + return None + + if rreg and (len(decl.sig.args) != 2 or not is_subtype(rreg.type, decl.sig.args[1].type)): + return None + + if rreg and is_subtype(not_implemented_op.type, decl.sig.ret_type): + # If the method is able to return NotImplemented, we should not optimize it. + # We can just let go so it will be handled through the python api. + return None + + args = [rreg] if rreg else [] + return self.gen_method_call(lreg, method_name, args, decl.sig.ret_type, line) + def check_tagged_short_int(self, val: Value, line: int, negated: bool = False) -> Value: """Check if a tagged integer is a short integer. @@ -1558,16 +1596,9 @@ def unary_op(self, value: Value, expr_op: str, line: int) -> Value: if isinstance(value, Float): return Float(-value.value, value.line) if isinstance(typ, RInstance): - if expr_op == "-": - method = "__neg__" - elif expr_op == "+": - method = "__pos__" - elif expr_op == "~": - method = "__invert__" - else: - method = "" - if method and typ.class_ir.has_method(method): - return self.gen_method_call(value, method, [], None, line) + result = self.dunder_op(value, None, expr_op, line) + if result is not None: + return result call_c_ops_candidates = unary_ops.get(expr_op, []) target = self.matching_call_c(call_c_ops_candidates, [value], line) assert target, "Unsupported unary operation: %s" % expr_op diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 90ce0e16c741..78e54aceed3d 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -160,7 +160,7 @@ def get_arg_rtype(self, typ: Type, kind: ArgKind) -> RType: else: return self.type_to_rtype(typ) - def fdef_to_sig(self, fdef: FuncDef) -> FuncSignature: + def fdef_to_sig(self, fdef: FuncDef, strict_dunders_typing: bool) -> FuncSignature: if isinstance(fdef.type, CallableType): arg_types = [ self.get_arg_rtype(typ, kind) @@ -199,11 +199,14 @@ def fdef_to_sig(self, fdef: FuncDef) -> FuncSignature: ) ] - # We force certain dunder methods to return objects to support letting them - # return NotImplemented. It also avoids some pointless boxing and unboxing, - # since tp_richcompare needs an object anyways. - if fdef.name in ("__eq__", "__ne__", "__lt__", "__gt__", "__le__", "__ge__"): - ret = object_rprimitive + if not strict_dunders_typing: + # We force certain dunder methods to return objects to support letting them + # return NotImplemented. It also avoids some pointless boxing and unboxing, + # since tp_richcompare needs an object anyways. + # However, it also prevents some optimizations. + if fdef.name in ("__eq__", "__ne__", "__lt__", "__gt__", "__le__", "__ge__"): + ret = object_rprimitive + return FuncSignature(args, ret) def is_native_module(self, module: str) -> bool: diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 40a40b79df49..bed9a1684326 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -99,9 +99,11 @@ def build_type_map( for module, cdef in classes: with catch_errors(module.path, cdef.line): if mapper.type_to_ir[cdef.info].is_ext_class: - prepare_class_def(module.path, module.fullname, cdef, errors, mapper) + prepare_class_def(module.path, module.fullname, cdef, errors, mapper, options) else: - prepare_non_ext_class_def(module.path, module.fullname, cdef, errors, mapper) + prepare_non_ext_class_def( + module.path, module.fullname, cdef, errors, mapper, options + ) # Prepare implicit attribute accessors as needed if an attribute overrides a property. for module, cdef in classes: @@ -114,7 +116,7 @@ def build_type_map( # is conditionally defined. for module in modules: for func in get_module_func_defs(module): - prepare_func_def(module.fullname, None, func, mapper) + prepare_func_def(module.fullname, None, func, mapper, options) # TODO: what else? # Check for incompatible attribute definitions that were not @@ -168,27 +170,39 @@ def get_module_func_defs(module: MypyFile) -> Iterable[FuncDef]: def prepare_func_def( - module_name: str, class_name: str | None, fdef: FuncDef, mapper: Mapper + module_name: str, + class_name: str | None, + fdef: FuncDef, + mapper: Mapper, + options: CompilerOptions, ) -> FuncDecl: kind = ( FUNC_STATICMETHOD if fdef.is_static else (FUNC_CLASSMETHOD if fdef.is_class else FUNC_NORMAL) ) - decl = FuncDecl(fdef.name, class_name, module_name, mapper.fdef_to_sig(fdef), kind) + sig = mapper.fdef_to_sig(fdef, options.strict_dunders_typing) + decl = FuncDecl(fdef.name, class_name, module_name, sig, kind) mapper.func_to_decl[fdef] = decl return decl def prepare_method_def( - ir: ClassIR, module_name: str, cdef: ClassDef, mapper: Mapper, node: FuncDef | Decorator + ir: ClassIR, + module_name: str, + cdef: ClassDef, + mapper: Mapper, + node: FuncDef | Decorator, + options: CompilerOptions, ) -> None: if isinstance(node, FuncDef): - ir.method_decls[node.name] = prepare_func_def(module_name, cdef.name, node, mapper) + ir.method_decls[node.name] = prepare_func_def( + module_name, cdef.name, node, mapper, options + ) elif isinstance(node, Decorator): # TODO: do something about abstract methods here. Currently, they are handled just like # normal methods. - decl = prepare_func_def(module_name, cdef.name, node.func, mapper) + decl = prepare_func_def(module_name, cdef.name, node.func, mapper, options) if not node.decorators: ir.method_decls[node.name] = decl elif isinstance(node.decorators[0], MemberExpr) and node.decorators[0].name == "setter": @@ -241,7 +255,12 @@ def can_subclass_builtin(builtin_base: str) -> bool: def prepare_class_def( - path: str, module_name: str, cdef: ClassDef, errors: Errors, mapper: Mapper + path: str, + module_name: str, + cdef: ClassDef, + errors: Errors, + mapper: Mapper, + options: CompilerOptions, ) -> None: """Populate the interface-level information in a class IR. @@ -308,7 +327,7 @@ def prepare_class_def( ir.mro = mro ir.base_mro = base_mro - prepare_methods_and_attributes(cdef, ir, path, module_name, errors, mapper) + prepare_methods_and_attributes(cdef, ir, path, module_name, errors, mapper, options) prepare_init_method(cdef, ir, module_name, mapper) for base in bases: @@ -320,7 +339,13 @@ def prepare_class_def( def prepare_methods_and_attributes( - cdef: ClassDef, ir: ClassIR, path: str, module_name: str, errors: Errors, mapper: Mapper + cdef: ClassDef, + ir: ClassIR, + path: str, + module_name: str, + errors: Errors, + mapper: Mapper, + options: CompilerOptions, ) -> None: """Populate attribute and method declarations.""" info = cdef.info @@ -342,20 +367,20 @@ def prepare_methods_and_attributes( add_setter_declaration(ir, name, attr_rtype, module_name) ir.attributes[name] = attr_rtype elif isinstance(node.node, (FuncDef, Decorator)): - prepare_method_def(ir, module_name, cdef, mapper, node.node) + prepare_method_def(ir, module_name, cdef, mapper, node.node, options) elif isinstance(node.node, OverloadedFuncDef): # Handle case for property with both a getter and a setter if node.node.is_property: if is_valid_multipart_property_def(node.node): for item in node.node.items: - prepare_method_def(ir, module_name, cdef, mapper, item) + prepare_method_def(ir, module_name, cdef, mapper, item, options) else: errors.error("Unsupported property decorator semantics", path, cdef.line) # Handle case for regular function overload else: assert node.node.impl - prepare_method_def(ir, module_name, cdef, mapper, node.node.impl) + prepare_method_def(ir, module_name, cdef, mapper, node.node.impl, options) if ir.builtin_base: ir.attributes.clear() @@ -440,7 +465,7 @@ def prepare_init_method(cdef: ClassDef, ir: ClassIR, module_name: str, mapper: M # Set up a constructor decl init_node = cdef.info["__init__"].node if not ir.is_trait and not ir.builtin_base and isinstance(init_node, FuncDef): - init_sig = mapper.fdef_to_sig(init_node) + init_sig = mapper.fdef_to_sig(init_node, True) defining_ir = mapper.type_to_ir.get(init_node.info) # If there is a nontrivial __init__ that wasn't defined in an @@ -467,24 +492,29 @@ def prepare_init_method(cdef: ClassDef, ir: ClassIR, module_name: str, mapper: M def prepare_non_ext_class_def( - path: str, module_name: str, cdef: ClassDef, errors: Errors, mapper: Mapper + path: str, + module_name: str, + cdef: ClassDef, + errors: Errors, + mapper: Mapper, + options: CompilerOptions, ) -> None: ir = mapper.type_to_ir[cdef.info] info = cdef.info for node in info.names.values(): if isinstance(node.node, (FuncDef, Decorator)): - prepare_method_def(ir, module_name, cdef, mapper, node.node) + prepare_method_def(ir, module_name, cdef, mapper, node.node, options) elif isinstance(node.node, OverloadedFuncDef): # Handle case for property with both a getter and a setter if node.node.is_property: if not is_valid_multipart_property_def(node.node): errors.error("Unsupported property decorator semantics", path, cdef.line) for item in node.node.items: - prepare_method_def(ir, module_name, cdef, mapper, item) + prepare_method_def(ir, module_name, cdef, mapper, item, options) # Handle case for regular function overload else: - prepare_method_def(ir, module_name, cdef, mapper, get_func_def(node.node)) + prepare_method_def(ir, module_name, cdef, mapper, get_func_def(node.node), options) if any(cls in mapper.type_to_ir and mapper.type_to_ir[cls].is_ext_class for cls in info.mro): errors.error( diff --git a/mypyc/options.py b/mypyc/options.py index 5f0cf12aeefe..24e68163bb11 100644 --- a/mypyc/options.py +++ b/mypyc/options.py @@ -14,6 +14,7 @@ def __init__( include_runtime_files: bool | None = None, capi_version: tuple[int, int] | None = None, python_version: tuple[int, int] | None = None, + strict_dunder_typing: bool = False, ) -> None: self.strip_asserts = strip_asserts self.multi_file = multi_file @@ -30,3 +31,10 @@ def __init__( # features are used. self.capi_version = capi_version or sys.version_info[:2] self.python_version = python_version + # Make possible to inline dunder methods in the generated code. + # Typically, the convention is the dunder methods can return `NotImplemented` + # even when its return type is just `bool`. + # By enabling this option, this convention is no longer valid and the dunder + # will assume the return type of the method strictly, which can lead to + # more optimization opportunities. + self.strict_dunders_typing = strict_dunder_typing diff --git a/mypyc/test-data/run-dunders-special.test b/mypyc/test-data/run-dunders-special.test new file mode 100644 index 000000000000..cd02cca65eef --- /dev/null +++ b/mypyc/test-data/run-dunders-special.test @@ -0,0 +1,9 @@ +[case testDundersNotImplemented] +# This case is special because it tests the behavior of NotImplemented +# used in a typed function which return type is bool. +# This is a convention that can be overriden by the user. +class UsesNotImplemented: + def __eq__(self, b: object) -> bool: + return NotImplemented + +assert UsesNotImplemented() != object() diff --git a/mypyc/test-data/run-dunders.test b/mypyc/test-data/run-dunders.test index 2845187de2c3..b8fb13c9dcec 100644 --- a/mypyc/test-data/run-dunders.test +++ b/mypyc/test-data/run-dunders.test @@ -32,10 +32,6 @@ class BoxedThing: class Subclass2(BoxedThing): pass -class UsesNotImplemented: - def __eq__(self, b: object) -> bool: - return NotImplemented - def index_into(x : Any, y : Any) -> Any: return x[y] @@ -81,8 +77,6 @@ assert is_truthy(Item('a')) assert not is_truthy(Subclass1('')) assert is_truthy(Subclass1('a')) -assert UsesNotImplemented() != object() - internal_index_into() [out] 7 7 @@ -943,3 +937,31 @@ def test_errors() -> None: pow(ForwardNotImplemented(), Child(), 3) # type: ignore with assertRaises(TypeError, "unsupported operand type(s) for ** or pow(): 'ForwardModRequired' and 'int'"): ForwardModRequired()**3 # type: ignore + +[case testDundersWithFinal] +from typing import final +class A: + def __init__(self, x: int) -> None: + self.x = x + + def __add__(self, y: int) -> int: + return self.x + y + + def __lt__(self, x: 'A') -> bool: + return self.x < x.x + +@final +class B(A): + def __add__(self, y: int) -> int: + return self.x + y + 1 + + def __lt__(self, x: 'A') -> bool: + return self.x < x.x + 1 + +def test_final() -> None: + a = A(5) + b = B(5) + assert a + 3 == 8 + assert b + 3 == 9 + assert (a < A(5)) is False + assert (b < A(5)) is True diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 668e5b124841..dd3c79da7b9b 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -63,6 +63,7 @@ "run-bench.test", "run-mypy-sim.test", "run-dunders.test", + "run-dunders-special.test", "run-singledispatch.test", "run-attrs.test", "run-python37.test", @@ -140,6 +141,7 @@ class TestRun(MypycDataSuite): optional_out = True multi_file = False separate = False # If True, using separate (incremental) compilation + strict_dunder_typing = False def run_case(self, testcase: DataDrivenTestCase) -> None: # setup.py wants to be run from the root directory of the package, which we accommodate @@ -232,7 +234,11 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) -> groups = construct_groups(sources, separate, len(module_names) > 1) try: - compiler_options = CompilerOptions(multi_file=self.multi_file, separate=self.separate) + compiler_options = CompilerOptions( + multi_file=self.multi_file, + separate=self.separate, + strict_dunder_typing=self.strict_dunder_typing, + ) result = emitmodule.parse_and_typecheck( sources=sources, options=options, @@ -401,6 +407,14 @@ class TestRunSeparate(TestRun): files = ["run-multimodule.test", "run-mypy-sim.test"] +class TestRunStrictDunderTyping(TestRun): + """Run the tests with strict dunder typing.""" + + strict_dunder_typing = True + test_name_suffix = "_dunder_typing" + files = ["run-dunders.test", "run-floats.test"] + + def fix_native_line_number(message: str, fnam: str, delta: int) -> str: """Update code locations in test case output to point to the .test file. From 603a3652b59c21e82927db407194b824b3ef7f7b Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 19 Oct 2024 16:55:31 +0200 Subject: [PATCH 237/247] Add initial support for new style TypeVar defaults (PEP 696) (#17985) Add initial support for TypeVar defaults using the new syntax. Similar to the old syntax, it doesn't fully work yet for ParamSpec, TypeVarTuple and recursive TypeVar defaults. Refs: #14851 --- mypy/checker.py | 15 ++ mypy/fastparse.py | 22 +- mypy/message_registry.py | 5 - mypy/nodes.py | 10 +- mypy/semanal.py | 132 ++++++----- mypy/strconv.py | 2 + mypy/test/testparse.py | 4 + test-data/unit/check-python313.test | 262 ++++++++++++++++++++- test-data/unit/check-typevar-defaults.test | 9 +- test-data/unit/parse-python313.test | 80 +++++++ 10 files changed, 456 insertions(+), 85 deletions(-) create mode 100644 test-data/unit/parse-python313.test diff --git a/mypy/checker.py b/mypy/checker.py index 74890aa86cda..4b3d6c3298b4 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1159,6 +1159,7 @@ def check_func_def( ) -> None: """Type check a function definition.""" # Expand type variables with value restrictions to ordinary types. + self.check_typevar_defaults(typ.variables) expanded = self.expand_typevars(defn, typ) original_typ = typ for item, typ in expanded: @@ -2483,6 +2484,8 @@ def visit_class_def(self, defn: ClassDef) -> None: context=defn, code=codes.TYPE_VAR, ) + if typ.defn.type_vars: + self.check_typevar_defaults(typ.defn.type_vars) if typ.is_protocol and typ.defn.type_vars: self.check_protocol_variance(defn) @@ -2546,6 +2549,15 @@ def check_init_subclass(self, defn: ClassDef) -> None: # all other bases have already been checked. break + def check_typevar_defaults(self, tvars: Sequence[TypeVarLikeType]) -> None: + for tv in tvars: + if not (isinstance(tv, TypeVarType) and tv.has_default()): + continue + if not is_subtype(tv.default, tv.upper_bound): + self.fail("TypeVar default must be a subtype of the bound type", tv) + if tv.values and not any(tv.default == value for value in tv.values): + self.fail("TypeVar default must be one of the constraint types", tv) + def check_enum(self, defn: ClassDef) -> None: assert defn.info.is_enum if defn.info.fullname not in ENUM_BASES: @@ -5365,6 +5377,9 @@ def remove_capture_conflicts(self, type_map: TypeMap, inferred_types: dict[Var, del type_map[expr] def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: + if o.alias_node: + self.check_typevar_defaults(o.alias_node.alias_tvars) + with self.msg.filter_errors(): self.expr_checker.accept(o.value) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index bc5b1ba8e57a..a47ed9b536da 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1196,19 +1196,17 @@ def validate_type_param(self, type_param: ast_TypeVar) -> None: def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: explicit_type_params = [] for p in type_params: - bound = None + bound: Type | None = None values: list[Type] = [] - if sys.version_info >= (3, 13) and p.default_value is not None: - self.fail( - message_registry.TYPE_PARAM_DEFAULT_NOT_SUPPORTED, - p.lineno, - p.col_offset, - blocker=False, - ) + default: Type | None = None + if sys.version_info >= (3, 13): + default = TypeConverter(self.errors, line=p.lineno).visit(p.default_value) if isinstance(p, ast_ParamSpec): # type: ignore[misc] - explicit_type_params.append(TypeParam(p.name, PARAM_SPEC_KIND, None, [])) + explicit_type_params.append(TypeParam(p.name, PARAM_SPEC_KIND, None, [], default)) elif isinstance(p, ast_TypeVarTuple): # type: ignore[misc] - explicit_type_params.append(TypeParam(p.name, TYPE_VAR_TUPLE_KIND, None, [])) + explicit_type_params.append( + TypeParam(p.name, TYPE_VAR_TUPLE_KIND, None, [], default) + ) else: if isinstance(p.bound, ast3.Tuple): if len(p.bound.elts) < 2: @@ -1224,7 +1222,9 @@ def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: elif p.bound is not None: self.validate_type_param(p) bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) - explicit_type_params.append(TypeParam(p.name, TYPE_VAR_KIND, bound, values)) + explicit_type_params.append( + TypeParam(p.name, TYPE_VAR_KIND, bound, values, default) + ) return explicit_type_params # Return(expr? value) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 507af6fdad74..29d539faaed6 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -362,8 +362,3 @@ def with_additional_msg(self, info: str) -> ErrorMessage: TYPE_ALIAS_WITH_AWAIT_EXPRESSION: Final = ErrorMessage( "Await expression cannot be used within a type alias", codes.SYNTAX ) - -TYPE_PARAM_DEFAULT_NOT_SUPPORTED: Final = ErrorMessage( - "Type parameter default types not supported when using Python 3.12 type parameter syntax", - codes.MISC, -) diff --git a/mypy/nodes.py b/mypy/nodes.py index c4d23ca7bff8..4806a7b0be7d 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -670,7 +670,7 @@ def set_line( class TypeParam: - __slots__ = ("name", "kind", "upper_bound", "values") + __slots__ = ("name", "kind", "upper_bound", "values", "default") def __init__( self, @@ -678,11 +678,13 @@ def __init__( kind: int, upper_bound: mypy.types.Type | None, values: list[mypy.types.Type], + default: mypy.types.Type | None, ) -> None: self.name = name self.kind = kind self.upper_bound = upper_bound self.values = values + self.default = default FUNCITEM_FLAGS: Final = FUNCBASE_FLAGS + [ @@ -782,7 +784,7 @@ class FuncDef(FuncItem, SymbolNode, Statement): "deco_line", "is_trivial_body", "is_mypy_only", - # Present only when a function is decorated with @typing.datasclass_transform or similar + # Present only when a function is decorated with @typing.dataclass_transform or similar "dataclass_transform_spec", "docstring", "deprecated", @@ -1657,7 +1659,7 @@ def accept(self, visitor: StatementVisitor[T]) -> T: class TypeAliasStmt(Statement): - __slots__ = ("name", "type_args", "value", "invalid_recursive_alias") + __slots__ = ("name", "type_args", "value", "invalid_recursive_alias", "alias_node") __match_args__ = ("name", "type_args", "value") @@ -1665,6 +1667,7 @@ class TypeAliasStmt(Statement): type_args: list[TypeParam] value: LambdaExpr # Return value will get translated into a type invalid_recursive_alias: bool + alias_node: TypeAlias | None def __init__(self, name: NameExpr, type_args: list[TypeParam], value: LambdaExpr) -> None: super().__init__() @@ -1672,6 +1675,7 @@ def __init__(self, name: NameExpr, type_args: list[TypeParam], value: LambdaExpr self.type_args = type_args self.value = value self.invalid_recursive_alias = False + self.alias_node = None def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_type_alias_stmt(self) diff --git a/mypy/semanal.py b/mypy/semanal.py index 5332c98c8f0d..a0d4daffca0c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1808,7 +1808,26 @@ def analyze_type_param( upper_bound = self.named_type("builtins.tuple", [self.object_type()]) else: upper_bound = self.object_type() - default = AnyType(TypeOfAny.from_omitted_generics) + if type_param.default: + default = self.anal_type( + type_param.default, + allow_placeholder=True, + allow_unbound_tvars=True, + report_invalid_types=False, + allow_param_spec_literals=type_param.kind == PARAM_SPEC_KIND, + allow_tuple_literal=type_param.kind == PARAM_SPEC_KIND, + allow_unpack=type_param.kind == TYPE_VAR_TUPLE_KIND, + ) + if default is None: + default = PlaceholderType(None, [], context.line) + elif type_param.kind == TYPE_VAR_KIND: + default = self.check_typevar_default(default, type_param.default) + elif type_param.kind == PARAM_SPEC_KIND: + default = self.check_paramspec_default(default, type_param.default) + elif type_param.kind == TYPE_VAR_TUPLE_KIND: + default = self.check_typevartuple_default(default, type_param.default) + else: + default = AnyType(TypeOfAny.from_omitted_generics) if type_param.kind == TYPE_VAR_KIND: values = [] if type_param.values: @@ -2243,21 +2262,7 @@ class Foo(Bar, Generic[T]): ... # grained incremental mode. defn.removed_base_type_exprs.append(defn.base_type_exprs[i]) del base_type_exprs[i] - tvar_defs: list[TypeVarLikeType] = [] - last_tvar_name_with_default: str | None = None - for name, tvar_expr in declared_tvars: - tvar_expr.default = tvar_expr.default.accept( - TypeVarDefaultTranslator(self, tvar_expr.name, context) - ) - tvar_def = self.tvar_scope.bind_new(name, tvar_expr) - if last_tvar_name_with_default is not None and not tvar_def.has_default(): - self.msg.tvar_without_default_type( - tvar_def.name, last_tvar_name_with_default, context - ) - tvar_def.default = AnyType(TypeOfAny.from_error) - elif tvar_def.has_default(): - last_tvar_name_with_default = tvar_def.name - tvar_defs.append(tvar_def) + tvar_defs = self.tvar_defs_from_tvars(declared_tvars, context) return base_type_exprs, tvar_defs, is_protocol def analyze_class_typevar_declaration(self, base: Type) -> tuple[TypeVarLikeList, bool] | None: @@ -2358,6 +2363,26 @@ def get_all_bases_tvars( tvars.extend(base_tvars) return remove_dups(tvars) + def tvar_defs_from_tvars( + self, tvars: TypeVarLikeList, context: Context + ) -> list[TypeVarLikeType]: + tvar_defs: list[TypeVarLikeType] = [] + last_tvar_name_with_default: str | None = None + for name, tvar_expr in tvars: + tvar_expr.default = tvar_expr.default.accept( + TypeVarDefaultTranslator(self, tvar_expr.name, context) + ) + tvar_def = self.tvar_scope.bind_new(name, tvar_expr) + if last_tvar_name_with_default is not None and not tvar_def.has_default(): + self.msg.tvar_without_default_type( + tvar_def.name, last_tvar_name_with_default, context + ) + tvar_def.default = AnyType(TypeOfAny.from_error) + elif tvar_def.has_default(): + last_tvar_name_with_default = tvar_def.name + tvar_defs.append(tvar_def) + return tvar_defs + def get_and_bind_all_tvars(self, type_exprs: list[Expression]) -> list[TypeVarLikeType]: """Return all type variable references in item type expressions. @@ -3833,21 +3858,8 @@ def analyze_alias( tvar_defs: list[TypeVarLikeType] = [] namespace = self.qualified_name(name) alias_type_vars = found_type_vars if declared_type_vars is None else declared_type_vars - last_tvar_name_with_default: str | None = None with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): - for name, tvar_expr in alias_type_vars: - tvar_expr.default = tvar_expr.default.accept( - TypeVarDefaultTranslator(self, tvar_expr.name, typ) - ) - tvar_def = self.tvar_scope.bind_new(name, tvar_expr) - if last_tvar_name_with_default is not None and not tvar_def.has_default(): - self.msg.tvar_without_default_type( - tvar_def.name, last_tvar_name_with_default, typ - ) - tvar_def.default = AnyType(TypeOfAny.from_error) - elif tvar_def.has_default(): - last_tvar_name_with_default = tvar_def.name - tvar_defs.append(tvar_def) + tvar_defs = self.tvar_defs_from_tvars(alias_type_vars, typ) if python_3_12_type_alias: with self.allow_unbound_tvars_set(): @@ -4615,6 +4627,40 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> bool: self.add_symbol(name, call.analyzed, s) return True + def check_typevar_default(self, default: Type, context: Context) -> Type: + typ = get_proper_type(default) + if isinstance(typ, AnyType) and typ.is_from_error: + self.fail( + message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format("TypeVar", "default"), context + ) + return default + + def check_paramspec_default(self, default: Type, context: Context) -> Type: + typ = get_proper_type(default) + if isinstance(typ, Parameters): + for i, arg_type in enumerate(typ.arg_types): + arg_ptype = get_proper_type(arg_type) + if isinstance(arg_ptype, AnyType) and arg_ptype.is_from_error: + self.fail(f"Argument {i} of ParamSpec default must be a type", context) + elif ( + isinstance(typ, AnyType) + and typ.is_from_error + or not isinstance(typ, (AnyType, UnboundType)) + ): + self.fail( + "The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec", + context, + ) + default = AnyType(TypeOfAny.from_error) + return default + + def check_typevartuple_default(self, default: Type, context: Context) -> Type: + typ = get_proper_type(default) + if not isinstance(typ, UnpackType): + self.fail("The default argument to TypeVarTuple must be an Unpacked tuple", context) + default = AnyType(TypeOfAny.from_error) + return default + def check_typevarlike_name(self, call: CallExpr, name: str, context: Context) -> bool: """Checks that the name of a TypeVar or ParamSpec matches its variable.""" name = unmangle(name) @@ -4822,23 +4868,7 @@ def process_paramspec_declaration(self, s: AssignmentStmt) -> bool: report_invalid_typevar_arg=False, ) default = tv_arg or AnyType(TypeOfAny.from_error) - if isinstance(tv_arg, Parameters): - for i, arg_type in enumerate(tv_arg.arg_types): - typ = get_proper_type(arg_type) - if isinstance(typ, AnyType) and typ.is_from_error: - self.fail( - f"Argument {i} of ParamSpec default must be a type", param_value - ) - elif ( - isinstance(default, AnyType) - and default.is_from_error - or not isinstance(default, (AnyType, UnboundType)) - ): - self.fail( - "The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec", - param_value, - ) - default = AnyType(TypeOfAny.from_error) + default = self.check_paramspec_default(default, param_value) else: # ParamSpec is different from a regular TypeVar: # arguments are not semantically valid. But, allowed in runtime. @@ -4899,12 +4929,7 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool: allow_unpack=True, ) default = tv_arg or AnyType(TypeOfAny.from_error) - if not isinstance(default, UnpackType): - self.fail( - "The default argument to TypeVarTuple must be an Unpacked tuple", - param_value, - ) - default = AnyType(TypeOfAny.from_error) + default = self.check_typevartuple_default(default, param_value) else: self.fail(f'Unexpected keyword argument "{param_name}" for "TypeVarTuple"', s) @@ -5503,6 +5528,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: eager=eager, python_3_12_type_alias=True, ) + s.alias_node = alias_node if ( existing diff --git a/mypy/strconv.py b/mypy/strconv.py index a96a27c45d75..d2ac71412f90 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -349,6 +349,8 @@ def type_param(self, p: mypy.nodes.TypeParam) -> list[Any]: a.append(p.upper_bound) if p.values: a.append(("Values", p.values)) + if p.default: + a.append(("Default", [p.default])) return [("TypeParam", a)] # Expressions diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py index e215920a6797..074ccfb379d0 100644 --- a/mypy/test/testparse.py +++ b/mypy/test/testparse.py @@ -25,6 +25,8 @@ class ParserSuite(DataSuite): files.remove("parse-python310.test") if sys.version_info < (3, 12): files.remove("parse-python312.test") + if sys.version_info < (3, 13): + files.remove("parse-python313.test") def run_case(self, testcase: DataDrivenTestCase) -> None: test_parser(testcase) @@ -43,6 +45,8 @@ def test_parser(testcase: DataDrivenTestCase) -> None: options.python_version = (3, 10) elif testcase.file.endswith("python312.test"): options.python_version = (3, 12) + elif testcase.file.endswith("python313.test"): + options.python_version = (3, 13) else: options.python_version = defaults.PYTHON3_VERSION diff --git a/test-data/unit/check-python313.test b/test-data/unit/check-python313.test index 0ba64ad67c91..2729ad3e21d1 100644 --- a/test-data/unit/check-python313.test +++ b/test-data/unit/check-python313.test @@ -1,11 +1,257 @@ -[case testPEP695TypeParameterDefaultNotSupported] -class C[T = None]: # E: Type parameter default types not supported when using Python 3.12 type parameter syntax - pass +[case testPEP695TypeParameterDefaultSupported] +class C[T = None]: ... +def f[T = list[int]]() -> None: ... +def g[**P = [int, str]]() -> None: ... +type A[T, S = int, U = str] = list[T] -def f[T = list[int]]() -> None: # E: Type parameter default types not supported when using Python 3.12 type parameter syntax - pass +[case testPEP695TypeParameterDefaultBasic] +from typing import Callable -def g[**P = [int, str]]() -> None: # E: Type parameter default types not supported when using Python 3.12 type parameter syntax - pass +def f1[T1 = int](a: T1) -> list[T1]: ... +reveal_type(f1) # N: Revealed type is "def [T1 = builtins.int] (a: T1`-1 = builtins.int) -> builtins.list[T1`-1 = builtins.int]" -type A[T, S = int, U = str] = list[T] # E: Type parameter default types not supported when using Python 3.12 type parameter syntax +def f2[**P1 = [int, str]](a: Callable[P1, None]) -> Callable[P1, None]: ... +reveal_type(f2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] (a: def (*P1.args, **P1.kwargs)) -> def (*P1.args, **P1.kwargs)" + +def f3[*Ts1 = *tuple[int, str]](a: tuple[*Ts1]) -> tuple[*Ts1]: ... +reveal_type(f3) # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] (a: Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]) -> Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]" + + +class ClassA1[T1 = int]: ... +class ClassA2[**P1 = [int, str]]: ... +class ClassA3[*Ts1 = *tuple[int, str]]: ... + +reveal_type(ClassA1) # N: Revealed type is "def [T1 = builtins.int] () -> __main__.ClassA1[T1`1 = builtins.int]" +reveal_type(ClassA2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] () -> __main__.ClassA2[P1`1 = [builtins.int, builtins.str]]" +reveal_type(ClassA3) # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] () -> __main__.ClassA3[Unpack[Ts1`1 = Unpack[Tuple[builtins.int, builtins.str]]]]" +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultValid] +from typing import Any + +class ClassT1[T = int]: ... +class ClassT2[T: float = int]: ... +class ClassT3[T: list[Any] = list[int]]: ... +class ClassT4[T: (int, str) = int]: ... + +class ClassP1[**P = []]: ... +class ClassP2[**P = ...]: ... +class ClassP3[**P = [int, str]]: ... + +class ClassTs1[*Ts = *tuple[int]]: ... +class ClassTs2[*Ts = *tuple[int, ...]]: ... +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultInvalid] +class ClassT1[T = 2]: ... # E: TypeVar "default" must be a type +class ClassT2[T = [int]]: ... # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? \ + # E: TypeVar "default" must be a type +class ClassT3[T: str = int]: ... # E: TypeVar default must be a subtype of the bound type +class ClassT4[T: list[str] = list[int]]: ... # E: TypeVar default must be a subtype of the bound type +class ClassT5[T: (int, str) = bytes]: ... # E: TypeVar default must be one of the constraint types +class ClassT6[T: (int, str) = int | str]: ... # E: TypeVar default must be one of the constraint types +class ClassT7[T: (float, str) = int]: ... # E: TypeVar default must be one of the constraint types + +class ClassP1[**P = int]: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +class ClassP2[**P = 2]: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +class ClassP3[**P = (2, int)]: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +class ClassP4[**P = [2, int]]: ... # E: Argument 0 of ParamSpec default must be a type + +class ClassTs1[*Ts = 2]: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +class ClassTs2[*Ts = int]: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +class ClassTs3[*Ts = tuple[int]]: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultInvalid2] +from typing import overload +def f1[T = 2]() -> None: ... # E: TypeVar "default" must be a type +def f2[T = [int]]() -> None: ... # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? \ + # E: TypeVar "default" must be a type +def f3[T: str = int](x: T) -> T: ... # E: TypeVar default must be a subtype of the bound type +def f4[T: list[str] = list[int]](x: T) -> T: ... # E: TypeVar default must be a subtype of the bound type +def f5[T: (int, str) = bytes](x: T) -> T: ... # E: TypeVar default must be one of the constraint types +def f6[T: (int, str) = int | str](x: T) -> T: ... # E: TypeVar default must be one of the constraint types +def f7[T: (float, str) = int](x: T) -> T: ... # E: TypeVar default must be one of the constraint types +def f8[T: str = int]() -> None: ... # TODO check unused TypeVars +@overload +def f9[T: str = int](x: T) -> T: ... # E: TypeVar default must be a subtype of the bound type +@overload +def f9[T: (int, str) = bytes](x: T) -> T: ... # E: TypeVar default must be one of the constraint types +def f9() -> None: ... # type: ignore[misc] + +def g1[**P = int]() -> None: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +def g2[**P = 2]() -> None: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +def g3[**P = (2, int)]() -> None: ... # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +def g4[**P = [2, int]]() -> None: ... # E: Argument 0 of ParamSpec default must be a type + +def h1[*Ts = 2]() -> None: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +def h2[*Ts = int]() -> None: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +def h3[*Ts = tuple[int]]() -> None: ... # E: The default argument to TypeVarTuple must be an Unpacked tuple +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultInvalid3] +from typing import Callable + +type TA1[T: str = 1] = list[T] # E: TypeVar "default" must be a type +type TA2[T: str = [int]] = list[T] # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? \ + # E: TypeVar "default" must be a type +type TA3[T: str = int] = list[T] # E: TypeVar default must be a subtype of the bound type +type TA4[T: list[str] = list[int]] = list[T] # E: TypeVar default must be a subtype of the bound type +type TA5[T: (int, str) = bytes] = list[T] # E: TypeVar default must be one of the constraint types +type TA6[T: (int, str) = int | str] = list[T] # E: TypeVar default must be one of the constraint types +type TA7[T: (float, str) = int] = list[T] # E: TypeVar default must be one of the constraint types + +type TB1[**P = int] = Callable[P, None] # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +type TB2[**P = 2] = Callable[P, None] # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +type TB3[**P = (2, int)] = Callable[P, None] # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +type TB4[**P = [2, int]] = Callable[P, None] # E: Argument 0 of ParamSpec default must be a type + +type TC1[*Ts = 2] = tuple[*Ts] # E: The default argument to TypeVarTuple must be an Unpacked tuple +type TC2[*Ts = int] = tuple[*Ts] # E: The default argument to TypeVarTuple must be an Unpacked tuple +type TC3[*Ts = tuple[int]] = tuple[*Ts] # E: The default argument to TypeVarTuple must be an Unpacked tuple +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeParameterDefaultFunctions] +from typing import Callable + +def callback1(x: str) -> None: ... + +def func_a1[T = str](x: int | T) -> T: ... +reveal_type(func_a1(2)) # N: Revealed type is "builtins.str" +reveal_type(func_a1(2.1)) # N: Revealed type is "builtins.float" + +def func_a2[T = str](x: int | T) -> list[T]: ... +reveal_type(func_a2(2)) # N: Revealed type is "builtins.list[builtins.str]" +reveal_type(func_a2(2.1)) # N: Revealed type is "builtins.list[builtins.float]" + + +def func_a3[T: str = str](x: int | T) -> T: ... +reveal_type(func_a3(2)) # N: Revealed type is "builtins.str" + +def func_a4[T: (bytes, str) = str](x: int | T) -> T: ... +reveal_type(func_a4(2)) # N: Revealed type is "builtins.str" + +def func_b1[**P = [int, str]](x: int | Callable[P, None]) -> Callable[P, None]: ... +reveal_type(func_b1(callback1)) # N: Revealed type is "def (x: builtins.str)" +reveal_type(func_b1(2)) # N: Revealed type is "def (builtins.int, builtins.str)" + +def func_c1[*Ts = *tuple[int, str]](x: int | Callable[[*Ts], None]) -> tuple[*Ts]: ... +# reveal_type(func_c1(callback1)) # Revealed type is "Tuple[str]" # TODO +reveal_type(func_c1(2)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultClass1] +# flags: --disallow-any-generics + +class ClassA1[T2 = int, T3 = str]: ... + +def func_a1( + a: ClassA1, + b: ClassA1[float], + c: ClassA1[float, float], + d: ClassA1[float, float, float], # E: "ClassA1" expects between 0 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.str]" + reveal_type(c) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultClass2] +# flags: --disallow-any-generics + +class ClassB1[**P2 = [int, str], **P3 = ...]: ... + +def func_b1( + a: ClassB1, + b: ClassB1[[float]], + c: ClassB1[[float], [float]], + d: ClassB1[[float], [float], [float]], # E: "ClassB1" expects between 0 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], ...]" + reveal_type(b) # N: Revealed type is "__main__.ClassB1[[builtins.float], ...]" + reveal_type(c) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], ...]" + + k = ClassB1() + reveal_type(k) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" + l = ClassB1[[float]]() + reveal_type(l) # N: Revealed type is "__main__.ClassB1[[builtins.float], [*Any, **Any]]" + m = ClassB1[[float], [float]]() + reveal_type(m) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + n = ClassB1[[float], [float], [float]]() # E: Type application has too many types (expected between 0 and 2) + reveal_type(n) # N: Revealed type is "Any" + +[case testPEP695TypeParameterDefaultClass3] +# flags: --disallow-any-generics + +class ClassC1[*Ts = *tuple[int, str]]: ... + +def func_c1( + a: ClassC1, + b: ClassC1[float], +) -> None: + # reveal_type(a) # Revealed type is "__main__.ClassC1[builtins.int, builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "__main__.ClassC1[builtins.float]" + + k = ClassC1() + reveal_type(k) # N: Revealed type is "__main__.ClassC1[builtins.int, builtins.str]" + l = ClassC1[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassC1[builtins.float]" +[builtins fixtures/tuple.pyi] + +[case testPEP695TypeParameterDefaultTypeAlias1] +# flags: --disallow-any-generics + +type TA1[T2 = int, T3 = str] = dict[T2, T3] + +def func_a1( + a: TA1, + b: TA1[float], + c: TA1[float, float], + d: TA1[float, float, float], # E: Bad number of arguments for type alias, expected between 0 and 2, given 3 +) -> None: + reveal_type(a) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.float, builtins.str]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeParameterDefaultTypeAlias2] +# flags: --disallow-any-generics + +class ClassB1[**P2, **P3]: ... +type TB1[**P2 = [int, str], **P3 = ...] = ClassB1[P2, P3] + +def func_b1( + a: TB1, + b: TB1[[float]], + c: TB1[[float], [float]], + d: TB1[[float], [float], [float]], # E: Bad number of arguments for type alias, expected between 0 and 2, given 3 +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" + reveal_type(b) # N: Revealed type is "__main__.ClassB1[[builtins.float], [*Any, **Any]]" + reveal_type(c) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testPEP695TypeParameterDefaultTypeAlias3] +# flags: --disallow-any-generics + +type TC1[*Ts = *tuple[int, str]] = tuple[*Ts] + +def func_c1( + a: TC1, + b: TC1[float], +) -> None: + # reveal_type(a) # Revealed type is "Tuple[builtins.int, builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "Tuple[builtins.float]" + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test index 22e2594eb38b..3cd94f4a46d2 100644 --- a/test-data/unit/check-typevar-defaults.test +++ b/test-data/unit/check-typevar-defaults.test @@ -1,5 +1,4 @@ [case testTypeVarDefaultsBasic] -import builtins from typing import Generic, TypeVar, ParamSpec, Callable, Tuple, List from typing_extensions import TypeVarTuple, Unpack @@ -10,7 +9,7 @@ Ts1 = TypeVarTuple("Ts1", default=Unpack[Tuple[int, str]]) def f1(a: T1) -> List[T1]: ... reveal_type(f1) # N: Revealed type is "def [T1 = builtins.int] (a: T1`-1 = builtins.int) -> builtins.list[T1`-1 = builtins.int]" -def f2(a: Callable[P1, None] ) -> Callable[P1, None]: ... +def f2(a: Callable[P1, None]) -> Callable[P1, None]: ... reveal_type(f2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] (a: def (*P1.args, **P1.kwargs)) -> def (*P1.args, **P1.kwargs)" def f3(a: Tuple[Unpack[Ts1]]) -> Tuple[Unpack[Ts1]]: ... @@ -68,7 +67,7 @@ P2 = ParamSpec("P2", default=2) # E: The default argument to ParamSpec must be P3 = ParamSpec("P3", default=(2, int)) # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec P4 = ParamSpec("P4", default=[2, int]) # E: Argument 0 of ParamSpec default must be a type -Ts1 = TypeVarTuple("Ts1", default=2) # E: The default argument to TypeVarTuple must be an Unpacked tuple +Ts1 = TypeVarTuple("Ts1", default=2) # E: The default argument to TypeVarTuple must be an Unpacked tuple Ts2 = TypeVarTuple("Ts2", default=int) # E: The default argument to TypeVarTuple must be an Unpacked tuple Ts3 = TypeVarTuple("Ts3", default=Tuple[int]) # E: The default argument to TypeVarTuple must be an Unpacked tuple [builtins fixtures/tuple.pyi] @@ -181,8 +180,8 @@ reveal_type(func_b1(callback1)) # N: Revealed type is "def (x: builtins.str)" reveal_type(func_b1(2)) # N: Revealed type is "def (builtins.int, builtins.str)" def func_c1(x: Union[int, Callable[[Unpack[Ts1]], None]]) -> Tuple[Unpack[Ts1]]: ... -# reveal_type(func_c1(callback1)) # Revealed type is "builtins.tuple[str]" # TODO -# reveal_type(func_c1(2)) # Revealed type is "builtins.tuple[builtins.int, builtins.str]" # TODO +# reveal_type(func_c1(callback1)) # Revealed type is "Tuple[str]" # TODO +reveal_type(func_c1(2)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" [builtins fixtures/tuple.pyi] [case testTypeVarDefaultsClass1] diff --git a/test-data/unit/parse-python313.test b/test-data/unit/parse-python313.test new file mode 100644 index 000000000000..efbafb0766f5 --- /dev/null +++ b/test-data/unit/parse-python313.test @@ -0,0 +1,80 @@ +[case testPEP696TypeAlias] +type A[T = int] = C[T] +[out] +MypyFile:1( + TypeAliasStmt:1( + NameExpr(A) + TypeParam( + T + Default( + int?)) + LambdaExpr:1( + Block:-1( + ReturnStmt:1( + IndexExpr:1( + NameExpr(C) + NameExpr(T))))))) + +[case testPEP696GenericFunction] +def f[T = int](): pass +class C[T = int]: pass +[out] +MypyFile:1( + FuncDef:1( + f + TypeParam( + T + Default( + int?)) + Block:1( + PassStmt:1())) + ClassDef:2( + C + TypeParam( + T + Default( + int?)) + PassStmt:2())) + +[case testPEP696ParamSpec] +def f[**P = [int, str]](): pass +class C[**P = [int, str]]: pass +[out] +[out] +MypyFile:1( + FuncDef:1( + f + TypeParam( + **P + Default( + )) + Block:1( + PassStmt:1())) + ClassDef:2( + C + TypeParam( + **P + Default( + )) + PassStmt:2())) + +[case testPEP696TypeVarTuple] +def f[*Ts = *tuple[str, int]](): pass +class C[*Ts = *tuple[str, int]]: pass +[out] +MypyFile:1( + FuncDef:1( + f + TypeParam( + *Ts + Default( + Unpack[tuple?[str?, int?]])) + Block:1( + PassStmt:1())) + ClassDef:2( + C + TypeParam( + *Ts + Default( + Unpack[tuple?[str?, int?]])) + PassStmt:2())) From 49e3a0ff8d6597663aac6709d8e6a022e0424718 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 19 Oct 2024 16:45:24 -0700 Subject: [PATCH 238/247] Update changelog for 1.12.1 (#17999) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4522465a484d..d0cbe466a5d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -261,6 +261,12 @@ This feature was contributed by Ivan Levkivskyi (PR [17457](https://github.com/p Please see [git log](https://github.com/python/typeshed/commits/main?after=91a58b07cdd807b1d965e04ba85af2adab8bf924+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. +### Mypy 1.12.1 + * Fix crash when showing partially analyzed type in error message (Ivan Levkivskyi, PR [17961](https://github.com/python/mypy/pull/17961)) + * Fix iteration over union (when self type is involved) (Shantanu, PR [17976](https://github.com/python/mypy/pull/17976)) + * Fix type object with type var default in union context (Jukka Lehtosalo, PR [17991](https://github.com/python/mypy/pull/17991)) + * Revert change to `os.path` stubs affecting use of `os.PathLike[Any]` (Shantanu, PR [17995](https://github.com/python/mypy/pull/17995)) + ### Acknowledgements Thanks to all mypy contributors who contributed to this release: From e3dbce48601ccfdf97be6b378ee31939c7558143 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 19 Oct 2024 16:49:53 -0700 Subject: [PATCH 239/247] Bump version to 1.14.0+dev (#17998) I'll be making a 1.13 (that's cut from the 1.12 branch), so we should bump the version on master as well --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index e0671c9feb06..4510cc56f32b 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.13.0+dev" +__version__ = "1.14.0+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From b16c19277b13aebedbae5542af4dcf107a475125 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 19 Oct 2024 18:03:16 -0700 Subject: [PATCH 240/247] Sync typeshed (#17994) Source commit: https://github.com/python/typeshed/commit/7fb84668fc66e41a3886e460abe54af0bfc5dc6f --- .../stdlib/distutils/command/bdist.pyi | 16 ++--- .../stdlib/distutils/command/bdist_dumb.pyi | 17 ++--- .../stdlib/distutils/command/bdist_msi.pyi | 35 ++++----- .../stdlib/distutils/command/bdist_rpm.pyi | 71 ++++++++++--------- .../stdlib/distutils/command/build.pyi | 22 +++--- .../stdlib/distutils/command/build_clib.pyi | 20 +++--- .../stdlib/distutils/command/build_ext.pyi | 48 ++++++------- .../stdlib/distutils/command/build_py.pyi | 23 +++--- .../distutils/command/build_scripts.pyi | 15 ++-- .../stdlib/distutils/command/check.pyi | 5 +- .../stdlib/distutils/command/clean.pyi | 15 ++-- .../stdlib/distutils/command/config.pyi | 6 +- .../stdlib/distutils/command/install.pyi | 41 +++++------ .../stdlib/distutils/command/install_data.pyi | 11 +-- .../distutils/command/install_egg_info.pyi | 9 +-- .../distutils/command/install_headers.pyi | 7 +- .../stdlib/distutils/command/install_lib.pyi | 13 ++-- .../distutils/command/install_scripts.pyi | 11 +-- .../stdlib/distutils/command/register.pyi | 3 +- .../stdlib/distutils/command/sdist.pyi | 18 ++--- .../stdlib/distutils/command/upload.pyi | 9 +-- mypy/typeshed/stdlib/distutils/core.pyi | 7 +- mypy/typeshed/stdlib/distutils/log.pyi | 1 + mypy/typeshed/stdlib/itertools.pyi | 38 +++++----- 24 files changed, 239 insertions(+), 222 deletions(-) diff --git a/mypy/typeshed/stdlib/distutils/command/bdist.pyi b/mypy/typeshed/stdlib/distutils/command/bdist.pyi index 43d77087f7d8..6f996207077e 100644 --- a/mypy/typeshed/stdlib/distutils/command/bdist.pyi +++ b/mypy/typeshed/stdlib/distutils/command/bdist.pyi @@ -1,6 +1,6 @@ -from _typeshed import Unused +from _typeshed import Incomplete, Unused from collections.abc import Callable -from typing import Any, ClassVar +from typing import ClassVar from ..cmd import Command @@ -15,13 +15,13 @@ class bdist(Command): default_format: ClassVar[dict[str, str]] format_commands: ClassVar[list[str]] format_command: ClassVar[dict[str, tuple[str, str]]] - bdist_base: Any - plat_name: Any - formats: Any - dist_dir: Any + bdist_base: Incomplete + plat_name: Incomplete + formats: Incomplete + dist_dir: Incomplete skip_build: int - group: Any - owner: Any + group: Incomplete + owner: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi index 19997882dd53..297a0c39ed43 100644 --- a/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi +++ b/mypy/typeshed/stdlib/distutils/command/bdist_dumb.pyi @@ -1,4 +1,5 @@ -from typing import Any, ClassVar +from _typeshed import Incomplete +from typing import ClassVar from ..cmd import Command @@ -7,15 +8,15 @@ class bdist_dumb(Command): user_options: ClassVar[list[tuple[str, str | None, str]]] boolean_options: ClassVar[list[str]] default_format: ClassVar[dict[str, str]] - bdist_dir: Any - plat_name: Any - format: Any + bdist_dir: Incomplete + plat_name: Incomplete + format: Incomplete keep_temp: int - dist_dir: Any - skip_build: Any + dist_dir: Incomplete + skip_build: Incomplete relative: int - owner: Any - group: Any + owner: Incomplete + group: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi index d0eac1a3be5b..baeee7d3eccb 100644 --- a/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi +++ b/mypy/typeshed/stdlib/distutils/command/bdist_msi.pyi @@ -1,42 +1,43 @@ import sys -from typing import Any, ClassVar, Literal +from _typeshed import Incomplete +from typing import ClassVar, Literal from ..cmd import Command if sys.platform == "win32": - from msilib import Dialog + from msilib import Control, Dialog class PyDialog(Dialog): def __init__(self, *args, **kw) -> None: ... def title(self, title) -> None: ... - def back(self, title, next, name: str = "Back", active: bool | Literal[0, 1] = 1): ... - def cancel(self, title, next, name: str = "Cancel", active: bool | Literal[0, 1] = 1): ... - def next(self, title, next, name: str = "Next", active: bool | Literal[0, 1] = 1): ... - def xbutton(self, name, title, next, xpos): ... + def back(self, title, next, name: str = "Back", active: bool | Literal[0, 1] = 1) -> Control: ... + def cancel(self, title, next, name: str = "Cancel", active: bool | Literal[0, 1] = 1) -> Control: ... + def next(self, title, next, name: str = "Next", active: bool | Literal[0, 1] = 1) -> Control: ... + def xbutton(self, name, title, next, xpos) -> Control: ... class bdist_msi(Command): description: str user_options: ClassVar[list[tuple[str, str | None, str]]] boolean_options: ClassVar[list[str]] - all_versions: Any + all_versions: Incomplete other_version: str if sys.version_info >= (3, 9): def __init__(self, *args, **kw) -> None: ... - bdist_dir: Any - plat_name: Any + bdist_dir: Incomplete + plat_name: Incomplete keep_temp: int no_target_compile: int no_target_optimize: int - target_version: Any - dist_dir: Any - skip_build: Any - install_script: Any - pre_install_script: Any - versions: Any + target_version: Incomplete + dist_dir: Incomplete + skip_build: Incomplete + install_script: Incomplete + pre_install_script: Incomplete + versions: Incomplete def initialize_options(self) -> None: ... - install_script_key: Any + install_script_key: Incomplete def finalize_options(self) -> None: ... - db: Any + db: Incomplete def run(self) -> None: ... def add_files(self) -> None: ... def add_find_python(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi b/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi index 89c43e1b974c..83b4161094c5 100644 --- a/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi +++ b/mypy/typeshed/stdlib/distutils/command/bdist_rpm.pyi @@ -1,4 +1,5 @@ -from typing import Any, ClassVar +from _typeshed import Incomplete +from typing import ClassVar from ..cmd import Command @@ -7,44 +8,44 @@ class bdist_rpm(Command): user_options: ClassVar[list[tuple[str, str | None, str]]] boolean_options: ClassVar[list[str]] negative_opt: ClassVar[dict[str, str]] - bdist_base: Any - rpm_base: Any - dist_dir: Any - python: Any - fix_python: Any - spec_only: Any - binary_only: Any - source_only: Any - use_bzip2: Any - distribution_name: Any - group: Any - release: Any - serial: Any - vendor: Any - packager: Any - doc_files: Any - changelog: Any - icon: Any - prep_script: Any - build_script: Any - install_script: Any - clean_script: Any - verify_script: Any - pre_install: Any - post_install: Any - pre_uninstall: Any - post_uninstall: Any - prep: Any - provides: Any - requires: Any - conflicts: Any - build_requires: Any - obsoletes: Any + bdist_base: Incomplete + rpm_base: Incomplete + dist_dir: Incomplete + python: Incomplete + fix_python: Incomplete + spec_only: Incomplete + binary_only: Incomplete + source_only: Incomplete + use_bzip2: Incomplete + distribution_name: Incomplete + group: Incomplete + release: Incomplete + serial: Incomplete + vendor: Incomplete + packager: Incomplete + doc_files: Incomplete + changelog: Incomplete + icon: Incomplete + prep_script: Incomplete + build_script: Incomplete + install_script: Incomplete + clean_script: Incomplete + verify_script: Incomplete + pre_install: Incomplete + post_install: Incomplete + pre_uninstall: Incomplete + post_uninstall: Incomplete + prep: Incomplete + provides: Incomplete + requires: Incomplete + conflicts: Incomplete + build_requires: Incomplete + obsoletes: Incomplete keep_temp: int use_rpm_opt_flags: int rpm3_mode: int no_autoreq: int - force_arch: Any + force_arch: Incomplete quiet: int def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/build.pyi b/mypy/typeshed/stdlib/distutils/command/build.pyi index 78ba6b7042dc..3ec0c9614d62 100644 --- a/mypy/typeshed/stdlib/distutils/command/build.pyi +++ b/mypy/typeshed/stdlib/distutils/command/build.pyi @@ -1,4 +1,4 @@ -from _typeshed import Unused +from _typeshed import Incomplete, Unused from collections.abc import Callable from typing import Any, ClassVar @@ -12,17 +12,17 @@ class build(Command): boolean_options: ClassVar[list[str]] help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] build_base: str - build_purelib: Any - build_platlib: Any - build_lib: Any - build_temp: Any - build_scripts: Any - compiler: Any - plat_name: Any - debug: Any + build_purelib: Incomplete + build_platlib: Incomplete + build_lib: Incomplete + build_temp: Incomplete + build_scripts: Incomplete + compiler: Incomplete + plat_name: Incomplete + debug: Incomplete force: int - executable: Any - parallel: Any + executable: Incomplete + parallel: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/build_clib.pyi b/mypy/typeshed/stdlib/distutils/command/build_clib.pyi index 1f66e2efc20c..69cfbe7120d8 100644 --- a/mypy/typeshed/stdlib/distutils/command/build_clib.pyi +++ b/mypy/typeshed/stdlib/distutils/command/build_clib.pyi @@ -1,6 +1,6 @@ -from _typeshed import Unused +from _typeshed import Incomplete, Unused from collections.abc import Callable -from typing import Any, ClassVar +from typing import ClassVar from ..cmd import Command @@ -11,15 +11,15 @@ class build_clib(Command): user_options: ClassVar[list[tuple[str, str, str]]] boolean_options: ClassVar[list[str]] help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] - build_clib: Any - build_temp: Any - libraries: Any - include_dirs: Any - define: Any - undef: Any - debug: Any + build_clib: Incomplete + build_temp: Incomplete + libraries: Incomplete + include_dirs: Incomplete + define: Incomplete + undef: Incomplete + debug: Incomplete force: int - compiler: Any + compiler: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/build_ext.pyi b/mypy/typeshed/stdlib/distutils/command/build_ext.pyi index a0813c314021..c5a9b5d508f0 100644 --- a/mypy/typeshed/stdlib/distutils/command/build_ext.pyi +++ b/mypy/typeshed/stdlib/distutils/command/build_ext.pyi @@ -1,40 +1,40 @@ -from _typeshed import Unused +from _typeshed import Incomplete, Unused from collections.abc import Callable -from typing import Any, ClassVar +from typing import ClassVar from ..cmd import Command -extension_name_re: Any +extension_name_re: Incomplete def show_compilers() -> None: ... class build_ext(Command): description: str - sep_by: Any + sep_by: Incomplete user_options: ClassVar[list[tuple[str, str | None, str]]] boolean_options: ClassVar[list[str]] help_options: ClassVar[list[tuple[str, str | None, str, Callable[[], Unused]]]] - extensions: Any - build_lib: Any - plat_name: Any - build_temp: Any + extensions: Incomplete + build_lib: Incomplete + plat_name: Incomplete + build_temp: Incomplete inplace: int - package: Any - include_dirs: Any - define: Any - undef: Any - libraries: Any - library_dirs: Any - rpath: Any - link_objects: Any - debug: Any - force: Any - compiler: Any - swig: Any - swig_cpp: Any - swig_opts: Any - user: Any - parallel: Any + package: Incomplete + include_dirs: Incomplete + define: Incomplete + undef: Incomplete + libraries: Incomplete + library_dirs: Incomplete + rpath: Incomplete + link_objects: Incomplete + debug: Incomplete + force: Incomplete + compiler: Incomplete + swig: Incomplete + swig_cpp: Incomplete + swig_opts: Incomplete + user: Incomplete + parallel: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/build_py.pyi b/mypy/typeshed/stdlib/distutils/command/build_py.pyi index 90f06751416a..23ed230bb2d8 100644 --- a/mypy/typeshed/stdlib/distutils/command/build_py.pyi +++ b/mypy/typeshed/stdlib/distutils/command/build_py.pyi @@ -1,4 +1,5 @@ -from typing import Any, ClassVar, Literal +from _typeshed import Incomplete +from typing import ClassVar, Literal from ..cmd import Command from ..util import Mixin2to3 as Mixin2to3 @@ -8,17 +9,17 @@ class build_py(Command): user_options: ClassVar[list[tuple[str, str | None, str]]] boolean_options: ClassVar[list[str]] negative_opt: ClassVar[dict[str, str]] - build_lib: Any - py_modules: Any - package: Any - package_data: Any - package_dir: Any + build_lib: Incomplete + py_modules: Incomplete + package: Incomplete + package_data: Incomplete + package_dir: Incomplete compile: int optimize: int - force: Any + force: Incomplete def initialize_options(self) -> None: ... - packages: Any - data_files: Any + packages: Incomplete + data_files: Incomplete def finalize_options(self) -> None: ... def run(self) -> None: ... def get_data_files(self): ... @@ -32,13 +33,13 @@ class build_py(Command): def find_all_modules(self): ... def get_source_files(self): ... def get_module_outfile(self, build_dir, package, module): ... - def get_outputs(self, include_bytecode: bool | Literal[0, 1] = 1): ... + def get_outputs(self, include_bytecode: bool | Literal[0, 1] = 1) -> list[str]: ... def build_module(self, module, module_file, package): ... def build_modules(self) -> None: ... def build_packages(self) -> None: ... def byte_compile(self, files) -> None: ... class build_py_2to3(build_py, Mixin2to3): - updated_files: Any + updated_files: Incomplete def run(self) -> None: ... def build_module(self, module, module_file, package): ... diff --git a/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi b/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi index 7871bb8a5719..8372919bbd53 100644 --- a/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi +++ b/mypy/typeshed/stdlib/distutils/command/build_scripts.pyi @@ -1,19 +1,20 @@ -from typing import Any, ClassVar +from _typeshed import Incomplete +from typing import ClassVar from ..cmd import Command from ..util import Mixin2to3 as Mixin2to3 -first_line_re: Any +first_line_re: Incomplete class build_scripts(Command): description: str user_options: ClassVar[list[tuple[str, str, str]]] boolean_options: ClassVar[list[str]] - build_dir: Any - scripts: Any - force: Any - executable: Any - outfiles: Any + build_dir: Incomplete + scripts: Incomplete + force: Incomplete + executable: Incomplete + outfiles: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... def get_source_files(self): ... diff --git a/mypy/typeshed/stdlib/distutils/command/check.pyi b/mypy/typeshed/stdlib/distutils/command/check.pyi index e69627d20c7a..2c807fd2c439 100644 --- a/mypy/typeshed/stdlib/distutils/command/check.pyi +++ b/mypy/typeshed/stdlib/distutils/command/check.pyi @@ -1,3 +1,4 @@ +from _typeshed import Incomplete from typing import Any, ClassVar, Final, Literal from typing_extensions import TypeAlias @@ -9,13 +10,13 @@ _Reporter: TypeAlias = Any # really docutils.utils.Reporter # Depends on a third-party stub. Since distutils is deprecated anyway, # it's easier to just suppress the "any subclassing" error. class SilentReporter(_Reporter): - messages: Any + messages: Incomplete def __init__( self, source, report_level, halt_level, - stream: Any | None = ..., + stream: Incomplete | None = ..., debug: bool | Literal[0, 1] = 0, encoding: str = ..., error_handler: str = ..., diff --git a/mypy/typeshed/stdlib/distutils/command/clean.pyi b/mypy/typeshed/stdlib/distutils/command/clean.pyi index 55f0a0eeaf10..0f3768d6dcf4 100644 --- a/mypy/typeshed/stdlib/distutils/command/clean.pyi +++ b/mypy/typeshed/stdlib/distutils/command/clean.pyi @@ -1,4 +1,5 @@ -from typing import Any, ClassVar +from _typeshed import Incomplete +from typing import ClassVar from ..cmd import Command @@ -6,12 +7,12 @@ class clean(Command): description: str user_options: ClassVar[list[tuple[str, str | None, str]]] boolean_options: ClassVar[list[str]] - build_base: Any - build_lib: Any - build_temp: Any - build_scripts: Any - bdist_base: Any - all: Any + build_base: Incomplete + build_lib: Incomplete + build_temp: Incomplete + build_scripts: Incomplete + bdist_base: Incomplete + all: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/config.pyi b/mypy/typeshed/stdlib/distutils/command/config.pyi index b0910091d5b6..562ff3a5271f 100644 --- a/mypy/typeshed/stdlib/distutils/command/config.pyi +++ b/mypy/typeshed/stdlib/distutils/command/config.pyi @@ -1,7 +1,7 @@ -from _typeshed import StrOrBytesPath +from _typeshed import Incomplete, StrOrBytesPath from collections.abc import Sequence from re import Pattern -from typing import Any, ClassVar, Final, Literal +from typing import ClassVar, Final, Literal from ..ccompiler import CCompiler from ..cmd import Command @@ -81,4 +81,4 @@ class config(Command): self, header: str, include_dirs: Sequence[str] | None = None, library_dirs: Sequence[str] | None = None, lang: str = "c" ) -> bool: ... -def dump_file(filename: StrOrBytesPath, head: Any | None = None) -> None: ... +def dump_file(filename: StrOrBytesPath, head: Incomplete | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/install.pyi b/mypy/typeshed/stdlib/distutils/command/install.pyi index 24a4eff2fb10..1714e01a2c28 100644 --- a/mypy/typeshed/stdlib/distutils/command/install.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install.pyi @@ -1,4 +1,5 @@ import sys +from _typeshed import Incomplete from collections.abc import Callable from typing import Any, ClassVar, Final, Literal @@ -18,33 +19,33 @@ class install(Command): boolean_options: ClassVar[list[str]] negative_opt: ClassVar[dict[str, str]] prefix: str | None - exec_prefix: Any + exec_prefix: Incomplete home: str | None user: bool - install_base: Any - install_platbase: Any + install_base: Incomplete + install_platbase: Incomplete root: str | None - install_purelib: Any - install_platlib: Any - install_headers: Any + install_purelib: Incomplete + install_platlib: Incomplete + install_headers: Incomplete install_lib: str | None - install_scripts: Any - install_data: Any - install_userbase: Any - install_usersite: Any - compile: Any - optimize: Any - extra_path: Any + install_scripts: Incomplete + install_data: Incomplete + install_userbase: Incomplete + install_usersite: Incomplete + compile: Incomplete + optimize: Incomplete + extra_path: Incomplete install_path_file: int force: int skip_build: int warn_dir: int - build_base: Any - build_lib: Any - record: Any + build_base: Incomplete + build_lib: Incomplete + record: Incomplete def initialize_options(self) -> None: ... - config_vars: Any - install_libbase: Any + config_vars: Incomplete + install_libbase: Incomplete def finalize_options(self) -> None: ... def dump_dirs(self, msg) -> None: ... def finalize_unix(self) -> None: ... @@ -53,8 +54,8 @@ class install(Command): def expand_basedirs(self) -> None: ... def expand_dirs(self) -> None: ... def convert_paths(self, *names) -> None: ... - path_file: Any - extra_dirs: Any + path_file: Incomplete + extra_dirs: Incomplete def handle_extra_path(self) -> None: ... def change_roots(self, *names) -> None: ... def create_home_path(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/install_data.pyi b/mypy/typeshed/stdlib/distutils/command/install_data.pyi index 342c7a7ccca4..609de62b04b5 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_data.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_data.pyi @@ -1,4 +1,5 @@ -from typing import Any, ClassVar +from _typeshed import Incomplete +from typing import ClassVar from ..cmd import Command @@ -6,11 +7,11 @@ class install_data(Command): description: str user_options: ClassVar[list[tuple[str, str | None, str]]] boolean_options: ClassVar[list[str]] - install_dir: Any - outfiles: Any - root: Any + install_dir: Incomplete + outfiles: Incomplete + root: Incomplete force: int - data_files: Any + data_files: Incomplete warn_dir: int def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi b/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi index 3fd54989d14f..75bb906ce582 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_egg_info.pyi @@ -1,14 +1,15 @@ -from typing import Any, ClassVar +from _typeshed import Incomplete +from typing import ClassVar from ..cmd import Command class install_egg_info(Command): description: ClassVar[str] user_options: ClassVar[list[tuple[str, str, str]]] - install_dir: Any + install_dir: Incomplete def initialize_options(self) -> None: ... - target: Any - outputs: Any + target: Incomplete + outputs: Incomplete def finalize_options(self) -> None: ... def run(self) -> None: ... def get_outputs(self) -> list[str]: ... diff --git a/mypy/typeshed/stdlib/distutils/command/install_headers.pyi b/mypy/typeshed/stdlib/distutils/command/install_headers.pyi index 7854d2393a98..3caad8a07dca 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_headers.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_headers.pyi @@ -1,4 +1,5 @@ -from typing import Any, ClassVar +from _typeshed import Incomplete +from typing import ClassVar from ..cmd import Command @@ -6,9 +7,9 @@ class install_headers(Command): description: str user_options: ClassVar[list[tuple[str, str, str]]] boolean_options: ClassVar[list[str]] - install_dir: Any + install_dir: Incomplete force: int - outfiles: Any + outfiles: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/install_lib.pyi b/mypy/typeshed/stdlib/distutils/command/install_lib.pyi index 149ecae89781..a537e254904a 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_lib.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_lib.pyi @@ -1,4 +1,5 @@ -from typing import Any, ClassVar, Final +from _typeshed import Incomplete +from typing import ClassVar, Final from ..cmd import Command @@ -9,12 +10,12 @@ class install_lib(Command): user_options: ClassVar[list[tuple[str, str | None, str]]] boolean_options: ClassVar[list[str]] negative_opt: ClassVar[dict[str, str]] - install_dir: Any - build_dir: Any + install_dir: Incomplete + build_dir: Incomplete force: int - compile: Any - optimize: Any - skip_build: Any + compile: Incomplete + optimize: Incomplete + skip_build: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... def run(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi b/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi index 5ee5589ad33d..658594f32e43 100644 --- a/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi +++ b/mypy/typeshed/stdlib/distutils/command/install_scripts.pyi @@ -1,4 +1,5 @@ -from typing import Any, ClassVar +from _typeshed import Incomplete +from typing import ClassVar from ..cmd import Command @@ -6,13 +7,13 @@ class install_scripts(Command): description: str user_options: ClassVar[list[tuple[str, str | None, str]]] boolean_options: ClassVar[list[str]] - install_dir: Any + install_dir: Incomplete force: int - build_dir: Any - skip_build: Any + build_dir: Incomplete + skip_build: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... - outfiles: Any + outfiles: Incomplete def run(self) -> None: ... def get_inputs(self): ... def get_outputs(self): ... diff --git a/mypy/typeshed/stdlib/distutils/command/register.pyi b/mypy/typeshed/stdlib/distutils/command/register.pyi index a5e251d2d01e..cf98e178a9ba 100644 --- a/mypy/typeshed/stdlib/distutils/command/register.pyi +++ b/mypy/typeshed/stdlib/distutils/command/register.pyi @@ -1,3 +1,4 @@ +from _typeshed import Incomplete from collections.abc import Callable from typing import Any, ClassVar @@ -17,4 +18,4 @@ class register(PyPIRCCommand): def verify_metadata(self) -> None: ... def send_metadata(self) -> None: ... def build_post_data(self, action): ... - def post_to_server(self, data, auth: Any | None = None): ... + def post_to_server(self, data, auth: Incomplete | None = None): ... diff --git a/mypy/typeshed/stdlib/distutils/command/sdist.pyi b/mypy/typeshed/stdlib/distutils/command/sdist.pyi index 5b7fe2419551..48a140714dda 100644 --- a/mypy/typeshed/stdlib/distutils/command/sdist.pyi +++ b/mypy/typeshed/stdlib/distutils/command/sdist.pyi @@ -1,4 +1,4 @@ -from _typeshed import Unused +from _typeshed import Incomplete, Unused from collections.abc import Callable from typing import Any, ClassVar @@ -16,22 +16,22 @@ class sdist(Command): # Any to work around variance issues sub_commands: ClassVar[list[tuple[str, Callable[[Any], bool] | None]]] READMES: ClassVar[tuple[str, ...]] - template: Any - manifest: Any + template: Incomplete + manifest: Incomplete use_defaults: int prune: int manifest_only: int force_manifest: int - formats: Any + formats: Incomplete keep_temp: int - dist_dir: Any - archive_files: Any + dist_dir: Incomplete + archive_files: Incomplete metadata_check: int - owner: Any - group: Any + owner: Incomplete + group: Incomplete def initialize_options(self) -> None: ... def finalize_options(self) -> None: ... - filelist: Any + filelist: Incomplete def run(self) -> None: ... def check_metadata(self) -> None: ... def get_file_list(self) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/command/upload.pyi b/mypy/typeshed/stdlib/distutils/command/upload.pyi index e6b77825c5f5..afcfbaf48677 100644 --- a/mypy/typeshed/stdlib/distutils/command/upload.pyi +++ b/mypy/typeshed/stdlib/distutils/command/upload.pyi @@ -1,4 +1,5 @@ -from typing import Any, ClassVar +from _typeshed import Incomplete +from typing import ClassVar from ..config import PyPIRCCommand @@ -8,10 +9,10 @@ class upload(PyPIRCCommand): password: str show_response: int sign: bool - identity: Any + identity: Incomplete def initialize_options(self) -> None: ... - repository: Any - realm: Any + repository: Incomplete + realm: Incomplete def finalize_options(self) -> None: ... def run(self) -> None: ... def upload_file(self, command: str, pyversion: str, filename: str) -> None: ... diff --git a/mypy/typeshed/stdlib/distutils/core.pyi b/mypy/typeshed/stdlib/distutils/core.pyi index a4d21f8ddd7b..174f24991351 100644 --- a/mypy/typeshed/stdlib/distutils/core.pyi +++ b/mypy/typeshed/stdlib/distutils/core.pyi @@ -1,4 +1,4 @@ -from _typeshed import StrOrBytesPath +from _typeshed import Incomplete, StrOrBytesPath from collections.abc import Mapping from distutils.cmd import Command as Command from distutils.dist import Distribution as Distribution @@ -32,7 +32,7 @@ def setup( distclass: type[Distribution] = ..., script_name: str = ..., script_args: list[str] = ..., - options: Mapping[str, Any] = ..., + options: Mapping[str, Incomplete] = ..., license: str = ..., keywords: list[str] | str = ..., platforms: list[str] | str = ..., @@ -43,7 +43,7 @@ def setup( provides: list[str] = ..., requires: list[str] = ..., command_packages: list[str] = ..., - command_options: Mapping[str, Mapping[str, tuple[Any, Any]]] = ..., + command_options: Mapping[str, Mapping[str, tuple[Incomplete, Incomplete]]] = ..., package_data: Mapping[str, list[str]] = ..., include_package_data: bool | Literal[0, 1] = ..., libraries: list[str] = ..., @@ -52,6 +52,7 @@ def setup( include_dirs: list[str] = ..., password: str = ..., fullname: str = ..., + # Custom Distributions could accept more params **attrs: Any, ) -> Distribution: ... def run_setup(script_name: str, script_args: list[str] | None = None, stop_after: str = "run") -> Distribution: ... diff --git a/mypy/typeshed/stdlib/distutils/log.pyi b/mypy/typeshed/stdlib/distutils/log.pyi index 0ea135c28371..7246dd6be0cd 100644 --- a/mypy/typeshed/stdlib/distutils/log.pyi +++ b/mypy/typeshed/stdlib/distutils/log.pyi @@ -8,6 +8,7 @@ FATAL: Final = 5 class Log: def __init__(self, threshold: int = 3) -> None: ... + # Arbitrary msg args' type depends on the format method def log(self, level: int, msg: str, *args: Any) -> None: ... def debug(self, msg: str, *args: Any) -> None: ... def info(self, msg: str, *args: Any) -> None: ... diff --git a/mypy/typeshed/stdlib/itertools.pyi b/mypy/typeshed/stdlib/itertools.pyi index 1635b6a0a072..18076011178f 100644 --- a/mypy/typeshed/stdlib/itertools.pyi +++ b/mypy/typeshed/stdlib/itertools.pyi @@ -28,7 +28,7 @@ _Predicate: TypeAlias = Callable[[_T], object] # Technically count can take anything that implements a number protocol and has an add method # but we can't enforce the add method -class count(Iterator[_N]): +class count(Generic[_N]): @overload def __new__(cls) -> count[int]: ... @overload @@ -38,12 +38,12 @@ class count(Iterator[_N]): def __next__(self) -> _N: ... def __iter__(self) -> Self: ... -class cycle(Iterator[_T]): +class cycle(Generic[_T]): def __init__(self, iterable: Iterable[_T], /) -> None: ... def __next__(self) -> _T: ... def __iter__(self) -> Self: ... -class repeat(Iterator[_T]): +class repeat(Generic[_T]): @overload def __init__(self, object: _T) -> None: ... @overload @@ -52,7 +52,7 @@ class repeat(Iterator[_T]): def __iter__(self) -> Self: ... def __length_hint__(self) -> int: ... -class accumulate(Iterator[_T]): +class accumulate(Generic[_T]): @overload def __init__(self, iterable: Iterable[_T], func: None = None, *, initial: _T | None = ...) -> None: ... @overload @@ -60,7 +60,7 @@ class accumulate(Iterator[_T]): def __iter__(self) -> Self: ... def __next__(self) -> _T: ... -class chain(Iterator[_T]): +class chain(Generic[_T]): def __init__(self, *iterables: Iterable[_T]) -> None: ... def __next__(self) -> _T: ... def __iter__(self) -> Self: ... @@ -70,22 +70,22 @@ class chain(Iterator[_T]): if sys.version_info >= (3, 9): def __class_getitem__(cls, item: Any, /) -> GenericAlias: ... -class compress(Iterator[_T]): +class compress(Generic[_T]): def __init__(self, data: Iterable[_T], selectors: Iterable[Any]) -> None: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... -class dropwhile(Iterator[_T]): +class dropwhile(Generic[_T]): def __init__(self, predicate: _Predicate[_T], iterable: Iterable[_T], /) -> None: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... -class filterfalse(Iterator[_T]): +class filterfalse(Generic[_T]): def __init__(self, predicate: _Predicate[_T] | None, iterable: Iterable[_T], /) -> None: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... -class groupby(Iterator[tuple[_T_co, Iterator[_S_co]]], Generic[_T_co, _S_co]): +class groupby(Generic[_T_co, _S_co]): @overload def __new__(cls, iterable: Iterable[_T1], key: None = None) -> groupby[_T1, _T1]: ... @overload @@ -93,7 +93,7 @@ class groupby(Iterator[tuple[_T_co, Iterator[_S_co]]], Generic[_T_co, _S_co]): def __iter__(self) -> Self: ... def __next__(self) -> tuple[_T_co, Iterator[_S_co]]: ... -class islice(Iterator[_T]): +class islice(Generic[_T]): @overload def __init__(self, iterable: Iterable[_T], stop: int | None, /) -> None: ... @overload @@ -101,19 +101,19 @@ class islice(Iterator[_T]): def __iter__(self) -> Self: ... def __next__(self) -> _T: ... -class starmap(Iterator[_T_co]): +class starmap(Generic[_T_co]): def __new__(cls, function: Callable[..., _T], iterable: Iterable[Iterable[Any]], /) -> starmap[_T]: ... def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... -class takewhile(Iterator[_T]): +class takewhile(Generic[_T]): def __init__(self, predicate: _Predicate[_T], iterable: Iterable[_T], /) -> None: ... def __iter__(self) -> Self: ... def __next__(self) -> _T: ... def tee(iterable: Iterable[_T], n: int = 2, /) -> tuple[Iterator[_T], ...]: ... -class zip_longest(Iterator[_T_co]): +class zip_longest(Generic[_T_co]): # one iterable (fillvalue doesn't matter) @overload def __new__(cls, iter1: Iterable[_T1], /, *, fillvalue: object = ...) -> zip_longest[tuple[_T1]]: ... @@ -191,7 +191,7 @@ class zip_longest(Iterator[_T_co]): def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... -class product(Iterator[_T_co]): +class product(Generic[_T_co]): @overload def __new__(cls, iter1: Iterable[_T1], /) -> product[tuple[_T1]]: ... @overload @@ -276,7 +276,7 @@ class product(Iterator[_T_co]): def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... -class permutations(Iterator[_T_co]): +class permutations(Generic[_T_co]): @overload def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> permutations[tuple[_T, _T]]: ... @overload @@ -290,7 +290,7 @@ class permutations(Iterator[_T_co]): def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... -class combinations(Iterator[_T_co]): +class combinations(Generic[_T_co]): @overload def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> combinations[tuple[_T, _T]]: ... @overload @@ -304,7 +304,7 @@ class combinations(Iterator[_T_co]): def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... -class combinations_with_replacement(Iterator[_T_co]): +class combinations_with_replacement(Generic[_T_co]): @overload def __new__(cls, iterable: Iterable[_T], r: Literal[2]) -> combinations_with_replacement[tuple[_T, _T]]: ... @overload @@ -319,13 +319,13 @@ class combinations_with_replacement(Iterator[_T_co]): def __next__(self) -> _T_co: ... if sys.version_info >= (3, 10): - class pairwise(Iterator[_T_co]): + class pairwise(Generic[_T_co]): def __new__(cls, iterable: Iterable[_T], /) -> pairwise[tuple[_T, _T]]: ... def __iter__(self) -> Self: ... def __next__(self) -> _T_co: ... if sys.version_info >= (3, 12): - class batched(Iterator[tuple[_T_co, ...]], Generic[_T_co]): + class batched(Generic[_T_co]): if sys.version_info >= (3, 13): def __new__(cls, iterable: Iterable[_T_co], n: int, *, strict: bool = False) -> Self: ... else: From cb9dd7d201f58bfce0f708b9d820da5c7fedf4c4 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 21 Oct 2024 00:25:20 -0700 Subject: [PATCH 241/247] [mypyc] fix relative imports in __init__.py (#17979) Fixes https://github.com/mypyc/mypyc/issues/1070 Missed in https://github.com/python/mypy/pull/10543 --- mypyc/irbuild/statement.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 4d828b1b9d82..ac3fa33f36ca 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -347,10 +347,10 @@ def transform_import_from(builder: IRBuilder, node: ImportFrom) -> None: return module_state = builder.graph[builder.module_name] - if module_state.ancestors is not None and module_state.ancestors: - module_package = module_state.ancestors[0] - elif builder.module_path.endswith("__init__.py"): + if builder.module_path.endswith("__init__.py"): module_package = builder.module_name + elif module_state.ancestors is not None and module_state.ancestors: + module_package = module_state.ancestors[0] else: module_package = "" From 5bae05dbf8c8a449dc320f8ba0da8727e78170f9 Mon Sep 17 00:00:00 2001 From: aatle <168398276+aatle@users.noreply.github.com> Date: Mon, 21 Oct 2024 00:28:20 -0700 Subject: [PATCH 242/247] [mypyc] fix name generation for modules with similar full names (#18001) Fixes https://github.com/mypyc/mypyc/issues/1071 Adds a test to cover this case Building certain package layouts now succeeds instead of failing. The behavior for all package layouts not affected by the error is unchanged. In `namegen.make_module_translation_map(names)`, if argument `names` have `"foo"` and `"foo.foo"`, all suffixes found for `"foo"` are also found for `"foo.foo"`. This means that module `foo` has no unique suffixes, which currently causes an `AssertionError`. The fix forces a module to take the last, fullest suffix if none are unique. It is guaranteed that no other module will also take the same suffix because they either will have a unique suffix to take, or they will take the fullest suffix for their name which is always going to be different. --- mypyc/namegen.py | 5 ++--- mypyc/test/test_namegen.py | 6 ++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mypyc/namegen.py b/mypyc/namegen.py index 675dae9001c7..ce84fde143d1 100644 --- a/mypyc/namegen.py +++ b/mypyc/namegen.py @@ -100,10 +100,9 @@ def make_module_translation_map(names: list[str]) -> dict[str, str]: for name in names: for suffix in candidate_suffixes(name): if num_instances[suffix] == 1: - result[name] = suffix break - else: - assert False, names + # Takes the last suffix if none are unique + result[name] = suffix return result diff --git a/mypyc/test/test_namegen.py b/mypyc/test/test_namegen.py index 509018b4c3bd..f88edbd00dce 100644 --- a/mypyc/test/test_namegen.py +++ b/mypyc/test/test_namegen.py @@ -35,6 +35,12 @@ def test_make_module_translation_map(self) -> None: "fu.bar": "fu.bar.", "foo.baz": "baz.", } + assert make_module_translation_map(["foo", "foo.foo", "bar.foo", "bar.foo.bar.foo"]) == { + "foo": "foo.", + "foo.foo": "foo.foo.", + "bar.foo": "bar.foo.", + "bar.foo.bar.foo": "foo.bar.foo.", + } def test_name_generator(self) -> None: g = NameGenerator([["foo", "foo.zar"]]) From 8a0dd7ad6c7a30170de3b87f997577b8c86e2451 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:26:55 +0200 Subject: [PATCH 243/247] Add timeout-minutes to ci config (#18003) Prevent blocking resources unnecessarily if something goes wrong and a CI task doesn't terminate. The default timeout would be 360 minutes. --- .github/workflows/docs.yml | 1 + .github/workflows/mypy_primer.yml | 1 + .github/workflows/sync_typeshed.yml | 1 + .github/workflows/test.yml | 2 ++ .github/workflows/test_stubgenc.yml | 1 + 5 files changed, 6 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 923d74a02f71..112102954dd3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,6 +28,7 @@ concurrency: jobs: docs: runs-on: ubuntu-latest + timeout-minutes: 10 env: TOXENV: docs TOX_SKIP_MISSING_INTERPRETERS: False diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index fee7e6d079f1..2b2327798a72 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -32,6 +32,7 @@ jobs: matrix: shard-index: [0, 1, 2, 3, 4] fail-fast: false + timeout-minutes: 60 steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/sync_typeshed.yml b/.github/workflows/sync_typeshed.yml index b545e7b0662b..84d246441f3d 100644 --- a/.github/workflows/sync_typeshed.yml +++ b/.github/workflows/sync_typeshed.yml @@ -14,6 +14,7 @@ jobs: name: Sync typeshed if: github.repository == 'python/mypy' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4150d1968bfb..9e6c9cd1d9b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -126,6 +126,7 @@ jobs: toxenv: lint name: ${{ matrix.name }} + timeout-minutes: 60 env: TOX_SKIP_MISSING_INTERPRETERS: False # Rich (pip) -- Disable color for windows + pytest @@ -205,6 +206,7 @@ jobs: python_32bits: runs-on: ubuntu-latest name: Test mypyc suite with 32-bit Python + timeout-minutes: 60 env: TOX_SKIP_MISSING_INTERPRETERS: False # Rich (pip) diff --git a/.github/workflows/test_stubgenc.yml b/.github/workflows/test_stubgenc.yml index 519f63ac2bd7..0652702a0fc0 100644 --- a/.github/workflows/test_stubgenc.yml +++ b/.github/workflows/test_stubgenc.yml @@ -25,6 +25,7 @@ jobs: # Check stub file generation for a small pybind11 project # (full text match is required to pass) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v4 From e106dd7a0653c24d67597adf3ae6939d7ff9a376 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:10:08 -0700 Subject: [PATCH 244/247] Changelog for 1.13 (#18000) --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0cbe466a5d5..801d592945c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,49 @@ ## Next release +## Mypy 1.13 + +We’ve just uploaded mypy 1.13 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). +Mypy is a static type checker for Python. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +Note that unlike typical releases, Mypy 1.13 does not have any changes to type checking semantics +from 1.12.1. + +### Improved performance + +Mypy 1.13 contains several performance improvements. Users can expect mypy to be 5-20% faster. +In environments with long search paths (such as environments using many editable installs), mypy +can be significantly faster, e.g. 2.2x faster in the use case targeted by these improvements. + +Mypy 1.13 allows use of the `orjson` library for handling the cache instead of the stdlib `json`, +for improved performance. You can ensure the presence of `orjson` using the `faster-cache` extra: + + python3 -m pip install -U mypy[faster-cache] + +Mypy may depend on `orjson` by default in the future. + +These improvements were contributed by Shantanu. + +List of changes: +* Significantly speed up file handling error paths (Shantanu, PR [17920](https://github.com/python/mypy/pull/17920)) +* Use fast path in modulefinder more often (Shantanu, PR [17950](https://github.com/python/mypy/pull/17950)) +* Let mypyc optimise os.path.join (Shantanu, PR [17949](https://github.com/python/mypy/pull/17949)) +* Make is_sub_path faster (Shantanu, PR [17962](https://github.com/python/mypy/pull/17962)) +* Speed up stubs suggestions (Shantanu, PR [17965](https://github.com/python/mypy/pull/17965)) +* Use sha1 for hashing (Shantanu, PR [17953](https://github.com/python/mypy/pull/17953)) +* Use orjson instead of json, when available (Shantanu, PR [17955](https://github.com/python/mypy/pull/17955)) +* Add faster-cache extra, test in CI (Shantanu, PR [17978](https://github.com/python/mypy/pull/17978)) + +### Acknowledgements +Thanks to all mypy contributors who contributed to this release: + +- Shantanu Jain +- Jukka Lehtosalo + ## Mypy 1.12 We’ve just uploaded mypy 1.12 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type From 60d1b3776229d3e3c332413ef49ad89be469a5e4 Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Tue, 22 Oct 2024 23:19:00 -0400 Subject: [PATCH 245/247] Fix compatibility checks for conditional function definitions using decorators (#18020) Fixes #17211, resolves this `# TODO`: https://github.com/python/mypy/blob/e106dd7a0653c24d67597adf3ae6939d7ff9a376/test-data/unit/check-functions.test#L1486-L1494 ### Before ```python from typing import Callable def dec(f: object) -> Callable[[int], None]: raise NotImplementedError if int(): def f(x: str) -> None: pass else: @dec def f() -> None: pass # uh oh! passes without error ``` ### After ```python from typing import Callable def dec(f: object) -> Callable[[int], None]: raise NotImplementedError if int(): def f(x: str) -> None: pass else: @dec def f() -> None: pass # E: All conditional function variants must have identical signatures \ # N: Original: \ # N: def f(x: str) -> None \ # N: Redefinition: \ # N: def f(int, /) -> None ``` --- mypy/checker.py | 88 +++++++++++++++------------ test-data/unit/check-functions.test | 11 ++-- test-data/unit/check-newsemanal.test | 4 +- test-data/unit/check-overloading.test | 6 +- 4 files changed, 60 insertions(+), 49 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 4b3d6c3298b4..f52bebdaa052 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1072,46 +1072,7 @@ def _visit_func_def(self, defn: FuncDef) -> None: if defn.original_def: # Override previous definition. new_type = self.function_type(defn) - if isinstance(defn.original_def, FuncDef): - # Function definition overrides function definition. - old_type = self.function_type(defn.original_def) - if not is_same_type(new_type, old_type): - self.msg.incompatible_conditional_function_def(defn, old_type, new_type) - else: - # Function definition overrides a variable initialized via assignment or a - # decorated function. - orig_type = defn.original_def.type - if orig_type is None: - # If other branch is unreachable, we don't type check it and so we might - # not have a type for the original definition - return - if isinstance(orig_type, PartialType): - if orig_type.type is None: - # Ah this is a partial type. Give it the type of the function. - orig_def = defn.original_def - if isinstance(orig_def, Decorator): - var = orig_def.var - else: - var = orig_def - partial_types = self.find_partial_types(var) - if partial_types is not None: - var.type = new_type - del partial_types[var] - else: - # Trying to redefine something like partial empty list as function. - self.fail(message_registry.INCOMPATIBLE_REDEFINITION, defn) - else: - name_expr = NameExpr(defn.name) - name_expr.node = defn.original_def - self.binder.assign_type(name_expr, new_type, orig_type) - self.check_subtype( - new_type, - orig_type, - defn, - message_registry.INCOMPATIBLE_REDEFINITION, - "redefinition with type", - "original type", - ) + self.check_func_def_override(defn, new_type) def check_func_item( self, @@ -1147,6 +1108,49 @@ def check_func_item( if dataclasses_plugin.is_processed_dataclass(defn.info): dataclasses_plugin.check_post_init(self, defn, defn.info) + def check_func_def_override(self, defn: FuncDef, new_type: FunctionLike) -> None: + assert defn.original_def is not None + if isinstance(defn.original_def, FuncDef): + # Function definition overrides function definition. + old_type = self.function_type(defn.original_def) + if not is_same_type(new_type, old_type): + self.msg.incompatible_conditional_function_def(defn, old_type, new_type) + else: + # Function definition overrides a variable initialized via assignment or a + # decorated function. + orig_type = defn.original_def.type + if orig_type is None: + # If other branch is unreachable, we don't type check it and so we might + # not have a type for the original definition + return + if isinstance(orig_type, PartialType): + if orig_type.type is None: + # Ah this is a partial type. Give it the type of the function. + orig_def = defn.original_def + if isinstance(orig_def, Decorator): + var = orig_def.var + else: + var = orig_def + partial_types = self.find_partial_types(var) + if partial_types is not None: + var.type = new_type + del partial_types[var] + else: + # Trying to redefine something like partial empty list as function. + self.fail(message_registry.INCOMPATIBLE_REDEFINITION, defn) + else: + name_expr = NameExpr(defn.name) + name_expr.node = defn.original_def + self.binder.assign_type(name_expr, new_type, orig_type) + self.check_subtype( + new_type, + orig_type, + defn, + message_registry.INCOMPATIBLE_REDEFINITION, + "redefinition with type", + "original type", + ) + @contextmanager def enter_attribute_inference_context(self) -> Iterator[None]: old_types = self.inferred_attribute_types @@ -5120,6 +5124,10 @@ def visit_decorator_inner(self, e: Decorator, allow_empty: bool = False) -> None if e.type and not isinstance(get_proper_type(e.type), (FunctionLike, AnyType)): self.fail(message_registry.BAD_CONSTRUCTOR_TYPE, e) + if e.func.original_def and isinstance(sig, FunctionLike): + # Function definition overrides function definition. + self.check_func_def_override(e.func, sig) + def check_for_untyped_decorator( self, func: FuncDef, dec_type: Type, dec_expr: Expression ) -> None: diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 96f9815019e6..b8a02a1ec7d4 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1474,7 +1474,7 @@ def dec(f) -> Callable[[int], None]: pass x = int() if x: - def f(x: int) -> None: pass + def f(x: int, /) -> None: pass else: @dec def f(): pass @@ -1489,9 +1489,12 @@ x = int() if x: def f(x: str) -> None: pass else: - # TODO: Complain about incompatible redefinition @dec - def f(): pass + def f(): pass # E: All conditional function variants must have identical signatures \ + # N: Original: \ + # N: def f(x: str) -> None \ + # N: Redefinition: \ + # N: def f(int, /) -> None [case testConditionalFunctionDefinitionUnreachable] def bar() -> None: @@ -1599,7 +1602,7 @@ else: def f(): yield [file m.py] -def f(): pass +def f() -> None: pass [case testDefineConditionallyAsImportedAndDecoratedWithInference] if int(): diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index 511c7b003015..fe02ac3ccd5e 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -1908,9 +1908,9 @@ else: @dec def f(x: int) -> None: 1() # E: "int" not callable -reveal_type(f) # N: Revealed type is "def (x: builtins.str)" +reveal_type(f) # N: Revealed type is "def (builtins.str)" [file m.py] -def f(x: str) -> None: pass +def f(x: str, /) -> None: pass [case testNewAnalyzerConditionallyDefineFuncOverVar] from typing import Callable diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index e414c1c9b0b6..9d01ce6bd480 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -6463,7 +6463,7 @@ class D: ... def f1(g: A) -> A: ... if True: @overload # E: Single overload definition, multiple required - def f1(g: B) -> B: ... + def f1(g: B) -> B: ... # E: Incompatible redefinition (redefinition with type "Callable[[B], B]", original type "Callable[[A], A]") if maybe_true: # E: Condition can't be inferred, unable to merge overloads \ # E: Name "maybe_true" is not defined @overload @@ -6480,14 +6480,14 @@ if True: def f2(g: B) -> B: ... elif maybe_true: # E: Name "maybe_true" is not defined @overload # E: Single overload definition, multiple required - def f2(g: C) -> C: ... + def f2(g: C) -> C: ... # E: Incompatible redefinition (redefinition with type "Callable[[C], C]", original type "Callable[[A], A]") def f2(g): ... # E: Name "f2" already defined on line 21 @overload # E: Single overload definition, multiple required def f3(g: A) -> A: ... if True: @overload # E: Single overload definition, multiple required - def f3(g: B) -> B: ... + def f3(g: B) -> B: ... # E: Incompatible redefinition (redefinition with type "Callable[[B], B]", original type "Callable[[A], A]") if True: pass # Some other node @overload # E: Name "f3" already defined on line 32 \ From 9e689593c9b99555631c2e2bcc05cb5481a557fb Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:52:29 +0200 Subject: [PATCH 246/247] Improve for loop index variable type narrowing (#18014) Preserve the literal type of index expressions a bit longer (until the next assignment) to support TypedDict lookups. ```py from typing import TypedDict class X(TypedDict): hourly: int daily: int def func(x: X) -> None: for var in ("hourly", "daily"): print(x[var]) ``` Closes #9230 --- mypy/checker.py | 9 +++++++++ mypy/nodes.py | 3 +++ mypy/semanal.py | 18 +++++++++++++++--- test-data/unit/check-inference.test | 29 +++++++++++++++++++++++++++++ test-data/unit/fixtures/for.pyi | 4 +++- 5 files changed, 59 insertions(+), 4 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f52bebdaa052..3e433408e40b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3175,6 +3175,14 @@ def check_assignment( # Don't use type binder for definitions of special forms, like named tuples. if not (isinstance(lvalue, NameExpr) and lvalue.is_special_form): self.binder.assign_type(lvalue, rvalue_type, lvalue_type, False) + if ( + isinstance(lvalue, NameExpr) + and isinstance(lvalue.node, Var) + and lvalue.node.is_inferred + and lvalue.node.is_index_var + and lvalue_type is not None + ): + lvalue.node.type = remove_instance_last_known_values(lvalue_type) elif index_lvalue: self.check_indexed_assignment(index_lvalue, rvalue, lvalue) @@ -3184,6 +3192,7 @@ def check_assignment( rvalue_type = self.expr_checker.accept(rvalue, type_context=type_context) if not ( inferred.is_final + or inferred.is_index_var or (isinstance(lvalue, NameExpr) and lvalue.name == "__match_args__") ): rvalue_type = remove_instance_last_known_values(rvalue_type) diff --git a/mypy/nodes.py b/mypy/nodes.py index 4806a7b0be7d..7b620bd52008 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -969,6 +969,7 @@ def is_dynamic(self) -> bool: "is_classvar", "is_abstract_var", "is_final", + "is_index_var", "final_unset_in_class", "final_set_in_init", "explicit_self_type", @@ -1005,6 +1006,7 @@ class Var(SymbolNode): "is_classvar", "is_abstract_var", "is_final", + "is_index_var", "final_unset_in_class", "final_set_in_init", "is_suppressed_import", @@ -1039,6 +1041,7 @@ def __init__(self, name: str, type: mypy.types.Type | None = None) -> None: self.is_settable_property = False self.is_classvar = False self.is_abstract_var = False + self.is_index_var = False # Set to true when this variable refers to a module we were unable to # parse for some reason (eg a silenced module) self.is_suppressed_import = False diff --git a/mypy/semanal.py b/mypy/semanal.py index a0d4daffca0c..e182322cfbe6 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4225,6 +4225,7 @@ def analyze_lvalue( is_final: bool = False, escape_comprehensions: bool = False, has_explicit_value: bool = False, + is_index_var: bool = False, ) -> None: """Analyze an lvalue or assignment target. @@ -4235,6 +4236,7 @@ def analyze_lvalue( escape_comprehensions: If we are inside a comprehension, set the variable in the enclosing scope instead. This implements https://www.python.org/dev/peps/pep-0572/#scope-of-the-target + is_index_var: If lval is the index variable in a for loop """ if escape_comprehensions: assert isinstance(lval, NameExpr), "assignment expression target must be NameExpr" @@ -4245,6 +4247,7 @@ def analyze_lvalue( is_final, escape_comprehensions, has_explicit_value=has_explicit_value, + is_index_var=is_index_var, ) elif isinstance(lval, MemberExpr): self.analyze_member_lvalue(lval, explicit_type, is_final, has_explicit_value) @@ -4271,6 +4274,7 @@ def analyze_name_lvalue( is_final: bool, escape_comprehensions: bool, has_explicit_value: bool, + is_index_var: bool, ) -> None: """Analyze an lvalue that targets a name expression. @@ -4309,7 +4313,9 @@ def analyze_name_lvalue( if (not existing or isinstance(existing.node, PlaceholderNode)) and not outer: # Define new variable. - var = self.make_name_lvalue_var(lvalue, kind, not explicit_type, has_explicit_value) + var = self.make_name_lvalue_var( + lvalue, kind, not explicit_type, has_explicit_value, is_index_var + ) added = self.add_symbol(name, var, lvalue, escape_comprehensions=escape_comprehensions) # Only bind expression if we successfully added name to symbol table. if added: @@ -4361,7 +4367,12 @@ def is_alias_for_final_name(self, name: str) -> bool: return existing is not None and is_final_node(existing.node) def make_name_lvalue_var( - self, lvalue: NameExpr, kind: int, inferred: bool, has_explicit_value: bool + self, + lvalue: NameExpr, + kind: int, + inferred: bool, + has_explicit_value: bool, + is_index_var: bool, ) -> Var: """Return a Var node for an lvalue that is a name expression.""" name = lvalue.name @@ -4380,6 +4391,7 @@ def make_name_lvalue_var( v._fullname = name v.is_ready = False # Type not inferred yet v.has_explicit_value = has_explicit_value + v.is_index_var = is_index_var return v def make_name_lvalue_point_to_existing_def( @@ -5290,7 +5302,7 @@ def visit_for_stmt(self, s: ForStmt) -> None: s.expr.accept(self) # Bind index variables and check if they define new names. - self.analyze_lvalue(s.index, explicit_type=s.index_type is not None) + self.analyze_lvalue(s.index, explicit_type=s.index_type is not None, is_index_var=True) if s.index_type: if self.is_classvar(s.index_type): self.fail_invalid_classvar(s.index) diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 0dbefbc774a3..b0ec590b54f9 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1238,6 +1238,35 @@ class B: pass [builtins fixtures/for.pyi] [out] +[case testForStatementIndexNarrowing] +from typing_extensions import TypedDict + +class X(TypedDict): + hourly: int + daily: int + +x: X +for a in ("hourly", "daily"): + reveal_type(a) # N: Revealed type is "Union[Literal['hourly']?, Literal['daily']?]" + reveal_type(x[a]) # N: Revealed type is "builtins.int" + reveal_type(a.upper()) # N: Revealed type is "builtins.str" + c = a + reveal_type(c) # N: Revealed type is "builtins.str" + a = "monthly" + reveal_type(a) # N: Revealed type is "builtins.str" + a = "yearly" + reveal_type(a) # N: Revealed type is "builtins.str" + a = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "str") + reveal_type(a) # N: Revealed type is "builtins.str" + d = a + reveal_type(d) # N: Revealed type is "builtins.str" + +b: str +for b in ("hourly", "daily"): + reveal_type(b) # N: Revealed type is "builtins.str" + reveal_type(b.upper()) # N: Revealed type is "builtins.str" +[builtins fixtures/for.pyi] + -- Regression tests -- ---------------- diff --git a/test-data/unit/fixtures/for.pyi b/test-data/unit/fixtures/for.pyi index 694f83e940b2..10f45e68cd7d 100644 --- a/test-data/unit/fixtures/for.pyi +++ b/test-data/unit/fixtures/for.pyi @@ -12,9 +12,11 @@ class type: pass class tuple(Generic[t]): def __iter__(self) -> Iterator[t]: pass class function: pass +class ellipsis: pass class bool: pass class int: pass # for convenience -class str: pass # for convenience +class str: # for convenience + def upper(self) -> str: ... class list(Iterable[t], Generic[t]): def __iter__(self) -> Iterator[t]: pass From eb0575eaa8720b2cdac62c7deb4c54c53035ea3b Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 23 Oct 2024 22:06:39 +0200 Subject: [PATCH 247/247] [stubtest] Verify __all__ exists in stub (#18005) Previously it wasn't an error if runtime included an `__all__` declaration, but the stubs did not. This PR changes this to reflect the consensus that it would be a good idea to ensure consistency in this case. Fixes #13300 --- mypy/stubtest.py | 2 ++ mypy/test/teststubtest.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 756f90dccf2e..0de5411b01de 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -348,6 +348,8 @@ def verify_mypyfile( # Only verify the contents of the stub's __all__ # if the stub actually defines __all__ yield from _verify_exported_names(object_path, stub, runtime_all_as_set) + else: + yield Error(object_path + ["__all__"], "is not present in stub", MISSING, runtime) else: runtime_all_as_set = None diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 70687b499651..4cab62875647 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1403,7 +1403,7 @@ def test_all_at_runtime_not_stub(self) -> Iterator[Case]: runtime=""" __all__ = [] Z = 5""", - error=None, + error="__all__", ) @collect_cases @@ -1443,7 +1443,7 @@ def h(x: str): ... runtime="", error="h", ) - yield Case(stub="", runtime="__all__ = []", error=None) # dummy case + yield Case(stub="", runtime="__all__ = []", error="__all__") # dummy case yield Case(stub="", runtime="__all__ += ['y']\ny = 5", error="y") yield Case(stub="", runtime="__all__ += ['g']\ndef g(): pass", error="g") # Here we should only check that runtime has B, since the stub explicitly re-exports it