From 8b73cc22c6a251682f777b104677fa0e1ed5fd67 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Sep 2023 23:23:25 +0100 Subject: [PATCH] Complete type analysis of variadic types (#15991) This PR closes the first part of support for `TypeVarTuple`: the "static" analysis of types (of course everything is static in mypy, but some parts are more static): `semanal`/`typeanal`, `expand_type()`, `map_instance_to_supertype()`, `erase_type()` (things that precede and/or form foundation for type inference and subtyping). This one was quite tricky, supporting unpacks of forward references required some thinking. What is included in this PR: * Moving argument count validation from `semanal_typeargs` to `typeanal`. In one of previous PRs I mentioned that `get_proper_type()` may be called during semantic analysis causing troubles if we have invalid aliases. So we need to move validation to early stage. For instances, this is not required, but I strongly prefer keeping instances and aliases similar. And ideally at some point we can combine the logic, since it gets more and more similar. At some point we may want to prohibit using `get_proper_type()` during semantic analysis, but I don't want to block `TypeVarTuple` support on this, since this may be a significant refactoring. * Fixing `map_instance_to_supertype()` and `erase_type()`. These two are straightforward, we either use `expand_type()` logic directly (by calling it), or following the same logic. * Few simplifications in `expandtype` and `typeops` following previous normalizations of representation, unless there is a flaw in my logic, removed branches should be all dead code. * Allow (only fixed) unpacks in argument lists for non-variadic types. They were prohibited for no good reason. * (Somewhat limited) support for forward references in unpacks. As I mentioned this one is tricky because of how forward references are represented. Usually they follow either a life cycle like: `Any` -> ``, or `` -> `` -> `` (second one is relatively rare and usually only appears for potentially recursive things like base classes or type alias targets). It looks like `` can never appear as a _valid_ unpack target, I don't have a proof for this, but I was not able to trigger this, so I am not handling it (possible downside is that there may be extra errors about invalid argument count for invalid unpack targets). If I am wrong and this can happen in some valid cases, we can add handling for unpacks of placeholders later. Currently, the handling for `Any` stage of forward references is following: if we detect it, we simply create a dummy valid alias or instance. This logic should work for the same reason having plain `Any` worked in the first place (and why all tests pass if we delete `visit_placeholder_type()`): because (almost) each time we analyze a type, it is either already complete, or we analyze it _from scratch_, i.e. we call `expr_to_unanalyzed_type()`, then `visit_unbound_type()` etc. We almost never store "partially analyzed" types (there are guards against incomplete references and placeholders in annotations), and when we do, it is done in a controlled way that guarantees a type will be re-analyzed again. Since this is such a tricky subject, I didn't add any complex logic to support more tricky use cases (like multiple forward references to fixed unpacks in single list). I propose that we release this, and then see what kind of bug reports we will get. * Additional validation for type arguments position to ensure that `TypeVarTuple`s are never split. Total count is not enough to ban case where we have type variables `[T, *Ts, S, U]` and arguments `[int, int, *Us, int]`. We need to explicitly ensure that actual suffix and prefix are longer or equal to formal ones. Such splitting would be very hard to support, and is explicitly banned by the PEP. * Few minor cleanups. Some random comments: * It is tricky to preserve valid parts of type arguments, if there is an argument count error involving an unpack. So after such error I simply set all arguments to `Any` (or `*tuple[Any, ...]` when needed). * I know there is some code duplication. I tried to factor it away, but it turned out non-trivial. I may do some de-duplication pass after everything is done, and it is easier to see the big picture. * Type applications (i.e. when we have `A[int, int]` in runtime context) are wild west currently. I decided to postpone variadic support for them to a separate PR, because there is already some support (we will just need to handle edge cases and more error conditions) and I wanted minimize size of this PR. * Something I wanted to mention in one of previous PRs but forgot: Long time ago I proposed to normalize away type aliases inside `Unpack`, but I abandoned this idea, it doesn't really give us any benefits. As I said, this is the last PR for the "static part", in the next PR I will work on fixing subtyping and inference for variadic instances. And then will continue with remaining items I mentioned in my master plan in https://github.com/python/mypy/pull/15924 Fixes https://github.com/python/mypy/issues/15978 --------- Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- mypy/erasetype.py | 34 ++++- mypy/expandtype.py | 66 ++++----- mypy/maptype.py | 22 +-- mypy/semanal_typeargs.py | 61 ++------ mypy/test/testtypes.py | 2 +- mypy/typeanal.py | 177 ++++++++++++++++++------ mypy/typeops.py | 4 +- test-data/unit/check-typevar-tuple.test | 123 +++++++++++++++- 8 files changed, 329 insertions(+), 160 deletions(-) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index fbbb4f80b578..d1a01fb6c779 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -165,9 +165,41 @@ def visit_type_var(self, t: TypeVarType) -> Type: return self.replacement return t + # TODO: below two methods duplicate some logic with expand_type(). + # In fact, we may want to refactor this whole visitor to use expand_type(). + def visit_instance(self, t: Instance) -> Type: + result = super().visit_instance(t) + assert isinstance(result, ProperType) and isinstance(result, Instance) + if t.type.fullname == "builtins.tuple": + # Normalize Tuple[*Tuple[X, ...], ...] -> Tuple[X, ...] + arg = result.args[0] + if isinstance(arg, UnpackType): + unpacked = get_proper_type(arg.type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + return unpacked + return result + + def visit_tuple_type(self, t: TupleType) -> Type: + result = super().visit_tuple_type(t) + assert isinstance(result, ProperType) and isinstance(result, TupleType) + if len(result.items) == 1: + # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...] + item = result.items[0] + if isinstance(item, UnpackType): + unpacked = get_proper_type(item.type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + if result.partial_fallback.type.fullname != "builtins.tuple": + # If it is a subtype (like named tuple) we need to preserve it, + # this essentially mimics the logic in tuple_fallback(). + return result.partial_fallback.accept(self) + return unpacked + return result + def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: if self.erase_id(t.id): - return self.replacement + return t.tuple_fallback.copy_modified(args=[self.replacement]) return t def visit_param_spec(self, t: ParamSpecType) -> Type: diff --git a/mypy/expandtype.py b/mypy/expandtype.py index be8ecb9ccfd9..c29fcb167777 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -212,10 +212,15 @@ def visit_erased_type(self, t: ErasedType) -> Type: def visit_instance(self, t: Instance) -> Type: args = self.expand_types_with_unpack(list(t.args)) - if isinstance(args, list): - return t.copy_modified(args=args) - else: - return args + if t.type.fullname == "builtins.tuple": + # Normalize Tuple[*Tuple[X, ...], ...] -> Tuple[X, ...] + arg = args[0] + if isinstance(arg, UnpackType): + unpacked = get_proper_type(arg.type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + args = list(unpacked.args) + return t.copy_modified(args=args) def visit_type_var(self, t: TypeVarType) -> Type: # Normally upper bounds can't contain other type variables, the only exception is @@ -285,7 +290,7 @@ def expand_unpack(self, t: UnpackType) -> list[Type]: ): return [UnpackType(typ=repl)] elif isinstance(repl, (AnyType, UninhabitedType)): - # Replace *Ts = Any with *Ts = *tuple[Any, ...] and some for Never. + # Replace *Ts = Any with *Ts = *tuple[Any, ...] and same for Never. # These types may appear here as a result of user error or failed inference. return [UnpackType(t.type.tuple_fallback.copy_modified(args=[repl]))] else: @@ -377,15 +382,8 @@ def visit_overloaded(self, t: Overloaded) -> Type: items.append(new_item) return Overloaded(items) - def expand_types_with_unpack( - self, typs: Sequence[Type] - ) -> list[Type] | AnyType | UninhabitedType: - """Expands a list of types that has an unpack. - - In corner cases, this can return a type rather than a list, in which case this - indicates use of Any or some error occurred earlier. In this case callers should - simply propagate the resulting type. - """ + def expand_types_with_unpack(self, typs: Sequence[Type]) -> list[Type]: + """Expands a list of types that has an unpack.""" items: list[Type] = [] for item in typs: if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType): @@ -396,24 +394,21 @@ def expand_types_with_unpack( def visit_tuple_type(self, t: TupleType) -> Type: items = self.expand_types_with_unpack(t.items) - if isinstance(items, list): - if len(items) == 1: - # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...] - item = items[0] - if isinstance(item, UnpackType): - unpacked = get_proper_type(item.type) - if isinstance(unpacked, Instance): - assert unpacked.type.fullname == "builtins.tuple" - if t.partial_fallback.type.fullname != "builtins.tuple": - # If it is a subtype (like named tuple) we need to preserve it, - # this essentially mimics the logic in tuple_fallback(). - return t.partial_fallback.accept(self) - return unpacked - fallback = t.partial_fallback.accept(self) - assert isinstance(fallback, ProperType) and isinstance(fallback, Instance) - return t.copy_modified(items=items, fallback=fallback) - else: - return items + if len(items) == 1: + # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...] + item = items[0] + if isinstance(item, UnpackType): + unpacked = get_proper_type(item.type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + if t.partial_fallback.type.fullname != "builtins.tuple": + # If it is a subtype (like named tuple) we need to preserve it, + # this essentially mimics the logic in tuple_fallback(). + return t.partial_fallback.accept(self) + return unpacked + fallback = t.partial_fallback.accept(self) + assert isinstance(fallback, ProperType) and isinstance(fallback, Instance) + return t.copy_modified(items=items, fallback=fallback) def visit_typeddict_type(self, t: TypedDictType) -> Type: fallback = t.fallback.accept(self) @@ -453,11 +448,8 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: # Target of the type alias cannot contain type variables (not bound by the type # alias itself), so we just expand the arguments. args = self.expand_types_with_unpack(t.args) - if isinstance(args, list): - # TODO: normalize if target is Tuple, and args are [*tuple[X, ...]]? - return t.copy_modified(args=args) - else: - return args + # TODO: normalize if target is Tuple, and args are [*tuple[X, ...]]? + return t.copy_modified(args=args) def expand_types(self, types: Iterable[Type]) -> list[Type]: a: list[Type] = [] diff --git a/mypy/maptype.py b/mypy/maptype.py index 4951306573c2..0d54a83127df 100644 --- a/mypy/maptype.py +++ b/mypy/maptype.py @@ -1,8 +1,8 @@ from __future__ import annotations -from mypy.expandtype import expand_type +from mypy.expandtype import expand_type_by_instance from mypy.nodes import TypeInfo -from mypy.types import AnyType, Instance, TupleType, Type, TypeOfAny, TypeVarId, has_type_vars +from mypy.types import AnyType, Instance, TupleType, TypeOfAny, has_type_vars def map_instance_to_supertype(instance: Instance, superclass: TypeInfo) -> Instance: @@ -25,8 +25,7 @@ def map_instance_to_supertype(instance: Instance, superclass: TypeInfo) -> Insta if not alias._is_recursive: # Unfortunately we can't support this for generic recursive tuples. # If we skip this special casing we will fall back to tuple[Any, ...]. - env = instance_to_type_environment(instance) - tuple_type = expand_type(instance.type.tuple_type, env) + tuple_type = expand_type_by_instance(instance.type.tuple_type, instance) if isinstance(tuple_type, TupleType): # Make the import here to avoid cyclic imports. import mypy.typeops @@ -91,8 +90,7 @@ def map_instance_to_direct_supertypes(instance: Instance, supertype: TypeInfo) - for b in typ.bases: if b.type == supertype: - env = instance_to_type_environment(instance) - t = expand_type(b, env) + t = expand_type_by_instance(b, instance) assert isinstance(t, Instance) result.append(t) @@ -103,15 +101,3 @@ def map_instance_to_direct_supertypes(instance: Instance, supertype: TypeInfo) - # type arguments implicitly. any_type = AnyType(TypeOfAny.unannotated) return [Instance(supertype, [any_type] * len(supertype.type_vars))] - - -def instance_to_type_environment(instance: Instance) -> dict[TypeVarId, Type]: - """Given an Instance, produce the resulting type environment for type - variables bound by the Instance's class definition. - - An Instance is a type application of a class (a TypeInfo) to its - required number of type arguments. So this environment consists - of the class's type variables mapped to the Instance's actual - arguments. The type variables are mapped by their `id`. - """ - return {binder.id: arg for binder, arg in zip(instance.type.defn.type_vars, instance.args)} diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index 749b02391e06..3e11951376c9 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -18,7 +18,6 @@ from mypy.options import Options from mypy.scope import Scope from mypy.subtypes import is_same_type, is_subtype -from mypy.typeanal import fix_type_var_tuple_argument, set_any_tvars from mypy.types import ( AnyType, CallableType, @@ -88,36 +87,7 @@ def visit_type_alias_type(self, t: TypeAliasType) -> None: # types, since errors there have already been reported. return self.seen_aliases.add(t) - # Some recursive aliases may produce spurious args. In principle this is not very - # important, as we would simply ignore them when expanding, but it is better to keep - # correct aliases. Also, variadic aliases are better to check when fully analyzed, - # so we do this here. assert t.alias is not None, f"Unfixed type alias {t.type_ref}" - # TODO: consider moving this validation to typeanal.py, expanding invalid aliases - # during semantic analysis may cause crashes. - if t.alias.tvar_tuple_index is not None: - correct = len(t.args) >= len(t.alias.alias_tvars) - 1 - if any( - isinstance(a, UnpackType) and isinstance(get_proper_type(a.type), Instance) - for a in t.args - ): - correct = True - else: - correct = len(t.args) == len(t.alias.alias_tvars) - if not correct: - if t.alias.tvar_tuple_index is not None: - exp_len = f"at least {len(t.alias.alias_tvars) - 1}" - else: - exp_len = f"{len(t.alias.alias_tvars)}" - self.fail( - "Bad number of arguments for type alias," - f" expected: {exp_len}, given: {len(t.args)}", - t, - code=codes.TYPE_ARG, - ) - t.args = set_any_tvars( - t.alias, t.line, t.column, self.options, from_error=True, fail=self.fail - ).args is_error = self.validate_args(t.alias.name, t.args, t.alias.alias_tvars, t) if not is_error: # If there was already an error for the alias itself, there is no point in checking @@ -144,34 +114,21 @@ def visit_callable_type(self, t: CallableType) -> None: t.arg_types[star_index] = p_type.args[0] def visit_instance(self, t: Instance) -> None: + super().visit_instance(t) # Type argument counts were checked in the main semantic analyzer pass. We assume # that the counts are correct here. info = t.type if isinstance(info, FakeInfo): return # https://github.com/python/mypy/issues/11079 - t.args = tuple(flatten_nested_tuples(t.args)) - if t.type.has_type_var_tuple_type: - # Regular Instances are already validated in typeanal.py. - # TODO: do something with partial overlap (probably just reject). - # also in other places where split_with_prefix_and_suffix() is used. - correct = len(t.args) >= len(t.type.type_vars) - 1 - if any( - isinstance(a, UnpackType) and isinstance(get_proper_type(a.type), Instance) - for a in t.args - ): - correct = True - if not correct: - exp_len = f"at least {len(t.type.type_vars) - 1}" - self.fail( - f"Bad number of arguments, expected: {exp_len}, given: {len(t.args)}", - t, - code=codes.TYPE_ARG, - ) - any_type = AnyType(TypeOfAny.from_error) - t.args = (any_type,) * len(t.type.type_vars) - fix_type_var_tuple_argument(any_type, t) self.validate_args(info.name, t.args, info.defn.type_vars, t) - super().visit_instance(t) + if t.type.fullname == "builtins.tuple" and len(t.args) == 1: + # Normalize Tuple[*Tuple[X, ...], ...] -> Tuple[X, ...] + arg = t.args[0] + if isinstance(arg, UnpackType): + unpacked = get_proper_type(arg.type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + t.args = unpacked.args def validate_args( self, name: str, args: Sequence[Type], type_vars: list[TypeVarLikeType], ctx: Context diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index 12e7b207b00a..59457dfa5d3b 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -1464,7 +1464,7 @@ def make_call(*items: tuple[str, str | None]) -> CallExpr: class TestExpandTypeLimitGetProperType(TestCase): # WARNING: do not increase this number unless absolutely necessary, # and you understand what you are doing. - ALLOWED_GET_PROPER_TYPES = 7 + ALLOWED_GET_PROPER_TYPES = 8 @skipUnless(mypy.expandtype.__file__.endswith(".py"), "Skip for compiled mypy") def test_count_get_proper_type(self) -> None: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index ed1a8073887b..e297f2bf1631 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -82,6 +82,7 @@ UnionType, UnpackType, callable_with_ellipsis, + find_unpack_in_list, flatten_nested_tuples, flatten_nested_unions, get_proper_type, @@ -404,7 +405,7 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) t.args, allow_param_spec=True, allow_param_spec_literals=node.has_param_spec_type, - allow_unpack=node.tvar_tuple_index is not None, + allow_unpack=True, # Fixed length unpacks can be used for non-variadic aliases. ) if node.has_param_spec_type and len(node.alias_tvars) == 1: an_args = self.pack_paramspec_args(an_args) @@ -425,9 +426,8 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) # when it is top-level instance, so no need to recurse. if ( isinstance(res, Instance) # type: ignore[misc] - and len(res.args) != len(res.type.type_vars) and not self.defining_alias - and not res.type.has_type_var_tuple_type + and not validate_instance(res, self.fail) ): fix_instance( res, @@ -510,9 +510,6 @@ def apply_concatenate_operator(self, t: UnboundType) -> Type: code=codes.VALID_TYPE, ) return AnyType(TypeOfAny.from_error) - - # TODO: this may not work well with aliases, if those worked. - # Those should be special-cased. elif isinstance(ps, ParamSpecType) and ps.prefix.arg_types: self.api.fail("Nested Concatenates are invalid", t, code=codes.VALID_TYPE) @@ -728,7 +725,7 @@ def analyze_type_with_type_info( args, allow_param_spec=True, allow_param_spec_literals=info.has_param_spec_type, - allow_unpack=info.has_type_var_tuple_type, + allow_unpack=True, # Fixed length tuples can be used for non-variadic types. ), ctx.line, ctx.column, @@ -736,19 +733,9 @@ def analyze_type_with_type_info( if len(info.type_vars) == 1 and info.has_param_spec_type: instance.args = tuple(self.pack_paramspec_args(instance.args)) - if info.has_type_var_tuple_type: - if instance.args: - # -1 to account for empty tuple - valid_arg_length = len(instance.args) >= len(info.type_vars) - 1 - # Empty case is special cased and we want to infer a Tuple[Any, ...] - # instead of the empty tuple, so no - 1 here. - else: - valid_arg_length = False - else: - valid_arg_length = len(instance.args) == len(info.type_vars) - # Check type argument count. - if not valid_arg_length and not self.defining_alias: + instance.args = tuple(flatten_nested_tuples(instance.args)) + if not self.defining_alias and not validate_instance(instance, self.fail): fix_instance( instance, self.fail, @@ -1342,9 +1329,7 @@ def analyze_callable_type(self, t: UnboundType) -> Type: callable_args, ret_type, fallback ) if isinstance(maybe_ret, CallableType): - maybe_ret = maybe_ret.copy_modified( - ret_type=ret_type.accept(self), variables=variables - ) + maybe_ret = maybe_ret.copy_modified(variables=variables) if maybe_ret is None: # Callable[?, RET] (where ? is something invalid) self.fail( @@ -1736,6 +1721,7 @@ def check_unpacks_in_list(self, items: list[Type]) -> list[Type]: num_unpacks = 0 final_unpack = None for item in items: + # TODO: handle forward references here, they appear as Unpack[Any]. if isinstance(item, UnpackType) and not isinstance( get_proper_type(item.type), TupleType ): @@ -1856,25 +1842,13 @@ def fix_instance( any_type = get_omitted_any(disallow_any, fail, note, t, options, fullname, unexpanded_type) t.args = (any_type,) * len(t.type.type_vars) fix_type_var_tuple_argument(any_type, t) - return - - if t.type.has_type_var_tuple_type: - # This can be only correctly analyzed when all arguments are fully - # analyzed, because there may be a variadic item among them, so we - # do this in semanal_typeargs.py. - return - - # Invalid number of type parameters. - fail( - wrong_type_arg_count(len(t.type.type_vars), str(len(t.args)), t.type.name), - t, - code=codes.TYPE_ARG, - ) # Construct the correct number of type arguments, as # otherwise the type checker may crash as it expects # things to be right. - t.args = tuple(AnyType(TypeOfAny.from_error) for _ in t.type.type_vars) + any_type = AnyType(TypeOfAny.from_error) + t.args = tuple(any_type for _ in t.type.type_vars) + fix_type_var_tuple_argument(any_type, t) t.invalid = True @@ -1903,6 +1877,15 @@ def instantiate_type_alias( ctx: context where expansion happens unexpanded_type, disallow_any, use_standard_error: used to customize error messages """ + # Type aliases are special, since they can be expanded during semantic analysis, + # so we need to normalize them as soon as possible. + # TODO: can this cause an infinite recursion? + args = flatten_nested_tuples(args) + if any(unknown_unpack(a) for a in args): + # This type is not ready to be validated, because of unknown total count. + # Note that we keep the kind of Any for consistency. + return set_any_tvars(node, ctx.line, ctx.column, options, special_form=True) + exp_len = len(node.alias_tvars) act_len = len(args) if ( @@ -1937,22 +1920,54 @@ def instantiate_type_alias( tp.line = ctx.line tp.column = ctx.column return tp - if act_len != exp_len and node.tvar_tuple_index is None: + if node.tvar_tuple_index is None: + if any(isinstance(a, UnpackType) for a in args): + # A variadic unpack in fixed size alias (fixed unpacks must be flattened by the caller) + fail(message_registry.INVALID_UNPACK_POSITION, ctx, code=codes.VALID_TYPE) + return set_any_tvars(node, ctx.line, ctx.column, options, from_error=True) + correct = act_len == exp_len + else: + correct = act_len >= exp_len - 1 + for a in args: + if isinstance(a, UnpackType): + unpacked = get_proper_type(a.type) + if isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple": + # Variadic tuple is always correct. + correct = True + if not correct: if use_standard_error: # This is used if type alias is an internal representation of another type, # for example a generic TypedDict or NamedTuple. msg = wrong_type_arg_count(exp_len, str(act_len), node.name) else: - msg = f"Bad number of arguments for type alias, expected: {exp_len}, given: {act_len}" + if node.tvar_tuple_index is not None: + exp_len_str = f"at least {exp_len - 1}" + else: + exp_len_str = str(exp_len) + msg = ( + "Bad number of arguments for type alias," + f" expected: {exp_len_str}, given: {act_len}" + ) fail(msg, ctx, code=codes.TYPE_ARG) return set_any_tvars(node, ctx.line, ctx.column, options, from_error=True) + elif node.tvar_tuple_index is not None: + # We also need to check if we are not performing a type variable tuple split. + unpack = find_unpack_in_list(args) + if unpack is not None: + unpack_arg = args[unpack] + assert isinstance(unpack_arg, UnpackType) + if isinstance(unpack_arg.type, TypeVarTupleType): + exp_prefix = node.tvar_tuple_index + act_prefix = unpack + exp_suffix = len(node.alias_tvars) - node.tvar_tuple_index - 1 + act_suffix = len(args) - unpack - 1 + if act_prefix < exp_prefix or act_suffix < exp_suffix: + fail("TypeVarTuple cannot be split", ctx, code=codes.TYPE_ARG) + return set_any_tvars(node, ctx.line, ctx.column, options, from_error=True) # TODO: we need to check args validity w.r.t alias.alias_tvars. # Otherwise invalid instantiations will be allowed in runtime context. # Note: in type context, these will be still caught by semanal_typeargs. - # Type aliases are special, since they can be expanded during semantic analysis, - # so we need to normalize them as soon as possible. - # TODO: can this cause an infinite recursion? - typ = TypeAliasType(node, flatten_nested_tuples(args), ctx.line, ctx.column) + typ = TypeAliasType(node, args, ctx.line, ctx.column) assert typ.alias is not None # HACK: Implement FlexibleAlias[T, typ] by expanding it to typ here. if ( @@ -1973,11 +1988,14 @@ def set_any_tvars( *, from_error: bool = False, disallow_any: bool = False, + special_form: bool = False, fail: MsgCallback | None = None, unexpanded_type: Type | None = None, ) -> TypeAliasType: if from_error or disallow_any: type_of_any = TypeOfAny.from_error + elif special_form: + type_of_any = TypeOfAny.special_form else: type_of_any = TypeOfAny.from_omitted_generics if disallow_any and node.alias_tvars: @@ -2227,6 +2245,63 @@ def make_optional_type(t: Type) -> Type: return UnionType([t, NoneType()], t.line, t.column) +def validate_instance(t: Instance, fail: MsgCallback) -> bool: + """Check if this is a well-formed instance with respect to argument count/positions.""" + # TODO: combine logic with instantiate_type_alias(). + if any(unknown_unpack(a) for a in t.args): + # This type is not ready to be validated, because of unknown total count. + # TODO: is it OK to fill with TypeOfAny.from_error instead of special form? + return False + if t.type.has_type_var_tuple_type: + correct = len(t.args) >= len(t.type.type_vars) - 1 + if any( + isinstance(a, UnpackType) and isinstance(get_proper_type(a.type), Instance) + for a in t.args + ): + correct = True + if not correct: + exp_len = f"at least {len(t.type.type_vars) - 1}" + fail( + f"Bad number of arguments, expected: {exp_len}, given: {len(t.args)}", + t, + code=codes.TYPE_ARG, + ) + return False + elif not t.args: + # The Any arguments should be set by the caller. + return False + else: + # We also need to check if we are not performing a type variable tuple split. + unpack = find_unpack_in_list(t.args) + if unpack is not None: + unpack_arg = t.args[unpack] + assert isinstance(unpack_arg, UnpackType) + if isinstance(unpack_arg.type, TypeVarTupleType): + assert t.type.type_var_tuple_prefix is not None + assert t.type.type_var_tuple_suffix is not None + exp_prefix = t.type.type_var_tuple_prefix + act_prefix = unpack + exp_suffix = t.type.type_var_tuple_suffix + act_suffix = len(t.args) - unpack - 1 + if act_prefix < exp_prefix or act_suffix < exp_suffix: + fail("TypeVarTuple cannot be split", t, code=codes.TYPE_ARG) + return False + elif any(isinstance(a, UnpackType) for a in t.args): + # A variadic unpack in fixed size instance (fixed unpacks must be flattened by the caller) + fail(message_registry.INVALID_UNPACK_POSITION, t, code=codes.VALID_TYPE) + return False + elif len(t.args) != len(t.type.type_vars): + # Invalid number of type parameters. + if t.args: + fail( + wrong_type_arg_count(len(t.type.type_vars), str(len(t.args)), t.type.name), + t, + code=codes.TYPE_ARG, + ) + return False + return True + + def fix_instance_types(t: Type, fail: MsgCallback, note: MsgCallback, options: Options) -> None: """Recursively fix all instance types (type argument count) in a given type. @@ -2244,7 +2319,7 @@ def __init__(self, fail: MsgCallback, note: MsgCallback, options: Options) -> No def visit_instance(self, typ: Instance) -> None: super().visit_instance(typ) - if len(typ.args) != len(typ.type.type_vars) and not typ.type.has_type_var_tuple_type: + if not validate_instance(typ, self.fail): fix_instance( typ, self.fail, @@ -2269,3 +2344,17 @@ def visit_unbound_type(self, t: UnboundType) -> bool: if sym and sym.fullname in SELF_TYPE_NAMES: return True return super().visit_unbound_type(t) + + +def unknown_unpack(t: Type) -> bool: + """Check if a given type is an unpack of an unknown type. + + Unfortunately, there is no robust way to distinguish forward references from + genuine undefined names here. But this worked well so far, although it looks + quite fragile. + """ + if isinstance(t, UnpackType): + unpacked = get_proper_type(t.type) + if isinstance(unpacked, AnyType) and unpacked.type_of_any == TypeOfAny.special_form: + return True + return False diff --git a/mypy/typeops.py b/mypy/typeops.py index f9c1914cc9a8..3efa3cc3e965 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -110,10 +110,8 @@ def tuple_fallback(typ: TupleType) -> Instance: and unpacked_type.type.fullname == "builtins.tuple" ): items.append(unpacked_type.args[0]) - elif isinstance(unpacked_type, (AnyType, UninhabitedType)): - continue else: - raise NotImplementedError(unpacked_type) + raise NotImplementedError else: items.append(item) # TODO: we should really use a union here, tuple types are special. diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index f7faab4818c9..2b47ff30cdfb 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -619,8 +619,7 @@ T = TypeVar("T") Ts = TypeVarTuple("Ts") A = List[Tuple[T, Unpack[Ts], T]] -B = A[Unpack[Ts]] -x: B[int, str, str] +x: A[int, str, str] reveal_type(x) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.str, builtins.str, builtins.int]]" [builtins fixtures/tuple.pyi] @@ -1052,8 +1051,7 @@ reveal_type(y.fn) # N: Revealed type is "def (builtins.int, builtins.str)" z: A[Unpack[Tuple[int, ...]]] reveal_type(z) # N: Revealed type is "__main__.A[Unpack[builtins.tuple[builtins.int, ...]]]" -# TODO: this requires fixing map_instance_to_supertype(). -# reveal_type(z[0]) +reveal_type(z[0]) # N: Revealed type is "builtins.int" reveal_type(z.fn) # N: Revealed type is "def (*builtins.int)" t: A[int, Unpack[Tuple[int, str]], str] @@ -1118,3 +1116,120 @@ reveal_type(td) # N: Revealed type is "TypedDict('__main__.A', {'fn': def (buil def bad() -> int: ... td2 = A({"fn": bad, "val": 42}) # E: Incompatible types (expression has type "Callable[[], int]", TypedDict item "fn" has type "Callable[[], None]") [builtins fixtures/tuple.pyi] + +[case testFixedUnpackWithRegularInstance] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack + +T1 = TypeVar("T1") +T2 = TypeVar("T2") +T3 = TypeVar("T3") +T4 = TypeVar("T4") + +class C(Generic[T1, T2, T3, T4]): ... +x: C[int, Unpack[Alias], str] +Alias = Tuple[int, str] +reveal_type(x) # N: Revealed type is "__main__.C[builtins.int, builtins.int, builtins.str, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testVariadicUnpackWithRegularInstance] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack + +T1 = TypeVar("T1") +T2 = TypeVar("T2") +T3 = TypeVar("T3") +T4 = TypeVar("T4") + +class C(Generic[T1, T2, T3, T4]): ... +x: C[int, Unpack[Alias], str, str] # E: Unpack is only valid in a variadic position +Alias = Tuple[int, ...] +reveal_type(x) # N: Revealed type is "__main__.C[Any, Any, Any, Any]" +y: C[int, Unpack[Undefined]] # E: Name "Undefined" is not defined +reveal_type(y) # N: Revealed type is "__main__.C[Any, Any, Any, Any]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasInvalidUnpackNoCrash] +from typing import Tuple, Generic, Union, List +from typing_extensions import Unpack, TypeVarTuple + +Ts = TypeVarTuple("Ts") +Alias = Tuple[int, Unpack[Ts], str] + +A = Union[int, str] +x: List[Alias[int, Unpack[A], str]] # E: "Union[int, str]" cannot be unpacked (must be tuple or TypeVarTuple) +reveal_type(x) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.int, Unpack[builtins.tuple[Any, ...]], builtins.str, builtins.str]]" +y: List[Alias[int, Unpack[Undefined], str]] # E: Name "Undefined" is not defined +reveal_type(y) # N: Revealed type is "builtins.list[Tuple[builtins.int, Unpack[builtins.tuple[Any, ...]], builtins.str]]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasForwardRefToFixedUnpack] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +Alias = Tuple[T, Unpack[Ts], S] +x: Alias[int, Unpack[Other]] +Other = Tuple[int, str] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasForwardRefToVariadicUnpack] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +Alias = Tuple[T, Unpack[Ts], S] +x: Alias[int, Unpack[Other]] +Other = Tuple[int, ...] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]], builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testVariadicInstanceStrictPrefixSuffixCheck] +from typing import Tuple, Generic, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +class C(Generic[T, Unpack[Ts], S]): ... + +def foo(x: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: + y: C[int, Unpack[Ts]] # E: TypeVarTuple cannot be split + z: C[Unpack[Ts], int] # E: TypeVarTuple cannot be split + return x +[builtins fixtures/tuple.pyi] + +[case testVariadicAliasStrictPrefixSuffixCheck] +from typing import Tuple, TypeVar +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +Alias = Tuple[T, Unpack[Ts], S] + +def foo(x: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: + y: Alias[int, Unpack[Ts]] # E: TypeVarTuple cannot be split + z: Alias[Unpack[Ts], int] # E: TypeVarTuple cannot be split + return x +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleWithIsInstance] +# flags: --warn-unreachable +from typing import Tuple +from typing_extensions import TypeVarTuple, Unpack + +TP = TypeVarTuple("TP") +class A(Tuple[Unpack[TP]]): ... + +def test(d: A[int, str]) -> None: + if isinstance(d, A): + reveal_type(d) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.A[builtins.int, builtins.str]]" + else: + reveal_type(d) # E: Statement is unreachable +[builtins fixtures/isinstancelist.pyi]