From ddf03d1372d6f9906d9d1161b009a5272053fa13 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Tue, 2 May 2017 09:05:10 -0700 Subject: [PATCH] Better callable: `Callable[[Arg('x', int), VarArg(str)], int]` now a thing you can do (#2607) Implements an experimental feature to allow Callable to have any kind of signature an actual function definition does. This should enable better typing of callbacks &c. Initial discussion: python/typing#239 Proposal, v. similar to this impl: python/typing#264 Relevant typeshed PR: python/typeshed#793 --- extensions/mypy_extensions.py | 36 +++ mypy/exprtotype.py | 72 ++++- mypy/fastparse.py | 102 ++++++-- mypy/fastparse2.py | 12 +- mypy/indirection.py | 3 + mypy/messages.py | 10 +- mypy/nodes.py | 46 +++- mypy/subtypes.py | 3 +- mypy/typeanal.py | 55 +++- mypy/types.py | 49 +++- test-data/unit/check-fastparse.test | 28 +- test-data/unit/check-functions.test | 276 +++++++++++++++++++- test-data/unit/check-incremental.test | 19 ++ test-data/unit/check-inference.test | 2 +- test-data/unit/check-varargs.test | 2 +- test-data/unit/fixtures/dict.pyi | 1 + test-data/unit/lib-stub/mypy_extensions.pyi | 19 +- test-data/unit/parse-python2.test | 4 +- test-data/unit/pythoneval.test | 2 +- 19 files changed, 660 insertions(+), 81 deletions(-) diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index 26e568cc0a27..82eea32a31d8 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -5,6 +5,8 @@ from mypy_extensions import TypedDict """ +from typing import Any + # NOTE: This module must support Python 2.7 in addition to Python 3.x import sys @@ -92,6 +94,40 @@ class Point2D(TypedDict): syntax forms work for Python 2.7 and 3.2+ """ +# Argument constructors for making more-detailed Callables. These all just +# return their type argument, to make them complete noops in terms of the +# `typing` module. + + +def Arg(type=Any, name=None): + """A normal positional argument""" + return type + + +def DefaultArg(type=Any, name=None): + """A positional argument with a default value""" + return type + + +def NamedArg(type=Any, name=None): + """A keyword-only argument""" + return type + + +def DefaultNamedArg(type=Any, name=None): + """A keyword-only argument with a default value""" + return type + + +def VarArg(type=Any): + """A *args-style variadic positional argument""" + return type + + +def KwArg(type=Any): + """A **kwargs-style variadic keyword argument""" + return type + # Return type that indicates a function does not return class NoReturn: pass diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index b99f1a653d05..db46cfd0f0c9 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -2,23 +2,38 @@ from mypy.nodes import ( Expression, NameExpr, MemberExpr, IndexExpr, TupleExpr, - ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr, - get_member_expr_fullname + ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr, CallExpr, + ARG_POS, ARG_NAMED, get_member_expr_fullname ) from mypy.fastparse import parse_type_comment -from mypy.types import Type, UnboundType, TypeList, EllipsisType +from mypy.types import ( + Type, UnboundType, TypeList, EllipsisType, AnyType, Optional, CallableArgument, +) class TypeTranslationError(Exception): """Exception raised when an expression is not valid as a type.""" -def expr_to_unanalyzed_type(expr: Expression) -> Type: +def _extract_argument_name(expr: Expression) -> Optional[str]: + if isinstance(expr, NameExpr) and expr.name == 'None': + return None + elif isinstance(expr, StrExpr): + return expr.value + elif isinstance(expr, UnicodeExpr): + return expr.value + else: + raise TypeTranslationError() + + +def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = None) -> Type: """Translate an expression to the corresponding type. The result is not semantically analyzed. It can be UnboundType or TypeList. Raise TypeTranslationError if the expression cannot represent a type. """ + # The `parent` paremeter is used in recursive calls to provide context for + # understanding whether an CallableArgument is ok. if isinstance(expr, NameExpr): name = expr.name return UnboundType(name, line=expr.line, column=expr.column) @@ -29,7 +44,7 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: else: raise TypeTranslationError() elif isinstance(expr, IndexExpr): - base = expr_to_unanalyzed_type(expr.base) + base = expr_to_unanalyzed_type(expr.base, expr) if isinstance(base, UnboundType): if base.args: raise TypeTranslationError() @@ -37,14 +52,57 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: args = expr.index.items else: args = [expr.index] - base.args = [expr_to_unanalyzed_type(arg) for arg in args] + base.args = [expr_to_unanalyzed_type(arg, expr) for arg in args] if not base.args: base.empty_tuple_index = True return base else: raise TypeTranslationError() + elif isinstance(expr, CallExpr) and isinstance(_parent, ListExpr): + c = expr.callee + names = [] + # Go through the dotted member expr chain to get the full arg + # constructor name to look up + while True: + if isinstance(c, NameExpr): + names.append(c.name) + break + elif isinstance(c, MemberExpr): + names.append(c.name) + c = c.expr + else: + raise TypeTranslationError() + arg_const = '.'.join(reversed(names)) + + # Go through the constructor args to get its name and type. + name = None + default_type = AnyType(implicit=True) + typ = default_type # type: Type + for i, arg in enumerate(expr.args): + if expr.arg_names[i] is not None: + if expr.arg_names[i] == "name": + if name is not None: + # Two names + raise TypeTranslationError() + name = _extract_argument_name(arg) + continue + elif expr.arg_names[i] == "type": + if typ is not default_type: + # Two types + raise TypeTranslationError() + typ = expr_to_unanalyzed_type(arg, expr) + continue + else: + raise TypeTranslationError() + elif i == 0: + typ = expr_to_unanalyzed_type(arg, expr) + elif i == 1: + name = _extract_argument_name(arg) + else: + raise TypeTranslationError() + return CallableArgument(typ, name, arg_const, expr.line, expr.column) elif isinstance(expr, ListExpr): - return TypeList([expr_to_unanalyzed_type(t) for t in expr.items], + return TypeList([expr_to_unanalyzed_type(t, expr) for t in expr.items], line=expr.line, column=expr.column) elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)): # Parse string literal type. diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 35efcb38eb49..fe31e4ae0fb2 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -19,10 +19,12 @@ StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument, AwaitExpr, TempNode, Expression, Statement, - ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR2 + ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR2, + check_arg_names, ) from mypy.types import ( Type, CallableType, AnyType, UnboundType, TupleType, TypeList, EllipsisType, + CallableArgument, ) from mypy import defaults from mypy import experiments @@ -444,24 +446,12 @@ def make_argument(arg: ast3.arg, default: Optional[ast3.expr], kind: int) -> Arg new_args.append(make_argument(args.kwarg, None, ARG_STAR2)) names.append(args.kwarg) - seen_names = set() # type: Set[str] - for name in names: - if name.arg in seen_names: - self.fail("duplicate argument '{}' in function definition".format(name.arg), - name.lineno, name.col_offset) - break - seen_names.add(name.arg) + def fail_arg(msg: str, arg: ast3.arg) -> None: + self.fail(msg, arg.lineno, arg.col_offset) - return new_args + check_arg_names([name.arg for name in names], names, fail_arg) - def stringify_name(self, n: ast3.AST) -> str: - if isinstance(n, ast3.Name): - return n.id - elif isinstance(n, ast3.Attribute): - sv = self.stringify_name(n.value) - if sv is not None: - return "{}.{}".format(sv, n.attr) - return None # Can't do it. + return new_args # ClassDef(identifier name, # expr* bases, @@ -474,7 +464,7 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: metaclass_arg = find(lambda x: x.arg == 'metaclass', n.keywords) metaclass = None if metaclass_arg: - metaclass = self.stringify_name(metaclass_arg.value) + metaclass = stringify_name(metaclass_arg.value) if metaclass is None: metaclass = '' # To be reported later @@ -965,6 +955,21 @@ class TypeConverter(ast3.NodeTransformer): # type: ignore # typeshed PR #931 def __init__(self, errors: Errors, line: int = -1) -> None: self.errors = errors self.line = line + self.node_stack = [] # type: List[ast3.AST] + + def visit(self, node: ast3.AST) -> Type: + """Modified visit -- keep track of the stack of nodes""" + self.node_stack.append(node) + try: + return super().visit(node) + finally: + self.node_stack.pop() + + def parent(self) -> ast3.AST: + """Return the AST node above the one we are processing""" + if len(self.node_stack) < 2: + return None + return self.node_stack[-2] def fail(self, msg: str, line: int, column: int) -> None: self.errors.report(line, column, msg) @@ -985,6 +990,55 @@ def visit_NoneType(self, n: Any) -> Type: def translate_expr_list(self, l: Sequence[ast3.AST]) -> List[Type]: return [self.visit(e) for e in l] + def visit_Call(self, e: ast3.Call) -> Type: + # Parse the arg constructor + if not isinstance(self.parent(), ast3.List): + return self.generic_visit(e) + f = e.func + constructor = stringify_name(f) + if not constructor: + self.fail("Expected arg constructor name", e.lineno, e.col_offset) + name = None # type: Optional[str] + default_type = AnyType(implicit=True) + typ = default_type # type: Type + for i, arg in enumerate(e.args): + if i == 0: + typ = self.visit(arg) + elif i == 1: + name = self._extract_argument_name(arg) + else: + self.fail("Too many arguments for argument constructor", + f.lineno, f.col_offset) + for k in e.keywords: + value = k.value + if k.arg == "name": + if name is not None: + self.fail('"{}" gets multiple values for keyword argument "name"'.format( + constructor), f.lineno, f.col_offset) + name = self._extract_argument_name(value) + elif k.arg == "type": + if typ is not default_type: + self.fail('"{}" gets multiple values for keyword argument "type"'.format( + constructor), f.lineno, f.col_offset) + typ = self.visit(value) + else: + self.fail( + 'Unexpected argument "{}" for argument constructor'.format(k.arg), + value.lineno, value.col_offset) + return CallableArgument(typ, name, constructor, e.lineno, e.col_offset) + + def translate_argument_list(self, l: Sequence[ast3.AST]) -> TypeList: + return TypeList([self.visit(e) for e in l], line=self.line) + + def _extract_argument_name(self, n: ast3.expr) -> str: + if isinstance(n, ast3.Str): + return n.s.strip() + elif isinstance(n, ast3.NameConstant) and str(n.value) == 'None': + return None + self.fail('Expected string literal for argument name, got {}'.format( + type(n).__name__), self.line, 0) + return None + def visit_Name(self, n: ast3.Name) -> Type: return UnboundType(n.id, line=self.line) @@ -1036,4 +1090,14 @@ def visit_Ellipsis(self, n: ast3.Ellipsis) -> Type: # List(expr* elts, expr_context ctx) def visit_List(self, n: ast3.List) -> Type: - return TypeList(self.translate_expr_list(n.elts), line=self.line) + return self.translate_argument_list(n.elts) + + +def stringify_name(n: ast3.AST) -> Optional[str]: + if isinstance(n, ast3.Name): + return n.id + elif isinstance(n, ast3.Attribute): + sv = stringify_name(n.value) + if sv is not None: + return "{}.{}".format(sv, n.attr) + return None # Can't do it. diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index aca04187e57c..b7d5e9d400db 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -33,7 +33,7 @@ UnaryExpr, LambdaExpr, ComparisonExpr, DictionaryComprehension, SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument, Expression, Statement, BackquoteExpr, PrintStmt, ExecStmt, - ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2, OverloadPart, + ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2, OverloadPart, check_arg_names, ) from mypy.types import ( Type, CallableType, AnyType, UnboundType, EllipsisType @@ -439,12 +439,10 @@ def get_type(i: int) -> Optional[Type]: new_args.append(Argument(Var(n.kwarg), typ, None, ARG_STAR2)) names.append(n.kwarg) - seen_names = set() # type: Set[str] - for name in names: - if name in seen_names: - self.fail("duplicate argument '{}' in function definition".format(name), line, 0) - break - seen_names.add(name) + # We don't have any context object to give, but we have closed around the line num + def fail_arg(msg: str, arg: None) -> None: + self.fail(msg, line, 0) + check_arg_names(names, [None] * len(names), fail_arg) return new_args, decompose_stmts diff --git a/mypy/indirection.py b/mypy/indirection.py index 332eb4357967..2e69c5ebd3ff 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -45,6 +45,9 @@ def visit_unbound_type(self, t: types.UnboundType) -> Set[str]: def visit_type_list(self, t: types.TypeList) -> Set[str]: return self._visit(*t.items) + def visit_callable_argument(self, t: types.CallableArgument) -> Set[str]: + return self._visit(t.typ) + def visit_any(self, t: types.AnyType) -> Set[str]: return set() diff --git a/mypy/messages.py b/mypy/messages.py index 78e17686d057..6d52765115d2 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -93,7 +93,7 @@ ARG_OPT: "DefaultArg", ARG_NAMED: "NamedArg", ARG_NAMED_OPT: "DefaultNamedArg", - ARG_STAR: "StarArg", + ARG_STAR: "VarArg", ARG_STAR2: "KwArg", } @@ -214,15 +214,15 @@ def format(self, typ: Type, verbosity: int = 0) -> str: verbosity = max(verbosity - 1, 0)))) else: constructor = ARG_CONSTRUCTOR_NAMES[arg_kind] - if arg_kind in (ARG_STAR, ARG_STAR2): + if arg_kind in (ARG_STAR, ARG_STAR2) or arg_name is None: arg_strings.append("{}({})".format( constructor, strip_quotes(self.format(arg_type)))) else: - arg_strings.append("{}('{}', {})".format( + arg_strings.append("{}({}, {})".format( constructor, - arg_name, - strip_quotes(self.format(arg_type)))) + strip_quotes(self.format(arg_type)), + repr(arg_name))) return 'Callable[[{}], {}]'.format(", ".join(arg_strings), return_type) else: diff --git a/mypy/nodes.py b/mypy/nodes.py index c5c354cfea55..11d6d215660a 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -4,7 +4,7 @@ from abc import abstractmethod from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional, Callable, ) import mypy.strconv @@ -2448,3 +2448,47 @@ def get_member_expr_fullname(expr: MemberExpr) -> Optional[str]: for key, obj in globals().items() if isinstance(obj, type) and issubclass(obj, SymbolNode) and obj is not SymbolNode } + + +def check_arg_kinds(arg_kinds: List[int], nodes: List[T], fail: Callable[[str, T], None]) -> None: + is_var_arg = False + is_kw_arg = False + seen_named = False + seen_opt = False + for kind, node in zip(arg_kinds, nodes): + if kind == ARG_POS: + if is_var_arg or is_kw_arg or seen_named or seen_opt: + fail("Required positional args may not appear " + "after default, named or var args", + node) + break + elif kind == ARG_OPT: + if is_var_arg or is_kw_arg or seen_named: + fail("Positional default args may not appear after named or var args", node) + break + seen_opt = True + elif kind == ARG_STAR: + if is_var_arg or is_kw_arg or seen_named: + fail("Var args may not appear after named or var args", node) + break + is_var_arg = True + elif kind == ARG_NAMED or kind == ARG_NAMED_OPT: + seen_named = True + if is_kw_arg: + fail("A **kwargs argument must be the last argument", node) + break + elif kind == ARG_STAR2: + if is_kw_arg: + fail("You may only have one **kwargs argument", node) + break + is_kw_arg = True + + +def check_arg_names(names: List[str], nodes: List[T], fail: Callable[[str, T], None], + description: str = 'function definition') -> None: + seen_names = set() # type: Set[str] + for name, node in zip(names, nodes): + if name is not None and name in seen_names: + fail("Duplicate argument '{}' in {}".format(name, description), node) + break + seen_names.add(name) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 6535dfd4496a..8ca6421a0a91 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -3,7 +3,8 @@ from mypy.types import ( Type, AnyType, UnboundType, TypeVisitor, FormalArgument, NoneTyp, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, - ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance + ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, + is_named_instance ) import mypy.applytype import mypy.constraints diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 21721325ebbc..2a506b7a5378 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -11,12 +11,14 @@ AnyType, CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, SyntheticTypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, - get_type_vars, TypeQuery, union_items, + CallableArgument, get_type_vars, TypeQuery, union_items ) + from mypy.nodes import ( TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression, - IndexExpr, RefExpr, nongen_builtins, TypeVarExpr + IndexExpr, RefExpr, nongen_builtins, check_arg_names, check_arg_kinds, + ARG_POS, ARG_NAMED, ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr ) from mypy.tvar_scope import TypeVarScope from mypy.sametypes import is_same_type @@ -37,6 +39,15 @@ 'typing.Union', } +ARG_KINDS_BY_CONSTRUCTOR = { + 'mypy_extensions.Arg': ARG_POS, + 'mypy_extensions.DefaultArg': ARG_OPT, + 'mypy_extensions.NamedArg': ARG_NAMED, + 'mypy_extensions.DefaultNamedArg': ARG_NAMED_OPT, + 'mypy_extensions.VarArg': ARG_STAR, + 'mypy_extensions.KwArg': ARG_STAR2, +} + def analyze_type_alias(node: Expression, lookup_func: Callable[[str, Context], SymbolTableNode], @@ -311,6 +322,10 @@ def visit_type_list(self, t: TypeList) -> Type: self.fail('Invalid type', t) return AnyType() + def visit_callable_argument(self, t: CallableArgument) -> Type: + self.fail('Invalid type', t) + return AnyType() + def visit_instance(self, t: Instance) -> Type: return t @@ -385,10 +400,40 @@ def analyze_callable_type(self, t: UnboundType) -> Type: ret_type = t.args[1] if isinstance(t.args[0], TypeList): # Callable[[ARG, ...], RET] (ordinary callable type) - args = t.args[0].items + args = [] # type: List[Type] + names = [] # type: List[str] + kinds = [] # type: List[int] + for arg in t.args[0].items: + if isinstance(arg, CallableArgument): + args.append(arg.typ) + names.append(arg.name) + if arg.constructor is None: + return AnyType() + found = self.lookup(arg.constructor, arg) + if found is None: + # Looking it up already put an error message in + return AnyType() + elif found.fullname not in ARG_KINDS_BY_CONSTRUCTOR: + self.fail('Invalid argument constructor "{}"'.format( + found.fullname), arg) + return AnyType() + else: + kind = ARG_KINDS_BY_CONSTRUCTOR[found.fullname] + kinds.append(kind) + if arg.name is not None and kind in {ARG_STAR, ARG_STAR2}: + self.fail("{} arguments should not have names".format( + arg.constructor), arg) + return AnyType() + else: + args.append(arg) + names.append(None) + kinds.append(ARG_POS) + + check_arg_names(names, [t] * len(args), self.fail, "Callable") + check_arg_kinds(kinds, [t] * len(args), self.fail) ret = CallableType(args, - [nodes.ARG_POS] * len(args), - [None] * len(args), + kinds, + names, ret_type=ret_type, fallback=fallback) elif isinstance(t.args[0], EllipsisType): diff --git a/mypy/types.py b/mypy/types.py index 91f75f6c592f..626d7ecb111c 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -223,8 +223,32 @@ def deserialize(cls, data: JsonDict) -> 'UnboundType': [deserialize_type(a) for a in data['args']]) +class CallableArgument(Type): + """Represents a Arg(type, 'name') inside a Callable's type list. + + Note that this is a synthetic type for helping parse ASTs, not a real type. + """ + typ = None # type: Type + name = None # type: Optional[str] + constructor = None # type: Optional[str] + + def __init__(self, typ: Type, name: Optional[str], constructor: Optional[str], + line: int = -1, column: int = -1) -> None: + super().__init__(line, column) + self.typ = typ + self.name = name + self.constructor = constructor + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + assert isinstance(visitor, SyntheticTypeVisitor) + return visitor.visit_callable_argument(self) + + def serialize(self) -> JsonDict: + assert False, "Synthetic types don't serialize" + + class TypeList(Type): - """A list of types [...]. + """Information about argument types and names [...]. This is only used for the arguments of a Callable type, i.e. for [arg, ...] in Callable[[arg, ...], ret]. This is not a real type @@ -514,10 +538,11 @@ class CallableType(FunctionLike): arg_types = None # type: List[Type] # Types of function arguments arg_kinds = None # type: List[int] # ARG_ constants arg_names = None # type: List[Optional[str]] # None if not a keyword argument - min_args = 0 # Minimum number of arguments; derived from arg_kinds - is_var_arg = False # Is it a varargs function? derived from arg_kinds - ret_type = None # type: Type # Return value type - name = '' # type: Optional[str] # Name (may be None; for error messages) + min_args = 0 # Minimum number of arguments; derived from arg_kinds + is_var_arg = False # Is it a varargs function? derived from arg_kinds + is_kw_arg = False + ret_type = None # type: Type # Return value type + name = '' # type: Optional[str] # Name (may be None; for error messages) definition = None # type: Optional[SymbolNode] # For error messages. May be None. # Type variables for a generic function variables = None # type: List[TypeVarDef] @@ -1266,6 +1291,10 @@ def visit_star_type(self, t: StarType) -> T: def visit_type_list(self, t: TypeList) -> T: pass + @abstractmethod + def visit_callable_argument(self, t: CallableArgument) -> T: + pass + @abstractmethod def visit_ellipsis_type(self, t: EllipsisType) -> T: pass @@ -1374,6 +1403,13 @@ def visit_unbound_type(self, t: UnboundType)-> str: def visit_type_list(self, t: TypeList) -> str: return ''.format(self.list_str(t.items)) + def visit_callable_argument(self, t: CallableArgument) -> str: + typ = t.typ.accept(self) + if t.name is None: + return "{}({})".format(t.constructor, typ) + else: + return "{}({}, {})".format(t.constructor, typ, t.name) + def visit_any(self, t: AnyType) -> str: return 'Any' @@ -1527,6 +1563,9 @@ def visit_unbound_type(self, t: UnboundType) -> T: def visit_type_list(self, t: TypeList) -> T: return self.query_types(t.items) + def visit_callable_argument(self, t: CallableArgument) -> T: + return t.typ.accept(self) + def visit_any(self, t: AnyType) -> T: return self.strategy([]) diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test index 763c255bbad1..bea5efbaeb8d 100644 --- a/test-data/unit/check-fastparse.test +++ b/test-data/unit/check-fastparse.test @@ -320,47 +320,47 @@ def f(x: int): # E: Function has duplicate type signatures def f(x, y, z): pass -def g(x, y, x): # E: duplicate argument 'x' in function definition +def g(x, y, x): # E: Duplicate argument 'x' in function definition pass -def h(x, y, *x): # E: duplicate argument 'x' in function definition +def h(x, y, *x): # E: Duplicate argument 'x' in function definition pass -def i(x, y, *z, **z): # E: duplicate argument 'z' in function definition +def i(x, y, *z, **z): # E: Duplicate argument 'z' in function definition pass -def j(x: int, y: int, *, x: int = 3): # E: duplicate argument 'x' in function definition +def j(x: int, y: int, *, x: int = 3): # E: Duplicate argument 'x' in function definition pass -def k(*, y, z, y): # E: duplicate argument 'y' in function definition +def k(*, y, z, y): # E: Duplicate argument 'y' in function definition pass -lambda x, y, x: ... # E: duplicate argument 'x' in function definition +lambda x, y, x: ... # E: Duplicate argument 'x' in function definition [case testFastParserDuplicateNames_python2] def f(x, y, z): pass -def g(x, y, x): # E: duplicate argument 'x' in function definition +def g(x, y, x): # E: Duplicate argument 'x' in function definition pass -def h(x, y, *x): # E: duplicate argument 'x' in function definition +def h(x, y, *x): # E: Duplicate argument 'x' in function definition pass -def i(x, y, *z, **z): # E: duplicate argument 'z' in function definition +def i(x, y, *z, **z): # E: Duplicate argument 'z' in function definition pass -def j(x, (y, y), z): # E: duplicate argument 'y' in function definition +def j(x, (y, y), z): # E: Duplicate argument 'y' in function definition pass -def k(x, (y, x)): # E: duplicate argument 'x' in function definition +def k(x, (y, x)): # E: Duplicate argument 'x' in function definition pass -def l((x, y), (z, x)): # E: duplicate argument 'x' in function definition +def l((x, y), (z, x)): # E: Duplicate argument 'x' in function definition pass -def m(x, ((x, y), z)): # E: duplicate argument 'x' in function definition +def m(x, ((x, y), z)): # E: Duplicate argument 'x' in function definition pass -lambda x, (y, x): None # E: duplicate argument 'x' in function definition +lambda x, (y, x): None # E: Duplicate argument 'x' in function definition diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 33fc51d3b046..f730de818242 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -79,7 +79,7 @@ h = h def l(x) -> None: ... def r(__, *, x) -> None: ... -r = l # E: Incompatible types in assignment (expression has type Callable[[Any], None], variable has type Callable[[Any, NamedArg('x', Any)], None]) +r = l # E: Incompatible types in assignment (expression has type Callable[[Any], None], variable has type Callable[[Any, NamedArg(Any, 'x')], None]) [case testSubtypingFunctionsRequiredLeftArgNotPresent] @@ -114,10 +114,10 @@ hh = h ff = gg ff_nonames = ff ff_nonames = f_nonames # reset -ff = ff_nonames # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg('a', int), Arg('b', str)], None]) +ff = ff_nonames # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg(int, 'a'), Arg(str, 'b')], None]) ff = f # reset -gg = ff # E: Incompatible types in assignment (expression has type Callable[[Arg('a', int), Arg('b', str)], None], variable has type Callable[[Arg('a', int), DefaultArg('b', str)], None]) -gg = hh # E: Incompatible types in assignment (expression has type Callable[[Arg('aa', int), DefaultArg('b', str)], None], variable has type Callable[[Arg('a', int), DefaultArg('b', str)], None]) +gg = ff # E: Incompatible types in assignment (expression has type Callable[[Arg(int, 'a'), Arg(str, 'b')], None], variable has type Callable[[Arg(int, 'a'), DefaultArg(str, 'b')], None]) +gg = hh # E: Incompatible types in assignment (expression has type Callable[[Arg(int, 'aa'), DefaultArg(str, 'b')], None], variable has type Callable[[Arg(int, 'a'), DefaultArg(str, 'b')], None]) [case testSubtypingFunctionsArgsKwargs] from typing import Any, Callable @@ -143,7 +143,7 @@ ee_var = everything ee_var = everywhere ee_var = specific_1 # The difference between Callable[..., blah] and one with a *args: Any, **kwargs: Any is that the ... goes loosely both ways. -ee_def = specific_1 # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[StarArg(Any), KwArg(Any)], None]) +ee_def = specific_1 # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[VarArg(Any), KwArg(Any)], None]) [builtins fixtures/dict.pyi] @@ -174,7 +174,7 @@ ff = f gg = g ff = g -gg = f # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg('a', int), Arg('b', str)], None]) +gg = f # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg(int, 'a'), Arg(str, 'b')], None]) [case testLackOfNamesFastparse] @@ -186,7 +186,7 @@ ff = f gg = g ff = g -gg = f # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg('a', int), Arg('b', str)], None]) +gg = f # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg(int, 'a'), Arg(str, 'b')], None]) [case testFunctionTypeCompatibilityWithOtherTypes] from typing import Callable @@ -1394,6 +1394,264 @@ f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" g('x') g(1) # E: Argument 1 to "g" has incompatible type "int"; expected "str" +-- Callable with specific arg list +-- ------------------------------- + +[case testCallableWithNamedArg] +from typing import Callable +from mypy_extensions import Arg + +def a(f: Callable[[Arg(int, 'x')], int]): + f(x=4) + f(5) + f(y=3) # E: Unexpected keyword argument "y" + +[builtins fixtures/dict.pyi] + +[case testCallableWithOptionalArg] +from typing import Callable +from mypy_extensions import DefaultArg + +def a(f: Callable[[DefaultArg(int, 'x')], int]): + f(x=4) + f(2) + f() + f(y=3) # E: Unexpected keyword argument "y" + f("foo") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallableWithNamedArgFromExpr] +from typing import Callable +from mypy_extensions import Arg + +F = Callable[[Arg(int, 'x')], int] + +def a(f: F): + f(x=4) + f(5) + f(y=3) # E: Unexpected keyword argument "y" + +[builtins fixtures/dict.pyi] + +[case testCallableWithOptionalArgFromExpr] +from typing import Callable +from mypy_extensions import DefaultArg + +F = Callable[[DefaultArg(int, 'x')], int] +def a(f: F): + f(x=4) + f(2) + f() + f(y=3) # E: Unexpected keyword argument "y" + f("foo") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallableParsingInInheritence] + +from collections import namedtuple +class C(namedtuple('t', 'x')): + pass + +[case testCallableParsingSameName] +from typing import Callable + +def Arg(x, y): pass + +F = Callable[[Arg(int, 'x')], int] # E: Invalid argument constructor "__main__.Arg" + +[case testCallableParsingFromExpr] +from typing import Callable, List +from mypy_extensions import Arg, VarArg, KwArg +import mypy_extensions + +def WrongArg(x, y): return y +# Note that for this test, the 'Value of type "int" is not indexable' errors are silly, +# and a consequence of Callable being set to an int in the test stub. We can't set it to +# something else sensible, because other tests require the stub not have anything +# that looks like a function call. +F = Callable[[WrongArg(int, 'x')], int] # E: Invalid argument constructor "__main__.WrongArg" +G = Callable[[Arg(1, 'x')], int] # E: Invalid type alias # E: Value of type "int" is not indexable +H = Callable[[VarArg(int, 'x')], int] # E: VarArg arguments should not have names +I = Callable[[VarArg(int)], int] # ok +J = Callable[[VarArg(), KwArg()], int] # ok +K = Callable[[VarArg(), int], int] # E: Required positional args may not appear after default, named or var args +L = Callable[[Arg(name='x', type=int)], int] # ok +# I have commented out the following test because I don't know how to expect the "defined here" note part of the error. +# M = Callable[[Arg(gnome='x', type=int)], int] E: Invalid type alias E: Unexpected keyword argument "gnome" for "Arg" +N = Callable[[Arg(name=None, type=int)], int] # ok +O = Callable[[List[Arg(int)]], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: Type expected within [...] # E: The type List[T] is not generic and not indexable +P = Callable[[mypy_extensions.VarArg(int)], int] # ok +Q = Callable[[Arg(int, type=int)], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "type" +R = Callable[[Arg(int, 'x', name='y')], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "name" + +[builtins fixtures/dict.pyi] + +[case testCallableParsing] +from typing import Callable +from mypy_extensions import Arg, VarArg, KwArg + +def WrongArg(x, y): return y + +def b(f: Callable[[Arg(1, 'x')], int]): pass # E: invalid type comment or annotation +def d(f: Callable[[VarArg(int)], int]): pass # ok +def e(f: Callable[[VarArg(), KwArg()], int]): pass # ok +def g(f: Callable[[Arg(name='x', type=int)], int]): pass # ok +def h(f: Callable[[Arg(gnome='x', type=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor +def i(f: Callable[[Arg(name=None, type=int)], int]): pass # ok +def j(f: Callable[[Arg(int, 'x', name='y')], int]): pass # E: "Arg" gets multiple values for keyword argument "name" +def k(f: Callable[[Arg(int, type=int)], int]): pass # E: "Arg" gets multiple values for keyword argument "type" + +[builtins fixtures/dict.pyi] + +[case testCallableTypeAnalysis] +from typing import Callable +from mypy_extensions import Arg, VarArg as VARG, KwArg +import mypy_extensions as ext + +def WrongArg(x, y): return y +def a(f: Callable[[WrongArg(int, 'x')], int]): pass # E: Invalid argument constructor "__main__.WrongArg" +def b(f: Callable[[BadArg(int, 'x')], int]): pass # E: Name 'BadArg' is not defined +def d(f: Callable[[ext.VarArg(int)], int]): pass # ok +def e(f: Callable[[VARG(), ext.KwArg()], int]): pass # ok +def g(f: Callable[[ext.Arg(name='x', type=int)], int]): pass # ok +def i(f: Callable[[Arg(name=None, type=int)], int]): pass # ok + +def f1(*args) -> int: pass +def f2(*args, **kwargs) -> int: pass + +d(f1) +e(f2) +d(f2) +e(f1) # E: Argument 1 to "e" has incompatible type Callable[[VarArg(Any)], int]; expected Callable[[VarArg(Any), KwArg(Any)], int] + +[builtins fixtures/dict.pyi] + +[case testCallableWrongTypeType] +from typing import Callable +from mypy_extensions import Arg +def b(f: Callable[[Arg(1, 'x')], int]): pass # E: invalid type comment or annotation +[builtins fixtures/dict.pyi] + +[case testCallableTooManyVarArg] +from typing import Callable +from mypy_extensions import VarArg +def c(f: Callable[[VarArg(int, 'x')], int]): pass # E: VarArg arguments should not have names +[builtins fixtures/dict.pyi] + +[case testCallableFastParseGood] +from typing import Callable +from mypy_extensions import VarArg, Arg, KwArg +def d(f: Callable[[VarArg(int)], int]): pass # ok +def e(f: Callable[[VarArg(), KwArg()], int]): pass # ok +def g(f: Callable[[Arg(name='x', type=int)], int]): pass # ok +def i(f: Callable[[Arg(name=None, type=int)], int]): pass # ok +[builtins fixtures/dict.pyi] + +[case testCallableFastParseBadArgArgName] +from typing import Callable +from mypy_extensions import Arg +def h(f: Callable[[Arg(gnome='x', type=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor +[builtins fixtures/dict.pyi] + +[case testCallableKindsOrdering] +from typing import Callable, Any +from mypy_extensions import Arg, VarArg, KwArg, DefaultArg, NamedArg + +def f(f: Callable[[VarArg(), int], int]): pass # E: Required positional args may not appear after default, named or var args +def g(f: Callable[[VarArg(), VarArg()], int]): pass # E: Var args may not appear after named or var args +def h(f: Callable[[KwArg(), KwArg()], int]): pass # E: You may only have one **kwargs argument +def i(f: Callable[[DefaultArg(), int], int]): pass # E: Required positional args may not appear after default, named or var args +def j(f: Callable[[NamedArg(Any, 'x'), DefaultArg(int, 'y')], int]): pass # E: Positional default args may not appear after named or var args +def k(f: Callable[[KwArg(), NamedArg(Any, 'x')], int]): pass # E: A **kwargs argument must be the last argument +[builtins fixtures/dict.pyi] + +[case testCallableDuplicateNames] +from typing import Callable +from mypy_extensions import Arg, VarArg, KwArg, DefaultArg + +def f(f: Callable[[Arg(int, 'x'), int, Arg(int, 'x')], int]): pass # E: Duplicate argument 'x' in Callable + +[builtins fixtures/dict.pyi] + + +[case testCallableWithKeywordOnlyArg] +from typing import Callable +from mypy_extensions import NamedArg + +def a(f: Callable[[NamedArg(int, 'x')], int]): + f(x=4) + f(2) # E: Too many positional arguments + f() # E: Missing named argument "x" + f(y=3) # E: Unexpected keyword argument "y" + f(x="foo") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallableWithKeywordOnlyOptionalArg] +from typing import Callable +from mypy_extensions import DefaultNamedArg + +def a(f: Callable[[DefaultNamedArg(int, 'x')], int]): + f(x=4) + f(2) # E: Too many positional arguments + f() + f(y=3) # E: Unexpected keyword argument "y" + f(x="foo") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallableWithKwargs] +from typing import Callable +from mypy_extensions import KwArg + +def a(f: Callable[[KwArg(int)], int]): + f(x=4) + f(2) # E: Too many arguments + f() + f(y=3) + f(x=4, y=3, z=10) + f(x="foo") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + + +[case testCallableWithVarArg] +from typing import Callable +from mypy_extensions import VarArg + +def a(f: Callable[[VarArg(int)], int]): + f(x=4) # E: Unexpected keyword argument "x" + f(2) + f() + f(3, 4, 5) + f("a") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + +[case testCallableArgKindSubtyping] +from typing import Callable +from mypy_extensions import Arg, DefaultArg + +int_str_fun = None # type: Callable[[int, str], str] +int_opt_str_fun = None # type: Callable[[int, DefaultArg(str, None)], str] +int_named_str_fun = None # type: Callable[[int, Arg(str, 's')], str] + +def isf(ii: int, ss: str) -> str: + return ss + +def iosf(i: int, s: str = "bar") -> str: + return s + +def isf_unnamed(__i: int, __s: str) -> str: + return __s + +int_str_fun = isf +int_str_fun = isf_unnamed +int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type Callable[[int, str], str], variable has type Callable[[int, Arg(str, 's')], str]) +int_opt_str_fun = iosf +int_str_fun = iosf +int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg(int, 'ii'), Arg(str, 'ss')], str], variable has type Callable[[int, DefaultArg(str)], str]) + +int_named_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg(int, 'ii'), Arg(str, 'ss')], str], variable has type Callable[[int, Arg(str, 's')], str]) +int_named_str_fun = iosf + +[builtins fixtures/dict.pyi] -- Callable[..., T] -- ---------------- @@ -1444,7 +1702,7 @@ def g4(*, y: int) -> str: pass f(g1) f(g2) f(g3) -f(g4) # E: Argument 1 to "f" has incompatible type Callable[[NamedArg('y', int)], str]; expected Callable[..., int] +f(g4) # E: Argument 1 to "f" has incompatible type Callable[[NamedArg(int, 'y')], str]; expected Callable[..., int] [case testCallableWithArbitraryArgsSubtypingWithGenericFunc] from typing import Callable, TypeVar @@ -1572,7 +1830,7 @@ def g(x, y): pass def h(x): pass def j(y) -> Any: pass f = h -f = j # E: Incompatible types in assignment (expression has type Callable[[Arg('y', Any)], Any], variable has type Callable[[Arg('x', Any)], Any]) +f = j # E: Incompatible types in assignment (expression has type Callable[[Arg(Any, 'y')], Any], variable has type Callable[[Arg(Any, 'x')], Any]) f = g # E: Incompatible types in assignment (expression has type Callable[[Any, Any], Any], variable has type Callable[[Any], Any]) [case testRedefineFunction2] diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index b6f92bd6a174..d000643f4a18 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -114,6 +114,25 @@ def func1() -> A: pass [out2] tmp/mod1.py:1: error: Name 'A' is not defined +[case testIncrementalCallable] +import mod1 + +[file mod1.py] +from typing import Callable +from mypy_extensions import Arg +def func1() -> Callable[[Arg(int, 'x')], int]: pass + +[file mod1.py.next] +from typing import Callable +from mypy_extensions import Arg +def func1() -> Callable[[Arg(int, 'x')], int]: ... + + +[rechecked mod1] +[stale] + +[builtins fixtures/dict.pyi] + [case testIncrementalSameNameChange] import mod1 diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 6eb38093f61a..9226a06d714a 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1119,7 +1119,7 @@ from typing import Callable def f(a: Callable[..., None] = lambda *a, **k: None): pass -def g(a: Callable[..., None] = lambda *a, **k: 1): # E: Incompatible types in assignment (expression has type Callable[[StarArg(Any), KwArg(Any)], int], variable has type Callable[..., None]) +def g(a: Callable[..., None] = lambda *a, **k: 1): # E: Incompatible types in assignment (expression has type Callable[[VarArg(Any), KwArg(Any)], int], variable has type Callable[..., None]) pass [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index 8ebf98076df4..db257478e76b 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -576,7 +576,7 @@ x = None # type: Callable[[int], None] def f(*x: int) -> None: pass def g(*x: str) -> None: pass x = f -x = g # E: Incompatible types in assignment (expression has type Callable[[StarArg(str)], None], variable has type Callable[[int], None]) +x = g # E: Incompatible types in assignment (expression has type Callable[[VarArg(str)], None], variable has type Callable[[int], None]) [builtins fixtures/list.pyi] [out] diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index dc89366c1133..e920512274dd 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -39,4 +39,5 @@ class function: pass class float: pass class bool: pass +class ellipsis: pass class BaseException: pass diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi index 6e1e3b0ed285..fa540b99f4cd 100644 --- a/test-data/unit/lib-stub/mypy_extensions.pyi +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -1,8 +1,21 @@ -from typing import Dict, Type, TypeVar +from typing import Dict, Type, TypeVar, Optional, Any -T = TypeVar('T') +_T = TypeVar('_T') -def TypedDict(typename: str, fields: Dict[str, Type[T]]) -> Type[dict]: pass +def Arg(type: _T = ..., name: Optional[str] = ...) -> _T: ... + +def DefaultArg(type: _T = ..., name: Optional[str] = ...) -> _T: ... + +def NamedArg(type: _T = ..., name: Optional[str] = ...) -> _T: ... + +def DefaultNamedArg(type: _T = ..., name: Optional[str] = ...) -> _T: ... + +def VarArg(type: _T = ...) -> _T: ... + +def KwArg(type: _T = ...) -> _T: ... + + +def TypedDict(typename: str, fields: Dict[str, Type[_T]]) -> Type[dict]: ... class NoReturn: pass diff --git a/test-data/unit/parse-python2.test b/test-data/unit/parse-python2.test index f3a88beb69a2..b654f6af7b45 100644 --- a/test-data/unit/parse-python2.test +++ b/test-data/unit/parse-python2.test @@ -374,8 +374,8 @@ def f(a, (a, b)): def g((x, (x, y))): pass [out] -main:1: error: duplicate argument 'a' in function definition -main:3: error: duplicate argument 'x' in function definition +main:1: error: Duplicate argument 'a' in function definition +main:3: error: Duplicate argument 'x' in function definition [case testBackquotesInPython2] `1 + 2` diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index c7e24e05bfd3..8a19d0010942 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -758,7 +758,7 @@ def f(*args: str) -> str: return args[0] map(f, ['x']) map(f, [1]) [out] -_program.py:4: error: Argument 1 to "map" has incompatible type Callable[[StarArg(str)], str]; expected Callable[[int], str] +_program.py:4: error: Argument 1 to "map" has incompatible type Callable[[VarArg(str)], str]; expected Callable[[int], str] [case testMapStr] import typing