From 200bd45310fa539e9f35382869ef9ca32652f41d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Oct 2022 14:46:47 +0100 Subject: [PATCH 01/23] WIP --- mypy/build.py | 3 ++ mypy/fastparse.py | 64 +++++++++++++++++++++++++++------ test-data/unit/check-basic.test | 30 ++++++++++++++++ 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 62367c35915e..a4627092b24d 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -842,6 +842,9 @@ def parse_file( Raise CompileError if there is a parse error. """ t0 = time.time() + #if ignore_errors: + # print('ignore_errors', path) + # self.errors.ignored_files.add(path) tree = parse(source, path, id, self.errors, options=options) tree._fullname = id self.add_stats( diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 209ebb89f36b..695ff990f905 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -115,6 +115,7 @@ UnionType, ) from mypy.util import bytes_to_human_readable_repr, unnamed_function +from mypy.traverser import TraverserVisitor try: # pull this into a final variable to make mypyc be quiet about the @@ -261,6 +262,11 @@ def parse( Return the parse tree. If errors is not provided, raise ParseError on failure. Otherwise, use the errors object to report parse errors. """ + if int(): + 1 + '' + ignore_errors = options.ignore_errors or (errors and fnam in errors.ignored_files) + if ignore_errors: + print(fnam, fnam in errors.ignored_files, len(errors.ignored_files)) raise_on_error = False if options is None: options = Options() @@ -282,7 +288,7 @@ def parse( warnings.filterwarnings("ignore", category=DeprecationWarning) ast = ast3_parse(source, fnam, "exec", feature_version=feature_version) - tree = ASTConverter(options=options, is_stub=is_stub_file, errors=errors).visit(ast) + tree = ASTConverter(options=options, is_stub=is_stub_file, errors=errors, ignore_errors=ignore_errors).visit(ast) tree.path = fnam tree.is_stub = is_stub_file except SyntaxError as e: @@ -402,14 +408,15 @@ def is_no_type_check_decorator(expr: ast3.expr) -> bool: class ASTConverter: - def __init__(self, options: Options, is_stub: bool, errors: Errors) -> None: - # 'C' for class, 'F' for function - self.class_and_function_stack: list[Literal["C", "F"]] = [] + def __init__(self, options: Options, is_stub: bool, errors: Errors, ignore_errors: bool) -> None: + # 'C' for class, 'F' for function, 'L' for lambda + self.class_and_function_stack: list[Literal["C", "F", "L"]] = [] self.imports: list[ImportBase] = [] self.options = options self.is_stub = is_stub self.errors = errors + self.ignore_errors = ignore_errors self.type_ignores: dict[int, list[str]] = {} @@ -504,11 +511,23 @@ def translate_stmt_list( mark_block_unreachable(block) return [block] + stack = self.class_and_function_stack + if self.ignore_errors and len(stack) == 1 and stack[0] == "F": + return [] + res: list[Statement] = [] for stmt in stmts: node = self.visit(stmt) res.append(node) + if self.ignore_errors and len(stack) == 2 and stack[-2:] == ["C", "F"]: + v = FindAttributeAssign() + for n in res: + n.accept(v) + if v.found: + break + else: + return [] return res def translate_type_comment( @@ -829,9 +848,6 @@ def _is_stripped_if_stmt(self, stmt: Statement) -> bool: # For elif, IfStmt are stored recursively in else_body return self._is_stripped_if_stmt(stmt.else_body.body[0]) - def in_method_scope(self) -> bool: - return self.class_and_function_stack[-2:] == ["C", "F"] - def translate_module_id(self, id: str) -> str: """Return the actual, internal module id for a source text id.""" if id == self.options.custom_typing_module: @@ -866,7 +882,6 @@ def do_func_def( self, n: ast3.FunctionDef | ast3.AsyncFunctionDef, is_coroutine: bool = False ) -> FuncDef | Decorator: """Helper shared between visit_FunctionDef and visit_AsyncFunctionDef.""" - self.class_and_function_stack.append("F") no_type_check = bool( n.decorator_list and any(is_no_type_check_decorator(d) for d in n.decorator_list) ) @@ -913,7 +928,8 @@ def do_func_def( return_type = TypeConverter(self.errors, line=lineno).visit(func_type_ast.returns) # add implicit self type - if self.in_method_scope() and len(arg_types) < len(args): + in_method_scope = self.class_and_function_stack[-2:] == ["C"] + if in_method_scope and len(arg_types) < len(args): arg_types.insert(0, AnyType(TypeOfAny.special_form)) except SyntaxError: stripped_type = n.type_comment.split("#", 2)[0].strip() @@ -963,7 +979,9 @@ def do_func_def( end_line = getattr(n, "end_lineno", None) end_column = getattr(n, "end_col_offset", None) - func_def = FuncDef(n.name, args, self.as_required_block(n.body, lineno), func_type) + self.class_and_function_stack.append("F") + body = self.as_required_block(n.body, lineno) + func_def = FuncDef(n.name, args, body, func_type) if isinstance(func_def.type, CallableType): # semanal.py does some in-place modifications we want to avoid func_def.unanalyzed_type = func_def.type.copy_modified() @@ -1400,9 +1418,11 @@ def visit_Lambda(self, n: ast3.Lambda) -> LambdaExpr: body.lineno = n.body.lineno body.col_offset = n.body.col_offset + self.class_and_function_stack.append("L") e = LambdaExpr( self.transform_args(n.args, n.lineno), self.as_required_block([body], n.lineno) ) + self.class_and_function_stack.pop() e.set_line(n.lineno, n.col_offset) # Overrides set_line -- can't use self.set_line return e @@ -2072,3 +2092,27 @@ def stringify_name(n: AST) -> str | None: if sv is not None: return f"{sv}.{n.attr}" return None # Can't do it. + + +class FindAttributeAssign(TraverserVisitor): + def __init__(self) -> None: + self.lval = False + self.found = False + + def visit_assignment_stmt(self, x: AssignmentStmt) -> None: + self.lval = True + for lv in x.lvalues: + lv.accept(self) + self.lval = False + + def visit_expression_stmt(self, e: ExpressionStmt) -> None: + # No need to look inside these + pass + + def visit_call_expr(self, e: CallExpr) -> None: + # No need to look inside these + pass + + def visit_member_expr(self, x: MemberExpr) -> None: + if self.lval: + self.found = True diff --git a/test-data/unit/check-basic.test b/test-data/unit/check-basic.test index a4056c8cb576..efda52300db7 100644 --- a/test-data/unit/check-basic.test +++ b/test-data/unit/check-basic.test @@ -520,3 +520,33 @@ class A: [file test.py] def foo(s: str) -> None: ... + +[case testXXX] +# mypy: ignore-errors=True + +def f() -> None: + while 1(): + pass + +[case testXXX2] +from m import C +c = C() +reveal_type(c.x) # N: Revealed type is "builtins.int" + +[file m.py] +# mypy: ignore-errors=True + +class C: + def f(self) -> None: + self.x = 1 + +[case testXXX3] +# mypy: ignore-errors=True + +def f(self, x=lambda: 1) -> None: + pass + +class C: + def f(self) -> None: + l = lambda: 1 + self.x = 1 From 5191aa246d792b253c35dfb6097bbaab6dd2c41d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 5 Nov 2022 10:02:42 +0000 Subject: [PATCH 02/23] more WIP --- mypy/fastparse.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 695ff990f905..4590d516a4a7 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -262,11 +262,9 @@ def parse( Return the parse tree. If errors is not provided, raise ParseError on failure. Otherwise, use the errors object to report parse errors. """ - if int(): - 1 + '' - ignore_errors = options.ignore_errors or (errors and fnam in errors.ignored_files) - if ignore_errors: - print(fnam, fnam in errors.ignored_files, len(errors.ignored_files)) + ignore_errors = (options is not None and options.ignore_errors) or (errors is not None and fnam in errors.ignored_files) + #if ignore_errors: + # print(fnam, fnam in errors.ignored_files, len(errors.ignored_files)) raise_on_error = False if options is None: options = Options() @@ -288,7 +286,8 @@ def parse( warnings.filterwarnings("ignore", category=DeprecationWarning) ast = ast3_parse(source, fnam, "exec", feature_version=feature_version) - tree = ASTConverter(options=options, is_stub=is_stub_file, errors=errors, ignore_errors=ignore_errors).visit(ast) + tree = ASTConverter(options=options, is_stub=is_stub_file, errors=errors, + ignore_errors=ignore_errors).visit(ast) tree.path = fnam tree.is_stub = is_stub_file except SyntaxError as e: From a19c4da5db357310a581eac374195dcee9adb560 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 5 Nov 2022 10:13:32 +0000 Subject: [PATCH 03/23] WIP fix ignoring errors --- mypy/build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index a4627092b24d..81e441c04ad2 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -842,9 +842,9 @@ def parse_file( Raise CompileError if there is a parse error. """ t0 = time.time() - #if ignore_errors: - # print('ignore_errors', path) - # self.errors.ignored_files.add(path) + if ignore_errors: + #print('ignore_errors', path) + self.errors.ignored_files.add(path) tree = parse(source, path, id, self.errors, options=options) tree._fullname = id self.add_stats( From 75077e8fba55ab582ec89a12ebb2312249d252e7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 13:45:05 +0000 Subject: [PATCH 04/23] Tweak docstring --- mypy/config_parser.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mypy/config_parser.py b/mypy/config_parser.py index 485d2f67f5de..3f8ad4251fff 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -529,10 +529,7 @@ def split_directive(s: str) -> tuple[list[str], list[str]]: def mypy_comments_to_config_map(line: str, template: Options) -> tuple[dict[str, str], list[str]]: - """Rewrite the mypy comment syntax into ini file syntax. - - Returns - """ + """Rewrite the mypy comment syntax into ini file syntax.""" options = {} entries, errors = split_directive(line) for entry in entries: From 88a1a586b299a2ab2f9a5f197f49000e6bc90ecd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 13:45:22 +0000 Subject: [PATCH 05/23] Add some parser tests --- mypy/test/testparse.py | 11 +++++- test-data/unit/parse.test | 80 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py index 6a2d1e145251..48330e07fb2d 100644 --- a/mypy/test/testparse.py +++ b/mypy/test/testparse.py @@ -10,6 +10,8 @@ from mypy.errors import CompileError from mypy.options import Options from mypy.parse import parse +from mypy.util import get_mypy_comments +from mypy.config_parser import parse_mypy_comments from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import assert_string_arrays_equal, find_test_files, parse_options @@ -39,9 +41,16 @@ def test_parser(testcase: DataDrivenTestCase) -> None: else: options.python_version = defaults.PYTHON3_VERSION + source = "\n".join(testcase.input) + + # Apply mypy: comments to options. + comments = get_mypy_comments(source) + changes, _ = parse_mypy_comments(comments, options) + options = options.apply_changes(changes) + try: n = parse( - bytes("\n".join(testcase.input), "ascii"), + bytes(source, "ascii"), fnam="main", module="__main__", errors=None, diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index ff892ce0ce05..b6907130284b 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -95,7 +95,6 @@ MypyFile:1( StrExpr(x\n\')) ExpressionStmt:2( StrExpr(x\n\"))) ---" fix syntax highlight [case testBytes] b'foo' @@ -128,7 +127,6 @@ MypyFile:1( MypyFile:1( ExpressionStmt:1( StrExpr('))) ---' [case testOctalEscapes] '\0\1\177\1234' @@ -3484,3 +3482,81 @@ MypyFile:1( NameExpr(y) NameExpr(y)) StrExpr())))))))))))) + +[case testStripFunctionBodiesIfIgnoringErrors] +# mypy: ignore-errors=True +def f(): + return 1 +[out] +MypyFile:1( + FuncDef:2( + f + Block:2())) + +[case testStripMethodBodiesIfIgnoringErrors] +# mypy: ignore-errors=True +class C: + def f(self): + x = self.x + while foo(): + bah() +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + f + Args( + Var(self)) + Block:3()))) + +[case testDoNotStripModuleTopLevelOrClassBody] +# mypy: ignore-errors=True +f() +class C: + x = 5 +[out] +MypyFile:1( + ExpressionStmt:2( + CallExpr:2( + NameExpr(f) + Args())) + ClassDef:3( + C + AssignmentStmt:4( + NameExpr(x) + IntExpr(5)))) + +[case testDoNotStripMethodThatAssignsToAnAttribute] +# mypy: ignore-errors=True +class C: + def m1(self): + self.x = 0 + def m2(self): + a, self.y = 0 +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + m1 + Args( + Var(self)) + Block:3( + AssignmentStmt:4( + MemberExpr:4( + NameExpr(self) + x) + IntExpr(0)))) + FuncDef:5( + m2 + Args( + Var(self)) + Block:5( + AssignmentStmt:6( + TupleExpr:6( + NameExpr(a) + MemberExpr:6( + NameExpr(self) + y)) + IntExpr(0)))))) From 9a1755f3cac8f635cac5d862956e607ace493cad Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 14:00:39 +0000 Subject: [PATCH 06/23] Handle more kinds of bodies --- mypy/fastparse.py | 39 ++++++++++---- test-data/unit/parse.test | 108 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 135 insertions(+), 12 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 4590d516a4a7..c880ac4cc5b2 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -2095,16 +2095,32 @@ def stringify_name(n: AST) -> str | None: class FindAttributeAssign(TraverserVisitor): def __init__(self) -> None: - self.lval = False + self.lvalue = False self.found = False - def visit_assignment_stmt(self, x: AssignmentStmt) -> None: - self.lval = True - for lv in x.lvalues: + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + self.lvalue = True + for lv in s.lvalues: lv.accept(self) - self.lval = False - - def visit_expression_stmt(self, e: ExpressionStmt) -> None: + self.lvalue = False + + def visit_with_stmt(self, s: WithStmt) -> None: + self.lvalue = True + for lv in s.target: + if lv is not None: + lv.accept(self) + self.lvalue = False + s.body.accept(self) + + def visit_for_stmt(self, s: ForStmt) -> None: + self.lvalue = True + s.index.accept(self) + self.lvalue = False + s.body.accept(self) + if s.else_body: + s.else_body.accept(self) + + def visit_expression_stmt(self, s: ExpressionStmt) -> None: # No need to look inside these pass @@ -2112,6 +2128,11 @@ def visit_call_expr(self, e: CallExpr) -> None: # No need to look inside these pass - def visit_member_expr(self, x: MemberExpr) -> None: - if self.lval: + def visit_index_expr(self, e: IndexExpr) -> None: + # No need to look inside these + pass + + def visit_member_expr(self, e: MemberExpr) -> None: + print(e, self.lvalue) + if self.lvalue: self.found = True diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index b6907130284b..bdab63e68457 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -3498,8 +3498,13 @@ MypyFile:1( class C: def f(self): x = self.x - while foo(): - bah() + for x in y: + pass + with a as y: + pass + while self.foo(): + self.bah() + a[self.x] = 1 [out] MypyFile:1( ClassDef:2( @@ -3527,7 +3532,7 @@ MypyFile:1( NameExpr(x) IntExpr(5)))) -[case testDoNotStripMethodThatAssignsToAnAttribute] +[case testDoNotStripMethodThatAssignsToAttribute] # mypy: ignore-errors=True class C: def m1(self): @@ -3560,3 +3565,100 @@ MypyFile:1( NameExpr(self) y)) IntExpr(0)))))) + +[case testDoNotStripMethodThatAssignsToAttributeWithinStatement] +# mypy: ignore-errors=True +class C: + def m1(self): + for x in y: + self.x = 0 + def m2(self): + with x: + self.y = 0 + def m3(self): + if x: + self.y = 0 +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + m1 + Args( + Var(self)) + Block:3( + ForStmt:4( + NameExpr(x) + NameExpr(y) + Block:4( + AssignmentStmt:5( + MemberExpr:5( + NameExpr(self) + x) + IntExpr(0)))))) + FuncDef:6( + m2 + Args( + Var(self)) + Block:6( + WithStmt:7( + Expr( + NameExpr(x)) + Block:7( + AssignmentStmt:8( + MemberExpr:8( + NameExpr(self) + y) + IntExpr(0)))))) + FuncDef:9( + m3 + Args( + Var(self)) + Block:9( + IfStmt:10( + If( + NameExpr(x)) + Then( + AssignmentStmt:11( + MemberExpr:11( + NameExpr(self) + y) + IntExpr(0)))))))) + +[case testDoNotStripMethodThatDefinesAttributeWithoutAssignment] +# mypy: ignore-errors=True +class C: + def m1(self): + with y as self.x: + pass + def m2(self): + for self.y in x: + pass +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + m1 + Args( + Var(self)) + Block:3( + WithStmt:4( + Expr( + NameExpr(y)) + Target( + MemberExpr:4( + NameExpr(self) + x)) + Block:4()))) + FuncDef:6( + m2 + Args( + Var(self)) + Block:6( + ForStmt:7( + MemberExpr:7( + NameExpr(self) + y) + NameExpr(x) + Block:7()))))) From 83abaf802f721ab535281ae695652cdbcad6e796 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 14:03:04 +0000 Subject: [PATCH 07/23] Improve test case --- test-data/unit/parse.test | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index bdab63e68457..0437297504fd 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -3485,12 +3485,15 @@ MypyFile:1( [case testStripFunctionBodiesIfIgnoringErrors] # mypy: ignore-errors=True -def f(): +def f(self): + self.x = 1 # Cannot define an attribute return 1 [out] MypyFile:1( FuncDef:2( f + Args( + Var(self)) Block:2())) [case testStripMethodBodiesIfIgnoringErrors] From f838466061d97c72d9250d6ed5d74dd0ee03cdab Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 14:24:54 +0000 Subject: [PATCH 08/23] Add tests --- test-data/unit/parse.test | 126 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index 0437297504fd..b56cc5fc9eb4 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -3665,3 +3665,129 @@ MypyFile:1( y) NameExpr(x) Block:7()))))) + +[case testStripDecoratedFunctionOrMethod] +# mypy: ignore-errors=True +@deco +def f(): + x = 0 + +class C: + @deco + def m1(self): + x = 0 + + @deco + def m2(self): + self.x = 0 +[out] +MypyFile:1( + Decorator:2( + Var(f) + NameExpr(deco) + FuncDef:3( + f + Block:3())) + ClassDef:6( + C + Decorator:7( + Var(m1) + NameExpr(deco) + FuncDef:8( + m1 + Args( + Var(self)) + Block:8())) + Decorator:11( + Var(m2) + NameExpr(deco) + FuncDef:12( + m2 + Args( + Var(self)) + Block:12( + AssignmentStmt:13( + MemberExpr:13( + NameExpr(self) + x) + IntExpr(0))))))) + +[case testStripOverloadedMethod] +# mypy: ignore-errors=True +class C: + @overload + def m1(self, x: int) -> None: ... + @overload + def m1(self, x: str) -> None: ... + def m1(self, x): + x = 0 + + @overload + def m2(self, x: int) -> None: ... + @overload + def m2(self, x: str) -> None: ... + def m2(self, x): + self.x = 0 +[out] +MypyFile:1( + ClassDef:2( + C + OverloadedFuncDef:3( + Decorator:3( + Var(m1) + NameExpr(overload) + FuncDef:4( + m1 + Args( + Var(self) + Var(x)) + def (self: Any, x: int?) -> None? + Block:4())) + Decorator:5( + Var(m1) + NameExpr(overload) + FuncDef:6( + m1 + Args( + Var(self) + Var(x)) + def (self: Any, x: str?) -> None? + Block:6())) + FuncDef:7( + m1 + Args( + Var(self) + Var(x)) + Block:7())) + OverloadedFuncDef:10( + Decorator:10( + Var(m2) + NameExpr(overload) + FuncDef:11( + m2 + Args( + Var(self) + Var(x)) + def (self: Any, x: int?) -> None? + Block:11())) + Decorator:12( + Var(m2) + NameExpr(overload) + FuncDef:13( + m2 + Args( + Var(self) + Var(x)) + def (self: Any, x: str?) -> None? + Block:13())) + FuncDef:14( + m2 + Args( + Var(self) + Var(x)) + Block:14( + AssignmentStmt:15( + MemberExpr:15( + NameExpr(self) + x) + IntExpr(0))))))) From 13d68cfd38be7810dfc04e4b707ed0595c4405ff Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 14:28:06 +0000 Subject: [PATCH 09/23] Clean up tests --- test-data/unit/check-basic.test | 30 ------------------------- test-data/unit/check-inline-config.test | 30 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/test-data/unit/check-basic.test b/test-data/unit/check-basic.test index efda52300db7..a4056c8cb576 100644 --- a/test-data/unit/check-basic.test +++ b/test-data/unit/check-basic.test @@ -520,33 +520,3 @@ class A: [file test.py] def foo(s: str) -> None: ... - -[case testXXX] -# mypy: ignore-errors=True - -def f() -> None: - while 1(): - pass - -[case testXXX2] -from m import C -c = C() -reveal_type(c.x) # N: Revealed type is "builtins.int" - -[file m.py] -# mypy: ignore-errors=True - -class C: - def f(self) -> None: - self.x = 1 - -[case testXXX3] -# mypy: ignore-errors=True - -def f(self, x=lambda: 1) -> None: - pass - -class C: - def f(self) -> None: - l = lambda: 1 - self.x = 1 diff --git a/test-data/unit/check-inline-config.test b/test-data/unit/check-inline-config.test index 1b2085e33e91..28de5a6dcd79 100644 --- a/test-data/unit/check-inline-config.test +++ b/test-data/unit/check-inline-config.test @@ -210,3 +210,33 @@ enable_error_code = ignore-without-code, truthy-bool \[mypy-tests.*] disable_error_code = ignore-without-code + +[case testIgnoreErrorsSimple] +# mypy: ignore-errors=True + +def f() -> None: + while 1(): + pass + +[case testIgnoreErrorsInImportedModule] +from m import C +c = C() +reveal_type(c.x) # N: Revealed type is "builtins.int" + +[file m.py] +# mypy: ignore-errors=True + +class C: + def f(self) -> None: + self.x = 1 + +[case testIgnoreErrorsWithLambda] +# mypy: ignore-errors=True + +def f(self, x=lambda: 1) -> None: + pass + +class C: + def f(self) -> None: + l = lambda: 1 + self.x = 1 From a64e92efbd97590b0711fdd3e3b20a5f0e0c72f1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 14:36:49 +0000 Subject: [PATCH 10/23] WIP add failing test case --- test-data/unit/check-inline-config.test | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test-data/unit/check-inline-config.test b/test-data/unit/check-inline-config.test index 28de5a6dcd79..209649e79475 100644 --- a/test-data/unit/check-inline-config.test +++ b/test-data/unit/check-inline-config.test @@ -240,3 +240,50 @@ class C: def f(self) -> None: l = lambda: 1 self.x = 1 + +[case testIgnoreErrorsWithUnsafeSuperCall_no_empty] +# flags: --strict-optional + +from m import C + +class D(C): + def m(self) -> None: + super().m1() + super().m2() \ + # E: Call to abstract method "m2" of "C" with trivial body via super() is unsafe + super().m3() \ + # E: Call to abstract method "m3" of "C" with trivial body via super() is unsafe + super().m4() \ + # E: Call to abstract method "m4" of "C" with trivial body via super() is unsafe + + def m1(self) -> int: + return 0 + + def m2(self) -> int: + return 0 + + def m3(self) -> int: + return 0 + + def m4(self) -> int: + return 0 + +[file m.py] +# mypy: ignore-errors=True +import abc + +class C: + @abc.abstractmethod + def m1(self) -> int: + return 0 + + @abc.abstractmethod + def m2(self) -> int: + """doc""" + + @abc.abstractmethod + def m3(self) -> int: + pass + + @abc.abstractmethod + def m4(self) -> int: ... From 20b45821f9ab565c522148bd2d7eea6e77976a08 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 15:03:08 +0000 Subject: [PATCH 11/23] Preserve trivial bodies and nested blocks --- mypy/fastparse.py | 32 +++++++++++++++++++++---- mypy/semanal.py | 11 +++++++-- test-data/unit/check-inline-config.test | 11 +++++++++ test-data/unit/parse.test | 30 +++++++++++++++++------ 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index c880ac4cc5b2..812a277bb554 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -483,7 +483,7 @@ def get_lineno(self, node: ast3.expr | ast3.stmt) -> int: return node.lineno def translate_stmt_list( - self, stmts: Sequence[ast3.stmt], ismodule: bool = False + self, stmts: Sequence[ast3.stmt], *, ismodule: bool = False, can_strip: bool = False ) -> list[Statement]: # A "# type: ignore" comment before the first statement of a module # ignores the whole module: @@ -519,7 +519,7 @@ def translate_stmt_list( node = self.visit(stmt) res.append(node) - if self.ignore_errors and len(stack) == 2 and stack[-2:] == ["C", "F"]: + if (self.ignore_errors and can_strip and len(stack) == 2 and stack[-2:] == ["C", "F"] and not is_possible_trivial_body(res)): v = FindAttributeAssign() for n in res: n.accept(v) @@ -591,9 +591,9 @@ def as_block(self, stmts: list[ast3.stmt], lineno: int) -> Block | None: b.set_line(lineno) return b - def as_required_block(self, stmts: list[ast3.stmt], lineno: int) -> Block: + def as_required_block(self, stmts: list[ast3.stmt], lineno: int, *, can_strip: bool = False) -> Block: assert stmts # must be non-empty - b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) + b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts, can_strip=can_strip))) # TODO: in most call sites line is wrong (includes first line of enclosing statement) # TODO: also we need to set the column, and the end position here. b.set_line(lineno) @@ -979,7 +979,7 @@ def do_func_def( end_column = getattr(n, "end_col_offset", None) self.class_and_function_stack.append("F") - body = self.as_required_block(n.body, lineno) + body = self.as_required_block(n.body, lineno, can_strip=True) func_def = FuncDef(n.name, args, body, func_type) if isinstance(func_def.type, CallableType): # semanal.py does some in-place modifications we want to avoid @@ -2136,3 +2136,25 @@ def visit_member_expr(self, e: MemberExpr) -> None: print(e, self.lvalue) if self.lvalue: self.found = True + + +def is_possible_trivial_body(s: list[Statement]) -> bool: + """Could the statements form a "trivial" function body, such as 'pass'? + + This mimics mypy.semanal.is_trivial_body, but this runs before + semantic analysis so some checks must be conservative. + """ + l = len(s) + if l == 0: + return False + i = 0 + if isinstance(s[0], ExpressionStmt) and isinstance(s[0].expr, StrExpr): + # Skip docstring + i += 1 + if i == l: + return True + if l > i + 1: + return False + stmt = s[i] + return isinstance(stmt, (PassStmt, RaiseStmt)) or (isinstance(stmt, ExpressionStmt) and + isinstance(stmt.expr, EllipsisExpr)) diff --git a/mypy/semanal.py b/mypy/semanal.py index 77555648ba7e..98aee8f65a48 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -6391,7 +6391,7 @@ def is_trivial_body(block: Block) -> bool: "..." (ellipsis), or "raise NotImplementedError()". A trivial body may also start with a statement containing just a string (e.g. a docstring). - Note: functions that raise other kinds of exceptions do not count as + Note: Functions that raise other kinds of exceptions do not count as "trivial". We use this function to help us determine when it's ok to relax certain checks on body, but functions that raise arbitrary exceptions are more likely to do non-trivial work. For example: @@ -6401,11 +6401,18 @@ def halt(self, reason: str = ...) -> NoReturn: A function that raises just NotImplementedError is much less likely to be this complex. + + Note: If you update this, you may also need to update + mypy.fastparse.is_possible_trivial_body! """ body = block.body + if not body: + # Functions have empty bodies only if the body is stripped or the function is + # generated or deserialized. In these cases the body is unknown. + return False # Skip a docstring - if body and isinstance(body[0], ExpressionStmt) and isinstance(body[0].expr, StrExpr): + if isinstance(body[0], ExpressionStmt) and isinstance(body[0].expr, StrExpr): body = block.body[1:] if len(body) == 0: diff --git a/test-data/unit/check-inline-config.test b/test-data/unit/check-inline-config.test index 209649e79475..9c31a1f9cc7d 100644 --- a/test-data/unit/check-inline-config.test +++ b/test-data/unit/check-inline-config.test @@ -255,6 +255,8 @@ class D(C): # E: Call to abstract method "m3" of "C" with trivial body via super() is unsafe super().m4() \ # E: Call to abstract method "m4" of "C" with trivial body via super() is unsafe + super().m5() \ + # E: Call to abstract method "m5" of "C" with trivial body via super() is unsafe def m1(self) -> int: return 0 @@ -268,6 +270,9 @@ class D(C): def m4(self) -> int: return 0 + def m5(self) -> int: + return 0 + [file m.py] # mypy: ignore-errors=True import abc @@ -275,6 +280,7 @@ import abc class C: @abc.abstractmethod def m1(self) -> int: + """x""" return 0 @abc.abstractmethod @@ -287,3 +293,8 @@ class C: @abc.abstractmethod def m4(self) -> int: ... + + @abc.abstractmethod + def m5(self) -> int: + """doc""" + ... diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index b56cc5fc9eb4..3987a3347767 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -3581,6 +3581,8 @@ class C: def m3(self): if x: self.y = 0 + else: + x = 4 [out] MypyFile:1( ClassDef:2( @@ -3626,7 +3628,11 @@ MypyFile:1( MemberExpr:11( NameExpr(self) y) - IntExpr(0)))))))) + IntExpr(0))) + Else( + AssignmentStmt:13( + NameExpr(x) + IntExpr(4)))))))) [case testDoNotStripMethodThatDefinesAttributeWithoutAssignment] # mypy: ignore-errors=True @@ -3653,7 +3659,8 @@ MypyFile:1( MemberExpr:4( NameExpr(self) x)) - Block:4()))) + Block:4( + PassStmt:5())))) FuncDef:6( m2 Args( @@ -3664,7 +3671,8 @@ MypyFile:1( NameExpr(self) y) NameExpr(x) - Block:7()))))) + Block:7( + PassStmt:8())))))) [case testStripDecoratedFunctionOrMethod] # mypy: ignore-errors=True @@ -3742,7 +3750,9 @@ MypyFile:1( Var(self) Var(x)) def (self: Any, x: int?) -> None? - Block:4())) + Block:4( + ExpressionStmt:4( + Ellipsis)))) Decorator:5( Var(m1) NameExpr(overload) @@ -3752,7 +3762,9 @@ MypyFile:1( Var(self) Var(x)) def (self: Any, x: str?) -> None? - Block:6())) + Block:6( + ExpressionStmt:6( + Ellipsis)))) FuncDef:7( m1 Args( @@ -3769,7 +3781,9 @@ MypyFile:1( Var(self) Var(x)) def (self: Any, x: int?) -> None? - Block:11())) + Block:11( + ExpressionStmt:11( + Ellipsis)))) Decorator:12( Var(m2) NameExpr(overload) @@ -3779,7 +3793,9 @@ MypyFile:1( Var(self) Var(x)) def (self: Any, x: str?) -> None? - Block:13())) + Block:13( + ExpressionStmt:13( + Ellipsis)))) FuncDef:14( m2 Args( From 89ddfd7e3fc38742481086549bed4816ddcfbcad Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 15:04:23 +0000 Subject: [PATCH 12/23] Remove debug print --- mypy/fastparse.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 812a277bb554..48a553cbb081 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -2133,7 +2133,6 @@ def visit_index_expr(self, e: IndexExpr) -> None: pass def visit_member_expr(self, e: MemberExpr) -> None: - print(e, self.lvalue) if self.lvalue: self.found = True From 45737b51af48fb85f91f1ab207ebc12b0f05f61f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 15:07:20 +0000 Subject: [PATCH 13/23] Update test case --- test-data/unit/check-inline-config.test | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test-data/unit/check-inline-config.test b/test-data/unit/check-inline-config.test index 9c31a1f9cc7d..3f456e97596f 100644 --- a/test-data/unit/check-inline-config.test +++ b/test-data/unit/check-inline-config.test @@ -257,6 +257,8 @@ class D(C): # E: Call to abstract method "m4" of "C" with trivial body via super() is unsafe super().m5() \ # E: Call to abstract method "m5" of "C" with trivial body via super() is unsafe + super().m6() \ + # E: Call to abstract method "m6" of "C" with trivial body via super() is unsafe def m1(self) -> int: return 0 @@ -273,6 +275,9 @@ class D(C): def m5(self) -> int: return 0 + def m6(self) -> int: + return 0 + [file m.py] # mypy: ignore-errors=True import abc @@ -298,3 +303,9 @@ class C: def m5(self) -> int: """doc""" ... + + @abc.abstractmethod + def m6(self) -> int: + raise NotImplementedError() + +[builtins fixtures/exception.pyi] From 538471666ebc59b61d93b15f1fe4f08c6b63d3e7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 15:08:52 +0000 Subject: [PATCH 14/23] Cleanup --- mypy/build.py | 1 - mypy/fastparse.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 81e441c04ad2..c05adff0b46b 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -843,7 +843,6 @@ def parse_file( """ t0 = time.time() if ignore_errors: - #print('ignore_errors', path) self.errors.ignored_files.add(path) tree = parse(source, path, id, self.errors, options=options) tree._fullname = id diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 48a553cbb081..75354034e571 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -263,8 +263,6 @@ def parse( on failure. Otherwise, use the errors object to report parse errors. """ ignore_errors = (options is not None and options.ignore_errors) or (errors is not None and fnam in errors.ignored_files) - #if ignore_errors: - # print(fnam, fnam in errors.ignored_files, len(errors.ignored_files)) raise_on_error = False if options is None: options = Options() From ad633799b272bd96909f4d89542903eca8298766 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 15:09:07 +0000 Subject: [PATCH 15/23] Black --- mypy/fastparse.py | 36 ++++++++++++++++++++++++++---------- mypy/test/testparse.py | 6 +----- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 75354034e571..65bc2558f40d 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -262,7 +262,9 @@ def parse( Return the parse tree. If errors is not provided, raise ParseError on failure. Otherwise, use the errors object to report parse errors. """ - ignore_errors = (options is not None and options.ignore_errors) or (errors is not None and fnam in errors.ignored_files) + ignore_errors = (options is not None and options.ignore_errors) or ( + errors is not None and fnam in errors.ignored_files + ) raise_on_error = False if options is None: options = Options() @@ -284,8 +286,9 @@ def parse( warnings.filterwarnings("ignore", category=DeprecationWarning) ast = ast3_parse(source, fnam, "exec", feature_version=feature_version) - tree = ASTConverter(options=options, is_stub=is_stub_file, errors=errors, - ignore_errors=ignore_errors).visit(ast) + tree = ASTConverter( + options=options, is_stub=is_stub_file, errors=errors, ignore_errors=ignore_errors + ).visit(ast) tree.path = fnam tree.is_stub = is_stub_file except SyntaxError as e: @@ -405,7 +408,9 @@ def is_no_type_check_decorator(expr: ast3.expr) -> bool: class ASTConverter: - def __init__(self, options: Options, is_stub: bool, errors: Errors, ignore_errors: bool) -> None: + def __init__( + self, options: Options, is_stub: bool, errors: Errors, ignore_errors: bool + ) -> None: # 'C' for class, 'F' for function, 'L' for lambda self.class_and_function_stack: list[Literal["C", "F", "L"]] = [] self.imports: list[ImportBase] = [] @@ -481,7 +486,7 @@ def get_lineno(self, node: ast3.expr | ast3.stmt) -> int: return node.lineno def translate_stmt_list( - self, stmts: Sequence[ast3.stmt], *, ismodule: bool = False, can_strip: bool = False + self, stmts: Sequence[ast3.stmt], *, ismodule: bool = False, can_strip: bool = False ) -> list[Statement]: # A "# type: ignore" comment before the first statement of a module # ignores the whole module: @@ -517,7 +522,13 @@ def translate_stmt_list( node = self.visit(stmt) res.append(node) - if (self.ignore_errors and can_strip and len(stack) == 2 and stack[-2:] == ["C", "F"] and not is_possible_trivial_body(res)): + if ( + self.ignore_errors + and can_strip + and len(stack) == 2 + and stack[-2:] == ["C", "F"] + and not is_possible_trivial_body(res) + ): v = FindAttributeAssign() for n in res: n.accept(v) @@ -589,9 +600,13 @@ def as_block(self, stmts: list[ast3.stmt], lineno: int) -> Block | None: b.set_line(lineno) return b - def as_required_block(self, stmts: list[ast3.stmt], lineno: int, *, can_strip: bool = False) -> Block: + def as_required_block( + self, stmts: list[ast3.stmt], lineno: int, *, can_strip: bool = False + ) -> Block: assert stmts # must be non-empty - b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts, can_strip=can_strip))) + b = Block( + self.fix_function_overloads(self.translate_stmt_list(stmts, can_strip=can_strip)) + ) # TODO: in most call sites line is wrong (includes first line of enclosing statement) # TODO: also we need to set the column, and the end position here. b.set_line(lineno) @@ -2153,5 +2168,6 @@ def is_possible_trivial_body(s: list[Statement]) -> bool: if l > i + 1: return False stmt = s[i] - return isinstance(stmt, (PassStmt, RaiseStmt)) or (isinstance(stmt, ExpressionStmt) and - isinstance(stmt.expr, EllipsisExpr)) + return isinstance(stmt, (PassStmt, RaiseStmt)) or ( + isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, EllipsisExpr) + ) diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py index 48330e07fb2d..568588f2ff83 100644 --- a/mypy/test/testparse.py +++ b/mypy/test/testparse.py @@ -50,11 +50,7 @@ def test_parser(testcase: DataDrivenTestCase) -> None: try: n = parse( - bytes(source, "ascii"), - fnam="main", - module="__main__", - errors=None, - options=options, + bytes(source, "ascii"), fnam="main", module="__main__", errors=None, options=options ) a = str(n).split("\n") except CompileError as e: From 24519fe9fbfb7cd8119fb4bd5ea699712328f343 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 15:21:48 +0000 Subject: [PATCH 16/23] isort --- mypy/test/testparse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py index 568588f2ff83..f9d7d02233cc 100644 --- a/mypy/test/testparse.py +++ b/mypy/test/testparse.py @@ -7,13 +7,13 @@ from pytest import skip from mypy import defaults +from mypy.config_parser import parse_mypy_comments from mypy.errors import CompileError from mypy.options import Options from mypy.parse import parse -from mypy.util import get_mypy_comments -from mypy.config_parser import parse_mypy_comments from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import assert_string_arrays_equal, find_test_files, parse_options +from mypy.util import get_mypy_comments class ParserSuite(DataSuite): From 79ab2076473ff8196794be85d6aadbab81fc2c5d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 15:21:56 +0000 Subject: [PATCH 17/23] Fix tests --- mypy/fastparse.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 65bc2558f40d..2c2c0f3137eb 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -99,6 +99,7 @@ ) from mypy.reachability import infer_reachability_of_if_statement, mark_block_unreachable from mypy.sharedparse import argument_elide_name, special_function_elide_names +from mypy.traverser import TraverserVisitor from mypy.types import ( AnyType, CallableArgument, @@ -115,7 +116,6 @@ UnionType, ) from mypy.util import bytes_to_human_readable_repr, unnamed_function -from mypy.traverser import TraverserVisitor try: # pull this into a final variable to make mypyc be quiet about the @@ -411,8 +411,8 @@ class ASTConverter: def __init__( self, options: Options, is_stub: bool, errors: Errors, ignore_errors: bool ) -> None: - # 'C' for class, 'F' for function, 'L' for lambda - self.class_and_function_stack: list[Literal["C", "F", "L"]] = [] + # 'C' for class, 'D' for function signature, 'F' for function, 'L' for lambda + self.class_and_function_stack: list[Literal["C", "D", "F", "L"]] = [] self.imports: list[ImportBase] = [] self.options = options @@ -894,6 +894,7 @@ def do_func_def( self, n: ast3.FunctionDef | ast3.AsyncFunctionDef, is_coroutine: bool = False ) -> FuncDef | Decorator: """Helper shared between visit_FunctionDef and visit_AsyncFunctionDef.""" + self.class_and_function_stack.append("D") no_type_check = bool( n.decorator_list and any(is_no_type_check_decorator(d) for d in n.decorator_list) ) @@ -940,7 +941,7 @@ def do_func_def( return_type = TypeConverter(self.errors, line=lineno).visit(func_type_ast.returns) # add implicit self type - in_method_scope = self.class_and_function_stack[-2:] == ["C"] + in_method_scope = self.class_and_function_stack[-2:] == ["C", "D"] if in_method_scope and len(arg_types) < len(args): arg_types.insert(0, AnyType(TypeOfAny.special_form)) except SyntaxError: @@ -991,6 +992,7 @@ def do_func_def( end_line = getattr(n, "end_lineno", None) end_column = getattr(n, "end_col_offset", None) + self.class_and_function_stack.pop() self.class_and_function_stack.append("F") body = self.as_required_block(n.body, lineno, can_strip=True) func_def = FuncDef(n.name, args, body, func_type) From 0a8e69dd5b9d53bf5852da73a7f7468294207ec5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 17:05:23 +0000 Subject: [PATCH 18/23] Minor test update --- test-data/unit/check-inline-config.test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test-data/unit/check-inline-config.test b/test-data/unit/check-inline-config.test index 3f456e97596f..2331acf21be4 100644 --- a/test-data/unit/check-inline-config.test +++ b/test-data/unit/check-inline-config.test @@ -259,6 +259,7 @@ class D(C): # E: Call to abstract method "m5" of "C" with trivial body via super() is unsafe super().m6() \ # E: Call to abstract method "m6" of "C" with trivial body via super() is unsafe + super().m7() def m1(self) -> int: return 0 @@ -308,4 +309,9 @@ class C: def m6(self) -> int: raise NotImplementedError() + @abc.abstractmethod + def m7(self) -> int: + raise NotImplementedError() + pass + [builtins fixtures/exception.pyi] From 3b8fc66011ff0c935107aad5903b9d244358fee4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 11 Nov 2022 17:07:18 +0000 Subject: [PATCH 19/23] Add docstring --- mypy/fastparse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 2c2c0f3137eb..ab71d91734eb 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -2109,6 +2109,8 @@ def stringify_name(n: AST) -> str | None: class FindAttributeAssign(TraverserVisitor): + """Check if an AST contains attribute assignments (e.g. self.x = 0).""" + def __init__(self) -> None: self.lvalue = False self.found = False From 683237f76518ae736a540757b90c6dd696fd98b5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 19 Nov 2022 12:42:20 +0000 Subject: [PATCH 20/23] Fix stubgen tests --- mypy/fastparse.py | 20 ++++++++++++++++---- mypy/stubgen.py | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index ab71d91734eb..173d73f93e0d 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -265,6 +265,7 @@ def parse( ignore_errors = (options is not None and options.ignore_errors) or ( errors is not None and fnam in errors.ignored_files ) + strip_function_bodies = ignore_errors and (options is None or not options.preserve_asts) raise_on_error = False if options is None: options = Options() @@ -287,7 +288,11 @@ def parse( ast = ast3_parse(source, fnam, "exec", feature_version=feature_version) tree = ASTConverter( - options=options, is_stub=is_stub_file, errors=errors, ignore_errors=ignore_errors + options=options, + is_stub=is_stub_file, + errors=errors, + ignore_errors=ignore_errors, + strip_function_bodies=strip_function_bodies, ).visit(ast) tree.path = fnam tree.is_stub = is_stub_file @@ -409,7 +414,13 @@ def is_no_type_check_decorator(expr: ast3.expr) -> bool: class ASTConverter: def __init__( - self, options: Options, is_stub: bool, errors: Errors, ignore_errors: bool + self, + options: Options, + is_stub: bool, + errors: Errors, + *, + ignore_errors: bool, + strip_function_bodies: bool, ) -> None: # 'C' for class, 'D' for function signature, 'F' for function, 'L' for lambda self.class_and_function_stack: list[Literal["C", "D", "F", "L"]] = [] @@ -419,6 +430,7 @@ def __init__( self.is_stub = is_stub self.errors = errors self.ignore_errors = ignore_errors + self.strip_function_bodies = strip_function_bodies self.type_ignores: dict[int, list[str]] = {} @@ -514,7 +526,7 @@ def translate_stmt_list( return [block] stack = self.class_and_function_stack - if self.ignore_errors and len(stack) == 1 and stack[0] == "F": + if self.strip_function_bodies and len(stack) == 1 and stack[0] == "F": return [] res: list[Statement] = [] @@ -523,7 +535,7 @@ def translate_stmt_list( res.append(node) if ( - self.ignore_errors + self.strip_function_bodies and can_strip and len(stack) == 2 and stack[-2:] == ["C", "F"] diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 8c7e24504270..16b7f468e579 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -1568,6 +1568,7 @@ def mypy_options(stubgen_options: Options) -> MypyOptions: options.python_version = stubgen_options.pyversion options.show_traceback = True options.transform_source = remove_misplaced_type_comments + options.preserve_asts = True return options From aa31f2c0710eccb9a465ccd22ddf0c2f11150e90 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 19 Nov 2022 13:17:40 +0000 Subject: [PATCH 21/23] Minor tweaks --- mypy/fastparse.py | 12 +++++++----- test-data/unit/parse.test | 30 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 173d73f93e0d..cbb18cdec1c8 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -265,6 +265,7 @@ def parse( ignore_errors = (options is not None and options.ignore_errors) or ( errors is not None and fnam in errors.ignored_files ) + # If errors are ignored, we can drop many function bodies to speed up type checking. strip_function_bodies = ignore_errors and (options is None or not options.preserve_asts) raise_on_error = False if options is None: @@ -537,14 +538,15 @@ def translate_stmt_list( if ( self.strip_function_bodies and can_strip - and len(stack) == 2 and stack[-2:] == ["C", "F"] and not is_possible_trivial_body(res) ): - v = FindAttributeAssign() - for n in res: - n.accept(v) - if v.found: + # We only strip method bodies if they don't assign to an attribute, as + # this may define an attribute which has an externally visible effect. + visitor = FindAttributeAssign() + for s in res: + s.accept(visitor) + if visitor.found: break else: return [] diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index 3987a3347767..22ebf31a7dbe 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -3807,3 +3807,33 @@ MypyFile:1( NameExpr(self) x) IntExpr(0))))))) + +[case testStripMethodInNestedClass] +# mypy: ignore-errors=True +class C: + class D: + def m1(self): + self.x = 1 + def m2(self): + return self.x +[out] +MypyFile:1( + ClassDef:2( + C + ClassDef:3( + D + FuncDef:4( + m1 + Args( + Var(self)) + Block:4( + AssignmentStmt:5( + MemberExpr:5( + NameExpr(self) + x) + IntExpr(1)))) + FuncDef:6( + m2 + Args( + Var(self)) + Block:6())))) From bc23a9236324252fc720fe2a730a52c520a39af8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 4 Dec 2022 11:44:26 +0000 Subject: [PATCH 22/23] Fix handling of yield and yield from These can have an externally visible effect within an async function. The existence of yield affects the inferred return type, and a yield from generates a blocking error. --- mypy/fastparse.py | 46 ++++++++++++++++++++++++--- test-data/unit/check-async-await.test | 44 +++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index cbb18cdec1c8..a2b3897ac2eb 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -499,7 +499,12 @@ def get_lineno(self, node: ast3.expr | ast3.stmt) -> int: return node.lineno def translate_stmt_list( - self, stmts: Sequence[ast3.stmt], *, ismodule: bool = False, can_strip: bool = False + self, + stmts: Sequence[ast3.stmt], + *, + ismodule: bool = False, + can_strip: bool = False, + is_coroutine: bool = False, ) -> list[Statement]: # A "# type: ignore" comment before the first statement of a module # ignores the whole module: @@ -549,7 +554,18 @@ def translate_stmt_list( if visitor.found: break else: - return [] + if is_coroutine: + # Yields inside an async function affect the return type and should not + # be stripped. + visitor = FindYield() + for s in res: + s.accept(visitor) + if visitor.found: + break + else: + return [] + else: + return [] return res def translate_type_comment( @@ -615,11 +631,18 @@ def as_block(self, stmts: list[ast3.stmt], lineno: int) -> Block | None: return b def as_required_block( - self, stmts: list[ast3.stmt], lineno: int, *, can_strip: bool = False + self, + stmts: list[ast3.stmt], + lineno: int, + *, + can_strip: bool = False, + is_coroutine: bool = False, ) -> Block: assert stmts # must be non-empty b = Block( - self.fix_function_overloads(self.translate_stmt_list(stmts, can_strip=can_strip)) + self.fix_function_overloads( + self.translate_stmt_list(stmts, can_strip=can_strip, is_coroutine=is_coroutine) + ) ) # TODO: in most call sites line is wrong (includes first line of enclosing statement) # TODO: also we need to set the column, and the end position here. @@ -1008,7 +1031,7 @@ def do_func_def( self.class_and_function_stack.pop() self.class_and_function_stack.append("F") - body = self.as_required_block(n.body, lineno, can_strip=True) + body = self.as_required_block(n.body, lineno, can_strip=True, is_coroutine=is_coroutine) func_def = FuncDef(n.name, args, body, func_type) if isinstance(func_def.type, CallableType): # semanal.py does some in-place modifications we want to avoid @@ -2168,6 +2191,19 @@ def visit_member_expr(self, e: MemberExpr) -> None: self.found = True +class FindYield(TraverserVisitor): + """Check if an AST contains yields or yield froms.""" + + def __init__(self) -> None: + self.found = False + + def visit_yield_expr(self, e: YieldExpr) -> None: + self.found = True + + def visit_yield_from_expr(self, e: YieldFromExpr) -> None: + self.found = True + + def is_possible_trivial_body(s: list[Statement]) -> bool: """Could the statements form a "trivial" function body, such as 'pass'? diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index d53cba2fc642..12dd2432dcdc 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -943,3 +943,47 @@ async def bar(x: Union[A, B]) -> None: [builtins fixtures/async_await.pyi] [typing fixtures/typing-async.pyi] + +[case testAsyncIteratorWithIgnoredErrors] +from m import L + +async def func(l: L) -> None: + reveal_type(l.get_iterator) # N: Revealed type is "def () -> typing.AsyncIterator[builtins.str]" + reveal_type(l.get_iterator2) # N: Revealed type is "def () -> typing.AsyncIterator[builtins.str]" + async for i in l.get_iterator(): + reveal_type(i) # N: Revealed type is "builtins.str" + +[file m.py] +# mypy: ignore-errors=True +from typing import AsyncIterator + +class L: + async def some_func(self, i: int) -> str: + return 'x' + + async def get_iterator(self) -> AsyncIterator[str]: + yield await self.some_func(0) + + async def get_iterator2(self) -> AsyncIterator[str]: + if self: + a = (yield 'x') + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] + +[case testAsyncIteratorWithIgnoredErrorsAndYieldFrom] +from m import L + +async def func(l: L) -> None: + reveal_type(l.get_iterator) + +[file m.py] +# mypy: ignore-errors=True +from typing import AsyncIterator + +class L: + async def get_iterator(self) -> AsyncIterator[str]: + yield from ['x'] # E: "yield from" in async function + +[builtins fixtures/async_await.pyi] +[typing fixtures/typing-async.pyi] From a94039d1fbeb975955816593a784965182478706 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 4 Dec 2022 11:49:36 +0000 Subject: [PATCH 23/23] Fix type check --- mypy/fastparse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index a2b3897ac2eb..a074d27e9dd9 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -557,10 +557,10 @@ def translate_stmt_list( if is_coroutine: # Yields inside an async function affect the return type and should not # be stripped. - visitor = FindYield() + yield_visitor = FindYield() for s in res: - s.accept(visitor) - if visitor.found: + s.accept(yield_visitor) + if yield_visitor.found: break else: return []