From 1912eab5ed4a400cece12390e516932c0e9a1b2a Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 7 Jul 2022 07:38:36 +0100 Subject: [PATCH 1/3] [3.11] gh-92228: disable the compiler's 'small exit block inlining' optimization for blocks that have a line number (GH-94592) Inlining of code that corresponds to source code lines, can make it hard to distinguish later between code which is only reachable from except handlers, and that which is reachable in normal control flow. This caused problems with the debugger's jump feature. This PR turns off the inlining optimisation for code which has line numbers. We still inline things like the implicit "return None".. (cherry picked from commit bde06e1b8381f140b296a397ddd1deb1c784ff8e) Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Lib/test/test_dis.py | 126 +++++++++++++++++- Lib/test/test_peepholer.py | 2 + Lib/test/test_sys_settrace.py | 9 ++ ...2-07-06-14-02-26.gh-issue-92228.44Cbly.rst | 1 + Python/compile.c | 14 ++ 5 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-07-06-14-02-26.gh-issue-92228.44Cbly.rst diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 4283c30e0a1c19..eb3ed0e0458019 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -367,13 +367,13 @@ def bug42562(): --> BINARY_OP 11 (/) POP_TOP -%3d LOAD_FAST 1 (tb) +%3d >> LOAD_FAST_CHECK 1 (tb) RETURN_VALUE >> PUSH_EXC_INFO %3d LOAD_GLOBAL 0 (Exception) CHECK_EXC_MATCH - POP_JUMP_FORWARD_IF_FALSE 18 (to 72) + POP_JUMP_FORWARD_IF_FALSE 22 (to 80) STORE_FAST 0 (e) %3d LOAD_FAST 0 (e) @@ -383,9 +383,7 @@ def bug42562(): LOAD_CONST 0 (None) STORE_FAST 0 (e) DELETE_FAST 0 (e) - -%3d LOAD_FAST 1 (tb) - RETURN_VALUE + JUMP_BACKWARD 29 (to 14) >> LOAD_CONST 0 (None) STORE_FAST 0 (e) DELETE_FAST 0 (e) @@ -402,7 +400,6 @@ def bug42562(): TRACEBACK_CODE.co_firstlineno + 5, TRACEBACK_CODE.co_firstlineno + 3, TRACEBACK_CODE.co_firstlineno + 4, - TRACEBACK_CODE.co_firstlineno + 5, TRACEBACK_CODE.co_firstlineno + 3) def _fstring(a, b, c, d): @@ -428,6 +425,123 @@ def _fstring(a, b, c, d): RETURN_VALUE """ % (_fstring.__code__.co_firstlineno, _fstring.__code__.co_firstlineno + 1) +def _with(c): + with c: + x = 1 + y = 2 + +dis_with = """\ +%3d RESUME 0 + +%3d LOAD_FAST 0 (c) + BEFORE_WITH + POP_TOP + +%3d LOAD_CONST 1 (1) + STORE_FAST 1 (x) + +%3d LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + CALL 2 + POP_TOP + +%3d >> LOAD_CONST 2 (2) + STORE_FAST 2 (y) + LOAD_CONST 0 (None) + RETURN_VALUE + +%3d >> PUSH_EXC_INFO + WITH_EXCEPT_START + POP_JUMP_FORWARD_IF_TRUE 1 (to 46) + RERAISE 2 + >> POP_TOP + POP_EXCEPT + POP_TOP + POP_TOP + JUMP_BACKWARD 13 (to 30) + >> COPY 3 + POP_EXCEPT + RERAISE 1 +ExceptionTable: +2 rows +""" % (_with.__code__.co_firstlineno, + _with.__code__.co_firstlineno + 1, + _with.__code__.co_firstlineno + 2, + _with.__code__.co_firstlineno + 1, + _with.__code__.co_firstlineno + 3, + _with.__code__.co_firstlineno + 1, + ) + +async def _asyncwith(c): + async with c: + x = 1 + y = 2 + +dis_asyncwith = """\ +%3d RETURN_GENERATOR + POP_TOP + RESUME 0 + +%3d LOAD_FAST 0 (c) + BEFORE_ASYNC_WITH + GET_AWAITABLE 1 + LOAD_CONST 0 (None) + >> SEND 3 (to 22) + YIELD_VALUE 3 + RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 4 (to 14) + >> POP_TOP + +%3d LOAD_CONST 1 (1) + STORE_FAST 1 (x) + +%3d LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + LOAD_CONST 0 (None) + CALL 2 + GET_AWAITABLE 2 + LOAD_CONST 0 (None) + >> SEND 3 (to 56) + YIELD_VALUE 2 + RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 4 (to 48) + >> POP_TOP + +%3d >> LOAD_CONST 2 (2) + STORE_FAST 2 (y) + LOAD_CONST 0 (None) + RETURN_VALUE + +%3d >> PUSH_EXC_INFO + WITH_EXCEPT_START + GET_AWAITABLE 2 + LOAD_CONST 0 (None) + >> SEND 3 (to 82) + YIELD_VALUE 6 + RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 4 (to 74) + >> POP_JUMP_FORWARD_IF_TRUE 1 (to 86) + RERAISE 2 + >> POP_TOP + POP_EXCEPT + POP_TOP + POP_TOP + JUMP_BACKWARD 19 (to 58) + >> COPY 3 + POP_EXCEPT + RERAISE 1 +ExceptionTable: +2 rows +""" % (_asyncwith.__code__.co_firstlineno, + _asyncwith.__code__.co_firstlineno + 1, + _asyncwith.__code__.co_firstlineno + 2, + _asyncwith.__code__.co_firstlineno + 1, + _asyncwith.__code__.co_firstlineno + 3, + _asyncwith.__code__.co_firstlineno + 1, + ) + + def _tryfinally(a, b): try: return a diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 6a2f550d67052a..241644ad7d2ff0 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -344,6 +344,8 @@ def f(x): self.assertEqual(len(returns), 1) self.check_lnotab(f) + @unittest.skip("Following gh-92228 the return has two predecessors " + "and that prevents jump elimination.") def test_elim_jump_to_return(self): # JUMP_FORWARD to RETURN --> RETURN def f(cond, true_value, false_value): diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 7ec290dbf04ad5..f03b03e19a2528 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2042,6 +2042,15 @@ def test_no_jump_within_except_block(output): output.append(6) output.append(7) + @jump_test(6, 1, [1, 5, 1, 5]) + def test_jump_over_try_except(output): + output.append(1) + try: + 1 / 0 + except ZeroDivisionError as e: + output.append(5) + x = 42 # has to be a two-instruction block + @jump_test(2, 4, [1, 4, 5, -4]) def test_jump_across_with(output): output.append(1) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-07-06-14-02-26.gh-issue-92228.44Cbly.rst b/Misc/NEWS.d/next/Core and Builtins/2022-07-06-14-02-26.gh-issue-92228.44Cbly.rst new file mode 100644 index 00000000000000..458ad897cefcb6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-07-06-14-02-26.gh-issue-92228.44Cbly.rst @@ -0,0 +1 @@ +Disable the compiler's inline-small-exit-blocks optimization for exit blocks that are associated with source code lines. This fixes a bug where the debugger cannot tell where an exception handler ends and the following code block begins. diff --git a/Python/compile.c b/Python/compile.c index cfe4b6ec04f9ed..a71e7d31eecd28 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -8976,6 +8976,16 @@ optimize_basic_block(struct compiler *c, basicblock *bb, PyObject *consts) return -1; } +static bool +basicblock_has_lineno(const basicblock *bb) { + for (int i = 0; i < bb->b_iused; i++) { + if (bb->b_instr[i].i_lineno > 0) { + return true; + } + } + return false; +} + /* If this block ends with an unconditional jump to an exit block, * then remove the jump and extend this block with the target. */ @@ -8992,6 +9002,10 @@ extend_block(basicblock *bb) { } if (last->i_target->b_exit && last->i_target->b_iused <= MAX_COPY_SIZE) { basicblock *to_copy = last->i_target; + if (basicblock_has_lineno(to_copy)) { + /* copy only blocks without line number (like implicit 'return None's) */ + return 0; + } last->i_opcode = NOP; for (int i = 0; i < to_copy->b_iused; i++) { int index = compiler_next_instr(bb); From 437c3b42133bfe2e203dc15bb75484b3b866b336 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 7 Jul 2022 08:57:27 +0200 Subject: [PATCH 2/3] Adjust dis_traceback --- Lib/test/test_dis.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index eb3ed0e0458019..a4797a069094a9 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -366,14 +366,12 @@ def bug42562(): LOAD_CONST 2 (0) --> BINARY_OP 11 (/) POP_TOP - -%3d >> LOAD_FAST_CHECK 1 (tb) - RETURN_VALUE + JUMP_FORWARD 30 (to 76) >> PUSH_EXC_INFO %3d LOAD_GLOBAL 0 (Exception) CHECK_EXC_MATCH - POP_JUMP_FORWARD_IF_FALSE 22 (to 80) + POP_JUMP_FORWARD_IF_FALSE 17 (to 68) STORE_FAST 0 (e) %3d LOAD_FAST 0 (e) @@ -383,7 +381,7 @@ def bug42562(): LOAD_CONST 0 (None) STORE_FAST 0 (e) DELETE_FAST 0 (e) - JUMP_BACKWARD 29 (to 14) + JUMP_FORWARD 8 (to 76) >> LOAD_CONST 0 (None) STORE_FAST 0 (e) DELETE_FAST 0 (e) @@ -393,14 +391,17 @@ def bug42562(): >> COPY 3 POP_EXCEPT RERAISE 1 + +%3d >> LOAD_FAST 1 (tb) + RETURN_VALUE ExceptionTable: """ % (TRACEBACK_CODE.co_firstlineno, TRACEBACK_CODE.co_firstlineno + 1, TRACEBACK_CODE.co_firstlineno + 2, - TRACEBACK_CODE.co_firstlineno + 5, TRACEBACK_CODE.co_firstlineno + 3, TRACEBACK_CODE.co_firstlineno + 4, - TRACEBACK_CODE.co_firstlineno + 3) + TRACEBACK_CODE.co_firstlineno + 3, + TRACEBACK_CODE.co_firstlineno + 5) def _fstring(a, b, c, d): return f'{a} {b:4} {c!r} {d!r:4}' From 6971a75939a53a8661c4e6956376ed5d8795c0f0 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 7 Jul 2022 10:31:30 +0100 Subject: [PATCH 3/3] do not backport the with/asyncwith dis tests --- Lib/test/test_dis.py | 117 ------------------------------------------- 1 file changed, 117 deletions(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index a4797a069094a9..b00f329e0068b6 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -426,123 +426,6 @@ def _fstring(a, b, c, d): RETURN_VALUE """ % (_fstring.__code__.co_firstlineno, _fstring.__code__.co_firstlineno + 1) -def _with(c): - with c: - x = 1 - y = 2 - -dis_with = """\ -%3d RESUME 0 - -%3d LOAD_FAST 0 (c) - BEFORE_WITH - POP_TOP - -%3d LOAD_CONST 1 (1) - STORE_FAST 1 (x) - -%3d LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - CALL 2 - POP_TOP - -%3d >> LOAD_CONST 2 (2) - STORE_FAST 2 (y) - LOAD_CONST 0 (None) - RETURN_VALUE - -%3d >> PUSH_EXC_INFO - WITH_EXCEPT_START - POP_JUMP_FORWARD_IF_TRUE 1 (to 46) - RERAISE 2 - >> POP_TOP - POP_EXCEPT - POP_TOP - POP_TOP - JUMP_BACKWARD 13 (to 30) - >> COPY 3 - POP_EXCEPT - RERAISE 1 -ExceptionTable: -2 rows -""" % (_with.__code__.co_firstlineno, - _with.__code__.co_firstlineno + 1, - _with.__code__.co_firstlineno + 2, - _with.__code__.co_firstlineno + 1, - _with.__code__.co_firstlineno + 3, - _with.__code__.co_firstlineno + 1, - ) - -async def _asyncwith(c): - async with c: - x = 1 - y = 2 - -dis_asyncwith = """\ -%3d RETURN_GENERATOR - POP_TOP - RESUME 0 - -%3d LOAD_FAST 0 (c) - BEFORE_ASYNC_WITH - GET_AWAITABLE 1 - LOAD_CONST 0 (None) - >> SEND 3 (to 22) - YIELD_VALUE 3 - RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 4 (to 14) - >> POP_TOP - -%3d LOAD_CONST 1 (1) - STORE_FAST 1 (x) - -%3d LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - LOAD_CONST 0 (None) - CALL 2 - GET_AWAITABLE 2 - LOAD_CONST 0 (None) - >> SEND 3 (to 56) - YIELD_VALUE 2 - RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 4 (to 48) - >> POP_TOP - -%3d >> LOAD_CONST 2 (2) - STORE_FAST 2 (y) - LOAD_CONST 0 (None) - RETURN_VALUE - -%3d >> PUSH_EXC_INFO - WITH_EXCEPT_START - GET_AWAITABLE 2 - LOAD_CONST 0 (None) - >> SEND 3 (to 82) - YIELD_VALUE 6 - RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 4 (to 74) - >> POP_JUMP_FORWARD_IF_TRUE 1 (to 86) - RERAISE 2 - >> POP_TOP - POP_EXCEPT - POP_TOP - POP_TOP - JUMP_BACKWARD 19 (to 58) - >> COPY 3 - POP_EXCEPT - RERAISE 1 -ExceptionTable: -2 rows -""" % (_asyncwith.__code__.co_firstlineno, - _asyncwith.__code__.co_firstlineno + 1, - _asyncwith.__code__.co_firstlineno + 2, - _asyncwith.__code__.co_firstlineno + 1, - _asyncwith.__code__.co_firstlineno + 3, - _asyncwith.__code__.co_firstlineno + 1, - ) - - def _tryfinally(a, b): try: return a