From e5ff5ec3ffb869ff410311c57aa05ff8fb17956a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 25 Jul 2022 12:11:06 +0100 Subject: [PATCH] [3.11] GH-94739: Backport GH-94958 to 3.11 (#94965) --- Include/internal/pycore_code.h | 13 ++ Lib/test/test_sys_settrace.py | 47 +++-- ...2-07-18-14-19-21.gh-issue-94739.NQJQi7.rst | 1 + Objects/frameobject.c | 190 ++++++++++++++++-- Python/ceval.c | 14 -- 5 files changed, 213 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-07-18-14-19-21.gh-issue-94739.NQJQi7.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 9528c8996a443c..3a24a65426985e 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -438,6 +438,19 @@ read_obj(uint16_t *p) return (PyObject *)val; } +/* See Objects/exception_handling_notes.txt for details. + */ +static inline unsigned char * +parse_varint(unsigned char *p, int *result) { + int val = p[0] & 63; + while (p[0] & 64) { + p++; + val = (val << 6) | (p[0] & 63); + } + *result = val; + return p+1; +} + static inline int write_varint(uint8_t *ptr, unsigned int val) { diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index d77deae32cd5cf..9f1aa81dbcd249 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1879,7 +1879,7 @@ def test_jump_out_of_block_backwards(output): output.append(6) output.append(7) - @async_jump_test(4, 5, [3], (ValueError, 'into')) + @async_jump_test(4, 5, [3, 5]) async def test_jump_out_of_async_for_block_forwards(output): for i in [1]: async for i in asynciter([1, 2]): @@ -1921,7 +1921,7 @@ def test_jump_in_nested_finally(output): output.append(8) output.append(9) - @jump_test(6, 7, [2], (ValueError, 'within')) + @jump_test(6, 7, [2, 7], (ZeroDivisionError, '')) def test_jump_in_nested_finally_2(output): try: output.append(2) @@ -1932,7 +1932,7 @@ def test_jump_in_nested_finally_2(output): output.append(7) output.append(8) - @jump_test(6, 11, [2], (ValueError, 'within')) + @jump_test(6, 11, [2, 11], (ZeroDivisionError, '')) def test_jump_in_nested_finally_3(output): try: output.append(2) @@ -2043,8 +2043,8 @@ def test_jump_backwards_out_of_try_except_block(output): output.append(5) raise - @jump_test(5, 7, [4], (ValueError, 'within')) - def test_no_jump_between_except_blocks(output): + @jump_test(5, 7, [4, 7, 8]) + def test_jump_between_except_blocks(output): try: 1/0 except ZeroDivisionError: @@ -2054,8 +2054,19 @@ def test_no_jump_between_except_blocks(output): output.append(7) output.append(8) - @jump_test(5, 6, [4], (ValueError, 'within')) - def test_no_jump_within_except_block(output): + @jump_test(5, 7, [4, 7, 8]) + def test_jump_from_except_to_finally(output): + try: + 1/0 + except ZeroDivisionError: + output.append(4) + output.append(5) + finally: + output.append(7) + output.append(8) + + @jump_test(5, 6, [4, 6, 7]) + def test_jump_within_except_block(output): try: 1/0 except: @@ -2290,7 +2301,7 @@ def test_no_jump_backwards_into_for_block(output): output.append(2) output.append(3) - @async_jump_test(3, 2, [2, 2], (ValueError, 'within')) + @async_jump_test(3, 2, [2, 2], (ValueError, "can't jump into the body of a for loop")) async def test_no_jump_backwards_into_async_for_block(output): async for i in asynciter([1, 2]): output.append(2) @@ -2355,8 +2366,8 @@ def test_jump_backwards_into_try_except_block(output): output.append(6) # 'except' with a variable creates an implicit finally block - @jump_test(5, 7, [4], (ValueError, 'within')) - def test_no_jump_between_except_blocks_2(output): + @jump_test(5, 7, [4, 7, 8]) + def test_jump_between_except_blocks_2(output): try: 1/0 except ZeroDivisionError: @@ -2392,7 +2403,7 @@ def test_jump_out_of_finally_block(output): finally: output.append(5) - @jump_test(1, 5, [], (ValueError, "into an exception")) + @jump_test(1, 5, [], (ValueError, "can't jump into an 'except' block as there's no exception")) def test_no_jump_into_bare_except_block(output): output.append(1) try: @@ -2400,7 +2411,7 @@ def test_no_jump_into_bare_except_block(output): except: output.append(5) - @jump_test(1, 5, [], (ValueError, "into an exception")) + @jump_test(1, 5, [], (ValueError, "can't jump into an 'except' block as there's no exception")) def test_no_jump_into_qualified_except_block(output): output.append(1) try: @@ -2408,7 +2419,7 @@ def test_no_jump_into_qualified_except_block(output): except Exception: output.append(5) - @jump_test(3, 6, [2, 5, 6], (ValueError, "into an exception")) + @jump_test(3, 6, [2, 5, 6], (ValueError, "can't jump into an 'except' block as there's no exception")) def test_no_jump_into_bare_except_block_from_try_block(output): try: output.append(2) @@ -2419,7 +2430,7 @@ def test_no_jump_into_bare_except_block_from_try_block(output): raise output.append(8) - @jump_test(3, 6, [2], (ValueError, "into an exception")) + @jump_test(3, 6, [2], (ValueError, "can't jump into an 'except' block as there's no exception")) def test_no_jump_into_qualified_except_block_from_try_block(output): try: output.append(2) @@ -2430,8 +2441,8 @@ def test_no_jump_into_qualified_except_block_from_try_block(output): raise output.append(8) - @jump_test(7, 1, [1, 3, 6], (ValueError, "within")) - def test_no_jump_out_of_bare_except_block(output): + @jump_test(7, 1, [1, 3, 6, 1, 3, 6, 7]) + def test_jump_out_of_bare_except_block(output): output.append(1) try: output.append(3) @@ -2440,8 +2451,8 @@ def test_no_jump_out_of_bare_except_block(output): output.append(6) output.append(7) - @jump_test(7, 1, [1, 3, 6], (ValueError, "within")) - def test_no_jump_out_of_qualified_except_block(output): + @jump_test(7, 1, [1, 3, 6, 1, 3, 6, 7]) + def test_jump_out_of_qualified_except_block(output): output.append(1) try: output.append(3) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-07-18-14-19-21.gh-issue-94739.NQJQi7.rst b/Misc/NEWS.d/next/Core and Builtins/2022-07-18-14-19-21.gh-issue-94739.NQJQi7.rst new file mode 100644 index 00000000000000..7476892c423b1a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-07-18-14-19-21.gh-issue-94739.NQJQi7.rst @@ -0,0 +1 @@ +Allow jumping within, out of, and across exception handlers in the debugger. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 47e108b9ae79b3..1ddfec2fb08ab7 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -138,6 +138,7 @@ typedef enum kind { Except = 2, Object = 3, Null = 4, + Lasti = 5, } Kind; static int @@ -162,6 +163,8 @@ compatible_kind(Kind from, Kind to) { #define MAX_STACK_ENTRIES (63/BITS_PER_BLOCK) #define WILL_OVERFLOW (1ULL<<((MAX_STACK_ENTRIES-1)*BITS_PER_BLOCK)) +#define EMPTY_STACK 0 + static inline int64_t push_value(int64_t stack, Kind kind) { @@ -179,12 +182,97 @@ pop_value(int64_t stack) return Py_ARITHMETIC_RIGHT_SHIFT(int64_t, stack, BITS_PER_BLOCK); } +#define MASK ((1<= 1); + return (stack>>(BITS_PER_BLOCK*(n-1))) & MASK; +} + +static Kind +stack_swap(int64_t stack, int n) +{ + assert(n >= 1); + Kind to_swap = peek(stack, n); + Kind top = top_of_stack(stack); + int shift = BITS_PER_BLOCK*(n-1); + int64_t replaced_low = (stack & ~(MASK << shift)) | (top << shift); + int64_t replaced_top = (replaced_low & ~MASK) | to_swap; + return replaced_top; +} + +static int64_t +pop_to_level(int64_t stack, int level) { + if (level == 0) { + return EMPTY_STACK; + } + int64_t max_item = (1< level_max_stack) { + stack = pop_value(stack); + } + return stack; +} + +#if 0 +/* These functions are useful for debugging the stack marking code */ + +static char +tos_char(int64_t stack) { + switch(top_of_stack(stack)) { + case Iterator: + return 'I'; + case Except: + return 'E'; + case Object: + return 'O'; + case Lasti: + return 'L'; + case Null: + return 'N'; + } + return '?'; } +static void +print_stack(int64_t stack) { + if (stack < 0) { + if (stack == UNINITIALIZED) { + printf("---"); + } + else if (stack == OVERFLOWED) { + printf("OVERFLOWED"); + } + else { + printf("??"); + } + return; + } + while (stack) { + printf("%c", tos_char(stack)); + stack = pop_value(stack); + } +} + +static void +print_stacks(int64_t *stacks, int n) { + for (int i = 0; i < n; i++) { + printf("%d: ", i); + print_stack(stacks[i]); + printf("\n"); + } +} + +#endif + static int64_t * mark_stacks(PyCodeObject *code_obj, int len) { @@ -204,7 +292,7 @@ mark_stacks(PyCodeObject *code_obj, int len) for (int i = 1; i <= len; i++) { stacks[i] = UNINITIALIZED; } - stacks[0] = 0; + stacks[0] = EMPTY_STACK; if (code_obj->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { // Generators get sent None while starting: @@ -213,6 +301,7 @@ mark_stacks(PyCodeObject *code_obj, int len) int todo = 1; while (todo) { todo = 0; + /* Scan instructions */ for (i = 0; i < len; i++) { int64_t next_stack = stacks[i]; if (next_stack == UNINITIALIZED) { @@ -300,23 +389,26 @@ mark_stacks(PyCodeObject *code_obj, int len) break; } case END_ASYNC_FOR: - next_stack = pop_value(pop_value(pop_value(next_stack))); + next_stack = pop_value(pop_value(next_stack)); stacks[i+1] = next_stack; break; case PUSH_EXC_INFO: + next_stack = push_value(next_stack, Except); + stacks[i+1] = next_stack; + break; case POP_EXCEPT: - /* These instructions only appear in exception handlers, which - * skip this switch ever since the move to zero-cost exceptions - * (their stack remains UNINITIALIZED because nothing sets it). - * - * Note that explain_incompatible_stack interprets an - * UNINITIALIZED stack as belonging to an exception handler. - */ - Py_UNREACHABLE(); + assert(top_of_stack(next_stack) == Except); + next_stack = pop_value(next_stack); + stacks[i+1] = next_stack; break; case RETURN_VALUE: + assert(pop_value(next_stack) == EMPTY_STACK); + assert(top_of_stack(next_stack) == Object); + break; case RAISE_VARARGS: + break; case RERAISE: + assert(top_of_stack(next_stack) == Except); /* End of block */ break; case PUSH_NULL: @@ -331,11 +423,33 @@ mark_stacks(PyCodeObject *code_obj, int len) stacks[i+1] = next_stack; break; case LOAD_METHOD: + assert(top_of_stack(next_stack) == Object); next_stack = pop_value(next_stack); next_stack = push_value(next_stack, Null); next_stack = push_value(next_stack, Object); stacks[i+1] = next_stack; break; + case CALL: + { + next_stack = pop_value(pop_value(next_stack)); + next_stack = push_value(next_stack, Object); + stacks[i+1] = next_stack; + break; + } + case SWAP: + { + int n = get_arg(code, i); + next_stack = stack_swap(next_stack, n); + stacks[i+1] = next_stack; + break; + } + case COPY: + { + int n = get_arg(code, i); + next_stack = push_value(next_stack, peek(next_stack, n)); + stacks[i+1] = next_stack; + break; + } default: { int delta = PyCompile_OpcodeStackEffect(opcode, get_arg(code, i)); @@ -352,6 +466,34 @@ mark_stacks(PyCodeObject *code_obj, int len) } } } + /* Scan exception table */ + unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code_obj->co_exceptiontable); + unsigned char *end = start + PyBytes_GET_SIZE(code_obj->co_exceptiontable); + unsigned char *scan = start; + while (scan < end) { + int start_offset, size, handler; + scan = parse_varint(scan, &start_offset); + assert(start_offset >= 0 && start_offset < len); + scan = parse_varint(scan, &size); + assert(size >= 0 && start_offset+size <= len); + scan = parse_varint(scan, &handler); + assert(handler >= 0 && handler < len); + int depth_and_lasti; + scan = parse_varint(scan, &depth_and_lasti); + int level = depth_and_lasti >> 1; + int lasti = depth_and_lasti & 1; + if (stacks[start_offset] != UNINITIALIZED) { + if (stacks[handler] == UNINITIALIZED) { + todo = 1; + uint64_t target_stack = pop_to_level(stacks[start_offset], level); + if (lasti) { + target_stack = push_value(target_stack, Lasti); + } + target_stack = push_value(target_stack, Except); + stacks[handler] = target_stack; + } + } + } } Py_DECREF(co_code); return stacks; @@ -392,6 +534,8 @@ explain_incompatible_stack(int64_t to_stack) switch(target_kind) { case Except: return "can't jump into an 'except' block as there's no exception"; + case Lasti: + return "can't jump into a re-raising block as there's no location"; case Object: case Null: return "incompatible stacks"; @@ -443,13 +587,6 @@ first_line_not_before(int *lines, int len, int line) return result; } -static void -frame_stack_pop(PyFrameObject *f) -{ - PyObject *v = _PyFrame_StackPop(f->f_frame); - Py_XDECREF(v); -} - static PyFrameState _PyFrame_GetState(PyFrameObject *frame) { @@ -617,7 +754,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore msg = "stack to deep to analyze"; } else if (start_stack == UNINITIALIZED) { - msg = "can't jump from within an exception handler"; + msg = "can't jump from unreachable code"; } else { msg = explain_incompatible_stack(target_stack); @@ -637,7 +774,20 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore start_stack = pop_value(start_stack); } while (start_stack > best_stack) { - frame_stack_pop(f); + if (top_of_stack(start_stack) == Except) { + /* Pop exception stack as well as the evaluation stack */ + PyThreadState *tstate = _PyThreadState_GET(); + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *value = exc_info->exc_value; + PyObject *exc = _PyFrame_StackPop(f->f_frame); + assert(PyExceptionInstance_Check(exc) || exc == Py_None); + exc_info->exc_value = exc; + Py_XDECREF(value); + } + else { + PyObject *v = _PyFrame_StackPop(f->f_frame); + Py_XDECREF(v); + } start_stack = pop_value(start_stack); } /* Finally set the new lasti and return OK. */ diff --git a/Python/ceval.c b/Python/ceval.c index d40bf275a17dae..31cdb554279e37 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -6056,20 +6056,6 @@ positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, } -/* Exception table parsing code. - * See Objects/exception_table_notes.txt for details. - */ - -static inline unsigned char * -parse_varint(unsigned char *p, int *result) { - int val = p[0] & 63; - while (p[0] & 64) { - p++; - val = (val << 6) | (p[0] & 63); - } - *result = val; - return p+1; -} static inline unsigned char * scan_back_to_entry_start(unsigned char *p) {