From 95a73917cd5a204979a78c13ba912621f1eeb2e3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 26 Jul 2024 14:35:57 +0100 Subject: [PATCH] GH-122029: Break INSTRUMENTED_CALL into micro-ops, so that its behavior is consistent with CALL (GH-122177) --- Include/internal/pycore_opcode_metadata.h | 6 +- Include/internal/pycore_uop_ids.h | 129 ++++++++-------- Include/internal/pycore_uop_metadata.h | 4 + Include/opcode_ids.h | 26 ++-- Lib/_opcode_metadata.py | 26 ++-- Lib/test/test_monitoring.py | 2 +- Python/bytecodes.c | 71 +++++---- Python/executor_cases.c.h | 34 ++++- Python/generated_cases.c.h | 176 +++++++++++++++++++--- Python/opcode_targets.h | 2 +- Python/optimizer_cases.c.h | 18 ++- 11 files changed, 341 insertions(+), 153 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index f5c439e81a6232..eaba280f1bf1cd 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -214,7 +214,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case IMPORT_NAME: return 2; case INSTRUMENTED_CALL: - return 0; + return 2 + oparg; case INSTRUMENTED_CALL_FUNCTION_EX: return 0; case INSTRUMENTED_CALL_KW: @@ -661,7 +661,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case IMPORT_NAME: return 1; case INSTRUMENTED_CALL: - return 0; + return 1; case INSTRUMENTED_CALL_FUNCTION_EX: return 0; case INSTRUMENTED_CALL_KW: @@ -1078,7 +1078,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 60de0573baf5f1..d6c910255eb87b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -33,12 +33,11 @@ extern "C" { #define _BUILD_SLICE BUILD_SLICE #define _BUILD_STRING BUILD_STRING #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL 312 #define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT -#define _CALL_BUILTIN_CLASS 313 -#define _CALL_BUILTIN_FAST 314 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 315 -#define _CALL_BUILTIN_O 316 +#define _CALL_BUILTIN_CLASS 312 +#define _CALL_BUILTIN_FAST 313 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 314 +#define _CALL_BUILTIN_O 315 #define _CALL_FUNCTION_EX CALL_FUNCTION_EX #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 @@ -46,38 +45,38 @@ extern "C" { #define _CALL_KW CALL_KW #define _CALL_LEN CALL_LEN #define _CALL_LIST_APPEND CALL_LIST_APPEND -#define _CALL_METHOD_DESCRIPTOR_FAST 317 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 318 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 319 -#define _CALL_METHOD_DESCRIPTOR_O 320 -#define _CALL_NON_PY_GENERAL 321 -#define _CALL_STR_1 322 -#define _CALL_TUPLE_1 323 +#define _CALL_METHOD_DESCRIPTOR_FAST 316 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 317 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 318 +#define _CALL_METHOD_DESCRIPTOR_O 319 +#define _CALL_NON_PY_GENERAL 320 +#define _CALL_STR_1 321 +#define _CALL_TUPLE_1 322 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 324 -#define _CHECK_ATTR_METHOD_LAZY_DICT 325 -#define _CHECK_ATTR_MODULE 326 -#define _CHECK_ATTR_WITH_HINT 327 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 328 +#define _CHECK_ATTR_CLASS 323 +#define _CHECK_ATTR_METHOD_LAZY_DICT 324 +#define _CHECK_ATTR_MODULE 325 +#define _CHECK_ATTR_WITH_HINT 326 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 327 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 329 -#define _CHECK_FUNCTION_EXACT_ARGS 330 -#define _CHECK_FUNCTION_VERSION 331 -#define _CHECK_IS_NOT_PY_CALLABLE 332 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 333 -#define _CHECK_METHOD_VERSION 334 -#define _CHECK_PEP_523 335 -#define _CHECK_PERIODIC 336 -#define _CHECK_STACK_SPACE 337 -#define _CHECK_STACK_SPACE_OPERAND 338 -#define _CHECK_VALIDITY 339 -#define _CHECK_VALIDITY_AND_SET_IP 340 -#define _COMPARE_OP 341 -#define _COMPARE_OP_FLOAT 342 -#define _COMPARE_OP_INT 343 -#define _COMPARE_OP_STR 344 -#define _CONTAINS_OP 345 +#define _CHECK_FUNCTION 328 +#define _CHECK_FUNCTION_EXACT_ARGS 329 +#define _CHECK_FUNCTION_VERSION 330 +#define _CHECK_IS_NOT_PY_CALLABLE 331 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 332 +#define _CHECK_METHOD_VERSION 333 +#define _CHECK_PEP_523 334 +#define _CHECK_PERIODIC 335 +#define _CHECK_STACK_SPACE 336 +#define _CHECK_STACK_SPACE_OPERAND 337 +#define _CHECK_VALIDITY 338 +#define _CHECK_VALIDITY_AND_SET_IP 339 +#define _COMPARE_OP 340 +#define _COMPARE_OP_FLOAT 341 +#define _COMPARE_OP_INT 342 +#define _COMPARE_OP_STR 343 +#define _CONTAINS_OP 344 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE @@ -89,9 +88,10 @@ extern "C" { #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 346 +#define _DEOPT 345 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE +#define _DO_CALL 346 #define _DYNAMIC_EXIT 347 #define _END_SEND END_SEND #define _ERROR_POP_N 348 @@ -138,7 +138,6 @@ extern "C" { #define _INIT_CALL_PY_EXACT_ARGS_2 378 #define _INIT_CALL_PY_EXACT_ARGS_3 379 #define _INIT_CALL_PY_EXACT_ARGS_4 380 -#define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER @@ -223,53 +222,55 @@ extern "C" { #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE +#define _MAYBE_EXPAND_METHOD 427 +#define _MONITOR_CALL 428 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 427 -#define _POP_JUMP_IF_TRUE 428 +#define _POP_JUMP_IF_FALSE 429 +#define _POP_JUMP_IF_TRUE 430 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 429 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 431 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 430 +#define _PUSH_FRAME 432 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 431 -#define _REPLACE_WITH_TRUE 432 +#define _PY_FRAME_GENERAL 433 +#define _REPLACE_WITH_TRUE 434 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 433 -#define _SEND 434 -#define _SEND_GEN_FRAME 435 +#define _SAVE_RETURN_OFFSET 435 +#define _SEND 436 +#define _SEND_GEN_FRAME 437 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 436 -#define _STORE_ATTR 437 -#define _STORE_ATTR_INSTANCE_VALUE 438 -#define _STORE_ATTR_SLOT 439 -#define _STORE_ATTR_WITH_HINT 440 +#define _START_EXECUTOR 438 +#define _STORE_ATTR 439 +#define _STORE_ATTR_INSTANCE_VALUE 440 +#define _STORE_ATTR_SLOT 441 +#define _STORE_ATTR_WITH_HINT 442 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 441 -#define _STORE_FAST_0 442 -#define _STORE_FAST_1 443 -#define _STORE_FAST_2 444 -#define _STORE_FAST_3 445 -#define _STORE_FAST_4 446 -#define _STORE_FAST_5 447 -#define _STORE_FAST_6 448 -#define _STORE_FAST_7 449 +#define _STORE_FAST 443 +#define _STORE_FAST_0 444 +#define _STORE_FAST_1 445 +#define _STORE_FAST_2 446 +#define _STORE_FAST_3 447 +#define _STORE_FAST_4 448 +#define _STORE_FAST_5 449 +#define _STORE_FAST_6 450 +#define _STORE_FAST_7 451 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 450 +#define _STORE_SUBSCR 452 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 451 -#define _TO_BOOL 452 +#define _TIER2_RESUME_CHECK 453 +#define _TO_BOOL 454 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -279,13 +280,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 453 +#define _UNPACK_SEQUENCE 455 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 453 +#define MAX_UOP_ID 455 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 190c6fb2365cc4..fd0d4a67d93538 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -199,6 +199,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, + [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG, [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -464,6 +465,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_MATCH_KEYS] = "_MATCH_KEYS", [_MATCH_MAPPING] = "_MATCH_MAPPING", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", [_NOP] = "_NOP", [_POP_EXCEPT] = "_POP_EXCEPT", [_POP_TOP] = "_POP_TOP", @@ -888,6 +890,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_METHOD_LAZY_DICT: return 1; + case _MAYBE_EXPAND_METHOD: + return 2 + oparg; case _CHECK_PERIODIC: return 0; case _PY_FRAME_GENERAL: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index d14b48f4289285..54dd76158bf84d 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -206,19 +206,19 @@ extern "C" { #define INSTRUMENTED_END_SEND 238 #define INSTRUMENTED_LOAD_SUPER_ATTR 239 #define INSTRUMENTED_FOR_ITER 240 -#define INSTRUMENTED_CALL 241 -#define INSTRUMENTED_CALL_KW 242 -#define INSTRUMENTED_CALL_FUNCTION_EX 243 -#define INSTRUMENTED_INSTRUCTION 244 -#define INSTRUMENTED_JUMP_FORWARD 245 -#define INSTRUMENTED_JUMP_BACKWARD 246 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 247 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 248 -#define INSTRUMENTED_POP_JUMP_IF_NONE 249 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 250 -#define INSTRUMENTED_RETURN_VALUE 251 -#define INSTRUMENTED_RETURN_CONST 252 -#define INSTRUMENTED_YIELD_VALUE 253 +#define INSTRUMENTED_CALL_KW 241 +#define INSTRUMENTED_CALL_FUNCTION_EX 242 +#define INSTRUMENTED_INSTRUCTION 243 +#define INSTRUMENTED_JUMP_FORWARD 244 +#define INSTRUMENTED_JUMP_BACKWARD 245 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 246 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 247 +#define INSTRUMENTED_POP_JUMP_IF_NONE 248 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 249 +#define INSTRUMENTED_RETURN_VALUE 250 +#define INSTRUMENTED_RETURN_CONST 251 +#define INSTRUMENTED_YIELD_VALUE 252 +#define INSTRUMENTED_CALL 253 #define INSTRUMENTED_LINE 254 #define JUMP 256 #define JUMP_NO_INTERRUPT 257 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 01c22a89846e97..05ee1f29b58331 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -310,19 +310,19 @@ 'INSTRUMENTED_END_SEND': 238, 'INSTRUMENTED_LOAD_SUPER_ATTR': 239, 'INSTRUMENTED_FOR_ITER': 240, - 'INSTRUMENTED_CALL': 241, - 'INSTRUMENTED_CALL_KW': 242, - 'INSTRUMENTED_CALL_FUNCTION_EX': 243, - 'INSTRUMENTED_INSTRUCTION': 244, - 'INSTRUMENTED_JUMP_FORWARD': 245, - 'INSTRUMENTED_JUMP_BACKWARD': 246, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 247, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 248, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 249, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 250, - 'INSTRUMENTED_RETURN_VALUE': 251, - 'INSTRUMENTED_RETURN_CONST': 252, - 'INSTRUMENTED_YIELD_VALUE': 253, + 'INSTRUMENTED_CALL_KW': 241, + 'INSTRUMENTED_CALL_FUNCTION_EX': 242, + 'INSTRUMENTED_INSTRUCTION': 243, + 'INSTRUMENTED_JUMP_FORWARD': 244, + 'INSTRUMENTED_JUMP_BACKWARD': 245, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 246, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 247, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 248, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 249, + 'INSTRUMENTED_RETURN_VALUE': 250, + 'INSTRUMENTED_RETURN_CONST': 251, + 'INSTRUMENTED_YIELD_VALUE': 252, + 'INSTRUMENTED_CALL': 253, 'JUMP': 256, 'JUMP_NO_INTERRUPT': 257, 'LOAD_CLOSURE': 258, diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index a07be306986b43..1a129b9432e72d 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1575,7 +1575,7 @@ def f(): ('line', 'method', 2), ('line', 'method', 3), ('line', 'method', 2), - ('call', 'method', 1), + ('call', 'method', d["b"]), ('line', 'method', 1), ('line', 'method', 1), ('line', 'get_events', 11), diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 971397c955de09..871e2dbf358418 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3241,20 +3241,6 @@ dummy_func( unused/1 + _LOAD_ATTR_METHOD_LAZY_DICT; - inst(INSTRUMENTED_CALL, (unused/3 -- )) { - int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - ERROR_IF(err, error); - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL); - } - // Cache layout: counter/1, func_version/2 // CALL_INTRINSIC_1/2, CALL_KW, and CALL_FUNCTION_EX aren't members! family(CALL, INLINE_CACHE_ENTRIES_CALL) = { @@ -3292,27 +3278,33 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + } + // When calling Python, inline the call using DISPATCH_INLINED(). - op(_CALL, (callable, self_or_null, args[oparg] -- res)) { + op(_DO_CALL, (callable, self_or_null, args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { - args--; - total_args++; - } - else if (Py_TYPE(callable_o) == &PyMethod_Type) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; } // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && @@ -3376,7 +3368,28 @@ dummy_func( CHECK_EVAL_BREAKER(); } - macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC; + op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) { + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + ERROR_IF(err, error); + } + + macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC; + macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC; op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 288e0f9135c27e..1ced8b951b5ce9 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3584,15 +3584,45 @@ break; } - /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 because it is instrumented */ + case _MAYBE_EXPAND_METHOD: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef func; + _PyStackRef maybe_self; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + break; + } - /* _CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + /* _DO_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _CHECK_PERIODIC: { CHECK_EVAL_BREAKER(); break; } + /* _MONITOR_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _PY_FRAME_GENERAL: { _PyStackRef *args; _PyStackRef self_or_null; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 634053a93837c6..76d1cc7ad6cf95 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -773,6 +773,8 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; + _PyStackRef func; + _PyStackRef maybe_self; _PyStackRef res; // _SPECIALIZE_CALL self_or_null = stack_pointer[-1 - oparg]; @@ -792,26 +794,34 @@ #endif /* ENABLE_SPECIALIZATION */ } /* Skip 2 cache entries */ - // _CALL + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + } + // _DO_CALL + self_or_null = maybe_self; + callable = func; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { - args--; - total_args++; - } - else if (Py_TYPE(callable_o) == &PyMethod_Type) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; } // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && @@ -3504,18 +3514,134 @@ (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef res; /* Skip 3 cache entries */ - int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL); + // _MAYBE_EXPAND_METHOD + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + } + // _MONITOR_CALL + { + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + if (err) goto error; + } + // _DO_CALL + self_or_null = maybe_self; + callable = func; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); + DISPATCH(); } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 224aeb834935eb..6b5f231e13d15a 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -240,7 +240,6 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, &&TARGET_INSTRUMENTED_FOR_ITER, - &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_CALL_KW, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_INSTRUCTION, @@ -253,6 +252,7 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_RETURN_VALUE, &&TARGET_INSTRUMENTED_RETURN_CONST, &&TARGET_INSTRUMENTED_YIELD_VALUE, + &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_LINE, &&_unknown_opcode, }; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 3c9e6d3043cde1..166b1674bc3334 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1598,14 +1598,28 @@ break; } - /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + case _MAYBE_EXPAND_METHOD: { + _Py_UopsSymbol *func; + _Py_UopsSymbol *maybe_self; + _Py_UopsSymbol **args; + func = sym_new_not_null(ctx); + maybe_self = sym_new_not_null(ctx); + for (int _i = oparg; --_i >= 0;) { + args[_i] = sym_new_not_null(ctx); + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + break; + } - /* _CALL is not a viable micro-op for tier 2 */ + /* _DO_CALL is not a viable micro-op for tier 2 */ case _CHECK_PERIODIC: { break; } + /* _MONITOR_CALL is not a viable micro-op for tier 2 */ + case _PY_FRAME_GENERAL: { _Py_UopsSymbol **args; _Py_UopsSymbol *self_or_null;