From 01ef78c36673a2ca50f8bdf52ab16d7f1b308142 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Mar 2024 10:48:00 +0000 Subject: [PATCH] GH-116422: Modify a few uops so that they can be supported by tier 2 with hot/cold splitting (GH-116832) --- Include/internal/pycore_ceval.h | 9 +++++++-- Python/bytecodes.c | 34 +++++++++++++++------------------ Python/executor_cases.c.h | 32 +++++++++++++++---------------- Python/generated_cases.c.h | 32 +++++++++++++++---------------- 4 files changed, 52 insertions(+), 55 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 6eab2ba1daedf8f..946f82ae3c20e38 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -138,12 +138,12 @@ extern void _PyEval_DeactivateOpCache(void); /* With USE_STACKCHECK macro defined, trigger stack checks in _Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */ static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return (tstate->c_recursion_remaining-- <= 0 + return (tstate->c_recursion_remaining-- < 0 || (tstate->c_recursion_remaining & 63) == 0); } #else static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return tstate->c_recursion_remaining-- <= 0; + return tstate->c_recursion_remaining-- < 0; } #endif @@ -161,6 +161,11 @@ static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); } +static inline void _Py_EnterRecursiveCallTstateUnchecked(PyThreadState *tstate) { + assert(tstate->c_recursion_remaining > 0); + tstate->c_recursion_remaining--; +} + static inline int _Py_EnterRecursiveCall(const char *where) { PyThreadState *tstate = _PyThreadState_GET(); return _Py_EnterRecursiveCallTstate(tstate, where); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e8fcb6c6d0067c0..bf7782a65bc5ce6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3376,14 +3376,12 @@ dummy_func( DEOPT_IF(total_args != 1); DEOPT_IF(!PyCFunction_CheckExact(callable)); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } PyObject *arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3480,10 +3478,11 @@ dummy_func( } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(callable); Py_DECREF(arg); - ERROR_IF(res == NULL, error); } inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { @@ -3505,11 +3504,12 @@ dummy_func( } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - ERROR_IF(res == NULL, error); } // This is secretly a super-instruction @@ -3543,16 +3543,14 @@ dummy_func( DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0); PyObject *arg = args[1]; PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3616,13 +3614,11 @@ dummy_func( PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); DEOPT_IF(meth->ml_flags != METH_NOARGS); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 077499ebc016878..bdc9c0b1501c912 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3189,14 +3189,12 @@ if (total_args != 1) goto deoptimize; if (!PyCFunction_CheckExact(callable)) goto deoptimize; if (PyCFunction_GET_FLAGS(callable) != METH_O) goto deoptimize; + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } PyObject *arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3305,9 +3303,11 @@ } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3340,10 +3340,12 @@ } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3368,16 +3370,14 @@ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_O) goto deoptimize; + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) goto deoptimize; PyObject *arg = args[1]; PyObject *self = args[0]; if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3450,13 +3450,11 @@ PyObject *self = args[0]; if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; if (meth->ml_flags != METH_NOARGS) goto deoptimize; + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) goto deoptimize; STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 645d0fb510e6fb1..2fbd9ed403f5d11 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1174,14 +1174,12 @@ DEOPT_IF(total_args != 1, CALL); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } PyObject *arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1350,10 +1348,12 @@ } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; DISPATCH(); @@ -1481,9 +1481,11 @@ } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res == NULL) { + GOTO_ERROR(error); + } Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; DISPATCH(); @@ -1649,13 +1651,11 @@ PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1698,16 +1698,14 @@ DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); + // CPython promises to check all non-vectorcall function calls. + DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); PyObject *arg = args[1]; PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } + _Py_EnterRecursiveCallTstateUnchecked(tstate); res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));