diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index b472d5d446b246..0318c0d0d672a7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -263,7 +263,7 @@ PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subjec PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); -PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); +PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArrayBorrow(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); PyAPI_FUNC(void) _PyObjectArray_Free(PyObject **array, PyObject **scratch); diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index d3a5be000fbce7..d3d745574667aa 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -128,6 +128,13 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * // Don't leave a dangling pointer to the old frame when creating generators // and coroutines: dest->previous = NULL; + +#ifdef Py_GIL_DISABLED + PyCodeObject *co = (PyCodeObject *)dest->f_executable; + for (int i = stacktop; i < co->co_nlocalsplus + co->co_stacksize; i++) { + dest->localsplus[i] = PyStackRef_NULL; + } +#endif } /* Consumes reference to func and locals. @@ -153,6 +160,16 @@ _PyFrame_Initialize( for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = PyStackRef_NULL; } + +#ifdef Py_GIL_DISABLED + // On GIL disabled, we walk the entire stack in GC. Since stacktop + // is not always in sync with the real stack pointer, we have + // no choice but to traverse the entire stack. + // This just makes sure we don't pass the GC invalid stack values. + for (int i = code->co_nlocalsplus; i < code->co_nlocalsplus + code->co_stacksize; i++) { + frame->localsplus[i] = PyStackRef_NULL; + } +#endif } /* Gets the pointer to the locals array @@ -314,6 +331,13 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->instr_ptr = _PyCode_CODE(code); frame->owner = FRAME_OWNED_BY_THREAD; frame->return_offset = 0; + +#ifdef Py_GIL_DISABLED + assert(code->co_nlocalsplus == 0); + for (int i = 0; i < code->co_stacksize; i++) { + frame->localsplus[i] = PyStackRef_NULL; + } +#endif return frame; } diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 28e34d3809634c..615dc60fa27ac1 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -388,6 +388,9 @@ extern void _Py_RunGC(PyThreadState *tstate); extern void _PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp); #endif +// Functions to clear generator frames +extern int _PyGC_VisitFrameStack(struct _PyInterpreterFrame *frame, visitproc visit, void *arg); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 73695d10e0c372..30c9ffd4fe3f1b 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_freelist.h" // _PyFreeListState +#include "pycore_stackref.h" // _PyStackRef PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); @@ -59,6 +60,8 @@ typedef struct { } _PyListIterObject; PyAPI_FUNC(PyObject *)_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n); +PyAPI_FUNC(PyObject *)_PyList_FromStackRefSteal(const _PyStackRef *src, Py_ssize_t n); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 8d3d559814bfd9..44d5f171c04c3a 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -142,6 +142,8 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj) // Converts a PyObject * to a PyStackRef, with a new reference +// IMPORTANT: The result of this operation must be immediately assigned to localsplus. +// There must be no interfering Py_DECREF calls or such between this operation and that. #ifdef Py_GIL_DISABLED static inline _PyStackRef PyStackRef_FromPyObjectNew(PyObject *obj) @@ -149,8 +151,7 @@ PyStackRef_FromPyObjectNew(PyObject *obj) // Make sure we don't take an already tagged value. assert(((uintptr_t)obj & Py_TAG_BITS) == 0); assert(obj != NULL); - // TODO (gh-117139): Add deferred objects later. - if (_Py_IsImmortal(obj)) { + if (_Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj)) { return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; } else { @@ -219,7 +220,8 @@ PyStackRef_DUP(_PyStackRef stackref) { if (PyStackRef_IsDeferred(stackref)) { assert(PyStackRef_IsNull(stackref) || - _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref))); + _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)) || + _PyObject_HasDeferredRefcount(PyStackRef_AsPyObjectBorrow(stackref))); return stackref; } Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); diff --git a/Objects/listobject.c b/Objects/listobject.c index f29f58dc25be04..df5d710bc5fe9a 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3203,6 +3203,29 @@ _PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n) return (PyObject *)list; } +PyObject * +_PyList_FromStackRefSteal(const _PyStackRef *src, Py_ssize_t n) +{ + if (n == 0) { + return PyList_New(0); + } + + PyListObject *list = (PyListObject *)PyList_New(n); + if (list == NULL) { + for (Py_ssize_t i = 0; i < n; i++) { + PyStackRef_CLOSE(src[i]); + } + return NULL; + } + + PyObject **dst = list->ob_item; + for (Py_ssize_t i = 0; i < n; i++) { + dst[i] = PyStackRef_AsPyObjectSteal(src[i]); + } + + return (PyObject *)list; +} + /*[clinic input] list.index diff --git a/Objects/object.c b/Objects/object.c index e2f96af54778ce..1eddc55cc0ffa3 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2432,13 +2432,6 @@ _PyObject_SetDeferredRefcount(PyObject *op) assert(_Py_IsOwnedByCurrentThread(op)); assert(op->ob_ref_shared == 0); _PyObject_SET_GC_BITS(op, _PyGC_BITS_DEFERRED); - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (_Py_atomic_load_int_relaxed(&interp->gc.immortalize) == 1) { - // gh-117696: immortalize objects instead of using deferred reference - // counting for now. - _Py_SetImmortal(op); - return; - } op->ob_ref_local += 1; op->ob_ref_shared = _Py_REF_QUEUED; #endif diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 142f97daeb0820..bc4f57df2e8e01 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -843,12 +843,11 @@ dummy_func( } inst(STORE_SUBSCR_DICT, (unused/1, value, dict_st, sub_st -- )) { - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub_st), PyStackRef_AsPyObjectSteal(value)); PyStackRef_CLOSE(dict_st); ERROR_IF(err, error); } @@ -1767,7 +1766,7 @@ dummy_func( } inst(BUILD_STRING, (pieces[oparg] -- str)) { - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + STACKREFS_TO_PYOBJECTS_BORROW(pieces, oparg, pieces_o); if (CONVERSION_FAILED(pieces_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -1786,13 +1785,7 @@ dummy_func( } inst(BUILD_LIST, (values[oparg] -- list)) { - STACKREFS_TO_PYOBJECTS(values, oparg, values_o); - if (CONVERSION_FAILED(values_o)) { - DECREF_INPUTS(); - ERROR_IF(true, error); - } - PyObject *list_o = _PyList_FromArraySteal(values_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); ERROR_IF(list_o == NULL, error); list = PyStackRef_FromPyObjectSteal(list_o); } @@ -1832,11 +1825,11 @@ dummy_func( } int err = 0; for (int i = 0; i < oparg; i++) { - PyObject *item = PyStackRef_AsPyObjectSteal(values[i]); + PyObject *item = PyStackRef_AsPyObjectBorrow(values[i]); if (err == 0) { err = PySet_Add(set_o, item); } - Py_DECREF(item); + PyStackRef_CLOSE(values[i]); } if (err != 0) { Py_DECREF(set_o); @@ -1846,7 +1839,7 @@ dummy_func( } inst(BUILD_MAP, (values[oparg*2] -- map)) { - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + STACKREFS_TO_PYOBJECTS_BORROW(values, oparg*2, values_o); if (CONVERSION_FAILED(values_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -1889,7 +1882,7 @@ dummy_func( assert(PyTuple_CheckExact(keys_o)); assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); - STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + STACKREFS_TO_PYOBJECTS_BORROW(values, oparg, values_o); if (CONVERSION_FAILED(values_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -3103,7 +3096,7 @@ dummy_func( inst(LOAD_SPECIAL, (owner -- attr, self_or_null)) { assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *self_or_null_o; attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); @@ -3381,9 +3374,12 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } ERROR_IF(true, error); } PyObject *res_o = PyObject_Vectorcall( @@ -3513,7 +3509,7 @@ dummy_func( total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -3747,7 +3743,7 @@ dummy_func( PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -3817,7 +3813,7 @@ dummy_func( STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -3861,7 +3857,7 @@ dummy_func( (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -4025,7 +4021,7 @@ dummy_func( PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, nargs, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -4106,7 +4102,7 @@ dummy_func( (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, nargs, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -4190,7 +4186,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); diff --git a/Python/ceval.c b/Python/ceval.c index 97d4b82e05d74d..4cb595bc177cef 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -698,7 +698,7 @@ extern void _PyUOpPrint(const _PyUOpInstruction *uop); PyObject ** -_PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch) +_PyObjectArray_FromStackRefArrayBorrow(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch) { PyObject **result; if (nargs > MAX_STACKREF_SCRATCH) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 595b72bfaf9613..a0260b9d5349b0 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -455,12 +455,12 @@ do { \ #define MAX_STACKREF_SCRATCH 10 #ifdef Py_GIL_DISABLED -#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ +#define STACKREFS_TO_PYOBJECTS_BORROW(ARGS, ARG_COUNT, NAME) \ /* +1 because vectorcall might use -1 to write self */ \ PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ - PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); + PyObject **NAME = _PyObjectArray_FromStackRefArrayBorrow(ARGS, ARG_COUNT, NAME##_temp + 1); #else -#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ +#define STACKREFS_TO_PYOBJECTS_BORROW(ARGS, ARG_COUNT, NAME) \ PyObject **NAME = (PyObject **)ARGS; \ assert(NAME != NULL); #endif diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 375e1fb05840a4..2d5aba3bb311b9 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -173,6 +173,9 @@ _PyStackRef value; oparg = CURRENT_OPARG(); value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = value; + #endif /* flush specials */ stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1012,14 +1015,13 @@ sub_st = stack_pointer[-1]; dict_st = stack_pointer[-2]; value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub_st), PyStackRef_AsPyObjectSteal(value)); PyStackRef_CLOSE(dict_st); if (err) JUMP_TO_ERROR(); stack_pointer += -3; @@ -1400,7 +1402,13 @@ } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = val0; + #endif /* flush specials */ val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = val1; + #endif /* flush specials */ PyStackRef_CLOSE(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; @@ -1428,6 +1436,8 @@ PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); + #ifdef Py_GIL_DISABLED /* flush specials */ + #endif /* flush specials */ } PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; @@ -1454,6 +1464,8 @@ PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); + #ifdef Py_GIL_DISABLED /* flush specials */ + #endif /* flush specials */ } PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; @@ -1542,7 +1554,11 @@ "no locals found"); if (true) JUMP_TO_ERROR(); } - locals = PyStackRef_FromPyObjectNew(l);; + locals = PyStackRef_FromPyObjectNew(l); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = locals; + #endif /* flush specials */ + ; stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1791,7 +1807,7 @@ _PyStackRef str; oparg = CURRENT_OPARG(); pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + STACKREFS_TO_PYOBJECTS_BORROW(pieces, oparg, pieces_o); if (CONVERSION_FAILED(pieces_o)) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(pieces[_i]); @@ -1830,15 +1846,7 @@ _PyStackRef list; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(values, oparg, values_o); - if (CONVERSION_FAILED(values_o)) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - if (true) JUMP_TO_ERROR(); - } - PyObject *list_o = _PyList_FromArraySteal(values_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); if (list_o == NULL) JUMP_TO_ERROR(); list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; @@ -1897,7 +1905,7 @@ _PyStackRef map; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + STACKREFS_TO_PYOBJECTS_BORROW(values, oparg*2, values_o); if (CONVERSION_FAILED(values_o)) { for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); @@ -1954,7 +1962,7 @@ PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); assert(PyTuple_CheckExact(keys_o)); assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); - STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + STACKREFS_TO_PYOBJECTS_BORROW(values, oparg, values_o); if (CONVERSION_FAILED(values_o)) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); @@ -2371,6 +2379,9 @@ STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; @@ -2393,6 +2404,9 @@ STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; @@ -2430,6 +2444,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; @@ -2446,6 +2463,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; @@ -3075,6 +3095,9 @@ assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = next; + #endif /* flush specials */ stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3122,6 +3145,9 @@ assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = next; + #endif /* flush specials */ stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3207,7 +3233,7 @@ oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *self_or_null_o; attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); @@ -3329,6 +3355,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -3350,6 +3379,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -3369,6 +3401,9 @@ assert(descr != NULL); PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ stack_pointer[-1] = attr; break; } @@ -3385,6 +3420,9 @@ assert(descr != NULL); PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ stack_pointer[-1] = attr; break; } @@ -3415,6 +3453,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -3527,7 +3568,14 @@ assert(PyStackRef_IsNull(null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1 - oparg] = self; + #endif /* flush specials */ method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-2 - oparg] = method; + #endif /* flush specials */ + stack_pointer[-2 - oparg] = method; assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); PyStackRef_CLOSE(callable); stack_pointer[-2 - oparg] = method; @@ -3571,7 +3619,7 @@ total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -3624,7 +3672,13 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1 - oparg] = self; + #endif /* flush specials */ func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-2 - oparg] = func; + #endif /* flush specials */ PyStackRef_CLOSE(callable); stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = self; @@ -3979,7 +4033,7 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -4080,7 +4134,7 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -4137,7 +4191,7 @@ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -4346,7 +4400,7 @@ int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, nargs, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -4463,7 +4517,7 @@ PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, nargs, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -4874,6 +4928,9 @@ _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); value = PyStackRef_FromPyObjectNew(ptr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = value; + #endif /* flush specials */ stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4906,6 +4963,9 @@ _PyStackRef null; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); value = PyStackRef_FromPyObjectNew(ptr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = value; + #endif /* flush specials */ null = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer[1] = null; diff --git a/Python/frame.c b/Python/frame.c index 25fa2824630f9b..3192968a0fb1b5 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -15,15 +15,7 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) Py_VISIT(frame->f_locals); Py_VISIT(frame->f_funcobj); Py_VISIT(_PyFrame_GetCode(frame)); - /* locals */ - _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); - _PyStackRef *sp = frame->stackpointer; - /* locals and stack */ - while (sp > locals) { - sp--; - Py_VISIT(PyStackRef_AsPyObjectBorrow(*sp)); - } - return 0; + return _PyGC_VisitFrameStack(frame, visit, arg); } PyFrameObject * diff --git a/Python/gc.c b/Python/gc.c index 38a0da91a97510..291c1079f5f9c3 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -534,6 +534,17 @@ visit_decref(PyObject *op, void *parent) return 0; } +int +_PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) +{ + _PyStackRef *i = frame->localsplus; + /* locals and stack */ + for (; i < frame->stackpointer; i++) { + Py_VISIT(PyStackRef_AsPyObjectBorrow(*i)); + } + return 0; +} + /* Subtract internal references from gc_refs. After this, gc_refs is >= 0 * for all objects in containers, and is GC_REACHABLE for all tracked gc * objects not in containers. The ones with gc_refs > 0 are directly diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index d1d5664ab96f33..e86dda29dddd24 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -302,6 +302,40 @@ gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor, return err; } +static inline void +gc_visit_stackref(_PyStackRef stackref) +{ + // Note: we MUST check that it is deferred before checking the rest. + // Otherwise we might read into invalid memory due to non-deferred references + // being dead already. + if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNull(stackref)) { + PyObject *curr_o = PyStackRef_AsPyObjectBorrow(stackref); + if (_PyObject_GC_IS_TRACKED(curr_o)) { + gc_add_refs(curr_o, 1); + } + } +} + +static void +gc_visit_thread_stacks(PyInterpreterState *interp) +{ + HEAD_LOCK(&_PyRuntime); + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + _PyInterpreterFrame *curr_frame = p->current_frame; + while (curr_frame != NULL) { + PyCodeObject *co = (PyCodeObject *)curr_frame->f_executable; + if (co != NULL && PyCode_Check(co)) { + for (int i = 0; + i < co->co_nlocalsplus + co->co_stacksize; i++) { + gc_visit_stackref(curr_frame->localsplus[i]); + } + } + curr_frame = curr_frame->previous; + } + } + HEAD_UNLOCK(&_PyRuntime); +} + static void merge_queued_objects(_PyThreadStateImpl *tstate, struct collection_state *state) { @@ -604,6 +638,8 @@ deduce_unreachable_heap(PyInterpreterState *interp, gc_visit_heaps(interp, &validate_gc_objects, &state->base); #endif + gc_visit_thread_stacks(interp); + // Transitively mark reachable objects by clearing the // _PyGC_BITS_UNREACHABLE flag. if (gc_visit_heaps(interp, &mark_heap_visitor, &state->base) < 0) { @@ -659,6 +695,18 @@ clear_weakrefs(struct collection_state *state) { PyObject *op; WORKSTACK_FOR_EACH(&state->unreachable, op) { + if (PyGen_CheckExact(op) || + PyCoro_CheckExact(op) || + PyAsyncGen_CheckExact(op)) { + // Ensure any non-refcounted pointers to cyclic trash are converted + // to refcounted pointers. This prevents bugs where the generator is + // freed after its function object. + PyGenObject *gen = (PyGenObject *)op; + struct _PyInterpreterFrame *frame = &(gen->gi_iframe); + for (_PyStackRef *i = frame->localsplus; i < frame->stackpointer; i++) { + gc_visit_stackref(*i); + } + } if (PyWeakref_Check(op)) { // Clear weakrefs that are themselves unreachable to ensure their // callbacks will not be executed later from a `tp_clear()` @@ -879,7 +927,7 @@ show_stats_each_generations(GCState *gcstate) } // Traversal callback for handle_resurrected_objects. -static int +int visit_decref_unreachable(PyObject *op, void *data) { if (gc_is_unreachable(op) && _PyObject_GC_IS_TRACKED(op)) { @@ -888,6 +936,21 @@ visit_decref_unreachable(PyObject *op, void *data) return 0; } +int +_PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) +{ + _PyStackRef *i = frame->localsplus; + /* locals and stack */ + for (; i < frame->stackpointer; i++) { + if (PyStackRef_IsDeferred(*i) && + (visit == visit_decref || visit == visit_decref_unreachable)) { + continue; + } + Py_VISIT(PyStackRef_AsPyObjectBorrow(*i)); + } + return 0; +} + // Handle objects that may have resurrected after a call to 'finalize_garbage'. static int handle_resurrected_objects(struct collection_state *state) @@ -1834,32 +1897,6 @@ custom_visitor_wrapper(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } -// gh-117783: Immortalize objects that use deferred reference counting to -// temporarily work around scaling bottlenecks. -static bool -immortalize_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, - void *block, size_t block_size, void *args) -{ - PyObject *op = op_from_block(block, args, false); - if (op != NULL && _PyObject_HasDeferredRefcount(op)) { - _Py_SetImmortal(op); - op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED; - } - return true; -} - -void -_PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp) -{ - struct visitor_args args; - _PyEval_StopTheWorld(interp); - if (interp->gc.immortalize == 0) { - gc_visit_heaps(interp, &immortalize_visitor, &args); - interp->gc.immortalize = 1; - } - _PyEval_StartTheWorld(interp); -} - void PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 382288440a7a2c..4afcee4b40a336 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -601,7 +601,7 @@ PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); assert(PyTuple_CheckExact(keys_o)); assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); - STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + STACKREFS_TO_PYOBJECTS_BORROW(values, oparg, values_o); if (CONVERSION_FAILED(values_o)) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); @@ -632,15 +632,7 @@ _PyStackRef *values; _PyStackRef list; values = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(values, oparg, values_o); - if (CONVERSION_FAILED(values_o)) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - if (true) { stack_pointer += -oparg; goto error; } - } - PyObject *list_o = _PyList_FromArraySteal(values_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + PyObject *list_o = _PyList_FromStackRefSteal(values, oparg); if (list_o == NULL) { stack_pointer += -oparg; goto error; } list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; @@ -656,7 +648,7 @@ _PyStackRef *values; _PyStackRef map; values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + STACKREFS_TO_PYOBJECTS_BORROW(values, oparg*2, values_o); if (CONVERSION_FAILED(values_o)) { for (int _i = oparg*2; --_i >= 0;) { PyStackRef_CLOSE(values[_i]); @@ -692,11 +684,11 @@ } int err = 0; for (int i = 0; i < oparg; i++) { - PyObject *item = PyStackRef_AsPyObjectSteal(values[i]); + PyObject *item = PyStackRef_AsPyObjectBorrow(values[i]); if (err == 0) { err = PySet_Add(set_o, item); } - Py_DECREF(item); + PyStackRef_CLOSE(values[i]); } if (err != 0) { Py_DECREF(set_o); @@ -742,7 +734,7 @@ _PyStackRef *pieces; _PyStackRef str; pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + STACKREFS_TO_PYOBJECTS_BORROW(pieces, oparg, pieces_o); if (CONVERSION_FAILED(pieces_o)) { for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(pieces[_i]); @@ -859,12 +851,11 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); - PyStackRef_CLOSE(self_or_null); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); } if (true) { stack_pointer += -2 - oparg; goto error; } } @@ -1001,7 +992,13 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1 - oparg] = self; + #endif /* flush specials */ func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-2 - oparg] = func; + #endif /* flush specials */ PyStackRef_CLOSE(callable); } // flush @@ -1110,7 +1107,14 @@ assert(PyStackRef_IsNull(null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1 - oparg] = self; + #endif /* flush specials */ method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-2 - oparg] = method; + #endif /* flush specials */ + stack_pointer[-2 - oparg] = method; assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); PyStackRef_CLOSE(callable); } @@ -1197,7 +1201,7 @@ PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -1254,7 +1258,7 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -1317,7 +1321,7 @@ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -1631,7 +1635,7 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -1789,7 +1793,7 @@ PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, nargs, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -1852,7 +1856,7 @@ int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, nargs, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -2026,7 +2030,7 @@ total_args++; } /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + STACKREFS_TO_PYOBJECTS_BORROW(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null); @@ -2393,6 +2397,9 @@ assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-2] = value; + #endif /* flush specials */ PyStackRef_CLOSE(sub_iter_st); PyStackRef_CLOSE(last_sent_val_st); PyStackRef_CLOSE(exc_value_st); @@ -3185,6 +3192,9 @@ assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = next; + #endif /* flush specials */ } stack_pointer[0] = next; stack_pointer += 1; @@ -3278,6 +3288,9 @@ assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = next; + #endif /* flush specials */ } stack_pointer[0] = next; stack_pointer += 1; @@ -4120,6 +4133,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } @@ -4244,6 +4260,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ self = owner; } stack_pointer[-1] = attr; @@ -4280,6 +4299,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ self = owner; } stack_pointer[-1] = attr; @@ -4328,6 +4350,9 @@ assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ self = owner; } stack_pointer[-1] = attr; @@ -4406,6 +4431,9 @@ assert(descr != NULL); PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ } stack_pointer[-1] = attr; DISPATCH(); @@ -4448,6 +4476,9 @@ assert(descr != NULL); PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ } stack_pointer[-1] = attr; DISPATCH(); @@ -4514,6 +4545,9 @@ STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; attr = PyStackRef_FromPyObjectNew(attr_o); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = attr; + #endif /* flush specials */ PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ @@ -4629,6 +4663,9 @@ INSTRUCTION_STATS(LOAD_CONST); _PyStackRef value; value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = value; + #endif /* flush specials */ stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4960,7 +4997,11 @@ "no locals found"); if (true) goto error; } - locals = PyStackRef_FromPyObjectNew(l);; + locals = PyStackRef_FromPyObjectNew(l); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = locals; + #endif /* flush specials */ + ; stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5015,7 +5056,7 @@ _PyStackRef self_or_null; owner = stack_pointer[-1]; assert(oparg <= SPECIAL_MAX); - PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *self_or_null_o; attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); @@ -5644,6 +5685,9 @@ // _LOAD_CONST { value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = value; + #endif /* flush specials */ } // _RETURN_VALUE retval = value; @@ -6299,11 +6343,10 @@ sub_st = stack_pointer[-1]; dict_st = stack_pointer[-2]; value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub_st), PyStackRef_AsPyObjectSteal(value)); PyStackRef_CLOSE(dict_st); if (err) goto pop_3_error; stack_pointer += -3; @@ -6637,6 +6680,8 @@ PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); + #ifdef Py_GIL_DISABLED /* flush specials */ + #endif /* flush specials */ } PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; @@ -6661,6 +6706,8 @@ PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); + #ifdef Py_GIL_DISABLED /* flush specials */ + #endif /* flush specials */ } PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; @@ -6684,7 +6731,13 @@ DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[0] = val0; + #endif /* flush specials */ val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + #ifdef Py_GIL_DISABLED /* flush specials */ + stack_pointer[-1] = val1; + #endif /* flush specials */ PyStackRef_CLOSE(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; diff --git a/Python/pystate.c b/Python/pystate.c index 7a272de11ec761..4f2e0af3b3aa79 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1584,13 +1584,6 @@ new_threadstate(PyInterpreterState *interp, int whence) PyMem_RawFree(new_tstate); } else { -#ifdef Py_GIL_DISABLED - if (_Py_atomic_load_int(&interp->gc.immortalize) == 0) { - // Immortalize objects marked as using deferred reference counting - // the first time a non-main thread is created. - _PyGC_ImmortalizeDeferredObjects(interp); - } -#endif } #ifdef Py_GIL_DISABLED diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c6f044e066539e..7776b961789f26 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -458,12 +458,13 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "PyFloat_AS_DOUBLE", "_PyFrame_PushUnchecked", "Py_FatalError", - "STACKREFS_TO_PYOBJECTS", + "STACKREFS_TO_PYOBJECTS_BORROW", "STACKREFS_TO_PYOBJECTS_CLEANUP", "CONVERSION_FAILED", "_PyList_FromArraySteal", "_PyTuple_FromArraySteal", "_PyTuple_FromStackRefSteal", + "_PyList_FromStackRefSteal", ) ESCAPING_FUNCTIONS = ( diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 9314bb9e79687f..4517fcf31c1429 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -177,6 +177,35 @@ def replace_check_eval_breaker( if not uop.properties.ends_with_eval_breaker: out.emit_at("CHECK_EVAL_BREAKER();", tkn) +def replace_pystackref_frompyobjectnew( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + out.emit(tkn) + emit_to(out, tkn_iter, "SEMI") + out.emit(";\n") + target = uop.body.index(tkn) + assert uop.body[target-1].kind == "EQUALS", (f"{uop.name} Result of a specials that is flushed" + " must be written to a stack variable directly") + # Scan to look for the first thing it's assigned to + found_valid_assignment = "" + for assgn_target in reversed(uop.body[:target-1]): + out_names = [out.name for out in uop.stack.outputs] + if assgn_target.kind == "IDENTIFIER": + if assgn_target.text in out_names: + found_valid_assignment = assgn_target.text + break + + if found_valid_assignment: + out.start_line() + out.emit("#ifdef Py_GIL_DISABLED /* flush specials */\n") + stack.write_variable_to_stack(out, found_valid_assignment) + out.emit("#endif /* flush specials */\n") + REPLACEMENT_FUNCTIONS = { "EXIT_IF": replace_deopt, @@ -186,6 +215,7 @@ def replace_check_eval_breaker( "DECREF_INPUTS": replace_decrefs, "CHECK_EVAL_BREAKER": replace_check_eval_breaker, "SYNC_SP": replace_sync_sp, + "PyStackRef_FromPyObjectNew": replace_pystackref_frompyobjectnew, } ReplacementFunctionType = Callable[ diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index f497fa34dfced4..cc484c73fe7720 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -213,6 +213,24 @@ def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = self.peek_offset.clear() out.start_line() + def write_variable_to_stack(self, out: CWriter, var_name: str, cast_type: str = "uintptr_t", extract_bits: bool = False) -> None: + out.start_line() + var = [variable for variable in self.variables if variable.name == var_name] + assert len(var) == 1, f"{var_name} not found" + to_write_var = var[0] + for var in self.variables: + if not var.peek: + cast = f"({cast_type})" if var.type else "" + bits = ".bits" if cast and not extract_bits else "" + if var.name not in UNUSED and not var.is_array(): + if var == to_write_var: + out.emit( + f"stack_pointer[{self.base_offset.to_c()}]{bits} = {cast}{var.name};\n" + ) + self.base_offset.push(var) + for var in self.variables: + self.base_offset.pop(var) + def as_comment(self) -> str: return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */"