diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 853bda4036b5165..b472d5d446b2464 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -261,8 +261,11 @@ PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *c PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); -PyAPI_FUNC(int) _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); +PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); + +PyAPI_FUNC(void) _PyObjectArray_Free(PyObject **array, PyObject **scratch); /* Bits that can be set in PyThreadState.eval_breaker */ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 48ff2701c8471d0..81caaa5abb2e932 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_stackref.h" // _PyStackRef #include "pycore_lock.h" // PyMutex #include "pycore_backoff.h" // _Py_BackoffCounter @@ -317,30 +318,30 @@ extern void _PyCode_Clear_Executors(PyCodeObject *code); /* Specialization functions */ -extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, +extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, _Py_CODEUNIT *instr, int load_method); -extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, +extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, +extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, +extern void _Py_Specialize_BinarySubscr(_PyStackRef sub, _PyStackRef container, _Py_CODEUNIT *instr); -extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, +extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, +extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); -extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, - int oparg, PyObject **locals); -extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, +extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, + int oparg, _PyStackRef *locals); +extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, +extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ContainsOp(PyObject *value, _Py_CODEUNIT *instr); +extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg); +extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); +extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); +extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); #ifdef Py_STATS diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index bab92c771a76b1b..1e0368faa5b5104 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -11,6 +11,7 @@ extern "C" { #include #include // offsetof() #include "pycore_code.h" // STATS +#include "pycore_stackref.h" // _PyStackRef /* See Objects/frame_layout.md for an explanation of the frame stack * including explanation of the PyFrameObject and _PyInterpreterFrame @@ -67,7 +68,7 @@ typedef struct _PyInterpreterFrame { uint16_t return_offset; /* Only relevant during a function call */ char owner; /* Locals and stack */ - PyObject *localsplus[1]; + _PyStackRef localsplus[1]; } _PyInterpreterFrame; #define _PyInterpreterFrame_LASTI(IF) \ @@ -78,23 +79,23 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { return (PyCodeObject *)f->f_executable; } -static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) { - return f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus; +static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { + return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); } -static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) { +static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - assert(f->localsplus[f->stacktop-1] != NULL); + assert(!PyStackRef_IsNull(f->localsplus[f->stacktop-1])); return f->localsplus[f->stacktop-1]; } -static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) { +static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) { assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); f->stacktop--; return f->localsplus[f->stacktop]; } -static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { +static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, _PyStackRef value) { f->localsplus[f->stacktop] = value; f->stacktop++; } @@ -143,14 +144,14 @@ _PyFrame_Initialize( frame->owner = FRAME_OWNED_BY_THREAD; for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { - frame->localsplus[i] = NULL; + frame->localsplus[i] = PyStackRef_NULL; } } /* Gets the pointer to the locals array * that precedes this frame. */ -static inline PyObject** +static inline _PyStackRef* _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) { return frame->localsplus; @@ -160,16 +161,16 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) Having stacktop <= 0 ensures that invalid values are not visible to the cycle GC. We choose -1 rather than 0 to assist debugging. */ -static inline PyObject** +static inline _PyStackRef* _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - PyObject **sp = frame->localsplus + frame->stacktop; + _PyStackRef *sp = frame->localsplus + frame->stacktop; frame->stacktop = -1; return sp; } static inline void -_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) +_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { frame->stacktop = (int)(stack_pointer - frame->localsplus); } @@ -309,7 +310,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int PyAPI_FUNC(_PyInterpreterFrame *) _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, PyObject* const* args, + PyObject *locals, _PyStackRef const* args, size_t argcount, PyObject *kwnames); #ifdef __cplusplus diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 17bd23f0752be20..4d6cc35a7a3de74 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -11,7 +11,7 @@ extern "C" { #ifdef _Py_JIT -typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate); +typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index ebb0f30b5f73779..9c963d8970d6651 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -159,21 +159,6 @@ static inline void _Py_ClearImmortal(PyObject *op) op = NULL; \ } while (0) -// Mark an object as supporting deferred reference counting. This is a no-op -// in the default (with GIL) build. Objects that use deferred reference -// counting should be tracked by the GC so that they are eventually collected. -extern void _PyObject_SetDeferredRefcount(PyObject *op); - -static inline int -_PyObject_HasDeferredRefcount(PyObject *op) -{ -#ifdef Py_GIL_DISABLED - return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_DEFERRED); -#else - return 0; -#endif -} - #if !defined(Py_GIL_DISABLED) static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) diff --git a/Include/internal/pycore_object_deferred.h b/Include/internal/pycore_object_deferred.h new file mode 100644 index 000000000000000..c070d768b7d771c --- /dev/null +++ b/Include/internal/pycore_object_deferred.h @@ -0,0 +1,32 @@ +#ifndef Py_INTERNAL_OBJECT_DEFERRED_H +#define Py_INTERNAL_OBJECT_DEFERRED_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "pycore_gc.h" + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Mark an object as supporting deferred reference counting. This is a no-op +// in the default (with GIL) build. Objects that use deferred reference +// counting should be tracked by the GC so that they are eventually collected. +extern void _PyObject_SetDeferredRefcount(PyObject *op); + +static inline int +_PyObject_HasDeferredRefcount(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_DEFERRED); +#else + return 0; +#endif +} + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_OBJECT_DEFERRED_H diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 04a029db0387002..1007f838b7e3744 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -271,7 +271,7 @@ extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr); +PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyStackRef *stack_pointer, _PyExecutorObject **exec_ptr); #ifdef __cplusplus } diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 93898174789f7b3..32e445dd96f9a16 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -8,187 +8,264 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_object_deferred.h" + #include +/* + This file introduces a new API for handling references on the stack, called + _PyStackRef. This API is inspired by HPy. + + There are 3 main operations, that convert _PyStackRef to PyObject* and + vice versa: + + 1. Borrow (discouraged) + 2. Steal + 3. New + + Borrow means that the reference is converted without any change in ownership. + This is discouraged because it makes verification much harder. It also makes + unboxed integers harder in the future. + + Steal means that ownership is transferred to something else. The total + number of references to the object stays the same. + + New creates a new reference from the old reference. The old reference + is still valid. + + With these 3 API, a strict stack discipline must be maintained. All + _PyStackRef must be operated on by the new reference operations: + + 1. DUP + 2. CLOSE + + DUP is roughly equivalent to Py_NewRef. It creates a new reference from an old + reference. The old reference remains unchanged. + + CLOSE is roughly equivalent to Py_DECREF. It destroys a reference. + + Note that it is unsafe to borrow a _PyStackRef and then do normal + CPython refcounting operations on it! +*/ + typedef union { uintptr_t bits; } _PyStackRef; -static const _PyStackRef Py_STACKREF_NULL = { .bits = 0 }; #define Py_TAG_DEFERRED (1) +#define Py_TAG_PTR (0) +#define Py_TAG_BITS (1) + +#ifdef Py_GIL_DISABLED + static const _PyStackRef PyStackRef_NULL = { .bits = 0 | Py_TAG_DEFERRED}; +#else + static const _PyStackRef PyStackRef_NULL = { .bits = 0 }; +#endif + +#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) + + +#ifdef Py_GIL_DISABLED +# define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) +#else +# define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) }) +#endif + +#ifdef Py_GIL_DISABLED +# define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) +#else +# define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) }) +#endif + +#ifdef Py_GIL_DISABLED +# define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED }) +#else +# define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) }) +#endif + + +static inline int +PyStackRef_Is(_PyStackRef a, _PyStackRef b) { + return a.bits == b.bits; +} + +static inline int +PyStackRef_IsDeferred(_PyStackRef ref) +{ + return ((ref.bits & Py_TAG_BITS) == Py_TAG_DEFERRED); +} + // Gets a PyObject * from a _PyStackRef -#if defined(Py_GIL_DISABLED) static inline PyObject * -PyStackRef_Get(_PyStackRef tagged) +PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) { - PyObject *cleared = ((PyObject *)((tagged).bits & (~Py_TAG_DEFERRED))); +#ifdef Py_GIL_DISABLED + PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS))); return cleared; +#else + return ((PyObject *)(stackref).bits); +#endif } + +// Converts a PyStackRef back to a PyObject *, stealing the +// PyStackRef. +static inline PyObject * +PyStackRef_AsPyObjectSteal(_PyStackRef stackref) +{ +#ifdef Py_GIL_DISABLED + if (!PyStackRef_IsNull(stackref) && PyStackRef_IsDeferred(stackref)) { + return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)); + } + return PyStackRef_AsPyObjectBorrow(stackref); #else -# define PyStackRef_Get(tagged) ((PyObject *)((tagged).bits)) + return PyStackRef_AsPyObjectBorrow(stackref); #endif +} + +// Converts a PyStackRef back to a PyObject *, converting the +// stackref to a new reference. +static inline PyObject * +PyStackRef_AsPyObjectNew(_PyStackRef stackref) +{ + return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline PyTypeObject * +PyStackRef_TYPE(_PyStackRef stackref) +{ + return Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)); +} -// Converts a PyObject * to a PyStackRef, stealing the reference. -#if defined(Py_GIL_DISABLED) +// Converts a PyObject * to a PyStackRef, stealing the reference static inline _PyStackRef -_PyStackRef_StealRef(PyObject *obj) +_PyStackRef_FromPyObjectSteal(PyObject *obj) { +#ifdef Py_GIL_DISABLED // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); - return ((_PyStackRef){.bits = ((uintptr_t)(obj))}); -} -# define PyStackRef_StealRef(obj) _PyStackRef_StealRef(_PyObject_CAST(obj)) + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + int tag = (obj == NULL || _Py_IsImmortal(obj)) ? (Py_TAG_DEFERRED) : Py_TAG_PTR; + return ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag}); #else -# define PyStackRef_StealRef(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))}) + return ((_PyStackRef){.bits = ((uintptr_t)(obj))}); #endif +} + +#define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) + // Converts a PyObject * to a PyStackRef, with a new reference -#if defined(Py_GIL_DISABLED) static inline _PyStackRef -_PyStackRef_NewRefDeferred(PyObject *obj) +PyStackRef_FromPyObjectNew(PyObject *obj) { +#ifdef Py_GIL_DISABLED // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); assert(obj != NULL); - if (_PyObject_HasDeferredRefcount(obj)) { + // TODO (gh-117139): Add deferred objects later. + if (_Py_IsImmortal(obj)) { return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; } else { - return (_PyStackRef){ .bits = (uintptr_t)Py_NewRef(obj) }; + return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; } -} -# define PyStackRef_NewRefDeferred(obj) _PyStackRef_NewRefDeferred(_PyObject_CAST(obj)) #else -# define PyStackRef_NewRefDeferred(obj) PyStackRef_NewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))})) + return ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) }); #endif +} -#if defined(Py_GIL_DISABLED) +#define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) + +// Same as PyStackRef_FromPyObjectNew but only for immortal objects. static inline _PyStackRef -_PyStackRef_XNewRefDeferred(PyObject *obj) +PyStackRef_FromPyObjectImmortal(PyObject *obj) { +#ifdef Py_GIL_DISABLED // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); - if (obj == NULL) { - return Py_STACKREF_NULL; - } - return _PyStackRef_NewRefDeferred(obj); -} -# define PyStackRef_XNewRefDeferred(obj) _PyStackRef_XNewRefDeferred(_PyObject_CAST(obj)) -#else -# define PyStackRef_XNewRefDeferred(obj) PyStackRef_XNewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))})) -#endif - -// Converts a PyStackRef back to a PyObject *. -#if defined(Py_GIL_DISABLED) -static inline PyObject * -PyStackRef_StealObject(_PyStackRef tagged) -{ - if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { - assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged))); - return Py_NewRef(PyStackRef_Get(tagged)); - } - return PyStackRef_Get(tagged); -} + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + assert(obj != NULL); + assert(_Py_IsImmortal(obj)); + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; #else -# define PyStackRef_StealObject(tagged) PyStackRef_Get(tagged) + assert(_Py_IsImmortal(obj)); + return ((_PyStackRef){ .bits = (uintptr_t)(obj) }); #endif - -static inline void -_Py_untag_stack_borrowed(PyObject **dst, const _PyStackRef *src, size_t length) -{ - for (size_t i = 0; i < length; i++) { - dst[i] = PyStackRef_Get(src[i]); - } -} - -static inline void -_Py_untag_stack_steal(PyObject **dst, const _PyStackRef *src, size_t length) -{ - for (size_t i = 0; i < length; i++) { - dst[i] = PyStackRef_StealObject(src[i]); - } } +#define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj)) -#define PyStackRef_XSETREF(dst, src) \ - do { \ - _PyStackRef *_tmp_dst_ptr = &(dst); \ - _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \ - *_tmp_dst_ptr = (src); \ - PyStackRef_XDECREF(_tmp_old_dst); \ - } while (0) - -#define PyStackRef_SETREF(dst, src) \ - do { \ - _PyStackRef *_tmp_dst_ptr = &(dst); \ - _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \ - *_tmp_dst_ptr = (src); \ - PyStackRef_DECREF(_tmp_old_dst); \ - } while (0) #define PyStackRef_CLEAR(op) \ do { \ _PyStackRef *_tmp_op_ptr = &(op); \ _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ - if (_tmp_old_op.bits != Py_STACKREF_NULL.bits) { \ - *_tmp_op_ptr = Py_STACKREF_NULL; \ - PyStackRef_DECREF(_tmp_old_op); \ + if (!PyStackRef_IsNull(_tmp_old_op)) { \ + *_tmp_op_ptr = PyStackRef_NULL; \ + PyStackRef_CLOSE(_tmp_old_op); \ } \ } while (0) -#if defined(Py_GIL_DISABLED) static inline void -PyStackRef_DECREF(_PyStackRef tagged) +PyStackRef_CLOSE(_PyStackRef stackref) { - if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { +#ifdef Py_GIL_DISABLED + if (PyStackRef_IsDeferred(stackref)) { + // No assert for being immortal or deferred here. + // The GC unsets deferred objects right before clearing. return; } - Py_DECREF(PyStackRef_Get(tagged)); -} + Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref)); #else -# define PyStackRef_DECREF(op) Py_DECREF(PyStackRef_Get(op)) + Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref)); #endif +} -#if defined(Py_GIL_DISABLED) -static inline void -PyStackRef_INCREF(_PyStackRef tagged) +#define PyStackRef_XCLOSE(stackref) \ + do { \ + _PyStackRef _tmp = (stackref); \ + if (!PyStackRef_IsNull(_tmp)) { \ + PyStackRef_CLOSE(_tmp); \ + } \ + } while (0); + + +static inline _PyStackRef +PyStackRef_DUP(_PyStackRef stackref) { - if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { - assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged))); - return; +#ifdef Py_GIL_DISABLED + if (PyStackRef_IsDeferred(stackref)) { + assert(PyStackRef_IsNull(stackref) || + _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref))); + return stackref; } - Py_INCREF(PyStackRef_Get(tagged)); -} + Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); + return stackref; #else -# define PyStackRef_INCREF(op) Py_INCREF(PyStackRef_Get(op)) + Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); + return stackref; #endif - -static inline void -PyStackRef_XDECREF(_PyStackRef op) -{ - if (op.bits != Py_STACKREF_NULL.bits) { - PyStackRef_DECREF(op); - } } static inline _PyStackRef -PyStackRef_NewRef(_PyStackRef obj) +PyStackRef_XDUP(_PyStackRef stackref) { - PyStackRef_INCREF(obj); - return obj; + if (!PyStackRef_IsNull(stackref)) { + return PyStackRef_DUP(stackref); + } + return stackref; } -static inline _PyStackRef -PyStackRef_XNewRef(_PyStackRef obj) + +static inline void +_PyObjectStack_FromStackRefStack(PyObject **dst, const _PyStackRef *src, size_t length) { - if (obj.bits == Py_STACKREF_NULL.bits) { - return obj; + for (size_t i = 0; i < length; i++) { + dst[i] = PyStackRef_AsPyObjectBorrow(src[i]); } - return PyStackRef_NewRef(obj); } + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 71a5711441807d4..30e39e7720e6d11 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -147,7 +147,7 @@ def test_inst_one_pop(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; spam(); stack_pointer += -1; @@ -168,7 +168,7 @@ def test_inst_one_push(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *res; + _PyStackRef res; spam(); stack_pointer[0] = res; stack_pointer += 1; @@ -189,8 +189,8 @@ def test_inst_one_push_one_pop(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; spam(); stack_pointer[-1] = res; @@ -210,9 +210,9 @@ def test_binary_op(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; spam(); @@ -235,9 +235,9 @@ def test_overlap(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *right; - PyObject *left; - PyObject *result; + _PyStackRef right; + _PyStackRef left; + _PyStackRef result; right = stack_pointer[-1]; left = stack_pointer[-2]; spam(); @@ -263,8 +263,8 @@ def test_predictions_and_eval_breaker(self): next_instr += 1; INSTRUCTION_STATS(OP1); PREDICTED(OP1); - PyObject *arg; - PyObject *rest; + _PyStackRef arg; + _PyStackRef rest; arg = stack_pointer[-1]; stack_pointer[-1] = rest; DISPATCH(); @@ -275,8 +275,8 @@ def test_predictions_and_eval_breaker(self): next_instr += 1; INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); - PyObject *arg; - PyObject *res; + _PyStackRef arg; + _PyStackRef res; arg = stack_pointer[-1]; DEOPT_IF(xxx, OP1); stack_pointer[-1] = res; @@ -332,9 +332,9 @@ def test_error_if_pop(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (cond) goto pop_2_label; @@ -357,7 +357,7 @@ def test_cache_effect(self): (void)this_instr; next_instr += 4; INSTRUCTION_STATS(OP); - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; @@ -408,10 +408,10 @@ def test_macro_instruction(self): PREDICTED(OP); _Py_CODEUNIT *this_instr = next_instr - 6; (void)this_instr; - PyObject *right; - PyObject *left; - PyObject *arg2; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef arg2; + _PyStackRef res; // _OP1 right = stack_pointer[-1]; left = stack_pointer[-2]; @@ -439,8 +439,8 @@ def test_macro_instruction(self): (void)this_instr; next_instr += 2; INSTRUCTION_STATS(OP1); - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); @@ -454,10 +454,10 @@ def test_macro_instruction(self): next_instr += 6; INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *arg2; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef arg2; + _PyStackRef res; /* Skip 5 cache entries */ right = stack_pointer[-1]; left = stack_pointer[-2]; @@ -539,9 +539,9 @@ def test_array_input(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *above; - PyObject **values; - PyObject *below; + _PyStackRef above; + _PyStackRef *values; + _PyStackRef below; above = stack_pointer[-1]; values = &stack_pointer[-1 - oparg*2]; below = stack_pointer[-2 - oparg*2]; @@ -564,9 +564,9 @@ def test_array_output(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *below; - PyObject **values; - PyObject *above; + _PyStackRef below; + _PyStackRef *values; + _PyStackRef above; values = &stack_pointer[-1]; spam(values, oparg); stack_pointer[-2] = below; @@ -589,8 +589,8 @@ def test_array_input_output(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject **values; - PyObject *above; + _PyStackRef *values; + _PyStackRef above; values = &stack_pointer[-oparg]; spam(values, oparg); stack_pointer[0] = above; @@ -612,8 +612,8 @@ def test_array_error_if(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject **values; - PyObject *extra; + _PyStackRef *values; + _PyStackRef extra; values = &stack_pointer[-oparg]; extra = stack_pointer[-1 - oparg]; if (oparg == 0) { stack_pointer += -1 - oparg; goto somewhere; } @@ -635,12 +635,12 @@ def test_cond_effect(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - PyObject *cc; - PyObject *input = NULL; - PyObject *aa; - PyObject *xx; - PyObject *output = NULL; - PyObject *zz; + _PyStackRef cc; + _PyStackRef input = PyStackRef_NULL; + _PyStackRef aa; + _PyStackRef xx; + _PyStackRef output = PyStackRef_NULL; + _PyStackRef zz; cc = stack_pointer[-1]; if ((oparg & 1) == 1) { input = stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)]; } aa = stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)]; @@ -670,12 +670,12 @@ def test_macro_cond_effect(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - PyObject *right; - PyObject *middle; - PyObject *left; - PyObject *deep; - PyObject *extra = NULL; - PyObject *res; + _PyStackRef right; + _PyStackRef middle; + _PyStackRef left; + _PyStackRef deep; + _PyStackRef extra = PyStackRef_NULL; + _PyStackRef res; // A right = stack_pointer[-1]; middle = stack_pointer[-2]; @@ -712,8 +712,8 @@ def test_macro_push_push(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - PyObject *val1; - PyObject *val2; + _PyStackRef val1; + _PyStackRef val2; // A { val1 = spam(); diff --git a/Makefile.pre.in b/Makefile.pre.in index e28f1720c7e1b04..da9f39fa0421a62 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1198,6 +1198,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_namespace.h \ $(srcdir)/Include/internal/pycore_object.h \ $(srcdir)/Include/internal/pycore_object_alloc.h \ + $(srcdir)/Include/internal/pycore_object_deferred.h \ $(srcdir)/Include/internal/pycore_object_stack.h \ $(srcdir)/Include/internal/pycore_object_state.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-01-05-09-16.gh-issue-117139.t41w_D.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-05-09-16.gh-issue-117139.t41w_D.rst new file mode 100644 index 000000000000000..07d5dd53aac30a8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-05-09-16.gh-issue-117139.t41w_D.rst @@ -0,0 +1,5 @@ +Convert the Python evaluation stack to use internal stack references. The +purpose is to support tagged pointers. In :pep:`703`, this will +allow for its form of deferred reference counting. For both +the default and free-threaded builds, this sets up the infrastructure +for unboxed integers in the future. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2cb113b3d01be10..3dc9ff058a5c9e7 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -21,10 +21,10 @@ static PyObject * framelocalsproxy_getval(_PyInterpreterFrame *frame, PyCodeObject *co, int i) { - PyObject **fast = _PyFrame_GetLocalsArray(frame); + _PyStackRef *fast = _PyFrame_GetLocalsArray(frame); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - PyObject *value = fast[i]; + PyObject *value = PyStackRef_AsPyObjectBorrow(fast[i]); PyObject *cell = NULL; if (value == NULL) { @@ -136,9 +136,9 @@ static int framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) { /* Merge locals into fast locals */ - PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; - PyObject** fast = _PyFrame_GetLocalsArray(frame->f_frame); - PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + _PyStackRef *fast = _PyFrame_GetLocalsArray(frame->f_frame); + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); @@ -151,26 +151,28 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - PyObject *oldvalue = fast[i]; + _PyStackRef oldvalue = fast[i]; PyObject *cell = NULL; if (kind == CO_FAST_FREE) { // The cell was set when the frame was created from // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } else if (kind & CO_FAST_CELL && oldvalue != NULL) { - if (PyCell_Check(oldvalue)) { - cell = oldvalue; + assert(oldvalue.bits != 0 && PyCell_Check(PyStackRef_AsPyObjectBorrow(oldvalue))); + cell = PyStackRef_AsPyObjectBorrow(oldvalue); + } else if (kind & CO_FAST_CELL && oldvalue.bits != 0) { + PyObject *as_obj = PyStackRef_AsPyObjectBorrow(oldvalue); + if (PyCell_Check(as_obj)) { + cell = as_obj; } } if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { + PyObject *oldvalue_o = PyCell_GET(cell); + if (value != oldvalue_o) { PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue); + Py_XDECREF(oldvalue_o); } - } else if (value != oldvalue) { - Py_XSETREF(fast[i], Py_NewRef(value)); + } else if (value != PyStackRef_AsPyObjectBorrow(oldvalue)) { + PyStackRef_XCLOSE(fast[i]); + fast[i] = PyStackRef_FromPyObjectNew(value); } return 0; } @@ -1511,7 +1513,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore for (int i = 0; i < code->co_nlocalsplus; i++) { // Counting every unbound local is overly-cautious, but a full flow // analysis (like we do in the compiler) is probably too expensive: - unbound += f->f_frame->localsplus[i] == NULL; + unbound += PyStackRef_IsNull(f->f_frame->localsplus[i]); } if (unbound) { const char *e = "assigning None to %d unbound local%s"; @@ -1522,8 +1524,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore // Do this in a second pass to avoid writing a bunch of Nones when // warnings are being treated as errors and the previous bit raises: for (int i = 0; i < code->co_nlocalsplus; i++) { - if (f->f_frame->localsplus[i] == NULL) { - f->f_frame->localsplus[i] = Py_NewRef(Py_None); + if (PyStackRef_IsNull(f->f_frame->localsplus[i])) { + f->f_frame->localsplus[i] = PyStackRef_None; unbound--; } } @@ -1536,14 +1538,13 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore while (start_stack > best_stack) { if (top_of_stack(start_stack) == Except) { /* Pop exception stack as well as the evaluation stack */ - PyObject *exc = _PyFrame_StackPop(f->f_frame); + PyObject *exc = PyStackRef_AsPyObjectBorrow(_PyFrame_StackPop(f->f_frame)); assert(PyExceptionInstance_Check(exc) || exc == Py_None); PyThreadState *tstate = _PyThreadState_GET(); Py_XSETREF(tstate->exc_info->exc_value, exc == Py_None ? NULL : exc); } else { - PyObject *v = _PyFrame_StackPop(f->f_frame); - Py_XDECREF(v); + PyStackRef_XCLOSE(_PyFrame_StackPop(f->f_frame)); } start_stack = pop_value(start_stack); } @@ -1618,9 +1619,9 @@ frame_dealloc(PyFrameObject *f) frame->f_executable = NULL; Py_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); - PyObject **locals = _PyFrame_GetLocalsArray(frame); + _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); for (int i = 0; i < frame->stacktop; i++) { - Py_CLEAR(locals[i]); + PyStackRef_CLEAR(locals[i]); } } Py_CLEAR(f->f_back); @@ -1651,10 +1652,10 @@ frame_tp_clear(PyFrameObject *f) Py_CLEAR(f->f_extra_locals); /* locals and stack */ - PyObject **locals = _PyFrame_GetLocalsArray(f->f_frame); + _PyStackRef *locals = _PyFrame_GetLocalsArray(f->f_frame); assert(f->f_frame->stacktop >= 0); for (int i = 0; i < f->f_frame->stacktop; i++) { - Py_CLEAR(locals[i]); + PyStackRef_CLEAR(locals[i]); } f->f_frame->stacktop = 0; Py_CLEAR(f->f_frame->f_locals); @@ -1848,7 +1849,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame) int offset = PyUnstable_Code_GetFirstFree(co); for (int i = 0; i < co->co_nfreevars; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } // COPY_FREE_VARS doesn't have inline CACHEs, either: frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)); @@ -1873,7 +1874,7 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, return 0; } - PyObject *value = frame->localsplus[i]; + PyObject *value = PyStackRef_AsPyObjectBorrow(frame->localsplus[i]); if (frame->stacktop) { if (kind & CO_FAST_FREE) { // The cell was set by COPY_FREE_VARS. diff --git a/Objects/genobject.c b/Objects/genobject.c index 9a3194d3de26149..37b40530589d57f 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -212,7 +212,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, /* Push arg onto the frame's value stack */ PyObject *arg_obj = arg ? arg : Py_None; - _PyFrame_StackPush(frame, Py_NewRef(arg_obj)); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectNew(arg_obj)); _PyErr_StackItem *prev_exc_info = tstate->exc_info; gen->gi_exc_state.previous_item = prev_exc_info; @@ -344,7 +344,7 @@ _PyGen_yf(PyGenObject *gen) _PyInterpreterFrame *frame = &gen->gi_iframe; assert(is_resume(frame->instr_ptr)); assert((frame->instr_ptr->op.arg & RESUME_OPARG_LOCATION_MASK) >= RESUME_AFTER_YIELD_FROM); - return Py_NewRef(_PyFrame_StackPeek(frame)); + return PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame)); } return NULL; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 71fcb6559fff736..51a7adf5c405035 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -11411,7 +11411,7 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, } assert(_PyFrame_GetCode(cframe)->co_nlocalsplus > 0); - PyObject *firstarg = _PyFrame_GetLocalsArray(cframe)[0]; + PyObject *firstarg = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[0]); // The first argument might be a cell. if (firstarg != NULL && (_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) { // "firstarg" is a cell here unless (very unlikely) super() @@ -11439,7 +11439,7 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); assert(PyUnicode_Check(name)); if (_PyUnicode_Equal(name, &_Py_ID(__class__))) { - PyObject *cell = _PyFrame_GetLocalsArray(cframe)[i]; + PyObject *cell = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[i]); if (cell == NULL || !PyCell_Check(cell)) { PyErr_SetString(PyExc_RuntimeError, "super(): bad __class__ cell"); diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c065d58d255279e..3378ed54203f188 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -264,6 +264,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index dd03d5608a411ac..742d88d9e1fa7a5 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -714,6 +714,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c00de88fc70a6ee..67061ac26323e88 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -218,41 +218,37 @@ dummy_func( }; inst(LOAD_FAST_CHECK, (-- value)) { - value = GETLOCAL(oparg); - if (value == NULL) { + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); ERROR_IF(1, error); } - Py_INCREF(value); + value = PyStackRef_DUP(value_s); } replicate(8) pure inst(LOAD_FAST, (-- value)) { - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); } inst(LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; + GETLOCAL(oparg) = PyStackRef_NULL; } inst(LOAD_FAST_LOAD_FAST, ( -- value1, value2)) { uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); } pure inst(LOAD_CONST, (-- value)) { - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); } replicate(8) inst(STORE_FAST, (value --)) { @@ -267,8 +263,7 @@ dummy_func( uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); } inst(STORE_FAST_STORE_FAST, (value2, value1 --)) { @@ -283,7 +278,7 @@ dummy_func( } pure inst(PUSH_NULL, (-- res)) { - res = NULL; + res = PyStackRef_NULL; } macro(END_FOR) = POP_TOP; @@ -291,8 +286,8 @@ dummy_func( tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + if (PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver))) { + if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { ERROR_NO_POP(); } } @@ -300,27 +295,31 @@ dummy_func( } pure inst(END_SEND, (receiver, value -- value)) { - Py_DECREF(receiver); + (void)receiver; + PyStackRef_CLOSE(receiver); } tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { ERROR_NO_POP(); } } - Py_DECREF(receiver); + PyStackRef_CLOSE(receiver); } inst(UNARY_NEGATIVE, (value -- res)) { - res = PyNumber_Negative(value); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } pure inst(UNARY_NOT, (value -- res)) { - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + res = PyStackRef_Is(value, PyStackRef_False) + ? PyStackRef_True : PyStackRef_False; } family(TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL) = { @@ -345,63 +344,66 @@ dummy_func( } op(_TO_BOOL, (value -- res)) { - int err = PyObject_IsTrue(value); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); ERROR_IF(err < 0, error); - res = err ? Py_True : Py_False; + res = err ? PyStackRef_True : PyStackRef_False; } macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL; inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { - EXIT_IF(!PyBool_Check(value)); + EXIT_IF(!PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); STAT_INC(TO_BOOL, hit); } inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) { - EXIT_IF(!PyLong_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyLong_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { DECREF_INPUTS(); - res = Py_True; + res = PyStackRef_True; } } inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) { - EXIT_IF(!PyList_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyList_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; + res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; DECREF_INPUTS(); } inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: - EXIT_IF(!Py_IsNone(value)); + EXIT_IF(!PyStackRef_Is(value, PyStackRef_None)); STAT_INC(TO_BOOL, hit); - res = Py_False; + res = PyStackRef_False; } inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) { - EXIT_IF(!PyUnicode_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyUnicode_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - assert(Py_SIZE(value)); + assert(Py_SIZE(value_o)); DECREF_INPUTS(); - res = Py_True; + res = PyStackRef_True; } } op(_REPLACE_WITH_TRUE, (value -- res)) { - Py_DECREF(value); - res = Py_True; + DECREF_INPUTS(); + res = PyStackRef_True; } macro(TO_BOOL_ALWAYS_TRUE) = @@ -410,9 +412,10 @@ dummy_func( _REPLACE_WITH_TRUE; inst(UNARY_INVERT, (value -- res)) { - res = PyNumber_Invert(value); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } family(BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP) = { @@ -427,40 +430,56 @@ dummy_func( }; op(_GUARD_BOTH_INT, (left, right -- left, right)) { - EXIT_IF(!PyLong_CheckExact(left)); - EXIT_IF(!PyLong_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + EXIT_IF(!PyLong_CheckExact(left_o)); + EXIT_IF(!PyLong_CheckExact(right_o)); } op(_GUARD_NOS_INT, (left, unused -- left, unused)) { - EXIT_IF(!PyLong_CheckExact(left)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + EXIT_IF(!PyLong_CheckExact(left_o)); } op(_GUARD_TOS_INT, (value -- value)) { - EXIT_IF(!PyLong_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyLong_CheckExact(value_o)); } pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - ERROR_IF(res == NULL, error); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_ADD_INT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - ERROR_IF(res == NULL, error); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - ERROR_IF(res == NULL, error); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_MULTIPLY_INT) = @@ -471,40 +490,59 @@ dummy_func( _GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT; op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - EXIT_IF(!PyFloat_CheckExact(left)); - EXIT_IF(!PyFloat_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + EXIT_IF(!PyFloat_CheckExact(left_o)); + EXIT_IF(!PyFloat_CheckExact(right_o)); } op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { - EXIT_IF(!PyFloat_CheckExact(left)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + EXIT_IF(!PyFloat_CheckExact(left_o)); } op(_GUARD_TOS_FLOAT, (value -- value)) { - EXIT_IF(!PyFloat_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyFloat_CheckExact(value_o)); } pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_MULTIPLY_FLOAT) = @@ -515,16 +553,23 @@ dummy_func( _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT; op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - EXIT_IF(!PyUnicode_CheckExact(left)); - EXIT_IF(!PyUnicode_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + EXIT_IF(!PyUnicode_CheckExact(left_o)); + EXIT_IF(!PyUnicode_CheckExact(right_o)); } pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - ERROR_IF(res == NULL, error); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_ADD_UNICODE) = @@ -537,9 +582,12 @@ dummy_func( // specializations, but there is no output. // At the end we just skip over the STORE_FAST. tier1 op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(next_instr->op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(next_instr->op.arg); - DEOPT_IF(*target_local != left); + _PyStackRef *target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(!PyStackRef_Is(*target_local, left)); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -552,11 +600,13 @@ dummy_func( * only the locals reference, so PyUnicode_Append knows * that the string is safe to mutate. */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - ERROR_IF(*target_local == NULL, error); + assert(Py_REFCNT(left_o) >= 2); + _Py_DECREF_NO_DEALLOC(left_o); + PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + ERROR_IF(PyStackRef_IsNull(*target_local), error); // The STORE_FAST is already done. assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); @@ -586,44 +636,55 @@ dummy_func( } op(_BINARY_SUBSCR, (container, sub -- res)) { - res = PyObject_GetItem(container, sub); + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + + PyObject *res_o = PyObject_GetItem(container_o, sub_o); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_SUBSCR) = _SPECIALIZE_BINARY_SUBSCR + _BINARY_SUBSCR; inst(BINARY_SLICE, (container, start, stop -- res)) { - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + PyObject *res_o; // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { - res = NULL; + res_o = NULL; } else { - res = PyObject_GetItem(container, slice); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); } - Py_DECREF(container); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(container); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } inst(STORE_SLICE, (v, container, start, stop -- )) { - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); int err; if (slice == NULL) { err = 1; } else { - err = PyObject_SetItem(container, slice, v); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v)); Py_DECREF(slice); } - Py_DECREF(v); - Py_DECREF(container); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); ERROR_IF(err, error); } - inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) { + inst(BINARY_SUBSCR_LIST_INT, (unused/1, list_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyList_CheckExact(list)); @@ -632,14 +693,18 @@ dummy_func( Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) { + inst(BINARY_SUBSCR_STR_INT, (unused/1, str_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyUnicode_CheckExact(str)); DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); @@ -649,12 +714,16 @@ dummy_func( Py_UCS4 c = PyUnicode_READ_CHAR(str, index); DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) { + inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyTuple_CheckExact(tuple)); @@ -663,25 +732,33 @@ dummy_func( Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple)); STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) { + inst(BINARY_SUBSCR_DICT, (unused/1, dict_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); + PyObject *res_o; + int rc = PyDict_GetItemRef(dict, sub, &res_o); if (rc == 0) { _PyErr_SetKeyError(sub); } DECREF_INPUTS(); ERROR_IF(rc <= 0, error); // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) { + inst(BINARY_SUBSCR_GETITEM, (unused/1, container_st, sub_st -- unused)) { + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); @@ -699,18 +776,20 @@ dummy_func( Py_INCREF(getitem); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; + new_frame->localsplus[0] = container_st; + new_frame->localsplus[1] = sub_st; frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } inst(LIST_APPEND, (list, unused[oparg-1], v -- list, unused[oparg-1])) { - ERROR_IF(_PyList_AppendTakeRef((PyListObject *)list, v) < 0, error); + ERROR_IF(_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)) < 0, error); } inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { - int err = PySet_Add(set, v); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -734,14 +813,17 @@ dummy_func( op(_STORE_SUBSCR, (v, container, sub -- )) { /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v)); DECREF_INPUTS(); ERROR_IF(err, error); } macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; - inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { + inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list_st, sub_st -- )) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyList_CheckExact(list)); @@ -753,50 +835,59 @@ dummy_func( STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyStackRef_CLOSE(list_st); } - inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { + inst(STORE_SUBSCR_DICT, (unused/1, value, dict_st, sub_st -- )) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + PyStackRef_CLOSE(dict_st); ERROR_IF(err, error); } inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ - int err = PyObject_DelItem(container, sub); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); DECREF_INPUTS(); ERROR_IF(err, error); } inst(CALL_INTRINSIC_1, (value -- res)) { assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_INTRINSIC_2, (value2, value1 -- res)) { + inst(CALL_INTRINSIC_2, (value2_st, value1_st -- res)) { assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } tier1 inst(RAISE_VARARGS, (args[oparg] -- )) { PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: - cause = args[1]; + cause = PyStackRef_AsPyObjectSteal(args[1]); /* fall through */ case 1: - exc = args[0]; + exc = PyStackRef_AsPyObjectSteal(args[0]); /* fall through */ case 0: if (do_raise(tstate, exc, cause)) { @@ -820,7 +911,7 @@ dummy_func( tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; + return PyStackRef_AsPyObjectSteal(retval); } // The stack effect here is ambiguous. @@ -848,7 +939,7 @@ dummy_func( inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); if (err) ERROR_NO_POP(); STACK_SHRINK(1); assert(EMPTY()); @@ -883,14 +974,16 @@ dummy_func( _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval)); LOAD_IP(frame->return_offset); goto resume_frame; } inst(GET_AITER, (obj -- iter)) { unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; @@ -905,30 +998,33 @@ dummy_func( ERROR_IF(true, error); } - iter = (*getter)(obj); + iter_o = (*getter)(obj_o); DECREF_INPUTS(); - ERROR_IF(iter == NULL, error); + ERROR_IF(iter_o == NULL, error); - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); ERROR_IF(true, error); } + iter = PyStackRef_FromPyObjectSteal(iter_o); } inst(GET_ANEXT, (aiter -- aiter, awaitable)) { unaryfunc getter = NULL; PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); + PyObject *awaitable_o; + PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter); + PyTypeObject *type = Py_TYPE(aiter_o); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { + if (PyAsyncGen_CheckExact(aiter_o)) { + awaitable_o = type->tp_as_async->am_anext(aiter_o); + if (awaitable_o == NULL) { ERROR_NO_POP(); } } else { @@ -937,7 +1033,7 @@ dummy_func( } if (getter != NULL) { - next_iter = (*getter)(aiter); + next_iter = (*getter)(aiter_o); if (next_iter == NULL) { ERROR_NO_POP(); } @@ -950,8 +1046,8 @@ dummy_func( ERROR_NO_POP(); } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { + awaitable_o = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable_o == NULL) { _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " @@ -964,32 +1060,35 @@ dummy_func( Py_DECREF(next_iter); } } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); } inst(GET_AWAITABLE, (iterable -- iter)) { - iter = _PyCoro_GetAwaitableIter(iterable); + PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable)); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + if (iter_o == NULL) { + _PyEval_FormatAwaitableError(tstate, + Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg); } DECREF_INPUTS(); - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (iter_o != NULL && PyCoro_CheckExact(iter_o)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter_o); if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ Py_DECREF(yf); - Py_CLEAR(iter); + Py_CLEAR(iter_o); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); /* The code below jumps to `error` if `iter` is NULL. */ } } - ERROR_IF(iter == NULL, error); + ERROR_IF(iter_o == NULL, error); + iter = PyStackRef_FromPyObjectSteal(iter_o); } family(SEND, INLINE_CACHE_ENTRIES_SEND) = { @@ -1009,12 +1108,15 @@ dummy_func( } op(_SEND, (receiver, v -- receiver, retval)) { + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + + PyObject *retval_o; assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { - PyGenObject *gen = (PyGenObject *)receiver; + PyGenObject *gen = (PyGenObject *)receiver_o; _PyInterpreterFrame *gen_frame = &gen->gi_iframe; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); @@ -1025,33 +1127,36 @@ dummy_func( frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); + if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); } else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); } - if (retval == NULL) { + if (retval_o == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { monitor_raise(tstate, frame, this_instr); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); + if (_PyGen_FetchStopIterationValue(&retval_o) == 0) { + assert(retval_o != NULL); JUMPBY(oparg); } else { ERROR_NO_POP(); } } - Py_DECREF(v); + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); } macro(SEND) = _SPECIALIZE_SEND + _SEND; inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { DEOPT_IF(tstate->interp->eval_frame); - PyGenObject *gen = (PyGenObject *)receiver; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(SEND, hit); @@ -1076,7 +1181,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); + frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); if (err) ERROR_NO_POP(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -1129,13 +1234,17 @@ dummy_func( inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + Py_XSETREF(exc_info->exc_value, + PyStackRef_AsPyObjectBorrow(exc_value) == Py_None + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); } - tier1 inst(RERAISE, (values[oparg], exc -- values[oparg])) { + tier1 inst(RERAISE, (values[oparg], exc_st -- values[oparg])) { + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(oparg >= 0 && oparg <= 2); if (oparg) { - PyObject *lasti = values[0]; + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); if (PyLong_Check(lasti)) { frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); @@ -1153,7 +1262,9 @@ dummy_func( goto exception_unwind; } - tier1 inst(END_ASYNC_FOR, (awaitable, exc -- )) { + tier1 inst(END_ASYNC_FOR, (awaitable_st, exc_st -- )) { + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { DECREF_INPUTS(); @@ -1166,13 +1277,15 @@ dummy_func( } } - tier1 inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) { + tier1 inst(CLEANUP_THROW, (sub_iter_st, last_sent_val_st, exc_value_st -- none, value)) { + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); DECREF_INPUTS(); - none = Py_None; + none = PyStackRef_None; } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); @@ -1185,10 +1298,10 @@ dummy_func( // Keep in sync with _common_constants in opcode.py switch(oparg) { case CONSTANT_ASSERTIONERROR: - value = PyExc_AssertionError; + value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); break; case CONSTANT_NOTIMPLEMENTEDERROR: - value = PyExc_NotImplementedError; + value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); break; default: Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); @@ -1196,12 +1309,14 @@ dummy_func( } inst(LOAD_BUILD_CLASS, ( -- bc)) { - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0, error); - if (bc == NULL) { + PyObject *bc_o; + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0, error); + if (bc_o == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); ERROR_IF(true, error); } + bc = PyStackRef_FromPyObjectSteal(bc_o); } inst(STORE_NAME, (v -- )) { @@ -1215,9 +1330,9 @@ dummy_func( ERROR_IF(true, error); } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); else - err = PyObject_SetItem(ns, name, v); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1262,8 +1377,8 @@ dummy_func( } op(_UNPACK_SEQUENCE, (seq -- unused[oparg])) { - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + _PyStackRef *top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1272,40 +1387,43 @@ dummy_func( inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- val1, val0)) { assert(oparg == 2); - DEOPT_IF(!PyTuple_CheckExact(seq)); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o)); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2); STAT_INC(UNPACK_SEQUENCE, hit); - val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); DECREF_INPUTS(); } inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyTuple_CheckExact(seq)); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o)); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } DECREF_INPUTS(); } inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyList_CheckExact(seq)); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyList_CheckExact(seq_o)); + DEOPT_IF(PyList_GET_SIZE(seq_o) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); + PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } DECREF_INPUTS(); } inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + _PyStackRef *top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1331,7 +1449,8 @@ dummy_func( op(_STORE_ATTR, (v, owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectSteal(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1340,14 +1459,14 @@ dummy_func( inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); DECREF_INPUTS(); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1367,28 +1486,29 @@ dummy_func( } inst(LOAD_LOCALS, ( -- locals)) { - locals = LOCALS(); - if (locals == NULL) { + PyObject *l = LOCALS(); + if (l == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); ERROR_IF(true, error); } - Py_INCREF(locals); + locals = PyStackRef_FromPyObjectNew(l);; } inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + PyObject *v_o; + if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) { ERROR_NO_POP(); } - if (v == NULL) { + if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - if (v == NULL) { + if (v_o == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1401,11 +1521,11 @@ dummy_func( else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0, error); - if (v == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0, error); + if (v_o == NULL) { /* namespace 2: builtins */ - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0, error); - if (v == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0, error); + if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1415,9 +1535,11 @@ dummy_func( } } DECREF_INPUTS(); + v = PyStackRef_FromPyObjectSteal(v_o); } inst(LOAD_NAME, (-- v)) { + PyObject *v_o; PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1425,18 +1547,18 @@ dummy_func( ERROR_IF(true, error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) { ERROR_NO_POP(); } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + if (v_o == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) { ERROR_NO_POP(); } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + if (v_o == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) { ERROR_NO_POP(); } - if (v == NULL) { + if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1444,6 +1566,7 @@ dummy_func( } } } + v = PyStackRef_FromPyObjectSteal(v_o); } family(LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL) = { @@ -1466,13 +1589,14 @@ dummy_func( op(_LOAD_GLOBAL, ( -- res, null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *res_o; if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - if (res == NULL) { + if (res_o == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1485,11 +1609,11 @@ dummy_func( else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0, error); - if (res == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0, error); + if (res_o == NULL) { /* namespace 2: builtins */ - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0, error); - if (res == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0, error); + if (res_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1497,7 +1621,8 @@ dummy_func( } } } - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } macro(LOAD_GLOBAL) = @@ -1524,21 +1649,23 @@ dummy_func( op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) { PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL); - Py_INCREF(res); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL); + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) { PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL); - Py_INCREF(res); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL); + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } macro(LOAD_GLOBAL_MODULE) = @@ -1554,30 +1681,30 @@ dummy_func( _LOAD_GLOBAL_BUILTINS; inst(DELETE_FAST, (--)) { - PyObject *v = GETLOCAL(oparg); - if (v == NULL) { + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); ERROR_IF(1, error); } - SETLOCAL(oparg, NULL); + SETLOCAL(oparg, PyStackRef_NULL); } inst(MAKE_CELL, (--)) { // "initial" is probably NULL but not if it's an arg (or set // via the f_locals proxy before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { ERROR_NO_POP(); } - SETLOCAL(oparg, cell); + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); } inst(DELETE_DEREF, (--)) { - PyObject *cell = GETLOCAL(oparg); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); @@ -1588,37 +1715,42 @@ dummy_func( Py_DECREF(oldobj); } - inst(LOAD_FROM_DICT_OR_DEREF, (class_dict -- value)) { + inst(LOAD_FROM_DICT_OR_DEREF, (class_dict_st -- value)) { + PyObject *value_o; PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { ERROR_NO_POP(); } - if (!value) { - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - value = PyCell_GetRef(cell); - if (value == NULL) { + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); ERROR_NO_POP(); } } - Py_DECREF(class_dict); + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); } inst(LOAD_DEREF, ( -- value)) { - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - value = PyCell_GetRef(cell); - if (value == NULL) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); ERROR_IF(true, error); } + value = PyStackRef_FromPyObjectSteal(value_o); } inst(STORE_DEREF, (v --)) { - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - PyCell_SetTakeRef(cell, v); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); } inst(COPY_FREE_VARS, (--)) { @@ -1630,27 +1762,51 @@ dummy_func( int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } } inst(BUILD_STRING, (pieces[oparg] -- str)) { - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); DECREF_INPUTS(); - ERROR_IF(str == NULL, error); + ERROR_IF(str_o == NULL, error); + str = PyStackRef_FromPyObjectSteal(str_o); } inst(BUILD_TUPLE, (values[oparg] -- tup)) { - tup = _PyTuple_FromArraySteal(values, oparg); - ERROR_IF(tup == NULL, error); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + ERROR_IF(tup_o == NULL, error); + tup = PyStackRef_FromPyObjectSteal(tup_o); } inst(BUILD_LIST, (values[oparg] -- list)) { - list = _PyList_FromArraySteal(values, oparg); - ERROR_IF(list == NULL, error); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *list_o = _PyList_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + ERROR_IF(list_o == NULL, error); + list = PyStackRef_FromPyObjectSteal(list_o); } - inst(LIST_EXTEND, (list, unused[oparg-1], iterable -- list, unused[oparg-1])) { + inst(LIST_EXTEND, (list_st, unused[oparg-1], iterable_st -- list_st, unused[oparg-1])) { + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1669,35 +1825,46 @@ dummy_func( } inst(SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1])) { - int err = _PySet_Update(set, iterable); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); DECREF_INPUTS(); ERROR_IF(err < 0, error); } inst(BUILD_SET, (values[oparg] -- set)) { - set = PySet_New(NULL); - if (set == NULL) + PyObject *set_o = PySet_New(NULL); + if (set_o == NULL) { ERROR_NO_POP(); + } int err = 0; for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); + PyObject *item = PyStackRef_AsPyObjectSteal(values[i]); + if (err == 0) { + err = PySet_Add(set_o, item); + } Py_DECREF(item); } if (err != 0) { - Py_DECREF(set); + Py_DECREF(set_o); ERROR_IF(true, error); } + set = PyStackRef_FromPyObjectSteal(set_o); } inst(BUILD_MAP, (values[oparg*2] -- map)) { - map = _PyDict_FromItems( - values, 2, - values+1, 2, + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); DECREF_INPUTS(); - ERROR_IF(map == NULL, error); + ERROR_IF(map_o == NULL, error); + map = PyStackRef_FromPyObjectSteal(map_o); } inst(SETUP_ANNOTATIONS, (--)) { @@ -1724,21 +1891,33 @@ dummy_func( } inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) { - assert(PyTuple_CheckExact(keys)); - assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); + + assert(PyTuple_CheckExact(keys_o)); + assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *map_o = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys_o, 0), 1, + values_o, 1, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); DECREF_INPUTS(); - ERROR_IF(map == NULL, error); + ERROR_IF(map_o == NULL, error); + map = PyStackRef_FromPyObjectSteal(map_o); } inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) { - if (PyDict_Update(dict, update) < 0) { + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + + if (PyDict_Update(dict_o, update_o) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + Py_TYPE(update_o)->tp_name); } DECREF_INPUTS(); ERROR_IF(true, error); @@ -1747,19 +1926,24 @@ dummy_func( } inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) { - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + + if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable_o, update_o); DECREF_INPUTS(); ERROR_IF(true, error); } DECREF_INPUTS(); } - inst(MAP_ADD, (dict, unused[oparg - 1], key, value -- dict, unused[oparg - 1])) { + inst(MAP_ADD, (dict_st, unused[oparg - 1], key, value -- dict_st, unused[oparg - 1])) { + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); + ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0, error); } inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) { @@ -1774,12 +1958,12 @@ dummy_func( LOAD_SUPER_ATTR_METHOD, }; - specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) { + specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) { #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); @@ -1787,7 +1971,11 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - tier1 op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) { + tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr, null if (oparg & 1))) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -1818,26 +2006,35 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(super == NULL, error); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = PyObject_GetAttr(super, name); + attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name)); Py_DECREF(super); - ERROR_IF(attr == NULL, error); - null = NULL; + ERROR_IF(PyStackRef_IsNull(attr), error); + null = PyStackRef_NULL; } macro(LOAD_SUPER_ATTR) = _SPECIALIZE_LOAD_SUPER_ATTR + _LOAD_SUPER_ATTR; - inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) { + inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super_st, class_st, self_st -- attr_st, unused if (0))) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type); DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); DECREF_INPUTS(); ERROR_IF(attr == NULL, error); + attr_st = PyStackRef_FromPyObjectSteal(attr); } - inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- attr, self_or_null)) { + inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super_st, class_st, self_st -- attr, self_or_null)) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type); DEOPT_IF(!PyType_Check(class)); @@ -1845,20 +2042,22 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, + PyObject *attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + if (attr_o == NULL) { + PyStackRef_CLOSE(self_st); ERROR_IF(true, error); } if (method_found) { - self_or_null = self; // transfer ownership + self_or_null = self_st; // transfer ownership } else { - Py_DECREF(self); - self_or_null = NULL; + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; } + + attr = PyStackRef_FromPyObjectSteal(attr_o); } family(LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR) = { @@ -1891,15 +2090,16 @@ dummy_func( op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + attr_o = NULL; + if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch + assert(attr_o != NULL); // No errors on this branch self_or_null = owner; // Transfer ownership } else { @@ -1910,16 +2110,17 @@ dummy_func( meth | NULL | arg1 | ... | argN */ DECREF_INPUTS(); - ERROR_IF(attr == NULL, error); - self_or_null = NULL; + ERROR_IF(attr_o == NULL, error); + self_or_null = PyStackRef_NULL; } } else { /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); DECREF_INPUTS(); - ERROR_IF(attr == NULL, error); + ERROR_IF(attr_o == NULL, error); } + attr = PyStackRef_FromPyObjectSteal(attr_o); } macro(LOAD_ATTR) = @@ -1928,23 +2129,26 @@ dummy_func( _LOAD_ATTR; op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); EXIT_IF(tp->tp_version_tag != type_version); } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner)->valid); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); } split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { - attr = _PyObject_InlineValues(owner)->values[index]; - DEOPT_IF(attr == NULL); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); DECREF_INPUTS(); } @@ -1956,22 +2160,25 @@ dummy_func( unused/5; // Skip over rest of cache op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { - DEOPT_IF(!PyModule_CheckExact(owner)); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyModule_CheckExact(owner_o)); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); DEOPT_IF(dict->ma_keys->dk_version != dict_version); } op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL); + PyObject *attr_o = ep->me_value; + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -1982,30 +2189,36 @@ dummy_func( unused/5; op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); - attr = ep->me_value; + attr_o = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); - attr = ep->me_value; + attr_o = ep->me_value; } - DEOPT_IF(attr == NULL); + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2017,12 +2230,14 @@ dummy_func( unused/5; split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); DECREF_INPUTS(); } @@ -2033,17 +2248,19 @@ dummy_func( unused/5; op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { - DEOPT_IF(!PyType_Check(owner)); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + DEOPT_IF(!PyType_Check(owner_o)); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version); } split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2054,10 +2271,12 @@ dummy_func( _LOAD_ATTR_CLASS; inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame); - PyTypeObject *cls = Py_TYPE(owner); + PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); DEOPT_IF(cls->tp_version_tag != type_version); assert(Py_IS_TYPE(fget, &PyFunction_Type)); @@ -2078,9 +2297,11 @@ dummy_func( } inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame); - PyTypeObject *cls = Py_TYPE(owner); + PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); DEOPT_IF(cls->tp_version_tag != type_version); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); @@ -2098,33 +2319,36 @@ dummy_func( // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } op(_GUARD_DORV_NO_DICT, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(_PyObject_GetManagedDict(owner)); - DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(_PyObject_GetManagedDict(owner_o)); + DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0); } op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner) == NULL); - PyDictValues *values = _PyObject_InlineValues(owner); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyDictValues *values = _PyObject_InlineValues(owner_o); PyObject *old_value = values->values[index]; - values->values[index] = value; + values->values[index] = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { _PyDictValues_AddToInsertionOrder(values, index); } else { Py_DECREF(old_value); } - - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } macro(STORE_ATTR_INSTANCE_VALUE) = @@ -2134,8 +2358,9 @@ dummy_func( _STORE_ATTR_INSTANCE_VALUE; op(_STORE_ATTR_WITH_HINT, (hint/1, value, owner --)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -2147,26 +2372,26 @@ dummy_func( DEOPT_IF(ep->me_key != name); old_value = ep->me_value; DEOPT_IF(old_value == NULL); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); old_value = ep->me_value; DEOPT_IF(old_value == NULL); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) { _PyObject_GC_TRACK(dict); } /* PEP 509 */ dict->ma_version_tag = new_version; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } macro(STORE_ATTR_WITH_HINT) = @@ -2175,12 +2400,14 @@ dummy_func( _STORE_ATTR_WITH_HINT; op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { - char *addr = (char *)owner + index; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; + *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); Py_XDECREF(old_value); - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } macro(STORE_ATTR_SLOT) = @@ -2207,15 +2434,21 @@ dummy_func( } op(_COMPARE_OP, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); ERROR_IF(res_bool < 0, error); - res = res_bool ? Py_True : Py_False; + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); } } @@ -2231,52 +2464,67 @@ dummy_func( _GUARD_BOTH_UNICODE + unused/1 + _COMPARE_OP_STR; op(_COMPARE_OP_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } // Similar to COMPARE_OP_FLOAT op(_COMPARE_OP_INT, (left, right -- res)) { - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left)); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } // Similar to COMPARE_OP_FLOAT, but for ==, != only op(_COMPARE_OP_STR, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); + int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } inst(IS_OP, (left, right -- b)) { - int res = Py_Is(left, right) ^ oparg; +#ifdef Py_GIL_DISABLED + // On free-threaded builds, objects are conditionally immortalized. + // So their bits don't always compare equally. + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; +#else + int res = PyStackRef_Is(left, right) ^ oparg; +#endif DECREF_INPUTS(); - b = res ? Py_True : Py_False; + b = res ? PyStackRef_True : PyStackRef_False; } family(CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP) = { @@ -2285,10 +2533,13 @@ dummy_func( }; op(_CONTAINS_OP, (left, right -- b)) { - int res = PySequence_Contains(right, left); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + int res = PySequence_Contains(right_o, left_o); DECREF_INPUTS(); ERROR_IF(res < 0, error); - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } specializing op(_SPECIALIZE_CONTAINS_OP, (counter/1, left, right -- left, right)) { @@ -2306,68 +2557,86 @@ dummy_func( macro(CONTAINS_OP) = _SPECIALIZE_CONTAINS_OP + _CONTAINS_OP; inst(CONTAINS_OP_SET, (unused/1, left, right -- b)) { - DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! - int res = _PySet_Contains((PySetObject *)right, left); + int res = _PySet_Contains((PySetObject *)right_o, left_o); DECREF_INPUTS(); ERROR_IF(res < 0, error); - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } inst(CONTAINS_OP_DICT, (unused/1, left, right -- b)) { - DEOPT_IF(!PyDict_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + DEOPT_IF(!PyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); - int res = PyDict_Contains(right, left); + int res = PyDict_Contains(right_o, left_o); DECREF_INPUTS(); ERROR_IF(res < 0, error); - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - inst(CHECK_EG_MATCH, (exc_value, match_type -- rest, match)) { + inst(CHECK_EG_MATCH, (exc_value_st, match_type_st -- rest, match)) { + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { DECREF_INPUTS(); ERROR_IF(true, error); } - match = NULL; - rest = NULL; + PyObject *match_o = NULL; + PyObject *rest_o = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); + &match_o, &rest_o); DECREF_INPUTS(); ERROR_IF(res < 0, error); - assert((match == NULL) == (rest == NULL)); - ERROR_IF(match == NULL, error); + assert((match_o == NULL) == (rest_o == NULL)); + ERROR_IF(match_o == NULL, error); - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + if (!Py_IsNone(match_o)) { + PyErr_SetHandledException(match_o); } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); } inst(CHECK_EXC_MATCH, (left, right -- left, b)) { - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + assert(PyExceptionInstance_Check(left_o)); + if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) { DECREF_INPUTS(); ERROR_IF(true, error); } - int res = PyErr_GivenExceptionMatches(left, right); + int res = PyErr_GivenExceptionMatches(left_o, right_o); DECREF_INPUTS(); - b = res ? Py_True : Py_False; + b = res ? PyStackRef_True : PyStackRef_False; } tier1 inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); + PyObject *res_o = import_name(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } tier1 inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - ERROR_IF(res == NULL, error); + PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } tier1 inst(JUMP_FORWARD, (--)) { @@ -2446,8 +2715,8 @@ dummy_func( } replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -2455,8 +2724,8 @@ dummy_func( } replaced op(_POP_JUMP_IF_TRUE, (cond -- )) { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -2464,11 +2733,11 @@ dummy_func( } op(_IS_NONE, (value -- b)) { - if (Py_IsNone(value)) { - b = Py_True; + if (PyStackRef_Is(value, PyStackRef_None)) { + b = PyStackRef_True; } else { - b = Py_False; + b = PyStackRef_False; DECREF_INPUTS(); } } @@ -2490,55 +2759,63 @@ dummy_func( JUMPBY(-oparg); } - inst(GET_LEN, (obj -- obj, len_o)) { + inst(GET_LEN, (obj -- obj, len)) { // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); ERROR_IF(len_i < 0, error); - len_o = PyLong_FromSsize_t(len_i); + PyObject *len_o = PyLong_FromSsize_t(len_i); ERROR_IF(len_o == NULL, error); + len = PyStackRef_FromPyObjectSteal(len_o); } inst(MATCH_CLASS, (subject, type, names -- attrs)) { // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); DECREF_INPUTS(); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { ERROR_IF(_PyErr_Occurred(tstate), error); // Error! - attrs = Py_None; // Failure! + attrs = PyStackRef_None; // Failure! } } inst(MATCH_MAPPING, (subject -- subject, res)) { - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; } inst(MATCH_SEQUENCE, (subject -- subject, res)) { - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; } inst(MATCH_KEYS, (subject, keys -- subject, keys, values_or_none)) { // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - ERROR_IF(values_or_none == NULL, error); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + ERROR_IF(values_or_none_o == NULL, error); + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); } inst(GET_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); DECREF_INPUTS(); - ERROR_IF(iter == NULL, error); + ERROR_IF(PyStackRef_IsNull(iter), error); } inst(GET_YIELD_FROM_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a @@ -2550,13 +2827,13 @@ dummy_func( } iter = iterable; } - else if (PyGen_CheckExact(iterable)) { + else if (PyGen_CheckExact(iterable_o)) { iter = iterable; } else { /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + if (PyStackRef_IsNull(iter)) { ERROR_NO_POP(); } DECREF_INPUTS(); @@ -2590,8 +2867,10 @@ dummy_func( replaced op(_FOR_ITER, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { + next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { ERROR_NO_POP(); @@ -2602,19 +2881,21 @@ dummy_func( /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ JUMPBY(oparg + 2); DISPATCH(); } + next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator } op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { ERROR_NO_POP(); @@ -2625,6 +2906,7 @@ dummy_func( /* The translator sets the deopt target just past the matching END_FOR */ DEOPT_IF(true); } + next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator } @@ -2632,10 +2914,11 @@ dummy_func( inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { _Py_CODEUNIT *target; - PyObject *iter = TOP(); + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next != NULL) { - PUSH(next); + PUSH(PyStackRef_FromPyObjectSteal(next)); target = next_instr; } else { @@ -2650,7 +2933,7 @@ dummy_func( assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); - Py_DECREF(iter); + PyStackRef_CLOSE(iter_stackref); /* Skip END_FOR and POP_TOP */ target = next_instr + oparg + 2; } @@ -2658,12 +2941,13 @@ dummy_func( } op(_ITER_CHECK_LIST, (iter -- iter)) { - EXIT_IF(Py_TYPE(iter) != &PyListIter_Type); + EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type); } replaced op(_ITER_JUMP_LIST, (iter -- iter)) { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { @@ -2674,7 +2958,7 @@ dummy_func( Py_DECREF(seq); } #endif - Py_DECREF(iter); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ JUMPBY(oparg + 2); @@ -2684,20 +2968,22 @@ dummy_func( // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_LIST, (iter -- iter)) { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; EXIT_IF(seq == NULL); EXIT_IF((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)); } op(_ITER_NEXT_LIST, (iter -- iter, next)) { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); } macro(FOR_ITER_LIST) = @@ -2707,12 +2993,13 @@ dummy_func( _ITER_NEXT_LIST; op(_ITER_CHECK_TUPLE, (iter -- iter)) { - EXIT_IF(Py_TYPE(iter) != &PyTupleIter_Type); + EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type); } replaced op(_ITER_JUMP_TUPLE, (iter -- iter)) { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); PyTupleObject *seq = it->it_seq; if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { @@ -2720,7 +3007,7 @@ dummy_func( it->it_seq = NULL; Py_DECREF(seq); } - Py_DECREF(iter); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ JUMPBY(oparg + 2); @@ -2730,20 +3017,22 @@ dummy_func( // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter -- iter)) { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; EXIT_IF(seq == NULL); EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq)); } op(_ITER_NEXT_TUPLE, (iter -- iter, next)) { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); } macro(FOR_ITER_TUPLE) = @@ -2753,17 +3042,17 @@ dummy_func( _ITER_NEXT_TUPLE; op(_ITER_CHECK_RANGE, (iter -- iter)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(r) != &PyRangeIter_Type); } replaced op(_ITER_JUMP_RANGE, (iter -- iter)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); if (r->len <= 0) { STACK_SHRINK(1); - Py_DECREF(r); + PyStackRef_CLOSE(iter); // Jump over END_FOR and POP_TOP instructions. JUMPBY(oparg + 2); DISPATCH(); @@ -2772,20 +3061,21 @@ dummy_func( // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_RANGE, (iter -- iter)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); EXIT_IF(r->len <= 0); } op(_ITER_NEXT_RANGE, (iter -- iter, next)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); long value = r->start; r->start = value + r->step; r->len--; - next = PyLong_FromLong(value); - ERROR_IF(next == NULL, error); + PyObject *res = PyLong_FromLong(value); + ERROR_IF(res == NULL, error); + next = PyStackRef_FromPyObjectSteal(res); } macro(FOR_ITER_RANGE) = @@ -2795,12 +3085,12 @@ dummy_func( _ITER_NEXT_RANGE; op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame: _PyInterpreterFrame*)) { - PyGenObject *gen = (PyGenObject *)iter; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); + _PyFrame_StackPush(gen_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; @@ -2816,16 +3106,19 @@ dummy_func( inst(LOAD_SPECIAL, (owner -- attr, self_or_null)) { assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); PyObject *name = _Py_SpecialMethods[oparg].name; - attr = _PyObject_LookupSpecialMethod(owner, name, &self_or_null); - if (attr == NULL) { + PyObject *self_or_null_o; + attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); + if (PyStackRef_IsNull(attr)) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, - Py_TYPE(owner)->tp_name); + Py_TYPE(owner_o)->tp_name); } } - ERROR_IF(attr == NULL, error); + ERROR_IF(PyStackRef_IsNull(attr), error); + self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); } inst(WITH_EXCEPT_START, (exit_func, exit_self, lasti, unused, val -- exit_func, exit_self, lasti, unused, val, res)) { @@ -2840,22 +3133,25 @@ dummy_func( */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); if (tb == NULL) { tb = Py_None; } else { Py_DECREF(tb); } - assert(PyLong_Check(lasti)); + assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti))); (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[5] = {NULL, exit_self, exc, val, tb}; - int has_self = (exit_self != NULL); - res = PyObject_Vectorcall(exit_func, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - ERROR_IF(res == NULL, error); + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); + ERROR_IF(PyStackRef_IsNull(res), error); } pseudo(SETUP_FINALLY, (-- unused), (HAS_ARG)) = { @@ -2883,24 +3179,26 @@ dummy_func( }; inst(PUSH_EXC_INFO, (new_exc -- prev_exc, new_exc)) { + _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } else { - prev_exc = Py_None; + prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); + assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc))); + exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); } op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner)->valid); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { - PyTypeObject *owner_cls = Py_TYPE(owner); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); } @@ -2910,8 +3208,8 @@ dummy_func( /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } @@ -2924,11 +3222,11 @@ dummy_func( op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } @@ -2943,7 +3241,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); } macro(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) = @@ -2955,11 +3253,11 @@ dummy_func( op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr, unused if (0))) { assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); } macro(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) = @@ -2969,7 +3267,7 @@ dummy_func( _LOAD_ATTR_NONDESCRIPTOR_NO_DICT; op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) { - char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL); @@ -2980,7 +3278,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } @@ -2992,11 +3290,11 @@ dummy_func( _LOAD_ATTR_METHOD_LAZY_DICT; inst(INSTRUMENTED_CALL, (unused/3 -- )) { - int is_meth = PEEK(oparg + 1) != NULL; + int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); @@ -3034,7 +3332,7 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); @@ -3044,31 +3342,35 @@ dummy_func( // When calling Python, inline the call using DISPATCH_INLINED(). op(_CALL, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - else if (Py_TYPE(callable) == &PyMethod_Type) { + else if (Py_TYPE(callable_o) == &PyMethod_Type) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; } // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, args, total_args, NULL ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -3082,33 +3384,40 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); if (err < 0) { - Py_CLEAR(res); + Py_CLEAR(res_o); } } } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } op(_CHECK_PERIODIC, (--)) { @@ -3118,17 +3427,20 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC; op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - assert(Py_TYPE(callable) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, args, total_args, NULL ); // The frame has stolen all the arguments from the stack, @@ -3140,8 +3452,9 @@ dummy_func( } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - EXIT_IF(!PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(!PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; EXIT_IF(func->func_version != func_version); } @@ -3154,23 +3467,25 @@ dummy_func( _PUSH_FRAME; op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { - EXIT_IF(Py_TYPE(callable) != &PyMethod_Type); - PyObject *func = ((PyMethodObject *)callable)->im_func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; EXIT_IF(!PyFunction_Check(func)); EXIT_IF(((PyFunctionObject *)func)->func_version != func_version); - EXIT_IF(null != NULL); + EXIT_IF(!PyStackRef_IsNull(null)); } op(_EXPAND_METHOD, (callable, null, unused[oparg] -- method, self, unused[oparg])) { - assert(null == NULL); - assert(Py_TYPE(callable) == &PyMethod_Type); - self = ((PyMethodObject *)callable)->im_self; - Py_INCREF(self); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL - method = ((PyMethodObject *)callable)->im_func; - assert(PyFunction_Check(method)); - Py_INCREF(method); - Py_DECREF(callable); + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); + PyStackRef_CLOSE(callable); } macro(CALL_BOUND_METHOD_GENERAL) = @@ -3183,30 +3498,41 @@ dummy_func( _PUSH_FRAME; op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - EXIT_IF(PyFunction_Check(callable)); - EXIT_IF(Py_TYPE(callable) == &PyMethod_Type); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(PyFunction_Check(callable_o)); + EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); } op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg] -- res)) { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_NON_PY_GENERAL) = @@ -3217,17 +3543,20 @@ dummy_func( _CHECK_PERIODIC; op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - EXIT_IF(null != NULL); - EXIT_IF(Py_TYPE(callable) != &PyMethod_Type); + EXIT_IF(!PyStackRef_IsNull(null)); + EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type); } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization + self = stack_pointer[-1 - oparg]; + func = stack_pointer[-2 - oparg]; + PyStackRef_CLOSE(callable); + // self may be unused in tier 1, so silence warnings. + (void)self; } op(_CHECK_PEP_523, (--)) { @@ -3235,25 +3564,28 @@ dummy_func( } op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - assert(PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - EXIT_IF(code->co_argcount != oparg + (self_or_null != NULL)); + EXIT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))); } op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); DEOPT_IF(tstate->py_recursion_remaining <= 1); } replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { - int has_self = (self_or_null != NULL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); - PyObject **first_non_self_local = new_frame->localsplus + has_self; + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; @@ -3298,22 +3630,28 @@ dummy_func( _PUSH_FRAME; inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, arg -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); - DEOPT_IF(null != NULL); - DEOPT_IF(callable != (PyObject *)&PyType_Type); + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type); STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(arg)); - Py_DECREF(arg); + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); } op(_CALL_STR_1, (callable, null, arg -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); - DEOPT_IF(null != NULL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); - res = PyObject_Str(arg); - Py_DECREF(arg); - ERROR_IF(res == NULL, error); + res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + PyStackRef_CLOSE(arg); + ERROR_IF(PyStackRef_IsNull(res), error); } macro(CALL_STR_1) = @@ -3323,13 +3661,16 @@ dummy_func( _CHECK_PERIODIC; op(_CALL_TUPLE_1, (callable, null, arg -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); - DEOPT_IF(null != NULL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type); + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); - res = PySequence_Tuple(arg); - Py_DECREF(arg); - ERROR_IF(res == NULL, error); + res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + PyStackRef_CLOSE(arg); + ERROR_IF(PyStackRef_IsNull(res), error); } macro(CALL_TUPLE_1) = @@ -3339,18 +3680,19 @@ dummy_func( _CHECK_PERIODIC; inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(null != NULL); - DEOPT_IF(!PyType_Check(callable)); - PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(!PyType_Check(callable_o)); + PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; DEOPT_IF(code->co_argcount != oparg+1); @@ -3360,17 +3702,17 @@ dummy_func( if (self == NULL) { ERROR_NO_POP(); } - Py_DECREF(tp); + PyStackRef_CLOSE(callable); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1); assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); - shim->localsplus[0] = self; + shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); Py_INCREF(init); _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; + init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } @@ -3391,31 +3733,40 @@ dummy_func( inst(EXIT_INIT_CHECK, (should_be_none -- )) { assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { + if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); ERROR_NO_POP(); } } op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyType_Check(callable)); - PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(!PyType_Check(callable_o)); + PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(tp); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_BUILTIN_CLASS) = @@ -3426,27 +3777,30 @@ dummy_func( op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1); - DEOPT_IF(!PyCFunction_CheckExact(callable)); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O); + DEOPT_IF(!PyCFunction_CheckExact(callable_o)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O); // CPython promises to check all non-vectorcall function calls. DEOPT_IF(tstate->c_recursion_remaining <= 0); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - PyObject *arg = args[0]; + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_BUILTIN_O) = @@ -3457,28 +3811,37 @@ dummy_func( op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable)); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - res = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_BUILTIN_FAST) = @@ -3489,27 +3852,38 @@ dummy_func( op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable)); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)); + DEOPT_IF(!PyCFunction_CheckExact(callable_o)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyCFunction_GET_FUNCTION(callable_o); + + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_BUILTIN_FAST_WITH_KEYWORDS) = @@ -3520,69 +3894,75 @@ dummy_func( inst(CALL_LEN, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len); + DEOPT_IF(callable_o != interp->callable_cache.len); STAT_INC(CALL, hit); - PyObject *arg = args[0]; + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { ERROR_NO_POP(); } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res == NULL) { + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { GOTO_ERROR(error); } - Py_DECREF(callable); - Py_DECREF(arg); + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(arg_stackref); + res = PyStackRef_FromPyObjectSteal(res_o); } inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 2); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance); + DEOPT_IF(callable_o != interp->callable_cache.isinstance); STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); + _PyStackRef cls_stackref = args[1]; + _PyStackRef inst_stackref = args[0]; + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); if (retval < 0) { ERROR_NO_POP(); } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res == NULL) { - GOTO_ERROR(error); - } - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(inst_stackref); + PyStackRef_CLOSE(cls_stackref); + PyStackRef_CLOSE(callable); } // This is secretly a super-instruction tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- unused)) { assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self)); + DEOPT_IF(callable_o != interp->callable_cache.list_append); + assert(self_o != NULL); + DEOPT_IF(!PyList_Check(self_o)); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { + if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } - Py_DECREF(self); - Py_DECREF(callable); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); STACK_SHRINK(3); // Skip POP_TOP assert(next_instr->op.code == POP_TOP); @@ -3591,31 +3971,38 @@ dummy_func( } op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(total_args != 2); DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O); // CPython promises to check all non-vectorcall function calls. DEOPT_IF(tstate->c_recursion_remaining <= 0); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, self, arg); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_METHOD_DESCRIPTOR_O) = @@ -3625,31 +4012,41 @@ dummy_func( _CHECK_PERIODIC; op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); DEOPT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) = @@ -3660,16 +4057,19 @@ dummy_func( op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); DEOPT_IF(meth->ml_flags != METH_NOARGS); // CPython promises to check all non-vectorcall function calls. @@ -3677,12 +4077,13 @@ dummy_func( STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_METHOD_DESCRIPTOR_NOARGS) = @@ -3692,30 +4093,41 @@ dummy_func( _CHECK_PERIODIC; op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL); - PyObject *self = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_METHOD_DESCRIPTOR_FAST) = @@ -3725,11 +4137,11 @@ dummy_func( _CHECK_PERIODIC; inst(INSTRUMENTED_CALL_KW, ( -- )) { - int is_meth = PEEK(oparg + 2) != NULL; + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); + : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); @@ -3738,35 +4150,40 @@ dummy_func( } inst(CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o ); - Py_DECREF(kwnames); + PyStackRef_CLOSE(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 3); // The frame has stolen all the arguments from the stack, @@ -3779,34 +4196,41 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + kwnames_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); if (err < 0) { - Py_CLEAR(res); + Py_CLEAR(res_o); } } } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + PyStackRef_CLOSE(kwnames); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); CHECK_EVAL_BREAKER(); } @@ -3814,7 +4238,11 @@ dummy_func( GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - inst(CALL_FUNCTION_EX, (func, unused, callargs, kwargs if (oparg & 1) -- result)) { + inst(CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -3826,7 +4254,9 @@ dummy_func( if (tuple == NULL) { ERROR_NO_POP(); } - Py_SETREF(callargs, tuple); + PyStackRef_CLOSE(callargs_st); + callargs_st = PyStackRef_FromPyObjectSteal(tuple); + callargs = tuple; } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); @@ -3837,10 +4267,10 @@ dummy_func( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); if (err) ERROR_NO_POP(); - result = PyObject_Call(func, callargs, kwargs); + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result == NULL) { + if (PyStackRef_IsNull(result)) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); @@ -3850,7 +4280,7 @@ dummy_func( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); if (err < 0) { - Py_CLEAR(result); + PyStackRef_CLEAR(result); } } } @@ -3865,7 +4295,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, + (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, nargs, callargs, kwargs); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); @@ -3876,30 +4306,34 @@ dummy_func( frame->return_offset = 1; DISPATCH_INLINED(new_frame); } - result = PyObject_Call(func, callargs, kwargs); + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); } DECREF_INPUTS(); - assert(PEEK(2 + (oparg & 1)) == NULL); - ERROR_IF(result == NULL, error); + assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); + ERROR_IF(PyStackRef_IsNull(result), error); CHECK_EVAL_BREAKER(); } - inst(MAKE_FUNCTION, (codeobj -- func)) { + inst(MAKE_FUNCTION, (codeobj_st -- func)) { + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); + PyStackRef_CLOSE(codeobj_st); if (func_obj == NULL) { ERROR_NO_POP(); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); } - inst(SET_FUNCTION_ATTRIBUTE, (attr, func -- func)) { + inst(SET_FUNCTION_ATTRIBUTE, (attr_st, func_st -- func_st)) { + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); + assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -3947,7 +4381,7 @@ dummy_func( gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - res = (PyObject *)gen; + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; @@ -3957,27 +4391,34 @@ dummy_func( } inst(BUILD_SLICE, (start, stop, step if (oparg == 3) -- slice)) { - slice = PySlice_New(start, stop, step); + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); DECREF_INPUTS(); - ERROR_IF(slice == NULL, error); + ERROR_IF(slice_o == NULL, error); + slice = PyStackRef_FromPyObjectSteal(slice_o); } inst(CONVERT_VALUE, (value -- result)) { conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; - result = conv_fn(value); - Py_DECREF(value); - ERROR_IF(result == NULL, error); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + ERROR_IF(result_o == NULL, error); + result = PyStackRef_FromPyObjectSteal(result_o); } inst(FORMAT_SIMPLE, (value -- res)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); /* If value is a unicode object, then we know the result * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - ERROR_IF(res == NULL, error); + if (!PyUnicode_CheckExact(value_o)) { + res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + PyStackRef_CLOSE(value); + ERROR_IF(PyStackRef_IsNull(res), error); } else { res = value; @@ -3985,15 +4426,16 @@ dummy_func( } inst(FORMAT_WITH_SPEC, (value, fmt_spec -- res)) { - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - ERROR_IF(res == NULL, error); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } pure inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); - top = Py_NewRef(bottom); + top = PyStackRef_DUP(bottom); } specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { @@ -4011,10 +4453,14 @@ dummy_func( } op(_BINARY_OP, (lhs, rhs -- res)) { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP; @@ -4047,9 +4493,9 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); + _PyStackRef cond = POP(); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); int offset = flag * oparg; #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -4058,9 +4504,9 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); + _PyStackRef cond = POP(); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); int offset = flag * oparg; #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -4069,14 +4515,14 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { - PyObject *value = POP(); - int flag = Py_IsNone(value); + _PyStackRef value_stackref = POP(); + int flag = PyStackRef_Is(value_stackref, PyStackRef_None); int offset; if (flag) { offset = oparg; } else { - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); offset = 0; } #if ENABLE_SPECIALIZATION @@ -4086,14 +4532,14 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { - PyObject *value = POP(); + _PyStackRef value_stackref = POP(); int offset; - int nflag = Py_IsNone(value); + int nflag = PyStackRef_Is(value_stackref, PyStackRef_None); if (nflag) { offset = 0; } else { - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); offset = oparg; } #if ENABLE_SPECIALIZATION @@ -4124,28 +4570,28 @@ dummy_func( op (_GUARD_IS_TRUE_POP, (flag -- )) { SYNC_SP(); - EXIT_IF(!Py_IsTrue(flag)); - assert(Py_IsTrue(flag)); + EXIT_IF(!PyStackRef_Is(flag, PyStackRef_True)); + assert(PyStackRef_Is(flag, PyStackRef_True)); } op (_GUARD_IS_FALSE_POP, (flag -- )) { SYNC_SP(); - EXIT_IF(!Py_IsFalse(flag)); - assert(Py_IsFalse(flag)); + EXIT_IF(!PyStackRef_Is(flag, PyStackRef_False)); + assert(PyStackRef_Is(flag, PyStackRef_False)); } op (_GUARD_IS_NONE_POP, (val -- )) { SYNC_SP(); - if (!Py_IsNone(val)) { - Py_DECREF(val); + if (!PyStackRef_Is(val, PyStackRef_None)) { + PyStackRef_CLOSE(val); EXIT_IF(1); } } op (_GUARD_IS_NOT_NONE_POP, (val -- )) { SYNC_SP(); - EXIT_IF(Py_IsNone(val)); - Py_DECREF(val); + EXIT_IF(PyStackRef_Is(val, PyStackRef_None)); + PyStackRef_CLOSE(val); } op(_JUMP_TO_TOP, (--)) { @@ -4180,26 +4626,26 @@ dummy_func( } tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = Py_NewRef(ptr); + value = PyStackRef_FromPyObjectNew(ptr); } tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = ptr; + value = PyStackRef_FromPyObjectImmortal(ptr); } tier2 pure op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - Py_DECREF(pop); - value = ptr; + PyStackRef_CLOSE(pop); + value = PyStackRef_FromPyObjectImmortal(ptr); } tier2 pure op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { - value = Py_NewRef(ptr); - null = NULL; + value = PyStackRef_FromPyObjectNew(ptr); + null = PyStackRef_NULL; } tier2 pure op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { - value = ptr; - null = NULL; + value = PyStackRef_FromPyObjectImmortal(ptr); + null = PyStackRef_NULL; } tier2 op(_CHECK_FUNCTION, (func_version/2 -- )) { @@ -4209,7 +4655,7 @@ dummy_func( /* Internal -- for testing executors */ op(_INTERNAL_INCREMENT_OPT_COUNTER, (opt --)) { - _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)opt; + _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt); exe->count++; } diff --git a/Python/ceval.c b/Python/ceval.c index d36e15e55f57ab5..2412256ace8c8db 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -39,6 +39,7 @@ #include "opcode.h" #include "pydtrace.h" #include "setobject.h" +#include "pycore_stackref.h" #include // bool @@ -104,33 +105,34 @@ #ifdef LLTRACE static void -dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) +dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { - PyObject **stack_base = _PyFrame_Stackbase(frame); + _PyStackRef *stack_base = _PyFrame_Stackbase(frame); PyObject *exc = PyErr_GetRaisedException(); printf(" stack=["); - for (PyObject **ptr = stack_base; ptr < stack_pointer; ptr++) { + for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) { if (ptr != stack_base) { printf(", "); } - if (*ptr == NULL) { + PyObject *obj = PyStackRef_AsPyObjectBorrow(*ptr); + if (obj == NULL) { printf(""); continue; } if ( - *ptr == Py_None - || PyBool_Check(*ptr) - || PyLong_CheckExact(*ptr) - || PyFloat_CheckExact(*ptr) - || PyUnicode_CheckExact(*ptr) + obj == Py_None + || PyBool_Check(obj) + || PyLong_CheckExact(obj) + || PyFloat_CheckExact(obj) + || PyUnicode_CheckExact(obj) ) { - if (PyObject_Print(*ptr, stdout, 0) == 0) { + if (PyObject_Print(obj, stdout, 0) == 0) { continue; } PyErr_Clear(); } // Don't call __repr__(), it might recurse into the interpreter. - printf("<%s at %p>", Py_TYPE(*ptr)->tp_name, (void *)(*ptr)); + printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)(ptr->bits)); } printf("]\n"); fflush(stdout); @@ -139,7 +141,7 @@ dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) static void lltrace_instruction(_PyInterpreterFrame *frame, - PyObject **stack_pointer, + _PyStackRef *stack_pointer, _Py_CODEUNIT *next_instr, int opcode, int oparg) @@ -695,6 +697,35 @@ extern void _PyUOpPrint(const _PyUOpInstruction *uop); #endif +PyObject ** +_PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch) +{ + PyObject **result; + if (nargs > MAX_STACKREF_SCRATCH) { + // +1 in case PY_VECTORCALL_ARGUMENTS_OFFSET is set. + result = PyMem_Malloc((nargs + 1) * sizeof(PyObject *)); + if (result == NULL) { + return NULL; + } + result++; + } + else { + result = scratch; + } + for (int i = 0; i < nargs; i++) { + result[i] = PyStackRef_AsPyObjectBorrow(input[i]); + } + return result; +} + +void +_PyObjectArray_Free(PyObject **array, PyObject **scratch) +{ + if (array != scratch) { + PyMem_Free(array); + } +} + /* _PyEval_EvalFrameDefault() is a *big* function, * so consume 3 units of C stack */ #define PY_EVAL_C_STACK_UNITS 2 @@ -773,7 +804,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Local "register" variables. * These are cached values from the frame and code object. */ _Py_CODEUNIT *next_instr; - PyObject **stack_pointer; + _PyStackRef *stack_pointer; #if defined(_Py_TIER2) && !defined(_Py_JIT) /* Tier 2 interpreter state */ @@ -916,10 +947,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(_PyErr_Occurred(tstate)); /* Pop remaining stack entries. */ - PyObject **stackbase = _PyFrame_Stackbase(frame); + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); while (stack_pointer > stackbase) { - PyObject *o = POP(); - Py_XDECREF(o); + PyStackRef_XCLOSE(POP()); } assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -928,10 +958,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } assert(STACK_LEVEL() >= level); - PyObject **new_top = _PyFrame_Stackbase(frame) + level; + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; while (stack_pointer > new_top) { - PyObject *v = POP(); - Py_XDECREF(v); + PyStackRef_XCLOSE(POP()); } if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); @@ -939,7 +968,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (lasti == NULL) { goto exception_unwind; } - PUSH(lasti); + PUSH(PyStackRef_FromPyObjectSteal(lasti)); } /* Make the raw exception data @@ -947,7 +976,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int so a program can emulate the Python main loop. */ PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(exc); + PUSH(PyStackRef_FromPyObjectSteal(exc)); next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; if (monitor_handled(tstate, frame, next_instr, exc) < 0) { @@ -1217,7 +1246,7 @@ format_missing(PyThreadState *tstate, const char *kind, static void missing_arguments(PyThreadState *tstate, PyCodeObject *co, Py_ssize_t missing, Py_ssize_t defcount, - PyObject **localsplus, PyObject *qualname) + _PyStackRef *localsplus, PyObject *qualname) { Py_ssize_t i, j = 0; Py_ssize_t start, end; @@ -1238,7 +1267,7 @@ missing_arguments(PyThreadState *tstate, PyCodeObject *co, end = start + co->co_kwonlyargcount; } for (i = start; i < end; i++) { - if (localsplus[i] == NULL) { + if (PyStackRef_IsNull(localsplus[i])) { PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i); PyObject *name = PyObject_Repr(raw); if (name == NULL) { @@ -1256,7 +1285,7 @@ missing_arguments(PyThreadState *tstate, PyCodeObject *co, static void too_many_positional(PyThreadState *tstate, PyCodeObject *co, Py_ssize_t given, PyObject *defaults, - PyObject **localsplus, PyObject *qualname) + _PyStackRef *localsplus, PyObject *qualname) { int plural; Py_ssize_t kwonly_given = 0; @@ -1267,7 +1296,7 @@ too_many_positional(PyThreadState *tstate, PyCodeObject *co, assert((co->co_flags & CO_VARARGS) == 0); /* Count missing keyword-only args. */ for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) { - if (localsplus[i] != NULL) { + if (PyStackRef_AsPyObjectBorrow(localsplus[i]) != NULL) { kwonly_given++; } } @@ -1445,7 +1474,7 @@ get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, i static int initialize_locals(PyThreadState *tstate, PyFunctionObject *func, - PyObject **localsplus, PyObject *const *args, + _PyStackRef *localsplus, _PyStackRef const *args, Py_ssize_t argcount, PyObject *kwnames) { PyCodeObject *co = (PyCodeObject*)func->func_code; @@ -1463,8 +1492,8 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (co->co_flags & CO_VARARGS) { i++; } - assert(localsplus[i] == NULL); - localsplus[i] = kwdict; + assert(PyStackRef_IsNull(localsplus[i])); + localsplus[i] = PyStackRef_FromPyObjectSteal(kwdict); } else { kwdict = NULL; @@ -1479,9 +1508,8 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, n = argcount; } for (j = 0; j < n; j++) { - PyObject *x = args[j]; - assert(localsplus[j] == NULL); - localsplus[j] = x; + assert(PyStackRef_IsNull(localsplus[j])); + localsplus[j] = args[j]; } /* Pack other positional arguments into the *args argument */ @@ -1492,18 +1520,23 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, } else { assert(args != NULL); - u = _PyTuple_FromArraySteal(args + n, argcount - n); + STACKREFS_TO_PYOBJECTS((_PyStackRef *)args, argcount, args_o); + if (args_o == NULL) { + goto fail_pre_positional; + } + u = _PyTuple_FromArraySteal((args_o + n), argcount - n); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); } if (u == NULL) { goto fail_post_positional; } - assert(localsplus[total_args] == NULL); - localsplus[total_args] = u; + assert(PyStackRef_AsPyObjectBorrow(localsplus[total_args]) == NULL); + localsplus[total_args] = PyStackRef_FromPyObjectSteal(u); } else if (argcount > n) { /* Too many positional args. Error is reported later */ for (j = n; j < argcount; j++) { - Py_DECREF(args[j]); + PyStackRef_CLOSE(args[j]); } } @@ -1513,7 +1546,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, for (i = 0; i < kwcount; i++) { PyObject **co_varnames; PyObject *keyword = PyTuple_GET_ITEM(kwnames, i); - PyObject *value = args[i+argcount]; + _PyStackRef value_stackref = args[i+argcount]; Py_ssize_t j; if (keyword == NULL || !PyUnicode_Check(keyword)) { @@ -1586,27 +1619,26 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, goto kw_fail; } - if (PyDict_SetItem(kwdict, keyword, value) == -1) { + if (PyDict_SetItem(kwdict, keyword, PyStackRef_AsPyObjectSteal(value_stackref)) == -1) { goto kw_fail; } - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); continue; kw_fail: for (;i < kwcount; i++) { - PyObject *value = args[i+argcount]; - Py_DECREF(value); + PyStackRef_CLOSE(args[i+argcount]); } goto fail_post_args; kw_found: - if (localsplus[j] != NULL) { + if (PyStackRef_AsPyObjectBorrow(localsplus[j]) != NULL) { _PyErr_Format(tstate, PyExc_TypeError, "%U() got multiple values for argument '%S'", func->func_qualname, keyword); goto kw_fail; } - localsplus[j] = value; + localsplus[j] = value_stackref; } } @@ -1623,7 +1655,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, Py_ssize_t m = co->co_argcount - defcount; Py_ssize_t missing = 0; for (i = argcount; i < m; i++) { - if (localsplus[i] == NULL) { + if (PyStackRef_IsNull(localsplus[i])) { missing++; } } @@ -1639,9 +1671,9 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (defcount) { PyObject **defs = &PyTuple_GET_ITEM(func->func_defaults, 0); for (; i < defcount; i++) { - if (localsplus[m+i] == NULL) { + if (PyStackRef_AsPyObjectBorrow(localsplus[m+i]) == NULL) { PyObject *def = defs[i]; - localsplus[m+i] = Py_NewRef(def); + localsplus[m+i] = PyStackRef_FromPyObjectNew(def); } } } @@ -1651,7 +1683,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (co->co_kwonlyargcount > 0) { Py_ssize_t missing = 0; for (i = co->co_argcount; i < total_args; i++) { - if (localsplus[i] != NULL) + if (PyStackRef_AsPyObjectBorrow(localsplus[i]) != NULL) continue; PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i); if (func->func_kwdefaults != NULL) { @@ -1660,7 +1692,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, goto fail_post_args; } if (def) { - localsplus[i] = def; + localsplus[i] = PyStackRef_FromPyObjectSteal(def); continue; } } @@ -1676,14 +1708,14 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, fail_pre_positional: for (j = 0; j < argcount; j++) { - Py_DECREF(args[j]); + PyStackRef_CLOSE(args[j]); } /* fall through */ fail_post_positional: if (kwnames) { Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); for (j = argcount; j < argcount+kwcount; j++) { - Py_DECREF(args[j]); + PyStackRef_CLOSE(args[j]); } } /* fall through */ @@ -1738,7 +1770,7 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) /* Consumes references to func, locals and all the args */ _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, PyObject* const* args, + PyObject *locals, _PyStackRef const* args, size_t argcount, PyObject *kwnames) { PyCodeObject * code = (PyCodeObject *)func->func_code; @@ -1759,18 +1791,45 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, Py_DECREF(func); Py_XDECREF(locals); for (size_t i = 0; i < argcount; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } if (kwnames) { Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); for (Py_ssize_t i = 0; i < kwcount; i++) { - Py_DECREF(args[i+argcount]); + PyStackRef_CLOSE(args[i+argcount]); } } PyErr_NoMemory(); return NULL; } +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, PyObject *const* args, + size_t argcount, PyObject *kwnames) +{ +#if defined(Py_GIL_DISABLED) + size_t kw_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); + size_t total_argcount = argcount + kw_count; + _PyStackRef *tagged_args_buffer = PyMem_Malloc(sizeof(_PyStackRef) * total_argcount); + if (tagged_args_buffer == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (size_t i = 0; i < argcount; i++) { + tagged_args_buffer[i] = PyStackRef_FromPyObjectSteal(args[i]); + } + for (size_t i = 0; i < kw_count; i++) { + tagged_args_buffer[argcount + i] = PyStackRef_FromPyObjectSteal(args[argcount + i]); + } + _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames); + PyMem_Free(tagged_args_buffer); + return res; +#else + return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames); +#endif +} + /* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. Steals references to func, callargs and kwargs. */ @@ -1795,7 +1854,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, Py_INCREF(PyTuple_GET_ITEM(callargs, i)); } } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged( tstate, (PyFunctionObject *)func, locals, newargs, nargs, kwnames ); @@ -1833,7 +1892,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, Py_INCREF(args[i+argcount]); } } - _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged( tstate, func, locals, args, argcount, kwnames); if (frame == NULL) { return NULL; @@ -2085,8 +2144,8 @@ _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, */ int -_PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, - int argcnt, int argcntafter, PyObject **sp) +_PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v_stackref, + int argcnt, int argcntafter, _PyStackRef *sp) { int i = 0, j = 0; Py_ssize_t ll = 0; @@ -2094,6 +2153,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, PyObject *w; PyObject *l = NULL; /* variable list */ + PyObject *v = PyStackRef_AsPyObjectBorrow(v_stackref); assert(v != NULL); it = PyObject_GetIter(v); @@ -2128,7 +2188,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, } goto Error; } - *--sp = w; + *--sp = PyStackRef_FromPyObjectSteal(w); } if (argcntafter == -1) { @@ -2150,7 +2210,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, l = PySequence_List(it); if (l == NULL) goto Error; - *--sp = l; + *--sp = PyStackRef_FromPyObjectSteal(l); i++; ll = PyList_GET_SIZE(l); @@ -2163,7 +2223,7 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, /* Pop the "after-variable" args off the list. */ for (j = argcntafter; j > 0; j--, i++) { - *--sp = PyList_GET_ITEM(l, ll - j); + *--sp = PyStackRef_FromPyObjectSteal(PyList_GET_ITEM(l, ll - j)); } /* Resize the list. */ Py_SET_SIZE(l, ll - argcntafter); @@ -2171,8 +2231,9 @@ _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, return 1; Error: - for (; i > 0; i--, sp++) - Py_DECREF(*sp); + for (; i > 0; i--, sp++) { + PyStackRef_CLOSE(*sp); + } Py_XDECREF(it); return 0; } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index e9247845b60bc0e..f6d055a1dfaa9fb 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -264,9 +264,9 @@ GETITEM(PyObject *v, Py_ssize_t i) { This is because it is possible that during the DECREF the frame is accessed by other code (e.g. a __del__ method or gc.collect()) and the variable would be pointing to already-freed memory. */ -#define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \ +#define SETLOCAL(i, value) do { _PyStackRef tmp = GETLOCAL(i); \ GETLOCAL(i) = value; \ - Py_XDECREF(tmp); } while (0) + PyStackRef_XCLOSE(tmp); } while (0) #define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) @@ -449,3 +449,34 @@ do { \ #define EXIT_TO_TRACE() goto exit_to_trace #define EXIT_TO_TIER1() goto exit_to_tier1 #define EXIT_TO_TIER1_DYNAMIC() goto exit_to_tier1_dynamic; + +/* Stackref macros */ + +/* How much scratch space to give stackref to PyObject* conversion. */ +#define MAX_STACKREF_SCRATCH 10 + +#ifdef Py_GIL_DISABLED +#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ + /* +1 because vectorcall might use -1 to write self */ \ + PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ + PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); +#else +#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ + PyObject **NAME = (PyObject **)ARGS; \ + assert(NAME != NULL); +#endif + +#ifdef Py_GIL_DISABLED +#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ + /* +1 because we +1 previously */ \ + _PyObjectArray_Free(NAME - 1, NAME##_temp); +#else +#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ + (void)(NAME); +#endif + +#ifdef Py_GIL_DISABLED +#define CONVERSION_FAILED(NAME) ((NAME) == NULL) +#else +#define CONVERSION_FAILED(NAME) (0) +#endif diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6bee9b03fc3c1f5..38437c6f2c087c8 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -33,17 +33,17 @@ /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 because it is instrumented */ case _LOAD_FAST_CHECK: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); - value = GETLOCAL(oparg); - if (value == NULL) { + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); if (1) JUMP_TO_ERROR(); } - Py_INCREF(value); + value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -51,12 +51,11 @@ } case _LOAD_FAST_0: { - PyObject *value; + _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -64,12 +63,11 @@ } case _LOAD_FAST_1: { - PyObject *value; + _PyStackRef value; oparg = 1; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -77,12 +75,11 @@ } case _LOAD_FAST_2: { - PyObject *value; + _PyStackRef value; oparg = 2; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -90,12 +87,11 @@ } case _LOAD_FAST_3: { - PyObject *value; + _PyStackRef value; oparg = 3; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -103,12 +99,11 @@ } case _LOAD_FAST_4: { - PyObject *value; + _PyStackRef value; oparg = 4; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -116,12 +111,11 @@ } case _LOAD_FAST_5: { - PyObject *value; + _PyStackRef value; oparg = 5; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -129,12 +123,11 @@ } case _LOAD_FAST_6: { - PyObject *value; + _PyStackRef value; oparg = 6; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -142,12 +135,11 @@ } case _LOAD_FAST_7: { - PyObject *value; + _PyStackRef value; oparg = 7; assert(oparg == CURRENT_OPARG()); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -155,11 +147,10 @@ } case _LOAD_FAST: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -167,11 +158,11 @@ } case _LOAD_FAST_AND_CLEAR: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; + GETLOCAL(oparg) = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -179,10 +170,9 @@ } case _LOAD_CONST: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -190,7 +180,7 @@ } case _STORE_FAST_0: { - PyObject *value; + _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; @@ -201,7 +191,7 @@ } case _STORE_FAST_1: { - PyObject *value; + _PyStackRef value; oparg = 1; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; @@ -212,7 +202,7 @@ } case _STORE_FAST_2: { - PyObject *value; + _PyStackRef value; oparg = 2; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; @@ -223,7 +213,7 @@ } case _STORE_FAST_3: { - PyObject *value; + _PyStackRef value; oparg = 3; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; @@ -234,7 +224,7 @@ } case _STORE_FAST_4: { - PyObject *value; + _PyStackRef value; oparg = 4; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; @@ -245,7 +235,7 @@ } case _STORE_FAST_5: { - PyObject *value; + _PyStackRef value; oparg = 5; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; @@ -256,7 +246,7 @@ } case _STORE_FAST_6: { - PyObject *value; + _PyStackRef value; oparg = 6; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; @@ -267,7 +257,7 @@ } case _STORE_FAST_7: { - PyObject *value; + _PyStackRef value; oparg = 7; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; @@ -278,7 +268,7 @@ } case _STORE_FAST: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; SETLOCAL(oparg, value); @@ -288,17 +278,17 @@ } case _POP_TOP: { - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - Py_DECREF(value); + PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_NULL: { - PyObject *res; - res = NULL; + _PyStackRef res; + res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -306,11 +296,12 @@ } case _END_SEND: { - PyObject *value; - PyObject *receiver; + _PyStackRef value; + _PyStackRef receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); + (void)receiver; + PyStackRef_CLOSE(receiver); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -318,42 +309,44 @@ } case _UNARY_NEGATIVE: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + res = PyStackRef_Is(value, PyStackRef_False) + ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; break; } case _TO_BOOL: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - int err = PyObject_IsTrue(value); - Py_DECREF(value); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); if (err < 0) JUMP_TO_ERROR(); - res = err ? Py_True : Py_False; + res = err ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - if (!PyBool_Check(value)) { + if (!PyBool_Check(PyStackRef_AsPyObjectBorrow(value))) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -362,109 +355,115 @@ } case _TO_BOOL_INT: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - Py_DECREF(value); - res = Py_True; + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; break; } case _TO_BOOL_LIST: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - if (!PyList_CheckExact(value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyList_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); + res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); stack_pointer[-1] = res; break; } case _TO_BOOL_NONE: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - if (!Py_IsNone(value)) { + if (!PyStackRef_Is(value, PyStackRef_None)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(TO_BOOL, hit); - res = Py_False; + res = PyStackRef_False; stack_pointer[-1] = res; break; } case _TO_BOOL_STR: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - if (!PyUnicode_CheckExact(value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; break; } case _REPLACE_WITH_TRUE: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - Py_DECREF(value); - res = Py_True; + PyStackRef_CLOSE(value); + res = PyStackRef_True; stack_pointer[-1] = res; break; } case _UNARY_INVERT: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; } case _GUARD_BOTH_INT: { - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (!PyLong_CheckExact(right)) { + if (!PyLong_CheckExact(right_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -472,9 +471,10 @@ } case _GUARD_NOS_INT: { - PyObject *left; + _PyStackRef left; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -482,9 +482,10 @@ } case _GUARD_TOS_INT: { - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -492,16 +493,19 @@ } case _BINARY_OP_MULTIPLY_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -509,16 +513,19 @@ } case _BINARY_OP_ADD_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -526,16 +533,19 @@ } case _BINARY_OP_SUBTRACT_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -543,15 +553,17 @@ } case _GUARD_BOTH_FLOAT: { - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (!PyFloat_CheckExact(right)) { + if (!PyFloat_CheckExact(right_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -559,9 +571,10 @@ } case _GUARD_NOS_FLOAT: { - PyObject *left; + _PyStackRef left; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -569,9 +582,10 @@ } case _GUARD_TOS_FLOAT: { - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - if (!PyFloat_CheckExact(value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -579,16 +593,20 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -596,16 +614,20 @@ } case _BINARY_OP_ADD_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -613,16 +635,20 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -630,15 +656,17 @@ } case _GUARD_BOTH_UNICODE: { - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyUnicode_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (!PyUnicode_CheckExact(right)) { + if (!PyUnicode_CheckExact(right_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -646,16 +674,19 @@ } case _BINARY_OP_ADD_UNICODE: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -663,15 +694,18 @@ } case _BINARY_SUBSCR: { - PyObject *sub; - PyObject *container; - PyObject *res; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef res; sub = stack_pointer[-1]; container = stack_pointer[-2]; - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -679,25 +713,28 @@ } case _BINARY_SLICE: { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *res; + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef res; stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + PyObject *res_o; // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { - res = NULL; + res_o = NULL; } else { - res = PyObject_GetItem(container, slice); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); } - Py_DECREF(container); - if (res == NULL) JUMP_TO_ERROR(); + PyStackRef_CLOSE(container); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -705,25 +742,26 @@ } case _STORE_SLICE: { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef v; stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); int err; if (slice == NULL) { err = 1; } else { - err = PyObject_SetItem(container, slice, v); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v)); Py_DECREF(slice); } - Py_DECREF(v); - Py_DECREF(container); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); if (err) JUMP_TO_ERROR(); stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); @@ -731,11 +769,13 @@ } case _BINARY_SUBSCR_LIST_INT: { - PyObject *sub; - PyObject *list; - PyObject *res; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); if (!PyLong_CheckExact(sub)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -755,11 +795,12 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -767,11 +808,13 @@ } case _BINARY_SUBSCR_STR_INT: { - PyObject *sub; - PyObject *str; - PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; + _PyStackRef sub_st; + _PyStackRef str_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); if (!PyLong_CheckExact(sub)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -796,9 +839,10 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -806,11 +850,13 @@ } case _BINARY_SUBSCR_TUPLE_INT: { - PyObject *sub; - PyObject *tuple; - PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); if (!PyLong_CheckExact(sub)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -830,11 +876,12 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -842,24 +889,28 @@ } case _BINARY_SUBSCR_DICT: { - PyObject *sub; - PyObject *dict; - PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef res; + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); + PyObject *res_o; + int rc = PyDict_GetItemRef(dict, sub, &res_o); if (rc == 0) { _PyErr_SetKeyError(sub); } - Py_DECREF(dict); - Py_DECREF(sub); + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); if (rc <= 0) JUMP_TO_ERROR(); // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -869,25 +920,27 @@ /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _LIST_APPEND: { - PyObject *v; - PyObject *list; + _PyStackRef v; + _PyStackRef list; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) JUMP_TO_ERROR(); + if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)) < 0) JUMP_TO_ERROR(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _SET_ADD: { - PyObject *v; - PyObject *set; + _PyStackRef v; + _PyStackRef set; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); + PyStackRef_CLOSE(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -895,17 +948,17 @@ } case _STORE_SUBSCR: { - PyObject *sub; - PyObject *container; - PyObject *v; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef v; sub = stack_pointer[-1]; container = stack_pointer[-2]; v = stack_pointer[-3]; /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v)); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); if (err) JUMP_TO_ERROR(); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -913,12 +966,14 @@ } case _STORE_SUBSCR_LIST_INT: { - PyObject *sub; - PyObject *list; - PyObject *value; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef value; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); if (!PyLong_CheckExact(sub)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -940,30 +995,32 @@ } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_SUBSCR_DICT: { - PyObject *sub; - PyObject *dict; - PyObject *value; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef value; + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + PyStackRef_CLOSE(dict_st); if (err) JUMP_TO_ERROR(); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -971,14 +1028,15 @@ } case _DELETE_SUBSCR: { - PyObject *sub; - PyObject *container; + _PyStackRef sub; + _PyStackRef container; sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); if (err) JUMP_TO_ERROR(); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -986,30 +1044,34 @@ } case _CALL_INTRINSIC_1: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - PyObject *value1; - PyObject *value2; - PyObject *res; + _PyStackRef value1_st; + _PyStackRef value2_st; + _PyStackRef res; oparg = CURRENT_OPARG(); - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1017,8 +1079,8 @@ } case _RETURN_VALUE: { - PyObject *retval; - PyObject *res; + _PyStackRef retval; + _PyStackRef res; retval = stack_pointer[-1]; #if TIER_ONE assert(frame != &entry_frame); @@ -1047,11 +1109,13 @@ /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 because it is instrumented */ case _GET_AITER: { - PyObject *obj; - PyObject *iter; + _PyStackRef obj; + _PyStackRef iter; obj = stack_pointer[-1]; unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } @@ -1060,35 +1124,38 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); + PyStackRef_CLOSE(obj); if (true) JUMP_TO_ERROR(); } - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) JUMP_TO_ERROR(); - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + iter_o = (*getter)(obj_o); + PyStackRef_CLOSE(obj); + if (iter_o == NULL) JUMP_TO_ERROR(); + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); if (true) JUMP_TO_ERROR(); } + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - PyObject *aiter; - PyObject *awaitable; + _PyStackRef aiter; + _PyStackRef awaitable; aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { + PyObject *awaitable_o; + PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter); + PyTypeObject *type = Py_TYPE(aiter_o); + if (PyAsyncGen_CheckExact(aiter_o)) { + awaitable_o = type->tp_as_async->am_anext(aiter_o); + if (awaitable_o == NULL) { JUMP_TO_ERROR(); } } else { @@ -1096,7 +1163,7 @@ getter = type->tp_as_async->am_anext; } if (getter != NULL) { - next_iter = (*getter)(aiter); + next_iter = (*getter)(aiter_o); if (next_iter == NULL) { JUMP_TO_ERROR(); } @@ -1108,8 +1175,8 @@ type->tp_name); JUMP_TO_ERROR(); } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { + awaitable_o = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable_o == NULL) { _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " @@ -1121,6 +1188,7 @@ Py_DECREF(next_iter); } } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1128,29 +1196,31 @@ } case _GET_AWAITABLE: { - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); - } - Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); + PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable)); + if (iter_o == NULL) { + _PyEval_FormatAwaitableError(tstate, + Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg); + } + PyStackRef_CLOSE(iterable); + if (iter_o != NULL && PyCoro_CheckExact(iter_o)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter_o); if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ Py_DECREF(yf); - Py_CLEAR(iter); + Py_CLEAR(iter_o); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) JUMP_TO_ERROR(); + if (iter_o == NULL) JUMP_TO_ERROR(); + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; break; } @@ -1162,8 +1232,8 @@ /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */ case _YIELD_VALUE: { - PyObject *retval; - PyObject *value; + _PyStackRef retval; + _PyStackRef value; oparg = CURRENT_OPARG(); retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! @@ -1207,25 +1277,27 @@ } case _POP_EXCEPT: { - PyObject *exc_value; + _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + Py_XSETREF(exc_info->exc_value, + PyStackRef_AsPyObjectBorrow(exc_value) == Py_None + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_COMMON_CONSTANT: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); // Keep in sync with _common_constants in opcode.py switch(oparg) { case CONSTANT_ASSERTIONERROR: - value = PyExc_AssertionError; + value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); break; case CONSTANT_NOTIMPLEMENTEDERROR: - value = PyExc_NotImplementedError; + value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); break; default: Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); @@ -1237,13 +1309,15 @@ } case _LOAD_BUILD_CLASS: { - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) JUMP_TO_ERROR(); - if (bc == NULL) { + _PyStackRef bc; + PyObject *bc_o; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0) JUMP_TO_ERROR(); + if (bc_o == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); if (true) JUMP_TO_ERROR(); } + bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1251,7 +1325,7 @@ } case _STORE_NAME: { - PyObject *v; + _PyStackRef v; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1260,14 +1334,14 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); + PyStackRef_CLOSE(v); if (true) JUMP_TO_ERROR(); } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); + PyStackRef_CLOSE(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1296,12 +1370,12 @@ } case _UNPACK_SEQUENCE: { - PyObject *seq; + _PyStackRef seq; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); + _PyStackRef *top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + PyStackRef_CLOSE(seq); if (res == 0) JUMP_TO_ERROR(); stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1309,24 +1383,25 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - PyObject *seq; - PyObject *val1; - PyObject *val0; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; assert(oparg == 2); - if (!PyTuple_CheckExact(seq)) { + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyTuple_CheckExact(seq_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (PyTuple_GET_SIZE(seq) != 2) { + if (PyTuple_GET_SIZE(seq_o) != 2) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(UNPACK_SEQUENCE, hit); - val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - Py_DECREF(seq); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -1335,63 +1410,65 @@ } case _UNPACK_SEQUENCE_TUPLE: { - PyObject *seq; - PyObject **values; + _PyStackRef seq; + _PyStackRef *values; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyTuple_CheckExact(seq)) { + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyTuple_CheckExact(seq_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (PyTuple_GET_SIZE(seq) != oparg) { + if (PyTuple_GET_SIZE(seq_o) != oparg) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } - Py_DECREF(seq); + PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_SEQUENCE_LIST: { - PyObject *seq; - PyObject **values; + _PyStackRef seq; + _PyStackRef *values; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyList_CheckExact(seq)) { + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + if (!PyList_CheckExact(seq_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (PyList_GET_SIZE(seq) != oparg) { + if (PyList_GET_SIZE(seq_o) != oparg) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); + PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } - Py_DECREF(seq); + PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_EX: { - PyObject *seq; + _PyStackRef seq; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); + _PyStackRef *top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + PyStackRef_CLOSE(seq); if (res == 0) JUMP_TO_ERROR(); stack_pointer += (oparg >> 8) + (oparg & 0xFF); assert(WITHIN_STACK_BOUNDS()); @@ -1399,15 +1476,16 @@ } case _STORE_ATTR: { - PyObject *owner; - PyObject *v; + _PyStackRef owner; + _PyStackRef v; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectSteal(v)); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); if (err) JUMP_TO_ERROR(); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1415,12 +1493,12 @@ } case _DELETE_ATTR: { - PyObject *owner; + _PyStackRef owner; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyStackRef_CLOSE(owner); if (err) JUMP_TO_ERROR(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1428,12 +1506,12 @@ } case _STORE_GLOBAL: { - PyObject *v; + _PyStackRef v; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1457,14 +1535,14 @@ } case _LOAD_LOCALS: { - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); if (true) JUMP_TO_ERROR(); } - Py_INCREF(locals); + locals = PyStackRef_FromPyObjectNew(l);; stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1476,17 +1554,18 @@ /* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _LOAD_GLOBAL: { - PyObject *res; - PyObject *null = NULL; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *res_o; if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - if (res == NULL) { + if (res_o == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1499,11 +1578,11 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR(); - if (res == NULL) { + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0) JUMP_TO_ERROR(); + if (res_o == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR(); - if (res == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0) JUMP_TO_ERROR(); + if (res_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1511,7 +1590,8 @@ } } } - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); @@ -1550,20 +1630,21 @@ } case _LOAD_GLOBAL_MODULE: { - PyObject *res; - PyObject *null = NULL; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - if (res == NULL) { + PyObject *res_o = entries[index].me_value; + if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - Py_INCREF(res); + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); @@ -1572,20 +1653,21 @@ } case _LOAD_GLOBAL_BUILTINS: { - PyObject *res; - PyObject *null = NULL; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - if (res == NULL) { + PyObject *res_o = entries[index].me_value; + if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - Py_INCREF(res); + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; stack_pointer += 1 + (oparg & 1); @@ -1595,15 +1677,15 @@ case _DELETE_FAST: { oparg = CURRENT_OPARG(); - PyObject *v = GETLOCAL(oparg); - if (v == NULL) { + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); if (1) JUMP_TO_ERROR(); } - SETLOCAL(oparg, NULL); + SETLOCAL(oparg, PyStackRef_NULL); break; } @@ -1611,18 +1693,18 @@ oparg = CURRENT_OPARG(); // "initial" is probably NULL but not if it's an arg (or set // via the f_locals proxy before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { JUMP_TO_ERROR(); } - SETLOCAL(oparg, cell); + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); break; } case _DELETE_DEREF: { oparg = CURRENT_OPARG(); - PyObject *cell = GETLOCAL(oparg); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); @@ -1635,39 +1717,43 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - PyObject *class_dict; - PyObject *value; + _PyStackRef class_dict_st; + _PyStackRef value; oparg = CURRENT_OPARG(); - class_dict = stack_pointer[-1]; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { JUMP_TO_ERROR(); } - if (!value) { - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - value = PyCell_GetRef(cell); - if (value == NULL) { + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); JUMP_TO_ERROR(); } } - Py_DECREF(class_dict); + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - PyObject *value; + _PyStackRef value; oparg = CURRENT_OPARG(); - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - value = PyCell_GetRef(cell); - if (value == NULL) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); if (true) JUMP_TO_ERROR(); } + value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1675,11 +1761,11 @@ } case _STORE_DEREF: { - PyObject *v; + _PyStackRef v; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - PyCell_SetTakeRef(cell, v); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1695,21 +1781,30 @@ int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } break; } case _BUILD_STRING: { - PyObject **pieces; - PyObject *str; + _PyStackRef *pieces; + _PyStackRef str; oparg = CURRENT_OPARG(); pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + PyStackRef_CLOSE(pieces[_i]); } - if (str == NULL) JUMP_TO_ERROR(); + if (str_o == NULL) JUMP_TO_ERROR(); + str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1717,12 +1812,21 @@ } case _BUILD_TUPLE: { - PyObject **values; - PyObject *tup; + _PyStackRef *values; + _PyStackRef tup; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) JUMP_TO_ERROR(); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + if (tup_o == NULL) JUMP_TO_ERROR(); + tup = PyStackRef_FromPyObjectSteal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1730,12 +1834,21 @@ } case _BUILD_LIST: { - PyObject **values; - PyObject *list; + _PyStackRef *values; + _PyStackRef list; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) JUMP_TO_ERROR(); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *list_o = _PyList_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + if (list_o == NULL) JUMP_TO_ERROR(); + list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1743,11 +1856,13 @@ } case _LIST_EXTEND: { - PyObject *iterable; - PyObject *list; + _PyStackRef iterable_st; + _PyStackRef list_st; oparg = CURRENT_OPARG(); - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1758,24 +1873,25 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - Py_DECREF(iterable); + PyStackRef_CLOSE(iterable_st); if (true) JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); - Py_DECREF(iterable); + PyStackRef_CLOSE(iterable_st); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _SET_UPDATE: { - PyObject *iterable; - PyObject *set; + _PyStackRef iterable; + _PyStackRef set; oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + PyStackRef_CLOSE(iterable); if (err < 0) JUMP_TO_ERROR(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1785,18 +1901,27 @@ /* _BUILD_SET is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _BUILD_MAP: { - PyObject **values; - PyObject *map; + _PyStackRef *values; + _PyStackRef map; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg*2]; - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + PyStackRef_CLOSE(values[_i]); } - if (map == NULL) JUMP_TO_ERROR(); + if (map_o == NULL) JUMP_TO_ERROR(); + map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; assert(WITHIN_STACK_BOUNDS()); @@ -1828,22 +1953,33 @@ } case _BUILD_CONST_KEY_MAP: { - PyObject *keys; - PyObject **values; - PyObject *map; + _PyStackRef keys; + _PyStackRef *values; + _PyStackRef map; oparg = CURRENT_OPARG(); keys = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - assert(PyTuple_CheckExact(keys)); - assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); + assert(PyTuple_CheckExact(keys_o)); + assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + PyStackRef_CLOSE(keys); + if (true) JUMP_TO_ERROR(); + } + PyObject *map_o = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys_o, 0), 1, + values_o, 1, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + PyStackRef_CLOSE(values[_i]); } - Py_DECREF(keys); - if (map == NULL) JUMP_TO_ERROR(); + PyStackRef_CLOSE(keys); + if (map_o == NULL) JUMP_TO_ERROR(); + map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1851,57 +1987,63 @@ } case _DICT_UPDATE: { - PyObject *update; - PyObject *dict; + _PyStackRef update; + _PyStackRef dict; oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + if (PyDict_Update(dict_o, update_o) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + Py_TYPE(update_o)->tp_name); } - Py_DECREF(update); + PyStackRef_CLOSE(update); if (true) JUMP_TO_ERROR(); } - Py_DECREF(update); + PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _DICT_MERGE: { - PyObject *update; - PyObject *dict; - PyObject *callable; + _PyStackRef update; + _PyStackRef dict; + _PyStackRef callable; oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + PyStackRef_CLOSE(update); if (true) JUMP_TO_ERROR(); } - Py_DECREF(update); + PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _MAP_ADD: { - PyObject *value; - PyObject *key; - PyObject *dict; + _PyStackRef value; + _PyStackRef key; + _PyStackRef dict_st; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) JUMP_TO_ERROR(); + if (_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0) JUMP_TO_ERROR(); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -1910,14 +2052,17 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */ case _LOAD_SUPER_ATTR_ATTR: { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr_st; oparg = CURRENT_OPARG(); - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); if (global_super != (PyObject *)&PySuper_Type) { UOP_STAT_INC(uopcode, miss); @@ -1929,27 +2074,31 @@ } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); if (attr == NULL) JUMP_TO_ERROR(); - stack_pointer[-3] = attr; + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_SUPER_ATTR_METHOD: { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr; + _PyStackRef self_or_null; oparg = CURRENT_OPARG(); - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); if (global_super != (PyObject *)&PySuper_Type) { UOP_STAT_INC(uopcode, miss); @@ -1963,20 +2112,21 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + if (attr_o == NULL) { + PyStackRef_CLOSE(self_st); if (true) JUMP_TO_ERROR(); } if (method_found) { - self_or_null = self; // transfer ownership + self_or_null = self_st; // transfer ownership } else { - Py_DECREF(self); - self_or_null = NULL; + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; } + attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; @@ -1985,21 +2135,22 @@ } case _LOAD_ATTR: { - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + attr_o = NULL; + if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch + assert(attr_o != NULL); // No errors on this branch self_or_null = owner; // Transfer ownership } else { @@ -2009,17 +2160,18 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ - Py_DECREF(owner); - if (attr == NULL) JUMP_TO_ERROR(); - self_or_null = NULL; + PyStackRef_CLOSE(owner); + if (attr_o == NULL) JUMP_TO_ERROR(); + self_or_null = PyStackRef_NULL; } } else { /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) JUMP_TO_ERROR(); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) JUMP_TO_ERROR(); } + attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; stack_pointer += (oparg & 1); @@ -2028,10 +2180,10 @@ } case _GUARD_TYPE_VERSION: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (tp->tp_version_tag != type_version) { UOP_STAT_INC(uopcode, miss); @@ -2041,11 +2193,12 @@ } case _CHECK_MANAGED_OBJECT_HAS_VALUES: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner)->valid) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!_PyObject_InlineValues(owner_o)->valid) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -2053,41 +2206,45 @@ } case _LOAD_ATTR_INSTANCE_VALUE_0: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - attr = _PyObject_InlineValues(owner)->values[index]; - if (attr == NULL) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_INSTANCE_VALUE_1: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - attr = _PyObject_InlineValues(owner)->values[index]; - if (attr == NULL) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; @@ -2098,14 +2255,15 @@ /* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */ case _CHECK_ATTR_MODULE: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); - if (!PyModule_CheckExact(owner)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyModule_CheckExact(owner_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); if (dict->ma_keys->dk_version != dict_version) { UOP_STAT_INC(uopcode, miss); @@ -2115,25 +2273,27 @@ } case _LOAD_ATTR_MODULE: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - if (attr == NULL) { + PyObject *attr_o = ep->me_value; + if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -2142,10 +2302,11 @@ } case _CHECK_ATTR_WITH_HINT: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); if (dict == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2155,13 +2316,15 @@ } case _LOAD_ATTR_WITH_HINT: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t hint = (uint16_t)CURRENT_OPERAND(); - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); if (hint >= (size_t)dict->ma_keys->dk_nentries) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2173,7 +2336,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - attr = ep->me_value; + attr_o = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; @@ -2181,16 +2344,17 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - attr = ep->me_value; + attr_o = ep->me_value; } - if (attr == NULL) { + if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -2199,43 +2363,45 @@ } case _LOAD_ATTR_SLOT_0: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - if (attr == NULL) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_SLOT_1: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - if (attr == NULL) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; @@ -2246,15 +2412,16 @@ /* _LOAD_ATTR_SLOT is split on (oparg & 1) */ case _CHECK_ATTR_CLASS: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - if (!PyType_Check(owner)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } assert(type_version != 0); - if (((PyTypeObject *)owner)->tp_version_tag != type_version) { + if (((PyTypeObject *)owner_o)->tp_version_tag != type_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -2262,33 +2429,33 @@ } case _LOAD_ATTR_CLASS_0: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_CLASS_1: { - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; (void)null; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; @@ -2303,15 +2470,16 @@ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _GUARD_DORV_NO_DICT: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (_PyObject_InlineValues(owner)->valid == 0) { + if (_PyObject_InlineValues(owner_o)->valid == 0) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -2319,37 +2487,39 @@ } case _STORE_ATTR_INSTANCE_VALUE: { - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner) == NULL); - PyDictValues *values = _PyObject_InlineValues(owner); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyDictValues *values = _PyObject_InlineValues(owner_o); PyObject *old_value = values->values[index]; - values->values[index] = value; + values->values[index] = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { _PyDictValues_AddToInsertionOrder(values, index); } else { Py_DECREF(old_value); } - Py_DECREF(owner); + PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_ATTR_WITH_HINT: { - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; value = stack_pointer[-2]; uint16_t hint = (uint16_t)CURRENT_OPERAND(); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); if (dict == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2373,8 +2543,8 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; @@ -2387,57 +2557,63 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) { _PyObject_GC_TRACK(dict); } /* PEP 509 */ dict->ma_version_tag = new_version; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } case _STORE_ATTR_SLOT: { - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; owner = stack_pointer[-1]; value = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - char *addr = (char *)owner + index; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; + *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); Py_XDECREF(old_value); - Py_DECREF(owner); + PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; } case _COMPARE_OP: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) JUMP_TO_ERROR(); if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); if (res_bool < 0) JUMP_TO_ERROR(); - res = res_bool ? Py_True : Py_False; + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -2446,20 +2622,22 @@ } case _COMPARE_OP_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; @@ -2468,30 +2646,32 @@ } case _COMPARE_OP_INT: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!_PyLong_IsCompact((PyLongObject *)left)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (!_PyLong_IsCompact((PyLongObject *)right)) { + if (!_PyLong_IsCompact((PyLongObject *)right_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; @@ -2500,21 +2680,23 @@ } case _COMPARE_OP_STR: { - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); + int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; stack_pointer += -1; @@ -2523,16 +2705,22 @@ } case _IS_OP: { - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; + #ifdef Py_GIL_DISABLED + // On free-threaded builds, objects are conditionally immortalized. + // So their bits don't always compare equally. + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + #else + int res = PyStackRef_Is(left, right) ^ oparg; + #endif + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2540,17 +2728,19 @@ } case _CONTAINS_OP: { - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int res = PySequence_Contains(right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); if (res < 0) JUMP_TO_ERROR(); - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2558,23 +2748,25 @@ } case _CONTAINS_OP_SET: { - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! - int res = _PySet_Contains((PySetObject *)right, left); - Py_DECREF(left); - Py_DECREF(right); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); if (res < 0) JUMP_TO_ERROR(); - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2582,22 +2774,24 @@ } case _CONTAINS_OP_DICT: { - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyDict_CheckExact(right)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyDict_CheckExact(right_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CONTAINS_OP, hit); - int res = PyDict_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + int res = PyDict_Contains(right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); if (res < 0) JUMP_TO_ERROR(); - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2605,48 +2799,54 @@ } case _CHECK_EG_MATCH: { - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; + _PyStackRef match_type_st; + _PyStackRef exc_value_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); if (true) JUMP_TO_ERROR(); } - match = NULL; - rest = NULL; + PyObject *match_o = NULL; + PyObject *rest_o = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); + &match_o, &rest_o); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); if (res < 0) JUMP_TO_ERROR(); - assert((match == NULL) == (rest == NULL)); - if (match == NULL) JUMP_TO_ERROR(); - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) JUMP_TO_ERROR(); + if (!Py_IsNone(match_o)) { + PyErr_SetHandledException(match_o); } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); stack_pointer[-2] = rest; stack_pointer[-1] = match; break; } case _CHECK_EXC_MATCH: { - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; right = stack_pointer[-1]; left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) { + PyStackRef_CLOSE(right); if (true) JUMP_TO_ERROR(); } - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; + int res = PyErr_GivenExceptionMatches(left_o, right_o); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = b; break; } @@ -2656,58 +2856,63 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ case _IS_NONE: { - PyObject *value; - PyObject *b; + _PyStackRef value; + _PyStackRef b; value = stack_pointer[-1]; - if (Py_IsNone(value)) { - b = Py_True; + if (PyStackRef_Is(value, PyStackRef_None)) { + b = PyStackRef_True; } else { - b = Py_False; - Py_DECREF(value); + b = PyStackRef_False; + PyStackRef_CLOSE(value); } stack_pointer[-1] = b; break; } case _GET_LEN: { - PyObject *obj; - PyObject *len_o; + _PyStackRef obj; + _PyStackRef len; obj = stack_pointer[-1]; // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); if (len_i < 0) JUMP_TO_ERROR(); - len_o = PyLong_FromSsize_t(len_i); + PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) JUMP_TO_ERROR(); - stack_pointer[0] = len_o; + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _MATCH_CLASS: { - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; + _PyStackRef names; + _PyStackRef type; + _PyStackRef subject; + _PyStackRef attrs; oparg = CURRENT_OPARG(); names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR(); // Error! - attrs = Py_None; // Failure! + attrs = PyStackRef_None; // Failure! } stack_pointer[-3] = attrs; stack_pointer += -2; @@ -2716,11 +2921,11 @@ } case _MATCH_MAPPING: { - PyObject *subject; - PyObject *res; + _PyStackRef subject; + _PyStackRef res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2728,11 +2933,11 @@ } case _MATCH_SEQUENCE: { - PyObject *subject; - PyObject *res; + _PyStackRef subject; + _PyStackRef res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2740,14 +2945,16 @@ } case _MATCH_KEYS: { - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; + _PyStackRef keys; + _PyStackRef subject; + _PyStackRef values_or_none; keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) JUMP_TO_ERROR(); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + if (values_or_none_o == NULL) JUMP_TO_ERROR(); + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2755,23 +2962,24 @@ } case _GET_ITER: { - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) JUMP_TO_ERROR(); + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); + PyStackRef_CLOSE(iterable); + if (PyStackRef_IsNull(iter)) JUMP_TO_ERROR(); stack_pointer[-1] = iter; break; } case _GET_YIELD_FROM_ITER: { - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a @@ -2783,16 +2991,16 @@ } iter = iterable; } - else if (PyGen_CheckExact(iterable)) { + else if (PyGen_CheckExact(iterable_o)) { iter = iterable; } else { /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + if (PyStackRef_IsNull(iter)) { JUMP_TO_ERROR(); } - Py_DECREF(iterable); + PyStackRef_CLOSE(iterable); } stack_pointer[-1] = iter; break; @@ -2801,12 +3009,13 @@ /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ case _FOR_ITER_TIER_TWO: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { JUMP_TO_ERROR(); @@ -2820,6 +3029,7 @@ JUMP_TO_JUMP_TARGET(); } } + next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator stack_pointer[0] = next; stack_pointer += 1; @@ -2830,9 +3040,9 @@ /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ case _ITER_CHECK_LIST: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyListIter_Type) { + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -2842,10 +3052,11 @@ /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_LIST: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; if (seq == NULL) { UOP_STAT_INC(uopcode, miss); @@ -2859,15 +3070,16 @@ } case _ITER_NEXT_LIST: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2875,9 +3087,9 @@ } case _ITER_CHECK_TUPLE: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyTupleIter_Type) { + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -2887,10 +3099,11 @@ /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_TUPLE: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; if (seq == NULL) { UOP_STAT_INC(uopcode, miss); @@ -2904,15 +3117,16 @@ } case _ITER_NEXT_TUPLE: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2920,9 +3134,9 @@ } case _ITER_CHECK_RANGE: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2933,9 +3147,9 @@ /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_RANGE: { - PyObject *iter; + _PyStackRef iter; iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); if (r->len <= 0) { UOP_STAT_INC(uopcode, miss); @@ -2945,17 +3159,18 @@ } case _ITER_NEXT_RANGE: { - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; iter = stack_pointer[-1]; - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); long value = r->start; r->start = value + r->step; r->len--; - next = PyLong_FromLong(value); - if (next == NULL) JUMP_TO_ERROR(); + PyObject *res = PyLong_FromLong(value); + if (res == NULL) JUMP_TO_ERROR(); + next = PyStackRef_FromPyObjectSteal(res); stack_pointer[0] = next; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2963,11 +3178,11 @@ } case _FOR_ITER_GEN_FRAME: { - PyObject *iter; + _PyStackRef iter; _PyInterpreterFrame *gen_frame; oparg = CURRENT_OPARG(); iter = stack_pointer[-1]; - PyGenObject *gen = (PyGenObject *)iter; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2978,35 +3193,38 @@ } STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); + _PyFrame_StackPush(gen_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - stack_pointer[0] = (PyObject *)gen_frame; + stack_pointer[0].bits = (uintptr_t)gen_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_SPECIAL: { - PyObject *owner; - PyObject *attr; - PyObject *self_or_null; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); PyObject *name = _Py_SpecialMethods[oparg].name; - attr = _PyObject_LookupSpecialMethod(owner, name, &self_or_null); - if (attr == NULL) { + PyObject *self_or_null_o; + attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); + if (PyStackRef_IsNull(attr)) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, - Py_TYPE(owner)->tp_name); + Py_TYPE(owner_o)->tp_name); } } - if (attr == NULL) JUMP_TO_ERROR(); + if (PyStackRef_IsNull(attr)) JUMP_TO_ERROR(); + self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); stack_pointer[-1] = attr; stack_pointer[0] = self_or_null; stack_pointer += 1; @@ -3015,11 +3233,11 @@ } case _WITH_EXCEPT_START: { - PyObject *val; - PyObject *lasti; - PyObject *exit_self; - PyObject *exit_func; - PyObject *res; + _PyStackRef val; + _PyStackRef lasti; + _PyStackRef exit_self; + _PyStackRef exit_func; + _PyStackRef res; val = stack_pointer[-1]; lasti = stack_pointer[-3]; exit_self = stack_pointer[-4]; @@ -3034,22 +3252,24 @@ Then we push the __exit__ return value. */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); if (tb == NULL) { tb = Py_None; } else { Py_DECREF(tb); } - assert(PyLong_Check(lasti)); + assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti))); (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[5] = {NULL, exit_self, exc, val, tb}; - int has_self = (exit_self != NULL); - res = PyObject_Vectorcall(exit_func, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); + if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3057,18 +3277,18 @@ } case _PUSH_EXC_INFO: { - PyObject *new_exc; - PyObject *prev_exc; + _PyStackRef new_exc; + _PyStackRef prev_exc; new_exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } else { - prev_exc = Py_None; + prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); + assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc))); + exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -3077,10 +3297,11 @@ } case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner)->valid) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!_PyObject_InlineValues(owner_o)->valid) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3088,10 +3309,10 @@ } case _GUARD_KEYS_VERSION: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); - PyTypeObject *owner_cls = Py_TYPE(owner); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; if (owner_heap_type->ht_cached_keys->dk_version != keys_version) { UOP_STAT_INC(uopcode, miss); @@ -3101,9 +3322,9 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); @@ -3111,8 +3332,8 @@ /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -3122,18 +3343,18 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -3143,41 +3364,41 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - PyObject *owner; - PyObject *attr; + _PyStackRef owner; + _PyStackRef attr; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - PyObject *owner; - PyObject *attr; + _PyStackRef owner; + _PyStackRef attr; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[-1] = attr; break; } case _CHECK_ATTR_METHOD_LAZY_DICT: { - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; uint16_t dictoffset = (uint16_t)CURRENT_OPERAND(); - char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ if (dict != NULL) { @@ -3188,9 +3409,9 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND(); @@ -3198,7 +3419,7 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; stack_pointer[-1] = attr; stack_pointer[0] = self; @@ -3217,25 +3438,27 @@ } case _PY_FRAME_GENERAL: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - assert(Py_TYPE(callable) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, args, total_args, NULL ); // The frame has stolen all the arguments from the stack, @@ -3245,22 +3468,23 @@ if (new_frame == NULL) { JUMP_TO_ERROR(); } - stack_pointer[0] = (PyObject *)new_frame; + stack_pointer[0].bits = (uintptr_t)new_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _CHECK_FUNCTION_VERSION: { - PyObject *callable; + _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - if (!PyFunction_Check(callable)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3269,17 +3493,18 @@ } case _CHECK_METHOD_VERSION: { - PyObject *null; - PyObject *callable; + _PyStackRef null; + _PyStackRef callable; oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - if (Py_TYPE(callable) != &PyMethod_Type) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyObject *func = ((PyMethodObject *)callable)->im_func; + PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3288,7 +3513,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (null != NULL) { + if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3296,36 +3521,36 @@ } case _EXPAND_METHOD: { - PyObject *null; - PyObject *callable; - PyObject *method; - PyObject *self; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef method; + _PyStackRef self; oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(null == NULL); - assert(Py_TYPE(callable) == &PyMethod_Type); - self = ((PyMethodObject *)callable)->im_self; - Py_INCREF(self); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL - method = ((PyMethodObject *)callable)->im_func; - assert(PyFunction_Check(method)); - Py_INCREF(method); - Py_DECREF(callable); + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); + PyStackRef_CLOSE(callable); stack_pointer[-2 - oparg] = method; stack_pointer[-1 - oparg] = self; break; } case _CHECK_IS_NOT_PY_CALLABLE: { - PyObject *callable; + _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; - if (PyFunction_Check(callable)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (Py_TYPE(callable) == &PyMethod_Type) { + if (Py_TYPE(callable_o) == &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3333,10 +3558,10 @@ } case _CALL_NON_PY_GENERAL: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -3344,22 +3569,35 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - if (res == NULL) JUMP_TO_ERROR(); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3367,16 +3605,16 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - PyObject *null; - PyObject *callable; + _PyStackRef null; + _PyStackRef callable; oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - if (null != NULL) { + if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (Py_TYPE(callable) != &PyMethod_Type) { + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3384,17 +3622,20 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - PyObject *callable; - PyObject *func; - PyObject *self; + _PyStackRef callable; + _PyStackRef func; + _PyStackRef self; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization + self = stack_pointer[-1 - oparg]; + func = stack_pointer[-2 - oparg]; + PyStackRef_CLOSE(callable); + // self may be unused in tier 1, so silence warnings. + (void)self; stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = self; break; @@ -3409,15 +3650,16 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - PyObject *self_or_null; - PyObject *callable; + _PyStackRef self_or_null; + _PyStackRef callable; oparg = CURRENT_OPARG(); self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (self_or_null != NULL)) { + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3425,10 +3667,11 @@ } case _CHECK_STACK_SPACE: { - PyObject *callable; + _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UOP_STAT_INC(uopcode, miss); @@ -3442,149 +3685,155 @@ } case _INIT_CALL_PY_EXACT_ARGS_0: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = 0; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int has_self = (self_or_null != NULL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); - PyObject **first_non_self_local = new_frame->localsplus + has_self; + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _INIT_CALL_PY_EXACT_ARGS_1: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = 1; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int has_self = (self_or_null != NULL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); - PyObject **first_non_self_local = new_frame->localsplus + has_self; + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _INIT_CALL_PY_EXACT_ARGS_2: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = 2; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int has_self = (self_or_null != NULL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); - PyObject **first_non_self_local = new_frame->localsplus + has_self; + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _INIT_CALL_PY_EXACT_ARGS_3: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = 3; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int has_self = (self_or_null != NULL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); - PyObject **first_non_self_local = new_frame->localsplus + has_self; + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _INIT_CALL_PY_EXACT_ARGS_4: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = 4; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int has_self = (self_or_null != NULL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); - PyObject **first_non_self_local = new_frame->localsplus + has_self; + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _INIT_CALL_PY_EXACT_ARGS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; _PyInterpreterFrame *new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int has_self = (self_or_null != NULL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); - PyObject **first_non_self_local = new_frame->localsplus + has_self; + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg] = (PyObject *)new_frame; + stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -3592,7 +3841,7 @@ case _PUSH_FRAME: { _PyInterpreterFrame *new_frame; - new_frame = (_PyInterpreterFrame *)stack_pointer[-1]; + new_frame = (_PyInterpreterFrame *)stack_pointer[-1].bits; // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); @@ -3610,26 +3859,28 @@ } case _CALL_TYPE_1: { - PyObject *arg; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - if (null != NULL) { + if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (callable != (PyObject *)&PyType_Type) { + if (callable_o != (PyObject *)&PyType_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(arg)); - Py_DECREF(arg); + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3637,27 +3888,29 @@ } case _CALL_STR_1: { - PyObject *arg; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - if (null != NULL) { + if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (callable != (PyObject *)&PyUnicode_Type) { + if (callable_o != (PyObject *)&PyUnicode_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - res = PyObject_Str(arg); - Py_DECREF(arg); - if (res == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + PyStackRef_CLOSE(arg); + if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3665,27 +3918,29 @@ } case _CALL_TUPLE_1: { - PyObject *arg; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - if (null != NULL) { + if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (callable != (PyObject *)&PyTuple_Type) { + if (callable_o != (PyObject *)&PyTuple_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - res = PySequence_Tuple(arg); - Py_DECREF(arg); - if (res == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + PyStackRef_CLOSE(arg); + if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3695,13 +3950,13 @@ /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _EXIT_INIT_CHECK: { - PyObject *should_be_none; + _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { + if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); JUMP_TO_ERROR(); } stack_pointer += -1; @@ -3710,36 +3965,48 @@ } case _CALL_BUILTIN_CLASS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - if (!PyType_Check(callable)) { + if (!PyType_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyTypeObject *tp = (PyTypeObject *)callable; + PyTypeObject *tp = (PyTypeObject *)callable_o; if (tp->tp_vectorcall == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(tp); - if (res == NULL) JUMP_TO_ERROR(); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3747,17 +4014,18 @@ } case _CALL_BUILTIN_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } @@ -3765,11 +4033,11 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (!PyCFunction_CheckExact(callable)) { + if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (PyCFunction_GET_FLAGS(callable) != METH_O) { + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3779,15 +4047,16 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - PyObject *arg = args[0]; + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3795,42 +4064,54 @@ } case _CALL_BUILTIN_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) { + if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) { + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - res = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3838,25 +4119,26 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) { + if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) { + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3864,15 +4146,26 @@ /* res = func(self, args, nargs, kwnames) */ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyCFunction_GET_FUNCTION(callable_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3880,17 +4173,18 @@ } case _CALL_LEN: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } @@ -3899,23 +4193,25 @@ JUMP_TO_JUMP_TARGET(); } PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.len) { + if (callable_o != interp->callable_cache.len) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - PyObject *arg = args[0]; + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { JUMP_TO_ERROR(); } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res == NULL) { + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { GOTO_ERROR(error); } - Py_DECREF(callable); - Py_DECREF(arg); + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(arg_stackref); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3923,17 +4219,18 @@ } case _CALL_ISINSTANCE: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } @@ -3942,25 +4239,22 @@ JUMP_TO_JUMP_TARGET(); } PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.isinstance) { + if (callable_o != interp->callable_cache.isinstance) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); + _PyStackRef cls_stackref = args[1]; + _PyStackRef inst_stackref = args[0]; + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); if (retval < 0) { JUMP_TO_ERROR(); } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res == NULL) { - GOTO_ERROR(error); - } - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(inst_stackref); + PyStackRef_CLOSE(cls_stackref); + PyStackRef_CLOSE(callable); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3968,20 +4262,21 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (total_args != 2) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4000,22 +4295,26 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyObject *arg = args[1]; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) { + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, self, arg); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4023,20 +4322,21 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4047,7 +4347,7 @@ JUMP_TO_JUMP_TARGET(); } PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); if (!Py_IS_TYPE(self, d_type)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4056,14 +4356,25 @@ int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4071,17 +4382,18 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } @@ -4089,13 +4401,14 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4112,12 +4425,13 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4125,20 +4439,21 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); @@ -4149,7 +4464,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - PyObject *self = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4158,14 +4473,25 @@ PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) JUMP_TO_ERROR(); + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); + PyStackRef_CLOSE(callable); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4181,28 +4507,31 @@ /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _MAKE_FUNCTION: { - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); + PyStackRef_CLOSE(codeobj_st); if (func_obj == NULL) { JUMP_TO_ERROR(); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - PyObject *func; - PyObject *attr; + _PyStackRef func_st; + _PyStackRef attr_st; oparg = CURRENT_OPARG(); - func = stack_pointer[-1]; - attr = stack_pointer[-2]; + func_st = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -4232,14 +4561,14 @@ default: Py_UNREACHABLE(); } - stack_pointer[-2] = func; + stack_pointer[-2] = func_st; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } case _RETURN_GENERATOR: { - PyObject *res; + _PyStackRef res; assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4255,7 +4584,7 @@ gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - res = (PyObject *)gen; + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; @@ -4269,19 +4598,23 @@ } case _BUILD_SLICE: { - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; + _PyStackRef step = PyStackRef_NULL; + _PyStackRef stop; + _PyStackRef start; + _PyStackRef slice; oparg = CURRENT_OPARG(); if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) JUMP_TO_ERROR(); + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); + if (slice_o == NULL) JUMP_TO_ERROR(); + slice = PyStackRef_FromPyObjectSteal(slice_o); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); @@ -4289,30 +4622,32 @@ } case _CONVERT_VALUE: { - PyObject *value; - PyObject *result; + _PyStackRef value; + _PyStackRef result; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) JUMP_TO_ERROR(); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (result_o == NULL) JUMP_TO_ERROR(); + result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); /* If value is a unicode object, then we know the result * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) JUMP_TO_ERROR(); + if (!PyUnicode_CheckExact(value_o)) { + res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + PyStackRef_CLOSE(value); + if (PyStackRef_IsNull(res)) JUMP_TO_ERROR(); } else { res = value; @@ -4322,15 +4657,16 @@ } case _FORMAT_WITH_SPEC: { - PyObject *fmt_spec; - PyObject *value; - PyObject *res; + _PyStackRef fmt_spec; + _PyStackRef value; + _PyStackRef res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -4338,12 +4674,12 @@ } case _COPY: { - PyObject *bottom; - PyObject *top; + _PyStackRef bottom; + _PyStackRef top; oparg = CURRENT_OPARG(); bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); - top = Py_NewRef(bottom); + top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4351,17 +4687,20 @@ } case _BINARY_OP: { - PyObject *rhs; - PyObject *lhs; - PyObject *res; + _PyStackRef rhs; + _PyStackRef lhs; + _PyStackRef res; oparg = CURRENT_OPARG(); rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) JUMP_TO_ERROR(); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -4369,8 +4708,8 @@ } case _SWAP: { - PyObject *top; - PyObject *bottom; + _PyStackRef top; + _PyStackRef bottom; oparg = CURRENT_OPARG(); top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; @@ -4395,38 +4734,38 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */ case _GUARD_IS_TRUE_POP: { - PyObject *flag; + _PyStackRef flag; flag = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - if (!Py_IsTrue(flag)) { + if (!PyStackRef_Is(flag, PyStackRef_True)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(Py_IsTrue(flag)); + assert(PyStackRef_Is(flag, PyStackRef_True)); break; } case _GUARD_IS_FALSE_POP: { - PyObject *flag; + _PyStackRef flag; flag = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - if (!Py_IsFalse(flag)) { + if (!PyStackRef_Is(flag, PyStackRef_False)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(Py_IsFalse(flag)); + assert(PyStackRef_Is(flag, PyStackRef_False)); break; } case _GUARD_IS_NONE_POP: { - PyObject *val; + _PyStackRef val; val = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - if (!Py_IsNone(val)) { - Py_DECREF(val); + if (!PyStackRef_Is(val, PyStackRef_None)) { + PyStackRef_CLOSE(val); if (1) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4436,15 +4775,15 @@ } case _GUARD_IS_NOT_NONE_POP: { - PyObject *val; + _PyStackRef val; val = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - if (Py_IsNone(val)) { + if (PyStackRef_Is(val, PyStackRef_None)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - Py_DECREF(val); + PyStackRef_CLOSE(val); break; } @@ -4498,9 +4837,9 @@ } case _LOAD_CONST_INLINE: { - PyObject *value; + _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - value = Py_NewRef(ptr); + value = PyStackRef_FromPyObjectNew(ptr); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4508,9 +4847,9 @@ } case _LOAD_CONST_INLINE_BORROW: { - PyObject *value; + _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - value = ptr; + value = PyStackRef_FromPyObjectImmortal(ptr); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4518,22 +4857,22 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - PyObject *pop; - PyObject *value; + _PyStackRef pop; + _PyStackRef value; pop = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - Py_DECREF(pop); - value = ptr; + PyStackRef_CLOSE(pop); + value = PyStackRef_FromPyObjectImmortal(ptr); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_WITH_NULL: { - PyObject *value; - PyObject *null; + _PyStackRef value; + _PyStackRef null; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - value = Py_NewRef(ptr); - null = NULL; + value = PyStackRef_FromPyObjectNew(ptr); + null = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; @@ -4542,11 +4881,11 @@ } case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { - PyObject *value; - PyObject *null; + _PyStackRef value; + _PyStackRef null; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - value = ptr; - null = NULL; + value = PyStackRef_FromPyObjectImmortal(ptr); + null = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer[1] = null; stack_pointer += 2; @@ -4565,9 +4904,9 @@ } case _INTERNAL_INCREMENT_OPT_COUNTER: { - PyObject *opt; + _PyStackRef opt; opt = stack_pointer[-1]; - _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)opt; + _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt); exe->count++; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/frame.c b/Python/frame.c index 7299a395efad28b..9c7e59601e6faf9 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -16,11 +16,11 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) Py_VISIT(frame->f_funcobj); Py_VISIT(_PyFrame_GetCode(frame)); /* locals */ - PyObject **locals = _PyFrame_GetLocalsArray(frame); + _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); int i = 0; /* locals and stack */ for (; i stacktop; i++) { - Py_VISIT(locals[i]); + Py_VISIT(PyStackRef_AsPyObjectBorrow(locals[i])); } return 0; } @@ -101,7 +101,7 @@ _PyFrame_ClearLocals(_PyInterpreterFrame *frame) int stacktop = frame->stacktop; frame->stacktop = 0; for (int i = 0; i < stacktop; i++) { - Py_XDECREF(frame->localsplus[i]); + PyStackRef_XCLOSE(frame->localsplus[i]); } Py_CLEAR(frame->f_locals); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2d23b43fe0daed4..4a5468040f5cc8b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -16,9 +16,9 @@ PREDICTED(BINARY_OP); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - PyObject *rhs; - PyObject *lhs; - PyObject *res; + _PyStackRef rhs; + _PyStackRef lhs; + _PyStackRef res; // _SPECIALIZE_BINARY_OP rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; @@ -39,11 +39,14 @@ } // _BINARY_OP { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -56,24 +59,30 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _GUARD_BOTH_FLOAT right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_FLOAT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -86,24 +95,29 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _GUARD_BOTH_INT right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_INT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -116,24 +130,29 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _GUARD_BOTH_UNICODE right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_ADD_UNICODE { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error; + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -146,21 +165,25 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; + _PyStackRef right; + _PyStackRef left; // _GUARD_BOTH_UNICODE right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_INPLACE_ADD_UNICODE { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(next_instr->op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(next_instr->op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); + _PyStackRef *target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(!PyStackRef_Is(*target_local, left), BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -173,11 +196,13 @@ * only the locals reference, so PyUnicode_Append knows * that the string is safe to mutate. */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (*target_local == NULL) goto pop_2_error; + assert(Py_REFCNT(left_o) >= 2); + _Py_DECREF_NO_DEALLOC(left_o); + PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + if (PyStackRef_IsNull(*target_local)) goto pop_2_error; // The STORE_FAST is already done. assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); @@ -192,24 +217,30 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _GUARD_BOTH_FLOAT right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_FLOAT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -222,24 +253,29 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _GUARD_BOTH_INT right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_MULTIPLY_INT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -252,24 +288,30 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _GUARD_BOTH_FLOAT right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_FLOAT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -282,24 +324,29 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _GUARD_BOTH_INT right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); } /* Skip 1 cache entry */ // _BINARY_OP_SUBTRACT_INT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error; + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -311,25 +358,28 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *res; + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef res; stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + PyObject *res_o; // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { - res = NULL; + res_o = NULL; } else { - res = PyObject_GetItem(container, slice); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); } - Py_DECREF(container); - if (res == NULL) goto pop_3_error; + PyStackRef_CLOSE(container); + if (res_o == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -343,9 +393,9 @@ PREDICTED(BINARY_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - PyObject *sub; - PyObject *container; - PyObject *res; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef res; // _SPECIALIZE_BINARY_SUBSCR sub = stack_pointer[-1]; container = stack_pointer[-2]; @@ -364,10 +414,13 @@ } // _BINARY_SUBSCR { - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error; + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; @@ -380,22 +433,26 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *dict; - PyObject *res; + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef res; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); + PyObject *res_o; + int rc = PyDict_GetItemRef(dict, sub, &res_o); if (rc == 0) { _PyErr_SetKeyError(sub); } - Py_DECREF(dict); - Py_DECREF(sub); + PyStackRef_CLOSE(dict_st); + PyStackRef_CLOSE(sub_st); if (rc <= 0) goto pop_2_error; // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -407,11 +464,12 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *container; + _PyStackRef sub_st; + _PyStackRef container_st; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - container = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + container_st = stack_pointer[-2]; + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); @@ -429,8 +487,8 @@ Py_INCREF(getitem); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; + new_frame->localsplus[0] = container_st; + new_frame->localsplus[1] = sub_st; frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } @@ -440,12 +498,14 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *list; - PyObject *res; + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - list = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyList_Size(list) @@ -453,11 +513,12 @@ Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -469,12 +530,14 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *str; - PyObject *res; + _PyStackRef sub_st; + _PyStackRef str_st; + _PyStackRef res; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - str = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + str_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); @@ -484,9 +547,10 @@ Py_UCS4 c = PyUnicode_READ_CHAR(str, index); DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -498,12 +562,14 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *tuple; - PyObject *res; + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); // Deopt unless 0 <= sub < PyTuple_Size(list) @@ -511,11 +577,12 @@ Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -526,21 +593,32 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); - PyObject *keys; - PyObject **values; - PyObject *map; + _PyStackRef keys; + _PyStackRef *values; + _PyStackRef map; keys = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - assert(PyTuple_CheckExact(keys)); - assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); + assert(PyTuple_CheckExact(keys_o)); + assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + PyStackRef_CLOSE(keys); + if (true) { stack_pointer += -1 - oparg; goto error; } + } + PyObject *map_o = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys_o, 0), 1, + values_o, 1, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + PyStackRef_CLOSE(values[_i]); } - Py_DECREF(keys); - if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + PyStackRef_CLOSE(keys); + if (map_o == NULL) { stack_pointer += -1 - oparg; goto error; } + map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); @@ -551,11 +629,20 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); - PyObject **values; - PyObject *list; + _PyStackRef *values; + _PyStackRef list; values = &stack_pointer[-oparg]; - list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { stack_pointer += -oparg; goto error; } + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) { stack_pointer += -oparg; goto error; } + } + PyObject *list_o = _PyList_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + if (list_o == NULL) { stack_pointer += -oparg; goto error; } + list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -566,17 +653,26 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); - PyObject **values; - PyObject *map; + _PyStackRef *values; + _PyStackRef map; values = &stack_pointer[-oparg*2]; - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg*2; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) { stack_pointer += -oparg*2; goto error; } + } + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + PyStackRef_CLOSE(values[_i]); } - if (map == NULL) { stack_pointer += -oparg*2; goto error; } + if (map_o == NULL) { stack_pointer += -oparg*2; goto error; } + map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; assert(WITHIN_STACK_BOUNDS()); @@ -587,23 +683,26 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); - PyObject **values; - PyObject *set; + _PyStackRef *values; + _PyStackRef set; values = &stack_pointer[-oparg]; - set = PySet_New(NULL); - if (set == NULL) - goto error; + PyObject *set_o = PySet_New(NULL); + if (set_o == NULL) { + goto error; + } int err = 0; for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); + PyObject *item = PyStackRef_AsPyObjectSteal(values[i]); + if (err == 0) { + err = PySet_Add(set_o, item); + } Py_DECREF(item); } if (err != 0) { - Py_DECREF(set); + Py_DECREF(set_o); if (true) { stack_pointer += -oparg; goto error; } } + set = PyStackRef_FromPyObjectSteal(set_o); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -614,18 +713,22 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; + _PyStackRef step = PyStackRef_NULL; + _PyStackRef stop; + _PyStackRef start; + _PyStackRef slice; if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; } + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); + if (slice_o == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; } + slice = PyStackRef_FromPyObjectSteal(slice_o); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); @@ -636,14 +739,23 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); - PyObject **pieces; - PyObject *str; + _PyStackRef *pieces; + _PyStackRef str; pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(pieces[_i]); + } + if (true) { stack_pointer += -oparg; goto error; } + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + PyStackRef_CLOSE(pieces[_i]); } - if (str == NULL) { stack_pointer += -oparg; goto error; } + if (str_o == NULL) { stack_pointer += -oparg; goto error; } + str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -654,11 +766,20 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); - PyObject **values; - PyObject *tup; + _PyStackRef *values; + _PyStackRef tup; values = &stack_pointer[-oparg]; - tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { stack_pointer += -oparg; goto error; } + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) { stack_pointer += -oparg; goto error; } + } + PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + if (tup_o == NULL) { stack_pointer += -oparg; goto error; } + tup = PyStackRef_FromPyObjectSteal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -681,10 +802,10 @@ PREDICTED(CALL); _Py_CODEUNIT *this_instr = next_instr - 4; (void)this_instr; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; // _SPECIALIZE_CALL args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -695,7 +816,7 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); @@ -705,31 +826,34 @@ /* Skip 2 cache entries */ // _CALL { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - else if (Py_TYPE(callable) == &PyMethod_Type) { + else if (Py_TYPE(callable_o) == &PyMethod_Type) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; } // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, args, total_args, NULL ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -743,33 +867,44 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); if (err < 0) { - Py_CLEAR(res); + Py_CLEAR(res_o); } } } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC { @@ -786,26 +921,27 @@ next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *null; - PyObject *callable; + _PyStackRef *args; + _PyStackRef null; + _PyStackRef callable; /* Skip 1 cache entry */ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; DEOPT_IF(code->co_argcount != oparg+1, CALL); @@ -815,17 +951,17 @@ if (self == NULL) { goto error; } - Py_DECREF(tp); + PyStackRef_CLOSE(callable); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1); assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); - shim->localsplus[0] = self; + shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); Py_INCREF(init); _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; + init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } @@ -849,12 +985,12 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *null; - PyObject *callable; - PyObject *func; - PyObject *self; - PyObject *self_or_null; - PyObject **args; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef func; + _PyStackRef self; + _PyStackRef self_or_null; + _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 @@ -865,37 +1001,43 @@ null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization + self = stack_pointer[-1 - oparg]; + func = stack_pointer[-2 - oparg]; + PyStackRef_CLOSE(callable); + // self may be unused in tier 1, so silence warnings. + (void)self; } // _CHECK_FUNCTION_VERSION callable = func; { uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS self_or_null = stack_pointer[-1 - oparg]; { - assert(PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)), CALL); } // _CHECK_STACK_SPACE { - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); @@ -904,11 +1046,12 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; { - int has_self = (self_or_null != NULL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); - PyObject **first_non_self_local = new_frame->localsplus + has_self; + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; @@ -947,12 +1090,12 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *null; - PyObject *callable; - PyObject *method; - PyObject *self; - PyObject **args; - PyObject *self_or_null; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef method; + _PyStackRef self; + _PyStackRef *args; + _PyStackRef self_or_null; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 @@ -964,40 +1107,42 @@ callable = stack_pointer[-2 - oparg]; { uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); - PyObject *func = ((PyMethodObject *)callable)->im_func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; DEOPT_IF(!PyFunction_Check(func), CALL); DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(null != NULL, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); } // _EXPAND_METHOD { - assert(null == NULL); - assert(Py_TYPE(callable) == &PyMethod_Type); - self = ((PyMethodObject *)callable)->im_self; - Py_INCREF(self); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL - method = ((PyMethodObject *)callable)->im_func; - assert(PyFunction_Check(method)); - Py_INCREF(method); - Py_DECREF(callable); + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); + PyStackRef_CLOSE(callable); } // _PY_FRAME_GENERAL args = &stack_pointer[-oparg]; self_or_null = self; callable = method; { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - assert(Py_TYPE(callable) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, args, total_args, NULL ); // The frame has stolen all the arguments from the stack, @@ -1039,10 +1184,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_CLASS @@ -1050,22 +1195,34 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(!PyType_Check(callable_o), CALL); + PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC { @@ -1082,10 +1239,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_FAST @@ -1094,27 +1251,39 @@ callable = stack_pointer[-2 - oparg]; { /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - res = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC { @@ -1131,10 +1300,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_FAST_WITH_KEYWORDS @@ -1143,26 +1312,38 @@ callable = stack_pointer[-2 - oparg]; { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyCFunction_GET_FUNCTION(callable_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC { @@ -1179,10 +1360,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_BUILTIN_O @@ -1191,26 +1372,28 @@ callable = stack_pointer[-2 - oparg]; { /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); // CPython promises to check all non-vectorcall function calls. DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - PyObject *arg = args[0]; + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC { @@ -1229,13 +1412,16 @@ PREDICTED(CALL_FUNCTION_EX); _Py_CODEUNIT *this_instr = next_instr - 1; (void)this_instr; - PyObject *kwargs = NULL; - PyObject *callargs; - PyObject *func; - PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1)]; } - callargs = stack_pointer[-1 - (oparg & 1)]; - func = stack_pointer[-3 - (oparg & 1)]; + _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef callargs_st; + _PyStackRef func_st; + _PyStackRef result; + if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; } + callargs_st = stack_pointer[-1 - (oparg & 1)]; + func_st = stack_pointer[-3 - (oparg & 1)]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -1247,7 +1433,9 @@ if (tuple == NULL) { goto error; } - Py_SETREF(callargs, tuple); + PyStackRef_CLOSE(callargs_st); + callargs_st = PyStackRef_FromPyObjectSteal(tuple); + callargs = tuple; } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); @@ -1258,9 +1446,9 @@ tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); if (err) goto error; - result = PyObject_Call(func, callargs, kwargs); + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result == NULL) { + if (PyStackRef_IsNull(result)) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); @@ -1270,7 +1458,7 @@ tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); if (err < 0) { - Py_CLEAR(result); + PyStackRef_CLEAR(result); } } } @@ -1284,7 +1472,7 @@ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, + (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, nargs, callargs, kwargs); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); @@ -1295,13 +1483,13 @@ frame->return_offset = 1; DISPATCH_INLINED(new_frame); } - result = PyObject_Call(func, callargs, kwargs); + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); } - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { stack_pointer += -3 - (oparg & 1); goto error; } + PyStackRef_CLOSE(func_st); + PyStackRef_CLOSE(callargs_st); + PyStackRef_XCLOSE(kwargs_st); + assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); + if (PyStackRef_IsNull(result)) { stack_pointer += -3 - (oparg & 1); goto error; } stack_pointer[-3 - (oparg & 1)] = result; stack_pointer += -2 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -1313,13 +1501,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); } @@ -1328,16 +1517,19 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); - PyObject *value1; - PyObject *value2; - PyObject *res; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; + _PyStackRef value1_st; + _PyStackRef value2_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - Py_DECREF(value2); - Py_DECREF(value1); - if (res == NULL) goto pop_2_error; + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + PyStackRef_CLOSE(value2_st); + PyStackRef_CLOSE(value1_st); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1349,39 +1541,37 @@ next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); + _PyStackRef cls_stackref = args[1]; + _PyStackRef inst_stackref = args[0]; + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); if (retval < 0) { goto error; } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res == NULL) { - GOTO_ERROR(error); - } - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(inst_stackref); + PyStackRef_CLOSE(cls_stackref); + PyStackRef_CLOSE(callable); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1395,44 +1585,48 @@ PREDICTED(CALL_KW); _Py_CODEUNIT *this_instr = next_instr - 1; (void)this_instr; - PyObject *kwnames; - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef kwnames; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o ); - Py_DECREF(kwnames); + PyStackRef_CLOSE(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 3); // The frame has stolen all the arguments from the stack, @@ -1445,34 +1639,46 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (true) { stack_pointer += -3 - oparg; goto error; } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); if (err < 0) { - Py_CLEAR(res); + Py_CLEAR(res_o); } } } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + PyStackRef_CLOSE(kwnames); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + if (res_o == NULL) { stack_pointer += -3 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1485,37 +1691,40 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1, CALL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); + DEOPT_IF(callable_o != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); - PyObject *arg = args[0]; + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { goto error; } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res == NULL) { + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { GOTO_ERROR(error); } - Py_DECREF(callable); - Py_DECREF(arg); + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(arg_stackref); + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1527,25 +1736,27 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *arg; - PyObject *self; - PyObject *callable; + _PyStackRef arg; + _PyStackRef self; + _PyStackRef callable; /* Skip 1 cache entry */ /* Skip 2 cache entries */ arg = stack_pointer[-1]; self = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); + DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + assert(self_o != NULL); + DEOPT_IF(!PyList_Check(self_o), CALL); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { + if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } - Py_DECREF(self); - Py_DECREF(callable); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); STACK_SHRINK(3); // Skip POP_TOP assert(next_instr->op.code == POP_TOP); @@ -1558,10 +1769,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_FAST @@ -1569,30 +1780,42 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC { @@ -1609,10 +1832,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS @@ -1620,30 +1843,42 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC { @@ -1660,10 +1895,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_NOARGS @@ -1672,16 +1907,18 @@ callable = stack_pointer[-2 - oparg]; { assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); // CPython promises to check all non-vectorcall function calls. @@ -1689,12 +1926,13 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC { @@ -1711,10 +1949,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_O @@ -1722,31 +1960,36 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(total_args != 2, CALL); DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); // CPython promises to check all non-vectorcall function calls. DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, self, arg); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable); + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC { @@ -1763,17 +2006,18 @@ next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *callable; - PyObject **args; - PyObject *self_or_null; - PyObject *res; + _PyStackRef callable; + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE callable = stack_pointer[-2 - oparg]; { - DEOPT_IF(PyFunction_Check(callable), CALL); - DEOPT_IF(Py_TYPE(callable) == &PyMethod_Type, CALL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(PyFunction_Check(callable_o), CALL); + DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); } // _CALL_NON_PY_GENERAL args = &stack_pointer[-oparg]; @@ -1782,22 +2026,35 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC { @@ -1814,9 +2071,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *callable; - PyObject *self_or_null; - PyObject **args; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 @@ -1827,21 +2084,24 @@ callable = stack_pointer[-2 - oparg]; { uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); } // _CHECK_FUNCTION_EXACT_ARGS self_or_null = stack_pointer[-1 - oparg]; { - assert(PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)), CALL); } // _CHECK_STACK_SPACE { - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); @@ -1850,11 +2110,12 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; { - int has_self = (self_or_null != NULL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); - PyObject **first_non_self_local = new_frame->localsplus + has_self; + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; @@ -1893,9 +2154,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *callable; - PyObject **args; - PyObject *self_or_null; + _PyStackRef callable; + _PyStackRef *args; + _PyStackRef self_or_null; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 @@ -1906,25 +2167,28 @@ callable = stack_pointer[-2 - oparg]; { uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + DEOPT_IF(!PyFunction_Check(callable_o), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable_o; DEOPT_IF(func->func_version != func_version, CALL); } // _PY_FRAME_GENERAL args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - assert(Py_TYPE(callable) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, args, total_args, NULL ); // The frame has stolen all the arguments from the stack, @@ -1966,10 +2230,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *arg; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_STR_1 @@ -1977,13 +2241,15 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); - res = PyObject_Str(arg); - Py_DECREF(arg); - if (res == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + PyStackRef_CLOSE(arg); + if (PyStackRef_IsNull(res)) goto pop_3_error; } // _CHECK_PERIODIC { @@ -2000,10 +2266,10 @@ next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *arg; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_TUPLE_1 @@ -2011,13 +2277,15 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); - res = PySequence_Tuple(arg); - Py_DECREF(arg); - if (res == NULL) goto pop_3_error; + res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + PyStackRef_CLOSE(arg); + if (PyStackRef_IsNull(res)) goto pop_3_error; } // _CHECK_PERIODIC { @@ -2034,21 +2302,23 @@ next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject *arg; - PyObject *null; - PyObject *callable; - PyObject *res; + _PyStackRef arg; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ arg = stack_pointer[-1]; null = stack_pointer[-2]; callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(!PyStackRef_IsNull(null), CALL); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(arg)); - Py_DECREF(arg); + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); stack_pointer[-3] = res; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2059,29 +2329,33 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; + _PyStackRef match_type_st; + _PyStackRef exc_value_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); if (true) goto pop_2_error; } - match = NULL; - rest = NULL; + PyObject *match_o = NULL; + PyObject *rest_o = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); + &match_o, &rest_o); + PyStackRef_CLOSE(exc_value_st); + PyStackRef_CLOSE(match_type_st); if (res < 0) goto pop_2_error; - assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error; - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) goto pop_2_error; + if (!Py_IsNone(match_o)) { + PyErr_SetHandledException(match_o); } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); stack_pointer[-2] = rest; stack_pointer[-1] = match; DISPATCH(); @@ -2091,19 +2365,21 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; right = stack_pointer[-1]; left = stack_pointer[-2]; - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) { + PyStackRef_CLOSE(right); if (true) goto pop_1_error; } - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); - b = res ? Py_True : Py_False; + int res = PyErr_GivenExceptionMatches(left_o, right_o); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = b; DISPATCH(); } @@ -2113,22 +2389,23 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); - PyObject *exc_value; - PyObject *last_sent_val; - PyObject *sub_iter; - PyObject *none; - PyObject *value; - exc_value = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; + _PyStackRef exc_value_st; + _PyStackRef last_sent_val_st; + _PyStackRef sub_iter_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val_st = stack_pointer[-2]; + sub_iter_st = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); - none = Py_None; + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + PyStackRef_CLOSE(sub_iter_st); + PyStackRef_CLOSE(last_sent_val_st); + PyStackRef_CLOSE(exc_value_st); + none = PyStackRef_None; } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); @@ -2149,9 +2426,9 @@ PREDICTED(COMPARE_OP); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _SPECIALIZE_COMPARE_OP right = stack_pointer[-1]; left = stack_pointer[-2]; @@ -2170,16 +2447,21 @@ } // _COMPARE_OP { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error; + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) goto pop_2_error; if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); if (res_bool < 0) goto pop_2_error; - res = res_bool ? Py_True : Py_False; + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); } } stack_pointer[-2] = res; @@ -2193,27 +2475,31 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _GUARD_BOTH_FLOAT right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } stack_pointer[-2] = res; @@ -2227,31 +2513,35 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _GUARD_BOTH_INT right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_INT { - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } stack_pointer[-2] = res; @@ -2265,28 +2555,32 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *res; + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; // _GUARD_BOTH_UNICODE right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); } /* Skip 1 cache entry */ // _COMPARE_OP_STR { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); + int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } stack_pointer[-2] = res; @@ -2302,9 +2596,9 @@ PREDICTED(CONTAINS_OP); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; // _SPECIALIZE_CONTAINS_OP right = stack_pointer[-1]; left = stack_pointer[-2]; @@ -2323,11 +2617,13 @@ } // _CONTAINS_OP { - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int res = PySequence_Contains(right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } stack_pointer[-2] = b; stack_pointer += -1; @@ -2340,19 +2636,21 @@ next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; /* Skip 1 cache entry */ right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyDict_CheckExact(right), CONTAINS_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); - int res = PyDict_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + int res = PyDict_Contains(right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2364,20 +2662,22 @@ next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; /* Skip 1 cache entry */ right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right)), CONTAINS_OP); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! - int res = _PySet_Contains((PySetObject *)right, left); - Py_DECREF(left); - Py_DECREF(right); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2388,15 +2688,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); - PyObject *value; - PyObject *result; + _PyStackRef value; + _PyStackRef result; value = stack_pointer[-1]; conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) goto pop_1_error; + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (result_o == NULL) goto pop_1_error; + result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[-1] = result; DISPATCH(); } @@ -2405,11 +2706,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); - PyObject *bottom; - PyObject *top; + _PyStackRef bottom; + _PyStackRef top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); - top = Py_NewRef(bottom); + top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -2428,7 +2729,7 @@ int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } DISPATCH(); } @@ -2437,11 +2738,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); - PyObject *owner; + _PyStackRef owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyStackRef_CLOSE(owner); if (err) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2452,7 +2753,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); - PyObject *cell = GETLOCAL(oparg); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); @@ -2468,15 +2769,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); - PyObject *v = GETLOCAL(oparg); - if (v == NULL) { + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); if (1) goto error; } - SETLOCAL(oparg, NULL); + SETLOCAL(oparg, PyStackRef_NULL); DISPATCH(); } @@ -2525,14 +2826,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); - PyObject *sub; - PyObject *container; + _PyStackRef sub; + _PyStackRef container; sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ - int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); if (err) goto pop_2_error; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2543,18 +2845,21 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); - PyObject *update; - PyObject *dict; - PyObject *callable; + _PyStackRef update; + _PyStackRef dict; + _PyStackRef callable; update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + PyStackRef_CLOSE(update); if (true) goto pop_1_error; } - Py_DECREF(update); + PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -2564,20 +2869,22 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); - PyObject *update; - PyObject *dict; + _PyStackRef update; + _PyStackRef dict; update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + if (PyDict_Update(dict_o, update_o) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + Py_TYPE(update_o)->tp_name); } - Py_DECREF(update); + PyStackRef_CLOSE(update); if (true) goto pop_1_error; } - Py_DECREF(update); + PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -2588,14 +2895,15 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); - PyObject *exc; - PyObject *awaitable; - exc = stack_pointer[-1]; - awaitable = stack_pointer[-2]; + _PyStackRef exc_st; + _PyStackRef awaitable_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - Py_DECREF(awaitable); - Py_DECREF(exc); + PyStackRef_CLOSE(awaitable_st); + PyStackRef_CLOSE(exc_st); } else { Py_INCREF(exc); @@ -2612,9 +2920,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_FOR); - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - Py_DECREF(value); + PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -2624,11 +2932,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); - PyObject *value; - PyObject *receiver; + _PyStackRef value; + _PyStackRef receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); + (void)receiver; + PyStackRef_CLOSE(receiver); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2672,13 +2981,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); - PyObject *should_be_none; + _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { + if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); goto error; } stack_pointer += -1; @@ -2701,15 +3010,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); /* If value is a unicode object, then we know the result * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + if (!PyUnicode_CheckExact(value_o)) { + res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + PyStackRef_CLOSE(value); + if (PyStackRef_IsNull(res)) goto pop_1_error; } else { res = value; @@ -2722,15 +3032,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); - PyObject *fmt_spec; - PyObject *value; - PyObject *res; + _PyStackRef fmt_spec; + _PyStackRef value; + _PyStackRef res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error; + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2744,8 +3055,8 @@ PREDICTED(FOR_ITER); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; // _SPECIALIZE_FOR_ITER iter = stack_pointer[-1]; { @@ -2764,8 +3075,10 @@ // _FOR_ITER { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { + next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; @@ -2776,12 +3089,13 @@ /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ JUMPBY(oparg + 2); DISPATCH(); } + next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator } stack_pointer[0] = next; @@ -2795,7 +3109,7 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; + _PyStackRef iter; _PyInterpreterFrame *gen_frame; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ @@ -2806,12 +3120,12 @@ // _FOR_ITER_GEN_FRAME iter = stack_pointer[-1]; { - PyGenObject *gen = (PyGenObject *)iter; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); + _PyFrame_StackPush(gen_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; @@ -2841,18 +3155,19 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_LIST iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { @@ -2863,7 +3178,7 @@ Py_DECREF(seq); } #endif - Py_DECREF(iter); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ JUMPBY(oparg + 2); @@ -2872,12 +3187,13 @@ } // _ITER_NEXT_LIST { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); } stack_pointer[0] = next; stack_pointer += 1; @@ -2890,23 +3206,23 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_RANGE iter = stack_pointer[-1]; { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); if (r->len <= 0) { STACK_SHRINK(1); - Py_DECREF(r); + PyStackRef_CLOSE(iter); // Jump over END_FOR and POP_TOP instructions. JUMPBY(oparg + 2); DISPATCH(); @@ -2914,14 +3230,15 @@ } // _ITER_NEXT_RANGE { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); long value = r->start; r->start = value + r->step; r->len--; - next = PyLong_FromLong(value); - if (next == NULL) goto error; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) goto error; + next = PyStackRef_FromPyObjectSteal(res); } stack_pointer[0] = next; stack_pointer += 1; @@ -2934,18 +3251,19 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter; - PyObject *next; + _PyStackRef iter; + _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); PyTupleObject *seq = it->it_seq; if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { @@ -2953,7 +3271,7 @@ it->it_seq = NULL; Py_DECREF(seq); } - Py_DECREF(iter); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ JUMPBY(oparg + 2); @@ -2962,12 +3280,13 @@ } // _ITER_NEXT_TUPLE { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); } stack_pointer[0] = next; stack_pointer += 1; @@ -2979,11 +3298,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); - PyObject *obj; - PyObject *iter; + _PyStackRef obj; + _PyStackRef iter; obj = stack_pointer[-1]; unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } @@ -2992,21 +3313,22 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); + PyStackRef_CLOSE(obj); if (true) goto pop_1_error; } - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) goto pop_1_error; - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + iter_o = (*getter)(obj_o); + PyStackRef_CLOSE(obj); + if (iter_o == NULL) goto pop_1_error; + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); if (true) goto pop_1_error; } + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); } @@ -3015,15 +3337,17 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); - PyObject *aiter; - PyObject *awaitable; + _PyStackRef aiter; + _PyStackRef awaitable; aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { + PyObject *awaitable_o; + PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter); + PyTypeObject *type = Py_TYPE(aiter_o); + if (PyAsyncGen_CheckExact(aiter_o)) { + awaitable_o = type->tp_as_async->am_anext(aiter_o); + if (awaitable_o == NULL) { goto error; } } else { @@ -3031,7 +3355,7 @@ getter = type->tp_as_async->am_anext; } if (getter != NULL) { - next_iter = (*getter)(aiter); + next_iter = (*getter)(aiter_o); if (next_iter == NULL) { goto error; } @@ -3043,8 +3367,8 @@ type->tp_name); goto error; } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { + awaitable_o = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable_o == NULL) { _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " @@ -3056,6 +3380,7 @@ Py_DECREF(next_iter); } } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3066,28 +3391,30 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); - } - Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); + PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable)); + if (iter_o == NULL) { + _PyEval_FormatAwaitableError(tstate, + Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg); + } + PyStackRef_CLOSE(iterable); + if (iter_o != NULL && PyCoro_CheckExact(iter_o)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter_o); if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ Py_DECREF(yf); - Py_CLEAR(iter); + Py_CLEAR(iter_o); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) goto pop_1_error; + if (iter_o == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; DISPATCH(); } @@ -3096,13 +3423,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error; + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); + PyStackRef_CLOSE(iterable); + if (PyStackRef_IsNull(iter)) goto pop_1_error; stack_pointer[-1] = iter; DISPATCH(); } @@ -3111,15 +3438,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); - PyObject *obj; - PyObject *len_o; + _PyStackRef obj; + _PyStackRef len; obj = stack_pointer[-1]; // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); if (len_i < 0) goto error; - len_o = PyLong_FromSsize_t(len_i); + PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - stack_pointer[0] = len_o; + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3129,11 +3457,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - PyObject *iterable; - PyObject *iter; + _PyStackRef iterable; + _PyStackRef iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a @@ -3145,16 +3474,16 @@ } iter = iterable; } - else if (PyGen_CheckExact(iterable)) { + else if (PyGen_CheckExact(iterable_o)) { iter = iterable; } else { /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + if (PyStackRef_IsNull(iter)) { goto error; } - Py_DECREF(iterable); + PyStackRef_CLOSE(iterable); } stack_pointer[-1] = iter; DISPATCH(); @@ -3164,12 +3493,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); - PyObject *from; - PyObject *res; + _PyStackRef from; + _PyStackRef res; from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - if (res == NULL) goto error; + PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name); + if (res_o == NULL) goto error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3180,16 +3510,19 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); - PyObject *fromlist; - PyObject *level; - PyObject *res; + _PyStackRef fromlist; + _PyStackRef level; + _PyStackRef res; fromlist = stack_pointer[-1]; level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); - Py_DECREF(level); - Py_DECREF(fromlist); - if (res == NULL) goto pop_2_error; + PyObject *res_o = import_name(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3202,11 +3535,11 @@ next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); /* Skip 3 cache entries */ - int is_meth = PEEK(oparg + 1) != NULL; + int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); @@ -3227,11 +3560,11 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - int is_meth = PEEK(oparg + 2) != NULL; + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); + : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); @@ -3244,18 +3577,18 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - PyObject *value; - PyObject *receiver; + _PyStackRef value; + _PyStackRef receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + if (PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver))) { + if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { goto error; } } - Py_DECREF(value); + PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3266,16 +3599,17 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - PyObject *value; - PyObject *receiver; + _PyStackRef value; + _PyStackRef receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { goto error; } } - Py_DECREF(receiver); + PyStackRef_CLOSE(receiver); stack_pointer[-2] = value; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3289,10 +3623,11 @@ INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ _Py_CODEUNIT *target; - PyObject *iter = TOP(); + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next != NULL) { - PUSH(next); + PUSH(PyStackRef_FromPyObjectSteal(next)); target = next_instr; } else { @@ -3307,7 +3642,7 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); - Py_DECREF(iter); + PyStackRef_CLOSE(iter_stackref); /* Skip END_FOR and POP_TOP */ target = next_instr + oparg + 2; } @@ -3370,9 +3705,9 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); + _PyStackRef cond = POP(); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); int offset = flag * oparg; #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -3387,14 +3722,14 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ - PyObject *value = POP(); - int flag = Py_IsNone(value); + _PyStackRef value_stackref = POP(); + int flag = PyStackRef_Is(value_stackref, PyStackRef_None); int offset; if (flag) { offset = oparg; } else { - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); offset = 0; } #if ENABLE_SPECIALIZATION @@ -3410,14 +3745,14 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ - PyObject *value = POP(); + _PyStackRef value_stackref = POP(); int offset; - int nflag = Py_IsNone(value); + int nflag = PyStackRef_Is(value_stackref, PyStackRef_None); if (nflag) { offset = 0; } else { - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); offset = oparg; } #if ENABLE_SPECIALIZATION @@ -3433,9 +3768,9 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); + _PyStackRef cond = POP(); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); int offset = flag * oparg; #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -3494,7 +3829,7 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval)); LOAD_IP(frame->return_offset); goto resume_frame; } @@ -3504,11 +3839,11 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - PyObject *retval; + _PyStackRef retval; retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); @@ -3529,7 +3864,7 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - PyObject *retval; + _PyStackRef retval; retval = stack_pointer[-1]; assert(frame != &entry_frame); frame->instr_ptr = next_instr; @@ -3540,7 +3875,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); + frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -3559,7 +3894,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); - PyObject *retval; + _PyStackRef retval; retval = stack_pointer[-1]; assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); @@ -3567,22 +3902,28 @@ tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; + return PyStackRef_AsPyObjectSteal(retval); } TARGET(IS_OP) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); - PyObject *right; - PyObject *left; - PyObject *b; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; right = stack_pointer[-1]; left = stack_pointer[-2]; - int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); - b = res ? Py_True : Py_False; + #ifdef Py_GIL_DISABLED + // On free-threaded builds, objects are conditionally immortalized. + // So their bits don't always compare equally. + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + #else + int res = PyStackRef_Is(left, right) ^ oparg; + #endif + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3653,11 +3994,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); - PyObject *v; - PyObject *list; + _PyStackRef v; + _PyStackRef list; v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)) < 0) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3667,10 +4009,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); - PyObject *iterable; - PyObject *list; - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; + _PyStackRef iterable_st; + _PyStackRef list_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -3681,11 +4025,11 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - Py_DECREF(iterable); + PyStackRef_CLOSE(iterable_st); if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); - Py_DECREF(iterable); + PyStackRef_CLOSE(iterable_st); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -3698,9 +4042,9 @@ PREDICTED(LOAD_ATTR); _Py_CODEUNIT *this_instr = next_instr - 10; (void)this_instr; - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null = PyStackRef_NULL; // _SPECIALIZE_LOAD_ATTR owner = stack_pointer[-1]; { @@ -3721,15 +4065,16 @@ // _LOAD_ATTR { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + attr_o = NULL; + if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch + assert(attr_o != NULL); // No errors on this branch self_or_null = owner; // Transfer ownership } else { @@ -3739,17 +4084,18 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - self_or_null = NULL; + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null = PyStackRef_NULL; } } else { /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; } + attr = PyStackRef_FromPyObjectSteal(attr_o); } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; @@ -3763,17 +4109,18 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -3781,9 +4128,9 @@ PyObject *descr = read_obj(&this_instr[6].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; - Py_DECREF(owner); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; @@ -3797,15 +4144,16 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; + _PyStackRef owner; /* Skip 1 cache entry */ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); + PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); @@ -3822,7 +4170,7 @@ // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } @@ -3832,33 +4180,36 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t index = read_u16(&this_instr[4].cache); - attr = _PyObject_InlineValues(owner)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); + PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -3873,22 +4224,22 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); @@ -3901,7 +4252,7 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } stack_pointer[-1] = attr; @@ -3916,15 +4267,15 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } @@ -3933,11 +4284,11 @@ { PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } stack_pointer[-1] = attr; @@ -3952,27 +4303,28 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *self = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } @@ -3983,8 +4335,8 @@ /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } stack_pointer[-1] = attr; @@ -3999,32 +4351,35 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; { uint32_t dict_version = read_u32(&this_instr[2].cache); - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyModule_CheckExact(owner_o), LOAD_ATTR); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR); } // _LOAD_ATTR_MODULE { uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); + PyObject *attr_o = ep->me_value; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4039,14 +4394,14 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; + _PyStackRef owner; + _PyStackRef attr; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } @@ -4055,11 +4410,11 @@ { PyObject *descr = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[-1] = attr; DISPATCH(); @@ -4070,26 +4425,27 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; + _PyStackRef owner; + _PyStackRef attr; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(owner); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } @@ -4099,8 +4455,8 @@ assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); - attr = Py_NewRef(descr); + PyStackRef_CLOSE(owner); + attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[-1] = attr; DISPATCH(); @@ -4111,15 +4467,16 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; + _PyStackRef owner; /* Skip 1 cache entry */ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *fget = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); + PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); assert(Py_IS_TYPE(fget, &PyFunction_Type)); @@ -4144,28 +4501,29 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { uint16_t index = read_u16(&this_instr[4].cache); - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); + PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4180,46 +4538,50 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); } // _LOAD_ATTR_WITH_HINT { uint16_t hint = read_u16(&this_instr[4].cache); - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; + attr_o = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - attr = ep->me_value; + attr_o = ep->me_value; } - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr_o == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; - Py_DECREF(owner); + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4233,13 +4595,15 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); - PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; - if (bc == NULL) { + _PyStackRef bc; + PyObject *bc_o; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0) goto error; + if (bc_o == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); if (true) goto error; } + bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4250,14 +4614,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - PyObject *value; + _PyStackRef value; // Keep in sync with _common_constants in opcode.py switch(oparg) { case CONSTANT_ASSERTIONERROR: - value = PyExc_AssertionError; + value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); break; case CONSTANT_NOTIMPLEMENTEDERROR: - value = PyExc_NotImplementedError; + value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); break; default: Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); @@ -4272,9 +4636,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PyObject *value; - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + _PyStackRef value; + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4285,13 +4648,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); - PyObject *value; - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - value = PyCell_GetRef(cell); - if (value == NULL) { + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); if (true) goto error; } + value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4302,10 +4666,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); - PyObject *value; - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + _PyStackRef value; + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4316,10 +4679,10 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - PyObject *value; + _PyStackRef value; value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; + GETLOCAL(oparg) = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4330,16 +4693,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); - PyObject *value; - value = GETLOCAL(oparg); - if (value == NULL) { + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); if (1) goto error; } - Py_INCREF(value); + value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4350,14 +4713,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; + _PyStackRef value1; + _PyStackRef value2; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); stack_pointer[0] = value1; stack_pointer[1] = value2; stack_pointer += 2; @@ -4369,25 +4730,28 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - PyObject *class_dict; - PyObject *value; - class_dict = stack_pointer[-1]; + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { goto error; } - if (!value) { - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - value = PyCell_GetRef(cell); - if (value == NULL) { + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); goto error; } } - Py_DECREF(class_dict); + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[-1] = value; DISPATCH(); } @@ -4396,21 +4760,22 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - PyObject *mod_or_class_dict; - PyObject *v; + _PyStackRef mod_or_class_dict; + _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + PyObject *v_o; + if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) { goto error; } - if (v == NULL) { + if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - if (v == NULL) { + if (v_o == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -4423,11 +4788,11 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto pop_1_error; - if (v == NULL) { + if (PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0) goto pop_1_error; + if (v_o == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto pop_1_error; - if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) goto pop_1_error; + if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -4436,7 +4801,8 @@ } } } - Py_DECREF(mod_or_class_dict); + PyStackRef_CLOSE(mod_or_class_dict); + v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[-1] = v; DISPATCH(); } @@ -4448,8 +4814,8 @@ PREDICTED(LOAD_GLOBAL); _Py_CODEUNIT *this_instr = next_instr - 5; (void)this_instr; - PyObject *res; - PyObject *null = NULL; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); @@ -4471,13 +4837,14 @@ // _LOAD_GLOBAL { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *res_o; if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - if (res == NULL) { + if (res_o == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -4490,11 +4857,11 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; - if (res == NULL) { + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0) goto error; + if (res_o == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; - if (res == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0) goto error; + if (res_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -4502,7 +4869,8 @@ } } } - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; @@ -4516,8 +4884,8 @@ next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *res; - PyObject *null = NULL; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { @@ -4540,11 +4908,12 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; @@ -4558,8 +4927,8 @@ next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - PyObject *res; - PyObject *null = NULL; + _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { @@ -4575,11 +4944,12 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); - Py_INCREF(res); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[0] = res; if (oparg & 1) stack_pointer[1] = null; @@ -4592,14 +4962,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); if (true) goto error; } - Py_INCREF(locals); + locals = PyStackRef_FromPyObjectNew(l);; stack_pointer[0] = locals; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4610,7 +4980,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); - PyObject *v; + _PyStackRef v; + PyObject *v_o; PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -4618,18 +4989,18 @@ if (true) goto error; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) { goto error; } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + if (v_o == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) { goto error; } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + if (v_o == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) { goto error; } - if (v == NULL) { + if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -4637,6 +5008,7 @@ } } } + v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4647,21 +5019,24 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); - PyObject *owner; - PyObject *attr; - PyObject *self_or_null; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; owner = stack_pointer[-1]; assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); PyObject *name = _Py_SpecialMethods[oparg].name; - attr = _PyObject_LookupSpecialMethod(owner, name, &self_or_null); - if (attr == NULL) { + PyObject *self_or_null_o; + attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); + if (PyStackRef_IsNull(attr)) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, - Py_TYPE(owner)->tp_name); + Py_TYPE(owner_o)->tp_name); } } - if (attr == NULL) goto pop_1_error; + if (PyStackRef_IsNull(attr)) goto pop_1_error; + self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); stack_pointer[-1] = attr; stack_pointer[0] = self_or_null; stack_pointer += 1; @@ -4676,14 +5051,14 @@ PREDICTED(LOAD_SUPER_ATTR); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - PyObject *class; - PyObject *global_super; - PyObject *self; - PyObject *attr; - PyObject *null = NULL; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_SUPER_ATTR - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; { uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; @@ -4691,7 +5066,7 @@ int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); @@ -4699,8 +5074,11 @@ #endif /* ENABLE_SPECIALIZATION */ } // _LOAD_SUPER_ATTR - self = stack_pointer[-1]; + self_st = stack_pointer[-1]; { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -4728,15 +5106,15 @@ } } } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); if (super == NULL) goto pop_3_error; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = PyObject_GetAttr(super, name); + attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name)); Py_DECREF(super); - if (attr == NULL) goto pop_3_error; - null = NULL; + if (PyStackRef_IsNull(attr)) goto pop_3_error; + null = PyStackRef_NULL; } stack_pointer[-3] = attr; if (oparg & 1) stack_pointer[-2] = null; @@ -4750,25 +5128,29 @@ next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr_st; /* Skip 1 cache entry */ - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); if (attr == NULL) goto pop_3_error; - stack_pointer[-3] = attr; + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[-3] = attr_st; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -4779,15 +5161,18 @@ next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr; + _PyStackRef self_or_null; /* Skip 1 cache entry */ - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -4795,20 +5180,21 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + if (attr_o == NULL) { + PyStackRef_CLOSE(self_st); if (true) goto pop_3_error; } if (method_found) { - self_or_null = self; // transfer ownership + self_or_null = self_st; // transfer ownership } else { - Py_DECREF(self); - self_or_null = NULL; + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; } + attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; @@ -4822,12 +5208,12 @@ INSTRUCTION_STATS(MAKE_CELL); // "initial" is probably NULL but not if it's an arg (or set // via the f_locals proxy before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { goto error; } - SETLOCAL(oparg, cell); + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); DISPATCH(); } @@ -4835,18 +5221,19 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); - PyObject *codeobj; - PyObject *func; - codeobj = stack_pointer[-1]; + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); + PyStackRef_CLOSE(codeobj_st); if (func_obj == NULL) { goto error; } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[-1] = func; DISPATCH(); } @@ -4855,16 +5242,17 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); - PyObject *value; - PyObject *key; - PyObject *dict; + _PyStackRef value; + _PyStackRef key; + _PyStackRef dict_st; value = stack_pointer[-1]; key = stack_pointer[-2]; - dict = stack_pointer[-3 - (oparg - 1)]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + if (_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0) goto pop_2_error; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -4874,27 +5262,31 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; + _PyStackRef names; + _PyStackRef type; + _PyStackRef subject; + _PyStackRef attrs; names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + PyStackRef_CLOSE(subject); + PyStackRef_CLOSE(type); + PyStackRef_CLOSE(names); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { if (_PyErr_Occurred(tstate)) goto pop_3_error; // Error! - attrs = Py_None; // Failure! + attrs = PyStackRef_None; // Failure! } stack_pointer[-3] = attrs; stack_pointer += -2; @@ -4906,14 +5298,16 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); - PyObject *keys; - PyObject *subject; - PyObject *values_or_none; + _PyStackRef keys; + _PyStackRef subject; + _PyStackRef values_or_none; keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error; + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + if (values_or_none_o == NULL) goto error; + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4924,11 +5318,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); - PyObject *subject; - PyObject *res; + _PyStackRef subject; + _PyStackRef res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4939,11 +5333,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); - PyObject *subject; - PyObject *res; + _PyStackRef subject; + _PyStackRef res; subject = stack_pointer[-1]; - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4961,10 +5355,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); - PyObject *exc_value; + _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + Py_XSETREF(exc_info->exc_value, + PyStackRef_AsPyObjectBorrow(exc_value) == Py_None + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -4975,11 +5371,11 @@ (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - PyObject *cond; + _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -4994,26 +5390,26 @@ (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; /* Skip 1 cache entry */ // _IS_NONE value = stack_pointer[-1]; { - if (Py_IsNone(value)) { - b = Py_True; + if (PyStackRef_Is(value, PyStackRef_None)) { + b = PyStackRef_True; } else { - b = Py_False; - Py_DECREF(value); + b = PyStackRef_False; + PyStackRef_CLOSE(value); } } // _POP_JUMP_IF_TRUE cond = b; { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -5029,26 +5425,26 @@ (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - PyObject *value; - PyObject *b; - PyObject *cond; + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; /* Skip 1 cache entry */ // _IS_NONE value = stack_pointer[-1]; { - if (Py_IsNone(value)) { - b = Py_True; + if (PyStackRef_Is(value, PyStackRef_None)) { + b = PyStackRef_True; } else { - b = Py_False; - Py_DECREF(value); + b = PyStackRef_False; + PyStackRef_CLOSE(value); } } // _POP_JUMP_IF_FALSE cond = b; { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -5064,11 +5460,11 @@ (void)this_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - PyObject *cond; + _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -5082,9 +5478,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; - Py_DECREF(value); + PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -5094,18 +5490,18 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); - PyObject *new_exc; - PyObject *prev_exc; + _PyStackRef new_exc; + _PyStackRef prev_exc; new_exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } else { - prev_exc = Py_None; + prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); + assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc))); + exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -5117,8 +5513,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); - PyObject *res; - res = NULL; + _PyStackRef res; + res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -5130,15 +5526,15 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); - PyObject **args; + _PyStackRef *args; args = &stack_pointer[-oparg]; PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: - cause = args[1]; + cause = PyStackRef_AsPyObjectSteal(args[1]); /* fall through */ case 1: - exc = args[0]; + exc = PyStackRef_AsPyObjectSteal(args[0]); /* fall through */ case 0: if (do_raise(tstate, exc, cause)) { @@ -5160,13 +5556,14 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); - PyObject *exc; - PyObject **values; - exc = stack_pointer[-1]; + _PyStackRef exc_st; + _PyStackRef *values; + exc_st = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(oparg >= 0 && oparg <= 2); if (oparg) { - PyObject *lasti = values[0]; + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); if (PyLong_Check(lasti)) { frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); @@ -5250,13 +5647,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_CONST); - PyObject *value; - PyObject *retval; - PyObject *res; + _PyStackRef value; + _PyStackRef retval; + _PyStackRef res; // _LOAD_CONST { - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); } // _RETURN_VALUE retval = value; @@ -5286,7 +5682,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); - PyObject *res; + _PyStackRef res; assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -5302,7 +5698,7 @@ gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - res = (PyObject *)gen; + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; @@ -5319,8 +5715,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); - PyObject *retval; - PyObject *res; + _PyStackRef retval; + _PyStackRef res; retval = stack_pointer[-1]; #if TIER_ONE assert(frame != &entry_frame); @@ -5351,9 +5747,9 @@ PREDICTED(SEND); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - PyObject *receiver; - PyObject *v; - PyObject *retval; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; // _SPECIALIZE_SEND receiver = stack_pointer[-2]; { @@ -5372,12 +5768,14 @@ // _SEND v = stack_pointer[-1]; { + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { - PyGenObject *gen = (PyGenObject *)receiver; + PyGenObject *gen = (PyGenObject *)receiver_o; _PyInterpreterFrame *gen_frame = &gen->gi_iframe; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); @@ -5388,26 +5786,29 @@ frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); + if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); } else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); } - if (retval == NULL) { + if (retval_o == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { monitor_raise(tstate, frame, this_instr); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); + if (_PyGen_FetchStopIterationValue(&retval_o) == 0) { + assert(retval_o != NULL); JUMPBY(oparg); } else { goto error; } } - Py_DECREF(v); + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); } stack_pointer[-1] = retval; DISPATCH(); @@ -5418,13 +5819,13 @@ next_instr += 2; INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v; - PyObject *receiver; + _PyStackRef v; + _PyStackRef receiver; /* Skip 1 cache entry */ v = stack_pointer[-1]; receiver = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)receiver; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); STAT_INC(SEND, hit); @@ -5470,12 +5871,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); - PyObject *v; - PyObject *set; + _PyStackRef v; + _PyStackRef set; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = PySet_Add(set, v); - Py_DECREF(v); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); + PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5486,10 +5888,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - PyObject *func; - PyObject *attr; - func = stack_pointer[-1]; - attr = stack_pointer[-2]; + _PyStackRef func_st; + _PyStackRef attr_st; + func_st = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -5519,7 +5923,7 @@ default: Py_UNREACHABLE(); } - stack_pointer[-2] = func; + stack_pointer[-2] = func_st; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -5529,12 +5933,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); - PyObject *iterable; - PyObject *set; + _PyStackRef iterable; + _PyStackRef set; iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + PyStackRef_CLOSE(iterable); if (err < 0) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5548,8 +5953,8 @@ PREDICTED(STORE_ATTR); _Py_CODEUNIT *this_instr = next_instr - 5; (void)this_instr; - PyObject *owner; - PyObject *v; + _PyStackRef owner; + _PyStackRef v; // _SPECIALIZE_STORE_ATTR owner = stack_pointer[-1]; { @@ -5571,9 +5976,10 @@ v = stack_pointer[-2]; { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectSteal(v)); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); if (err) goto pop_2_error; } stack_pointer += -2; @@ -5586,40 +5992,42 @@ next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _GUARD_DORV_NO_DICT { - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(_PyObject_GetManagedDict(owner), STORE_ATTR); - DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(_PyObject_GetManagedDict(owner_o), STORE_ATTR); + DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR); } // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; { uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner) == NULL); - PyDictValues *values = _PyObject_InlineValues(owner); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyDictValues *values = _PyObject_InlineValues(owner_o); PyObject *old_value = values->values[index]; - values->values[index] = value; + values->values[index] = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { _PyDictValues_AddToInsertionOrder(values, index); } else { Py_DECREF(old_value); } - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -5631,14 +6039,14 @@ next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } @@ -5646,12 +6054,13 @@ value = stack_pointer[-2]; { uint16_t index = read_u16(&this_instr[4].cache); - char *addr = (char *)owner + index; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; + *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); Py_XDECREF(old_value); - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -5663,14 +6072,14 @@ next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner; - PyObject *value; + _PyStackRef owner; + _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } @@ -5678,8 +6087,9 @@ value = stack_pointer[-2]; { uint16_t hint = read_u16(&this_instr[4].cache); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -5691,26 +6101,26 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) { _PyObject_GC_TRACK(dict); } /* PEP 509 */ dict->ma_version_tag = new_version; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -5721,10 +6131,10 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); - PyObject *v; + _PyStackRef v; v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - PyCell_SetTakeRef(cell, v); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -5734,7 +6144,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); - PyObject *value; + _PyStackRef value; value = stack_pointer[-1]; SETLOCAL(oparg, value); stack_pointer += -1; @@ -5746,14 +6156,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - PyObject *value1; - PyObject *value2; + _PyStackRef value1; + _PyStackRef value2; value1 = stack_pointer[-1]; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); stack_pointer[-1] = value2; DISPATCH(); } @@ -5762,8 +6171,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - PyObject *value1; - PyObject *value2; + _PyStackRef value1; + _PyStackRef value2; value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; uint32_t oparg1 = oparg >> 4; @@ -5779,11 +6188,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); - PyObject *v; + _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5794,7 +6203,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); - PyObject *v; + _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -5802,14 +6211,14 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); + PyStackRef_CLOSE(v); if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); + PyStackRef_CLOSE(v); if (err) goto pop_1_error; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5820,25 +6229,26 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef v; stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); int err; if (slice == NULL) { err = 1; } else { - err = PyObject_SetItem(container, slice, v); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v)); Py_DECREF(slice); } - Py_DECREF(v); - Py_DECREF(container); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); if (err) goto pop_4_error; stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); @@ -5852,9 +6262,9 @@ PREDICTED(STORE_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - PyObject *sub; - PyObject *container; - PyObject *v; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef v; // _SPECIALIZE_STORE_SUBSCR sub = stack_pointer[-1]; container = stack_pointer[-2]; @@ -5875,10 +6285,10 @@ v = stack_pointer[-3]; { /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v)); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); if (err) goto pop_3_error; } stack_pointer += -3; @@ -5891,17 +6301,19 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *dict; - PyObject *value; + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef value; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + dict_st = stack_pointer[-2]; value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + PyStackRef_CLOSE(dict_st); if (err) goto pop_3_error; stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); @@ -5913,13 +6325,15 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub; - PyObject *list; - PyObject *value; + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef value; /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - list = stack_pointer[-2]; + sub_st = stack_pointer[-1]; + list_st = stack_pointer[-2]; value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); // Ensure nonnegative, zero-or-one-digit ints. @@ -5929,11 +6343,11 @@ DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -5943,8 +6357,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); - PyObject *top; - PyObject *bottom; + _PyStackRef top; + _PyStackRef bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); @@ -5960,8 +6374,8 @@ PREDICTED(TO_BOOL); _Py_CODEUNIT *this_instr = next_instr - 4; (void)this_instr; - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; // _SPECIALIZE_TO_BOOL value = stack_pointer[-1]; { @@ -5980,10 +6394,10 @@ /* Skip 2 cache entries */ // _TO_BOOL { - int err = PyObject_IsTrue(value); - Py_DECREF(value); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); if (err < 0) goto pop_1_error; - res = err ? Py_True : Py_False; + res = err ? PyStackRef_True : PyStackRef_False; } stack_pointer[-1] = res; DISPATCH(); @@ -5994,23 +6408,23 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *owner; - PyObject *value; - PyObject *res; + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); } // _REPLACE_WITH_TRUE value = owner; { - Py_DECREF(value); - res = Py_True; + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; DISPATCH(); @@ -6021,11 +6435,11 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(value), TO_BOOL); + DEOPT_IF(!PyBool_Check(PyStackRef_AsPyObjectBorrow(value)), TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -6035,20 +6449,21 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - Py_DECREF(value); - res = Py_True; + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; DISPATCH(); @@ -6059,15 +6474,16 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); + res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); stack_pointer[-1] = res; DISPATCH(); } @@ -6077,15 +6493,15 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); + DEOPT_IF(!PyStackRef_Is(value, PyStackRef_None), TO_BOOL); STAT_INC(TO_BOOL, hit); - res = Py_False; + res = PyStackRef_False; stack_pointer[-1] = res; DISPATCH(); } @@ -6095,21 +6511,22 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - assert(Py_SIZE(value)); - Py_DECREF(value); - res = Py_True; + assert(Py_SIZE(value_o)); + PyStackRef_CLOSE(value); + res = PyStackRef_True; } stack_pointer[-1] = res; DISPATCH(); @@ -6119,12 +6536,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - res = PyNumber_Invert(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); } @@ -6133,12 +6551,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - res = PyNumber_Negative(value); - Py_DECREF(value); - if (res == NULL) goto pop_1_error; + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) goto pop_1_error; + res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; DISPATCH(); } @@ -6147,11 +6566,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); - PyObject *value; - PyObject *res; + _PyStackRef value; + _PyStackRef res; value = stack_pointer[-1]; - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + res = PyStackRef_Is(value, PyStackRef_False) + ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; DISPATCH(); } @@ -6160,12 +6580,12 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); - PyObject *seq; + _PyStackRef seq; seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); + _PyStackRef *top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + PyStackRef_CLOSE(seq); if (res == 0) goto pop_1_error; stack_pointer += (oparg >> 8) + (oparg & 0xFF); assert(WITHIN_STACK_BOUNDS()); @@ -6179,7 +6599,7 @@ PREDICTED(UNPACK_SEQUENCE); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - PyObject *seq; + _PyStackRef seq; // _SPECIALIZE_UNPACK_SEQUENCE seq = stack_pointer[-1]; { @@ -6199,9 +6619,9 @@ } // _UNPACK_SEQUENCE { - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); + _PyStackRef *top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + PyStackRef_CLOSE(seq); if (res == 0) goto pop_1_error; } stack_pointer += -1 + oparg; @@ -6214,19 +6634,20 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; + _PyStackRef seq; + _PyStackRef *values; /* Skip 1 cache entry */ seq = stack_pointer[-1]; values = &stack_pointer[-1]; - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyList_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); + PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } - Py_DECREF(seq); + PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6237,19 +6658,20 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject **values; + _PyStackRef seq; + _PyStackRef *values; /* Skip 1 cache entry */ seq = stack_pointer[-1]; values = &stack_pointer[-1]; - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } - Py_DECREF(seq); + PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -6260,18 +6682,19 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq; - PyObject *val1; - PyObject *val0; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; /* Skip 1 cache entry */ seq = stack_pointer[-1]; assert(oparg == 2); - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); - val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - Py_DECREF(seq); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + PyStackRef_CLOSE(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -6283,11 +6706,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); - PyObject *val; - PyObject *lasti; - PyObject *exit_self; - PyObject *exit_func; - PyObject *res; + _PyStackRef val; + _PyStackRef lasti; + _PyStackRef exit_self; + _PyStackRef exit_func; + _PyStackRef res; val = stack_pointer[-1]; lasti = stack_pointer[-3]; exit_self = stack_pointer[-4]; @@ -6302,22 +6725,24 @@ Then we push the __exit__ return value. */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); if (tb == NULL) { tb = Py_None; } else { Py_DECREF(tb); } - assert(PyLong_Check(lasti)); + assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti))); (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[5] = {NULL, exit_self, exc, val, tb}; - int has_self = (exit_self != NULL); - res = PyObject_Vectorcall(exit_func, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); + if (PyStackRef_IsNull(res)) goto error; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -6328,8 +6753,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); - PyObject *retval; - PyObject *value; + _PyStackRef retval; + _PyStackRef value; retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() diff --git a/Python/optimizer.c b/Python/optimizer.c index 960820ee06bf306..f7387dc0b27532e 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -196,7 +196,7 @@ _Py_SetTier2Optimizer(_PyOptimizerObject *optimizer) int _PyOptimizer_Optimize( _PyInterpreterFrame *frame, _Py_CODEUNIT *start, - PyObject **stack_pointer, _PyExecutorObject **executor_ptr) + _PyStackRef *stack_pointer, _PyExecutorObject **executor_ptr) { PyCodeObject *code = _PyFrame_GetCode(frame); assert(PyCode_Check(code)); @@ -1393,7 +1393,7 @@ counter_optimize( _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; _PyUOpInstruction buffer[4] = { { .opcode = _START_EXECUTOR, .jump_target = 3, .format=UOP_FORMAT_JUMP }, - { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, + { .opcode = _LOAD_CONST_INLINE, .operand = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } }; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c4f310dc20cbfa5..a414b04fb6a5b11 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -955,9 +955,9 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ case _LOAD_SUPER_ATTR_ATTR: { - _Py_UopsSymbol *attr; - attr = sym_new_not_null(ctx); - stack_pointer[-3] = attr; + _Py_UopsSymbol *attr_st; + attr_st = sym_new_not_null(ctx); + stack_pointer[-3] = attr_st; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); break; @@ -1319,9 +1319,9 @@ } case _GET_LEN: { - _Py_UopsSymbol *len_o; - len_o = sym_new_not_null(ctx); - stack_pointer[0] = len_o; + _Py_UopsSymbol *len; + len = sym_new_not_null(ctx); + stack_pointer[0] = len; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1898,9 +1898,9 @@ } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UopsSymbol *func; - func = sym_new_not_null(ctx); - stack_pointer[-2] = func; + _Py_UopsSymbol *func_st; + func_st = sym_new_not_null(ctx); + stack_pointer[-2] = func_st; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; diff --git a/Python/specialize.c b/Python/specialize.c index ad2f74788b3eadf..dc0e319880b9761 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -679,7 +679,10 @@ specialize_module_load_attr( /* Attribute specialization */ void -_Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, _Py_CODEUNIT *instr, int load_method) { +_Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _Py_CODEUNIT *instr, int load_method) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *cls = PyStackRef_AsPyObjectBorrow(cls_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); _PySuperAttrCache *cache = (_PySuperAttrCache *)(instr + 1); @@ -885,8 +888,10 @@ static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, P static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name); void -_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) +_Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); @@ -1081,8 +1086,10 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) } void -_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) +_Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); @@ -1521,8 +1528,11 @@ type_get_version(PyTypeObject *t, int opcode) void _Py_Specialize_BinarySubscr( - PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) + _PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) { + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[BINARY_SUBSCR] == INLINE_CACHE_ENTRIES_BINARY_SUBSCR); @@ -1621,8 +1631,11 @@ _Py_Specialize_BinarySubscr( } void -_Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) +_Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) { + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + assert(ENABLE_SPECIALIZATION); _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1); PyTypeObject *container_type = Py_TYPE(container); @@ -1939,8 +1952,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } void -_Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) +_Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { + PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); @@ -2056,9 +2071,11 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) #endif // Py_STATS void -_Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, - int oparg, PyObject **locals) +_Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, + int oparg, _PyStackRef *locals) { + PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st); + PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st); assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1); @@ -2071,7 +2088,7 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, if (PyUnicode_CheckExact(lhs)) { _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1]; bool to_store = (next.op.code == STORE_FAST); - if (to_store && locals[next.op.arg] == lhs) { + if (to_store && PyStackRef_AsPyObjectBorrow(locals[next.op.arg]) == lhs) { instr->op.code = BINARY_OP_INPLACE_ADD_UNICODE; goto success; } @@ -2163,9 +2180,12 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs) #endif // Py_STATS void -_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, +_Py_Specialize_CompareOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, int oparg) { + PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st); + PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); // All of these specializations compute boolean values, so they're all valid @@ -2226,8 +2246,10 @@ unpack_sequence_fail_kind(PyObject *seq) #endif // Py_STATS void -_Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) +_Py_Specialize_UnpackSequence(_PyStackRef seq_st, _Py_CODEUNIT *instr, int oparg) { + PyObject *seq = PyStackRef_AsPyObjectBorrow(seq_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[UNPACK_SEQUENCE] == INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); @@ -2337,12 +2359,12 @@ int #endif // Py_STATS void -_Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) +_Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) { assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); _PyForIterCache *cache = (_PyForIterCache *)(instr + 1); - PyTypeObject *tp = Py_TYPE(iter); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)); if (tp == &PyListIter_Type) { instr->op.code = FOR_ITER_LIST; goto success; @@ -2379,8 +2401,10 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) } void -_Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) +_Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) { + PyObject *receiver = PyStackRef_AsPyObjectBorrow(receiver_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND); _PySendCache *cache = (_PySendCache *)(instr + 1); @@ -2406,11 +2430,12 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) } void -_Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr) +_Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr) { assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[TO_BOOL] == INLINE_CACHE_ENTRIES_TO_BOOL); _PyToBoolCache *cache = (_PyToBoolCache *)(instr + 1); + PyObject *value = PyStackRef_AsPyObjectBorrow(value_o); if (PyBool_Check(value)) { instr->op.code = TO_BOOL_BOOL; goto success; @@ -2520,8 +2545,10 @@ static int containsop_fail_kind(PyObject *value) { #endif // Py_STATS void -_Py_Specialize_ContainsOp(PyObject *value, _Py_CODEUNIT *instr) +_Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr) { + PyObject *value = PyStackRef_AsPyObjectBorrow(value_st); + assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[CONTAINS_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); _PyContainsOpCache *cache = (_PyContainsOpCache *)(instr + 1); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 96e2fd57c745cbb..6b1af1b59f14d82 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -111,7 +111,7 @@ def __str__(self) -> str: return f"{type}{self.name}{size}{cond} {self.peek}" def is_array(self) -> bool: - return self.type == "PyObject **" + return self.type == "_PyStackRef *" @dataclass @@ -353,6 +353,21 @@ def has_error_without_pop(op: parser.InstDef) -> bool: NON_ESCAPING_FUNCTIONS = ( + "PyStackRef_FromPyObjectSteal", + "PyStackRef_AsPyObjectBorrow", + "PyStackRef_AsPyObjectSteal", + "PyStackRef_CLOSE", + "PyStackRef_DUP", + "PyStackRef_CLEAR", + "PyStackRef_IsNull", + "PyStackRef_TYPE", + "PyStackRef_False", + "PyStackRef_True", + "PyStackRef_None", + "PyStackRef_Is", + "PyStackRef_FromPyObjectNew", + "PyStackRef_AsPyObjectNew", + "PyStackRef_FromPyObjectImmortal", "Py_INCREF", "_PyManagedDictPointer_IsValues", "_PyObject_GetManagedDict", @@ -399,8 +414,6 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "_PyFrame_SetStackPointer", "_PyType_HasFeature", "PyUnicode_Concat", - "_PyList_FromArraySteal", - "_PyTuple_FromArraySteal", "PySlice_New", "_Py_LeaveRecursiveCallPy", "CALL_STAT_INC", @@ -413,6 +426,11 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "PyFloat_AS_DOUBLE", "_PyFrame_PushUnchecked", "Py_FatalError", + "STACKREFS_TO_PYOBJECTS", + "STACKREFS_TO_PYOBJECTS_CLEANUP", + "CONVERSION_FAILED", + "_PyList_FromArraySteal", + "_PyTuple_FromArraySteal", ) ESCAPING_FUNCTIONS = ( diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index efbfc94b4159764..e4e0c9b658c19d3 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -128,15 +128,15 @@ def replace_decrefs( continue if var.size != "1": out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"Py_DECREF({var.name}[_i]);\n") + out.emit(f"PyStackRef_CLOSE({var.name}[_i]);\n") out.emit("}\n") elif var.condition: if var.condition == "1": - out.emit(f"Py_DECREF({var.name});\n") + out.emit(f"PyStackRef_CLOSE({var.name});\n") elif var.condition != "0": - out.emit(f"Py_XDECREF({var.name});\n") + out.emit(f"PyStackRef_XCLOSE({var.name});\n") else: - out.emit(f"Py_DECREF({var.name});\n") + out.emit(f"PyStackRef_CLOSE({var.name});\n") def replace_sync_sp( diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index fb3e577de7346ad..277521678434c20 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -103,7 +103,7 @@ def write_uop( is_override = override is not None out.start_line() for var in reversed(prototype.stack.inputs): - res = stack.pop(var) + res = stack.pop(var, extract_bits=True) if not skip_inputs: out.emit(res) if not prototype.properties.stores_sp: @@ -140,7 +140,7 @@ def write_uop( if not var.peek or is_override: out.emit(stack.push(var)) out.start_line() - stack.flush(out, cast_type="_Py_UopsSymbol *") + stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True) except SizeMismatch as ex: raise analysis_error(ex.args[0], uop.body[0]) diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index cc897ff2cbe9aab..0bd4229e2beaf24 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -285,7 +285,7 @@ def stack_effect(self) -> StackEffect | None: if not (size := self.expression()): raise self.make_syntax_error("Expected expression") self.require(lx.RBRACKET) - type_text = "PyObject **" + type_text = "_PyStackRef *" size_text = size.text.strip() return StackEffect(tkn.text, type_text, cond_text, size_text) return None diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 7325f2f188777b9..c0e1278e5191431 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -125,7 +125,7 @@ def __init__(self) -> None: self.variables: list[StackItem] = [] self.defined: set[str] = set() - def pop(self, var: StackItem) -> str: + def pop(self, var: StackItem, extract_bits: bool = False) -> str: self.top_offset.pop(var) if not var.peek: self.peek_offset.pop(var) @@ -155,8 +155,9 @@ def pop(self, var: StackItem) -> str: else: self.defined.add(var.name) cast = f"({var.type})" if (not indirect and var.type) else "" + bits = ".bits" if cast and not extract_bits else "" assign = ( - f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}];" + f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}]{bits};" ) if var.condition: if var.condition == "1": @@ -178,11 +179,12 @@ def push(self, var: StackItem) -> str: self.top_offset.push(var) return "" - def flush(self, out: CWriter, cast_type: str = "PyObject *") -> None: + def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = False) -> None: out.start_line() for var in self.variables: if not var.peek: cast = f"({cast_type})" if var.type else "" + bits = ".bits" if cast and not extract_bits else "" if var.name not in UNUSED and not var.is_array(): if var.condition: if var.condition == "0": @@ -190,7 +192,7 @@ def flush(self, out: CWriter, cast_type: str = "PyObject *") -> None: elif var.condition != "1": out.emit(f"if ({var.condition}) ") out.emit( - f"stack_pointer[{self.base_offset.to_c()}] = {cast}{var.name};\n" + f"stack_pointer[{self.base_offset.to_c()}]{bits} = {cast}{var.name};\n" ) self.base_offset.push(var) if self.base_offset.to_c() != self.top_offset.to_c(): diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 5df4413e833d5a4..c9dce1d5f1804e4 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -37,20 +37,25 @@ def declare_variables(inst: Instruction, out: CWriter) -> None: if isinstance(uop, Uop): for var in reversed(uop.stack.inputs): if var.name not in variables: - type = var.type if var.type else "PyObject *" variables.add(var.name) + type, null = (var.type, "NULL") if var.type else ("_PyStackRef", "PyStackRef_NULL") + space = " " if type[-1].isalnum() else "" if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") + out.emit(f"{type}{space}{var.name} = {null};\n") else: - out.emit(f"{type}{var.name};\n") + if var.is_array(): + out.emit(f"{var.type}{space}{var.name};\n") + else: + out.emit(f"{type}{space}{var.name};\n") for var in uop.stack.outputs: if var.name not in variables: variables.add(var.name) - type = var.type if var.type else "PyObject *" + type, null = (var.type, "NULL") if var.type else ("_PyStackRef", "PyStackRef_NULL") + space = " " if type[-1].isalnum() else "" if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") + out.emit(f"{type}{space}{var.name} = {null};\n") else: - out.emit(f"{type}{var.name};\n") + out.emit(f"{type}{space}{var.name};\n") def write_uop( diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index a091870489fbaf4..f3769bd31c295d1 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -34,16 +34,17 @@ def declare_variable( ) -> None: if var.name in variables: return - type = var.type if var.type else "PyObject *" variables.add(var.name) + type, null = (var.type, "NULL") if var.type else ("_PyStackRef", "PyStackRef_NULL") + space = " " if type[-1].isalnum() else "" if var.condition: - out.emit(f"{type}{var.name} = NULL;\n") + out.emit(f"{type}{space}{var.name} = {null};\n") if uop.replicates: # Replicas may not use all their conditional variables # So avoid a compiler warning with a fake use out.emit(f"(void){var.name};\n") else: - out.emit(f"{type}{var.name};\n") + out.emit(f"{type}{space}{var.name};\n") def declare_variables(uop: Uop, out: CWriter) -> None: diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 5fdc812a00f0593..8aa74635aedb4f0 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -100,6 +100,8 @@ def _managed_dict_offset(): hexdigits = "0123456789abcdef" +USED_TAGS = 0b11 + ENCODING = locale.getpreferredencoding() FRAME_INFO_OPTIMIZED_OUT = '(frame information optimized out)' @@ -158,6 +160,8 @@ class PyObjectPtr(object): _typename = 'PyObject' def __init__(self, gdbval, cast_to=None): + # Clear the tagged pointer + gdbval = gdb.Value(int(gdbval) & (~USED_TAGS)).cast(gdbval.type) if cast_to: self._gdbval = gdbval.cast(cast_to) else: diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 39ce236bb60d44c..2bcbf8d615bd8a3 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -17,6 +17,7 @@ #include "pycore_setobject.h" #include "pycore_sliceobject.h" #include "pycore_descrobject.h" +#include "pycore_stackref.h" #include "ceval_macros.h" @@ -84,7 +85,7 @@ do { \ #define WITHIN_STACK_BOUNDS() 1 _Py_CODEUNIT * -_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) +_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { // Locals that the instruction implementations expect to exist: PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) diff --git a/Tools/jit/trampoline.c b/Tools/jit/trampoline.c index 01b3d63a6790ba3..a0a963f2a496565 100644 --- a/Tools/jit/trampoline.c +++ b/Tools/jit/trampoline.c @@ -8,7 +8,7 @@ // The actual change is patched in while the JIT compiler is being built, in // Tools/jit/_targets.py. On other platforms, this function compiles to nothing. _Py_CODEUNIT * -_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) +_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { // This is subtle. The actual trace will return to us once it exits, so we // need to make sure that we stay alive until then. If our trace side-exits