From bf4f5592f258c838c4b4090a6130ab1a68285e3c Mon Sep 17 00:00:00 2001 From: Karl Nelson Date: Sat, 14 Sep 2024 15:35:40 -0700 Subject: [PATCH] I give up --- native/python/pyjp_value.cpp | 19 +- native/python/pyjp_value2.cpp | 331 ++++++++++++++++++++++++++++++++++ 2 files changed, 345 insertions(+), 5 deletions(-) create mode 100644 native/python/pyjp_value2.cpp diff --git a/native/python/pyjp_value.cpp b/native/python/pyjp_value.cpp index 8b7b9de5a..629f8983b 100644 --- a/native/python/pyjp_value.cpp +++ b/native/python/pyjp_value.cpp @@ -66,14 +66,23 @@ PyObject* PyJPValue_alloc(PyTypeObject* type, Py_ssize_t nitems) if (PyType_IS_GC(type)) PyObject_GC_Track(obj); +#elif PY_VERSION_HEX<0x030c0000 + // At some point Python 3.11 backported changes corresponding to Python 3.12 + // without the support for allocation of method, and with bugs. We have + // to mutilate the base size of the object to get the extra memory we need + // And the size we must request appears to be larger tha needed. This + // coresponds to the same bug in PyUnstable_Object_GC_NewWithExtraData. + // + // This is a horrible hack and I can't guarantee anything about the stability + // of it! + type->tp_basicsize += 2*sizeof(JPValue); + PyObject* obj = PyType_GenericAlloc(type, nitems); + type->tp_basicsize -= 2*sizeof(JPValue); #else - - // 1) allocate memory (+pre +inline) - // 2) gc link - // 3) init (set type, ref type, set ob_size) - // 4) set up inline dict past the length of object (if inline) PyObject* obj = nullptr; if (type->tp_itemsize != 0) { + // There is no corresponding PyUnstable_VarObject_GC_NewWithExtraData method + // Without one we are just going to aske for our needed memory though backdoor methods. Py_ssize_t extra = (sizeof(JPValue))/type->tp_itemsize; if (extra == 0) extra = 1; diff --git a/native/python/pyjp_value2.cpp b/native/python/pyjp_value2.cpp new file mode 100644 index 000000000..0a28ee365 --- /dev/null +++ b/native/python/pyjp_value2.cpp @@ -0,0 +1,331 @@ +/***************************************************************************** + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + See NOTICE file for details. + *****************************************************************************/ +#include "jpype.h" +#include "pyjp.h" +#include "jp_stringtype.h" +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Allocate a new Python object with a slot for Java. + * + * We need extra space to store our values, but because there + * is no way to do so without disturbing the object layout. + * Fortunately, Python already handles this for dict and weakref. + * Python aligns the ends of the structure and increases the + * base type size to add additional slots to a standard object. + * + * We will use the same trick to add an additional slot for Java + * after the end of the object outside of where Python is looking. + * As the memory is aligned this is safe to do. We will use + * the alloc and finalize slot to recognize which objects have this + * extra slot appended. + */ +PyObject* PyJPValue_alloc(PyTypeObject* type, Py_ssize_t nitems) +{ + JP_PY_TRY("PyJPValue_alloc"); + +#if PY_VERSION_HEX<0x030c0000 + Py_ssize_t refcnt = ((PyObject*) type)->ob_refcnt; + // Modification from Python to add size elements + const size_t size = _PyObject_VAR_SIZE(type, nitems + 1) + sizeof (JPValue); + PyObject *obj = (PyType_IS_GC(type)) ? _PyObject_GC_Malloc(size) + : (PyObject *) PyObject_MALLOC(size); + if (obj == NULL) + return PyErr_NoMemory(); // GCOVR_EXCL_LINE + memset(obj, 0, size); + + if (type->tp_itemsize == 0) + PyObject_Init(obj, type); + else + PyObject_InitVar((PyVarObject *) obj, type, nitems); + + // This line is required to deal with Python bug (GH-11661) + // Some versions of Python fail to increment the reference counter of + // heap types properly. + if (refcnt == ((PyObject*) type)->ob_refcnt) + Py_INCREF(type); // GCOVR_EXCL_LINE + + if (PyType_IS_GC(type)) + PyObject_GC_Track(obj); + +#else + // 1) allocate memory (+pre +inline) + // 2) gc link + // 3) init (set type, ref type, set ob_size) + // 4) set up inline dict past the length of object (if inline) + type->tp_basicsize += sizeof(JPValue); + PyObject* obj = PyType_GenericAlloc(type, nitems); + type->tp_basicsize -= sizeof(JPValue); +#endif + + JP_TRACE("alloc", type->tp_name, obj); + return obj; + JP_PY_CATCH(nullptr); +} + +bool PyJPValue_hasJavaSlot(PyTypeObject* type) +{ + if (type == nullptr + || type->tp_alloc != (allocfunc) PyJPValue_alloc + || type->tp_finalize != (destructor) PyJPValue_finalize) + return false; // GCOVR_EXCL_LINE + return true; +} + +Py_ssize_t PyJPValue_getJavaSlotOffset(PyObject* self) +{ + PyTypeObject *type = Py_TYPE(self); + if (type == nullptr + || type->tp_alloc != (allocfunc) PyJPValue_alloc + || type->tp_finalize != (destructor) PyJPValue_finalize) + { + return 0; + } + + Py_ssize_t offset = 0; + Py_ssize_t sz = 0; + +#if PY_VERSION_HEX>=0x030c0000 + // starting in 3.12 there is no longer ob_size in PyLong + if (PyType_HasFeature(self->ob_type, Py_TPFLAGS_LONG_SUBCLASS)) + sz = (((PyLongObject*)self)->long_value.lv_tag) >> 3; // Private NON_SIZE_BITS + else +#endif + if (type->tp_itemsize != 0) + sz = Py_SIZE(self); + // PyLong abuses ob_size with negative values prior to 3.12 + if (sz < 0) + sz = -sz; + if (type->tp_itemsize == 0) + offset = _PyObject_VAR_SIZE(type, 1); + else + offset = _PyObject_VAR_SIZE(type, sz + 1); + return offset; +} + +/** + * Get the Java value if attached. + * + * The Java class is guaranteed not to be nullptr on success. + * + * @param obj + * @return the Java value or 0 if not found. + */ +JPValue* PyJPValue_getJavaSlot(PyObject* self) +{ + Py_ssize_t offset = PyJPValue_getJavaSlotOffset(self); + if (offset == 0) + return nullptr; + auto value = (JPValue*) (((char*) self) + offset); + if (value->getClass() == nullptr) + return nullptr; + return value; +} + +void PyJPValue_free(void* obj) +{ + JP_PY_TRY("PyJPValue_free", obj); + // Normally finalize is not run on simple classes. + PyTypeObject *type = Py_TYPE(obj); + if (type->tp_finalize != nullptr) + type->tp_finalize((PyObject*) obj); + if (type->tp_flags & Py_TPFLAGS_HAVE_GC) + PyObject_GC_Del(obj); + else + PyObject_Free(obj); // GCOVR_EXCL_LINE + JP_PY_CATCH_NONE(); +} + +void PyJPValue_finalize(void* obj) +{ + JP_PY_TRY("PyJPValue_finalize", obj); + JP_TRACE("type", Py_TYPE(obj)->tp_name); + JPValue* value = PyJPValue_getJavaSlot((PyObject*) obj); + if (value == nullptr) + return; + JPContext *context = JPContext_global; + if (context == nullptr || !context->isRunning()) + return; + JPJavaFrame frame = JPJavaFrame::outer(context); + JPClass* cls = value->getClass(); + // This one can't check for initialized because we may need to delete a stale + // resource after shutdown. + if (cls != nullptr && context->isRunning() && !cls->isPrimitive()) + { + JP_TRACE("Value", cls->getCanonicalName(), &(value->getValue())); + JP_TRACE("Dereference object"); + context->ReleaseGlobalRef(value->getValue().l); + *value = JPValue(); + } + JP_PY_CATCH_NONE(); +} + +/** This is the way to convert an object into a python string. */ +PyObject* PyJPValue_str(PyObject* self) +{ + JP_PY_TRY("PyJPValue_str", self); + JPContext *context = PyJPModule_getContext(); + JPJavaFrame frame = JPJavaFrame::outer(context); + JPValue* value = PyJPValue_getJavaSlot(self); + if (value == nullptr) + { + PyErr_SetString(PyExc_TypeError, "Not a Java value"); + return nullptr; + } + + JPClass* cls = value->getClass(); + if (cls->isPrimitive()) + { + PyErr_SetString(PyExc_TypeError, "toString requires a Java object"); + return nullptr; + } + + if (value->getValue().l == nullptr) + return JPPyString::fromStringUTF8("null").keep(); + + if (cls == context->_java_lang_String) + { + PyObject *cache; + JPPyObject dict = JPPyObject::accept(PyObject_GenericGetDict(self, nullptr)); + if (!dict.isNull()) + { + cache = PyDict_GetItemString(dict.get(), "_jstr"); + if (cache) + { + Py_INCREF(cache); + return cache; + } + auto jstr = (jstring) value->getValue().l; + string str; + str = frame.toStringUTF8(jstr); + cache = JPPyString::fromStringUTF8(str).keep(); + PyDict_SetItemString(dict.get(), "_jstr", cache); + return cache; + } + } + + // In general toString is not immutable, so we won't cache it. + return JPPyString::fromStringUTF8(frame.toString(value->getValue().l)).keep(); + JP_PY_CATCH(nullptr); +} + +PyObject *PyJPValue_getattro(PyObject *obj, PyObject *name) +{ + JP_PY_TRY("PyJPObject_getattro"); + if (!PyUnicode_Check(name)) + { + PyErr_Format(PyExc_TypeError, + "attribute name must be string, not '%.200s'", + Py_TYPE(name)->tp_name); + return nullptr; + } + + // Private members are accessed directly + PyObject* pyattr = PyBaseObject_Type.tp_getattro(obj, name); + if (pyattr == nullptr) + return nullptr; + JPPyObject attr = JPPyObject::accept(pyattr); + + // Private members go regardless + if (PyUnicode_GetLength(name) && PyUnicode_ReadChar(name, 0) == '_') + return attr.keep(); + + // Methods + if (Py_TYPE(attr.get()) == (PyTypeObject*) PyJPMethod_Type) + return attr.keep(); + + // Don't allow properties to be rewritten + if (!PyObject_IsInstance(attr.get(), (PyObject*) & PyProperty_Type)) + return attr.keep(); + + PyErr_Format(PyExc_AttributeError, "Field '%U' is static", name); + return nullptr; + JP_PY_CATCH(nullptr); +} + +int PyJPValue_setattro(PyObject *self, PyObject *name, PyObject *value) +{ + JP_PY_TRY("PyJPObject_setattro"); + + // Private members are accessed directly + if (PyUnicode_GetLength(name) && PyUnicode_ReadChar(name, 0) == '_') + return PyObject_GenericSetAttr(self, name, value); + JPPyObject f = JPPyObject::accept(PyJP_GetAttrDescriptor(Py_TYPE(self), name)); + if (f.isNull()) + { + PyErr_Format(PyExc_AttributeError, "Field '%U' is not found", name); + return -1; + } + descrsetfunc desc = Py_TYPE(f.get())->tp_descr_set; + if (desc != nullptr) + return desc(f.get(), self, value); + + // Not a descriptor + PyErr_Format(PyExc_AttributeError, + "Field '%U' is not settable on Java '%s' object", name, Py_TYPE(self)->tp_name); + return -1; + JP_PY_CATCH(-1); +} + +#ifdef __cplusplus +} +#endif + +// These are from the internal methods when we already have the jvalue + +void PyJPValue_assignJavaSlot(JPJavaFrame &frame, PyObject* self, const JPValue& value) +{ + Py_ssize_t offset = PyJPValue_getJavaSlotOffset(self); + // GCOVR_EXCL_START + if (offset == 0) + { + std::stringstream ss; + ss << "Missing Java slot on `" << Py_TYPE(self)->tp_name << "`"; + JP_RAISE(PyExc_SystemError, ss.str()); + } + // GCOVR_EXCL_STOP + + auto* slot = (JPValue*) (((char*) self) + offset); + // GCOVR_EXCL_START + // This is a sanity check that should never trigger in normal operations. + if (slot->getClass() != nullptr) + { + JP_RAISE(PyExc_SystemError, "Slot assigned twice"); + } + // GCOVR_EXCL_STOP + JPClass* cls = value.getClass(); + if (cls != nullptr && !cls->isPrimitive()) + { + jvalue q; + q.l = frame.NewGlobalRef(value.getValue().l); + *slot = JPValue(cls, q); + } else + *slot = value; +} + +bool PyJPValue_isSetJavaSlot(PyObject* self) +{ + Py_ssize_t offset = PyJPValue_getJavaSlotOffset(self); + if (offset == 0) + return false; // GCOVR_EXCL_LINE + auto* slot = (JPValue*) (((char*) self) + offset); + return slot->getClass() != nullptr; +}