From a0df9ee8fc77443510ab7e9ba8fd830f255a8fec Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 5 May 2023 17:53:07 +0100 Subject: [PATCH 1/9] GH-96803: Add three C-API functions to make _PyInterpreterFrame less opaque for users of PEP 523. (GH-96849) --- Include/cpython/frameobject.h | 17 +++++++++++++++++ Include/internal/pycore_frame.h | 2 -- ...22-09-15-15-21-34.gh-issue-96803.ynBKIS.rst | 6 ++++++ Modules/_tracemalloc.c | 3 ++- Objects/frameobject.c | 2 +- Objects/genobject.c | 3 ++- Python/ceval.c | 3 ++- Python/frame.c | 18 +++++++++++++++++- Python/traceback.c | 2 +- 9 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 4e19535c656f2c..6f3efe36ede5d8 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -4,6 +4,8 @@ # error "this header file must not be included directly" #endif +struct _PyInterpreterFrame; + /* Standard object interface */ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *, @@ -27,3 +29,18 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + +/* The following functions are for use by debuggers and other tools + * implementing custom frame evaluators with PEP 523. */ + +/* Returns the code object of the frame (strong reference). + * Does not raise an exception. */ +PyAPI_FUNC(PyCodeObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); + +/* Returns a byte ofsset into the last executed instruction. + * Does not raise an exception. */ +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); + +/* Returns the currently executing line number, or -1 if there is no line number. + * Does not raise an exception. */ +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index d8d7fe9ef2ebde..3d3cbbff7aae81 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -265,8 +265,6 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_l return new_frame; } -int _PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame); - static inline PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) { diff --git a/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst b/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst new file mode 100644 index 00000000000000..6fc56d2249f581 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst @@ -0,0 +1,6 @@ +Add unstable C-API functions to get the code object, lasti and line number from +the internal ``_PyInterpreterFrame`` in the limited API. The functions are: + +* ``PyCodeObject * PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame)`` diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index d69c5636486da9..c5714d5e7d5a0f 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -7,6 +7,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_traceback.h" #include +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include // malloc() @@ -257,7 +258,7 @@ static void tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) { frame->filename = &_Py_STR(anon_unknown); - int lineno = _PyInterpreterFrame_GetLine(pyframe); + int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); if (lineno < 0) { lineno = 0; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ef0070199ab2c0..d0eca447c012ea 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -38,7 +38,7 @@ PyFrame_GetLineNumber(PyFrameObject *f) return f->f_lineno; } else { - return _PyInterpreterFrame_GetLine(f->f_frame); + return PyUnstable_InterpreterFrame_GetLine(f->f_frame); } } diff --git a/Objects/genobject.c b/Objects/genobject.c index 6316fa9865fe65..937d497753e970 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -12,6 +12,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "structmember.h" // PyMemberDef #include "opcode.h" // SEND +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include "pystats.h" static PyObject *gen_close(PyGenObject *, PyObject *); @@ -1322,7 +1323,7 @@ compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame) frame = current_frame; for (int i = 0; i < frame_count; ++i) { PyCodeObject *code = frame->f_code; - int line = _PyInterpreterFrame_GetLine(frame); + int line = PyUnstable_InterpreterFrame_GetLine(frame); PyObject *frameinfo = Py_BuildValue("OiO", code->co_filename, line, code->co_name); if (!frameinfo) { diff --git a/Python/ceval.c b/Python/ceval.c index 958689debc87f8..56a3b123f46331 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -27,6 +27,7 @@ #include "pycore_dict.h" #include "dictobject.h" #include "pycore_frame.h" +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include "opcode.h" #include "pydtrace.h" #include "setobject.h" @@ -785,7 +786,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", frame->f_code->co_filename, - _PyInterpreterFrame_GetLine(frame), + PyUnstable_InterpreterFrame_GetLine(frame), opcode); goto error; diff --git a/Python/frame.c b/Python/frame.c index c2c0be30113912..d792b92fa57560 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -144,8 +144,24 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) Py_DECREF(frame->f_funcobj); } +/* Unstable API functions */ + +PyCodeObject * +PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame) +{ + PyCodeObject *code = frame->f_code; + Py_INCREF(code); + return code; +} + +int +PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame) +{ + return _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); +} + int -_PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame) +PyUnstable_InterpreterFrame_GetLine(_PyInterpreterFrame *frame) { int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); return PyCode_Addr2Line(frame->f_code, addr); diff --git a/Python/traceback.c b/Python/traceback.c index 097f69c76abfe1..b2479542047308 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1180,7 +1180,7 @@ dump_frame(int fd, _PyInterpreterFrame *frame) PUTS(fd, "???"); } - int lineno = _PyInterpreterFrame_GetLine(frame); + int lineno = PyUnstable_InterpreterFrame_GetLine(frame); PUTS(fd, ", line "); if (lineno >= 0) { _Py_DumpDecimal(fd, (size_t)lineno); From b9797417315cc2d1700cb2d427685016d3380711 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 5 May 2023 10:38:47 -0700 Subject: [PATCH 2/9] gh-103533: Use PEP 669 APIs for cprofile (GH-103534) --- Lib/test/test_cprofile.py | 19 +- ...-04-14-06-32-54.gh-issue-103533.n_AfcS.rst | 1 + Modules/_lsprof.c | 252 +++++++++++++----- Tools/c-analyzer/cpython/ignored.tsv | 1 + 4 files changed, 200 insertions(+), 73 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 98648528bc81f2..484b8f8e3a365c 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -25,7 +25,6 @@ def test_bad_counter_during_dealloc(self): with support.catch_unraisable_exception() as cm: obj = _lsprof.Profiler(lambda: int) obj.enable() - obj = _lsprof.Profiler(1) obj.disable() obj.clear() @@ -37,10 +36,11 @@ def test_profile_enable_disable(self): self.addCleanup(prof.disable) prof.enable() - self.assertIs(sys.getprofile(), prof) + self.assertEqual( + sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), "cProfile") prof.disable() - self.assertIs(sys.getprofile(), None) + self.assertIs(sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), None) def test_profile_as_context_manager(self): prof = self.profilerclass() @@ -53,10 +53,19 @@ def test_profile_as_context_manager(self): # profile should be set as the global profiler inside the # with-block - self.assertIs(sys.getprofile(), prof) + self.assertEqual( + sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), "cProfile") # profile shouldn't be set once we leave the with-block. - self.assertIs(sys.getprofile(), None) + self.assertIs(sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), None) + + def test_second_profiler(self): + pr = self.profilerclass() + pr2 = self.profilerclass() + pr.enable() + self.assertRaises(ValueError, pr2.enable) + pr.disable() + class TestCommandLine(unittest.TestCase): def test_sort(self): diff --git a/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst b/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst new file mode 100644 index 00000000000000..1008ea076c71a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst @@ -0,0 +1 @@ +Update :mod:`cProfile` to use PEP 669 API diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 83d034ae7eed78..a7ce328cb5307a 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -49,6 +49,8 @@ typedef struct { int flags; PyObject *externalTimer; double externalTimerUnit; + int tool_id; + PyObject* missing; } ProfilerObject; #define POF_ENABLED 0x001 @@ -399,64 +401,6 @@ ptrace_leave_call(PyObject *self, void *key) pObj->freelistProfilerContext = pContext; } -static int -profiler_callback(PyObject *self, PyFrameObject *frame, int what, - PyObject *arg) -{ - switch (what) { - - /* the 'frame' of a called function is about to start its execution */ - case PyTrace_CALL: - { - PyCodeObject *code = PyFrame_GetCode(frame); - ptrace_enter_call(self, (void *)code, (PyObject *)code); - Py_DECREF(code); - break; - } - - /* the 'frame' of a called function is about to finish - (either normally or with an exception) */ - case PyTrace_RETURN: - { - PyCodeObject *code = PyFrame_GetCode(frame); - ptrace_leave_call(self, (void *)code); - Py_DECREF(code); - break; - } - - /* case PyTrace_EXCEPTION: - If the exception results in the function exiting, a - PyTrace_RETURN event will be generated, so we don't need to - handle it. */ - - /* the Python function 'frame' is issuing a call to the built-in - function 'arg' */ - case PyTrace_C_CALL: - if ((((ProfilerObject *)self)->flags & POF_BUILTINS) - && PyCFunction_Check(arg)) { - ptrace_enter_call(self, - ((PyCFunctionObject *)arg)->m_ml, - arg); - } - break; - - /* the call to the built-in function 'arg' is returning into its - caller 'frame' */ - case PyTrace_C_RETURN: /* ...normally */ - case PyTrace_C_EXCEPTION: /* ...with an exception set */ - if ((((ProfilerObject *)self)->flags & POF_BUILTINS) - && PyCFunction_Check(arg)) { - ptrace_leave_call(self, - ((PyCFunctionObject *)arg)->m_ml); - } - break; - - default: - break; - } - return 0; -} - static int pending_exception(ProfilerObject *pObj) { @@ -650,6 +594,99 @@ setBuiltins(ProfilerObject *pObj, int nvalue) return 0; } +PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + PyObject* code = args[0]; + ptrace_enter_call((PyObject*)self, (void *)code, (PyObject *)code); + + Py_RETURN_NONE; +} + +PyObject* pyreturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + PyObject* code = args[0]; + ptrace_leave_call((PyObject*)self, (void *)code); + + Py_RETURN_NONE; +} + +PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObject* missing) +{ + // return a new reference + if (PyCFunction_Check(callable)) { + Py_INCREF(callable); + return (PyObject*)((PyCFunctionObject *)callable); + } + if (Py_TYPE(callable) == &PyMethodDescr_Type) { + /* For backwards compatibility need to + * convert to builtin method */ + + /* If no arg, skip */ + if (self_arg == missing) { + return NULL; + } + PyObject *meth = Py_TYPE(callable)->tp_descr_get( + callable, self_arg, (PyObject*)Py_TYPE(self_arg)); + if (meth == NULL) { + return NULL; + } + if (PyCFunction_Check(meth)) { + return (PyObject*)((PyCFunctionObject *)meth); + } + } + return NULL; +} + +PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + if (self->flags & POF_BUILTINS) { + PyObject* callable = args[2]; + PyObject* self_arg = args[3]; + + PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); + + if (cfunc) { + ptrace_enter_call((PyObject*)self, + ((PyCFunctionObject *)cfunc)->m_ml, + cfunc); + Py_DECREF(cfunc); + } + } + Py_RETURN_NONE; +} + +PyObject* creturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + if (self->flags & POF_BUILTINS) { + PyObject* callable = args[2]; + PyObject* self_arg = args[3]; + + PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); + + if (cfunc) { + ptrace_leave_call((PyObject*)self, + ((PyCFunctionObject *)cfunc)->m_ml); + Py_DECREF(cfunc); + } + } + Py_RETURN_NONE; +} + +static const struct { + int event; + const char* callback_method; +} callback_table[] = { + {PY_MONITORING_EVENT_PY_START, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_RESUME, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_RETURN, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_PY_YIELD, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_PY_UNWIND, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_CALL, "_ccall_callback"}, + {PY_MONITORING_EVENT_C_RETURN, "_creturn_callback"}, + {PY_MONITORING_EVENT_C_RAISE, "_creturn_callback"}, + {0, NULL} +}; + PyDoc_STRVAR(enable_doc, "\ enable(subcalls=True, builtins=True)\n\ \n\ @@ -666,6 +703,8 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) int subcalls = -1; int builtins = -1; static char *kwlist[] = {"subcalls", "builtins", 0}; + int all_events = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pp:enable", kwlist, &subcalls, &builtins)) return NULL; @@ -673,11 +712,37 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, profiler_callback, (PyObject*)self) < 0) { + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + if (!monitoring) { + return NULL; + } + + if (PyObject_CallMethod(monitoring, "use_tool_id", "is", self->tool_id, "cProfile") == NULL) { + PyErr_Format(PyExc_ValueError, "Another profiling tool is already active"); + Py_DECREF(monitoring); + return NULL; + } + + for (int i = 0; callback_table[i].callback_method; i++) { + PyObject* callback = PyObject_GetAttrString((PyObject*)self, callback_table[i].callback_method); + if (!callback) { + Py_DECREF(monitoring); + return NULL; + } + Py_XDECREF(PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id, + (1 << callback_table[i].event), + callback)); + Py_DECREF(callback); + all_events |= (1 << callback_table[i].event); + } + + if (!PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, all_events)) { + Py_DECREF(monitoring); return NULL; } + Py_DECREF(monitoring); + self->flags |= POF_ENABLED; Py_RETURN_NONE; } @@ -707,13 +772,44 @@ Stop collecting profiling information.\n\ static PyObject* profiler_disable(ProfilerObject *self, PyObject* noarg) { - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { - return NULL; + if (self->flags & POF_ENABLED) { + PyObject* result = NULL; + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + + if (!monitoring) { + return NULL; + } + + for (int i = 0; callback_table[i].callback_method; i++) { + result = PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id, + (1 << callback_table[i].event), Py_None); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + } + + result = PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, 0); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + + result = PyObject_CallMethod(monitoring, "free_tool_id", "i", self->tool_id); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + + Py_DECREF(monitoring); + + self->flags &= ~POF_ENABLED; + flush_unmatched(self); } - self->flags &= ~POF_ENABLED; - flush_unmatched(self); if (pending_exception(self)) { return NULL; } @@ -778,17 +874,37 @@ profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) return -1; pObj->externalTimerUnit = timeunit; Py_XSETREF(pObj->externalTimer, Py_XNewRef(timer)); + pObj->tool_id = PY_MONITORING_PROFILER_ID; + + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + if (!monitoring) { + return -1; + } + pObj->missing = PyObject_GetAttrString(monitoring, "MISSING"); + if (!pObj->missing) { + Py_DECREF(monitoring); + return -1; + } + Py_DECREF(monitoring); return 0; } static PyMethodDef profiler_methods[] = { _LSPROF_PROFILER_GETSTATS_METHODDEF - {"enable", _PyCFunction_CAST(profiler_enable), + {"enable", _PyCFunction_CAST(profiler_enable), METH_VARARGS | METH_KEYWORDS, enable_doc}, - {"disable", (PyCFunction)profiler_disable, + {"disable", (PyCFunction)profiler_disable, METH_NOARGS, disable_doc}, - {"clear", (PyCFunction)profiler_clear, + {"clear", (PyCFunction)profiler_clear, METH_NOARGS, clear_doc}, + {"_pystart_callback", _PyCFunction_CAST(pystart_callback), + METH_FASTCALL, NULL}, + {"_pyreturn_callback", _PyCFunction_CAST(pyreturn_callback), + METH_FASTCALL, NULL}, + {"_ccall_callback", _PyCFunction_CAST(ccall_callback), + METH_FASTCALL, NULL}, + {"_creturn_callback", _PyCFunction_CAST(creturn_callback), + METH_FASTCALL, NULL}, {NULL, NULL} }; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index fee493ff7f1666..7ba116dcb171cf 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -216,6 +216,7 @@ Modules/_io/_iomodule.c - static_types - Modules/_io/textio.c - encodefuncs - Modules/_io/winconsoleio.c - _PyWindowsConsoleIO_Type - Modules/_localemodule.c - langinfo_constants - +Modules/_lsprof.c - callback_table - Modules/_pickle.c - READ_WHOLE_LINE - Modules/_sqlite/module.c - error_codes - Modules/_sre/sre.c pattern_repr flag_names - From 1afe0e0320c6f19418d44d682ad95ba0c689c595 Mon Sep 17 00:00:00 2001 From: Alexey Namyotkin <62434915+nametkin@users.noreply.github.com> Date: Fri, 5 May 2023 21:52:24 +0300 Subject: [PATCH 3/9] gh-69152: Add _proxy_response_headers attribute to HTTPConnection (#26152) Add _proxy_response_headers attribute to HTTPConnection (#26152) --------- Co-authored-by: Senthil Kumaran --- Lib/http/client.py | 18 +++++++----------- Lib/test/test_httplib.py | 14 ++++++++++++++ .../2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst | 3 +++ 3 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index 74f7bcb68fb6bc..50f2b4680769c8 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -858,6 +858,7 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, self._tunnel_host = None self._tunnel_port = None self._tunnel_headers = {} + self._proxy_response_headers = None (self.host, self.port) = self._get_hostport(host, port) @@ -944,21 +945,16 @@ def _tunnel(self): try: (version, code, message) = response._read_status() + self._proxy_response_headers = parse_headers(response.fp) + + if self.debuglevel > 0: + for hdr, val in self._proxy_response_headers.items(): + print("header:", hdr + ":", val) + if code != http.HTTPStatus.OK: self.close() raise OSError(f"Tunnel connection failed: {code} {message.strip()}") - while True: - line = response.fp.readline(_MAXLINE + 1) - if len(line) > _MAXLINE: - raise LineTooLong("header line") - if not line: - # for sites which EOF without sending a trailer - break - if line in (b'\r\n', b'\n', b''): - break - if self.debuglevel > 0: - print('header:', line.decode()) finally: response.close() diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 37f77fe0a320c7..4b1d355f550b49 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2390,6 +2390,20 @@ def test_tunnel_debuglog(self): lines = output.getvalue().splitlines() self.assertIn('header: {}'.format(expected_header), lines) + def test_proxy_response_headers(self): + expected_header = ('X-Dummy', '1') + response_text = ( + 'HTTP/1.0 200 OK\r\n' + '{0}\r\n\r\n'.format(':'.join(expected_header)) + ) + + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + self.conn.request('PUT', '/', '') + headers = self.conn._proxy_response_headers + self.assertIn(expected_header, headers.items()) + def test_tunnel_leak(self): sock = None diff --git a/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst b/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst new file mode 100644 index 00000000000000..ba113673b7fbe5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-16-14-28-30.bpo-24964.Oa5Ie_.rst @@ -0,0 +1,3 @@ +Added attribute '_proxy_response_headers' to HTTPConnection class. This +attribute contains the headers of the proxy server response to the CONNECT +request. From d00d94214971621e6a3541425ee8c8072023ca1a Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 5 May 2023 20:04:53 +0100 Subject: [PATCH 4/9] GH-100479: Add `pathlib.PurePath.with_segments()` (GH-103975) Add `pathlib.PurePath.with_segments()`, which creates a path object from arguments. This method is called whenever a derivative path is created, such as from `pathlib.PurePath.parent`. Subclasses may override this method to share information between path objects. Co-authored-by: Alex Waygood --- Doc/library/pathlib.rst | 28 +++++++- Doc/whatsnew/3.12.rst | 5 ++ Lib/pathlib.py | 66 ++++++++++--------- Lib/test/test_pathlib.py | 52 ++++++++++----- ...-04-03-22-02-35.gh-issue-100479.kNBjQm.rst | 4 ++ 5 files changed, 108 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 14118127835bbe..5ffa33d4e61f19 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -530,10 +530,10 @@ Pure paths provide the following methods and properties: unintended effects. -.. method:: PurePath.joinpath(*other) +.. method:: PurePath.joinpath(*pathsegments) Calling this method is equivalent to combining the path with each of - the *other* arguments in turn:: + the given *pathsegments* in turn:: >>> PurePosixPath('/etc').joinpath('passwd') PurePosixPath('/etc/passwd') @@ -680,6 +680,30 @@ Pure paths provide the following methods and properties: PureWindowsPath('README') +.. method:: PurePath.with_segments(*pathsegments) + + Create a new path object of the same type by combining the given + *pathsegments*. This method is called whenever a derivative path is created, + such as from :attr:`parent` and :meth:`relative_to`. Subclasses may + override this method to pass information to derivative paths, for example:: + + from pathlib import PurePosixPath + + class MyPath(PurePosixPath): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + + etc = MyPath('/etc', session_id=42) + hosts = etc / 'hosts' + print(hosts.session_id) # 42 + + .. versionadded:: 3.12 + + .. _concrete-paths: diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 4f952e2a37ef2f..ccddc8bd832f29 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -348,6 +348,11 @@ inspect pathlib ------- +* Add support for subclassing :class:`pathlib.PurePath` and + :class:`~pathlib.Path`, plus their Posix- and Windows-specific variants. + Subclasses may override the :meth:`~pathlib.PurePath.with_segments` method + to pass information between path instances. + * Add :meth:`~pathlib.Path.walk` for walking the directory trees and generating all file or directory names within them, similar to :func:`os.walk`. (Contributed by Stanislav Zmiev in :gh:`90385`.) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f32e1e2d822834..9aa3c1e52447d1 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -204,11 +204,10 @@ def _select_from(self, parent_path, scandir): class _PathParents(Sequence): """This object provides sequence-like access to the logical ancestors of a path. Don't try to construct it yourself.""" - __slots__ = ('_pathcls', '_drv', '_root', '_tail') + __slots__ = ('_path', '_drv', '_root', '_tail') def __init__(self, path): - # We don't store the instance to avoid reference cycles - self._pathcls = type(path) + self._path = path self._drv = path.drive self._root = path.root self._tail = path._tail @@ -224,11 +223,11 @@ def __getitem__(self, idx): raise IndexError(idx) if idx < 0: idx += len(self) - return self._pathcls._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) def __repr__(self): - return "<{}.parents>".format(self._pathcls.__name__) + return "<{}.parents>".format(type(self._path).__name__) class PurePath(object): @@ -316,6 +315,13 @@ def __init__(self, *args): else: self._raw_path = self._flavour.join(*paths) + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + @classmethod def _parse_path(cls, path): if not path: @@ -342,15 +348,14 @@ def _load_parts(self): self._root = root self._tail_cached = tail - @classmethod - def _from_parsed_parts(cls, drv, root, tail): - path = cls._format_parsed_parts(drv, root, tail) - self = cls(path) - self._str = path or '.' - self._drv = drv - self._root = root - self._tail_cached = tail - return self + def _from_parsed_parts(self, drv, root, tail): + path_str = self._format_parsed_parts(drv, root, tail) + path = self.with_segments(path_str) + path._str = path_str or '.' + path._drv = drv + path._root = root + path._tail_cached = tail + return path @classmethod def _format_parsed_parts(cls, drv, root, tail): @@ -584,8 +589,7 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): "scheduled for removal in Python {remove}") warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, remove=(3, 14)) - path_cls = type(self) - other = path_cls(other, *_deprecated) + other = self.with_segments(other, *_deprecated) for step, path in enumerate([other] + list(other.parents)): if self.is_relative_to(path): break @@ -594,7 +598,7 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): if step and not walk_up: raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") parts = ['..'] * step + self._tail[len(path._tail):] - return path_cls(*parts) + return self.with_segments(*parts) def is_relative_to(self, other, /, *_deprecated): """Return True if the path is relative to another path or False. @@ -605,7 +609,7 @@ def is_relative_to(self, other, /, *_deprecated): "scheduled for removal in Python {remove}") warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", msg, remove=(3, 14)) - other = type(self)(other, *_deprecated) + other = self.with_segments(other, *_deprecated) return other == self or other in self.parents @property @@ -617,13 +621,13 @@ def parts(self): else: return tuple(self._tail) - def joinpath(self, *args): + def joinpath(self, *pathsegments): """Combine this path with one or several arguments, and return a new path representing either a subpath (if all arguments are relative paths) or a totally different path (if one of the arguments is anchored). """ - return self.__class__(self, *args) + return self.with_segments(self, *pathsegments) def __truediv__(self, key): try: @@ -633,7 +637,7 @@ def __truediv__(self, key): def __rtruediv__(self, key): try: - return type(self)(key, self) + return self.with_segments(key, self) except TypeError: return NotImplemented @@ -650,6 +654,8 @@ def parent(self): @property def parents(self): """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. return _PathParents(self) def is_absolute(self): @@ -680,7 +686,7 @@ def match(self, path_pattern): """ Return True if this path matches the given pattern. """ - pat = type(self)(path_pattern) + pat = self.with_segments(path_pattern) if not pat.parts: raise ValueError("empty pattern") pat_parts = pat._parts_normcase @@ -755,7 +761,7 @@ def _make_child_relpath(self, name): path_str = f'{path_str}{name}' else: path_str = name - path = type(self)(path_str) + path = self.with_segments(path_str) path._str = path_str path._drv = self.drive path._root = self.root @@ -805,7 +811,7 @@ def samefile(self, other_path): try: other_st = other_path.stat() except AttributeError: - other_st = self.__class__(other_path).stat() + other_st = self.with_segments(other_path).stat() return self._flavour.samestat(st, other_st) def iterdir(self): @@ -867,7 +873,7 @@ def absolute(self): cwd = self._flavour.abspath(self.drive) else: cwd = os.getcwd() - return type(self)(cwd, self) + return self.with_segments(cwd, self) def resolve(self, strict=False): """ @@ -885,7 +891,7 @@ def check_eloop(e): except OSError as e: check_eloop(e) raise - p = type(self)(s) + p = self.with_segments(s) # In non-strict mode, realpath() doesn't raise on symlink loops. # Ensure we get an exception by calling stat() @@ -975,7 +981,7 @@ def readlink(self): """ if not hasattr(os, "readlink"): raise NotImplementedError("os.readlink() not available on this system") - return type(self)(os.readlink(self)) + return self.with_segments(os.readlink(self)) def touch(self, mode=0o666, exist_ok=True): """ @@ -1064,7 +1070,7 @@ def rename(self, target): Returns the new Path instance pointing to the target path. """ os.rename(self, target) - return self.__class__(target) + return self.with_segments(target) def replace(self, target): """ @@ -1077,7 +1083,7 @@ def replace(self, target): Returns the new Path instance pointing to the target path. """ os.replace(self, target) - return self.__class__(target) + return self.with_segments(target) def symlink_to(self, target, target_is_directory=False): """ diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index a932e03df4236d..7586610833b063 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -29,11 +29,12 @@ # class _BasePurePathSubclass(object): - init_called = False + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id - def __init__(self, *args): - super().__init__(*args) - self.init_called = True + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) class _BasePurePathTest(object): @@ -121,20 +122,21 @@ def test_str_subclass_common(self): self._check_str_subclass('a/b.txt') self._check_str_subclass('/a/b.txt') - def test_init_called_common(self): + def test_with_segments_common(self): class P(_BasePurePathSubclass, self.cls): pass - p = P('foo', 'bar') - self.assertTrue((p / 'foo').init_called) - self.assertTrue(('foo' / p).init_called) - self.assertTrue(p.joinpath('foo').init_called) - self.assertTrue(p.with_name('foo').init_called) - self.assertTrue(p.with_stem('foo').init_called) - self.assertTrue(p.with_suffix('.foo').init_called) - self.assertTrue(p.relative_to('foo').init_called) - self.assertTrue(p.parent.init_called) + p = P('foo', 'bar', session_id=42) + self.assertEqual(42, (p / 'foo').session_id) + self.assertEqual(42, ('foo' / p).session_id) + self.assertEqual(42, p.joinpath('foo').session_id) + self.assertEqual(42, p.with_name('foo').session_id) + self.assertEqual(42, p.with_stem('foo').session_id) + self.assertEqual(42, p.with_suffix('.foo').session_id) + self.assertEqual(42, p.with_segments('foo').session_id) + self.assertEqual(42, p.relative_to('foo').session_id) + self.assertEqual(42, p.parent.session_id) for parent in p.parents: - self.assertTrue(parent.init_called) + self.assertEqual(42, parent.session_id) def _get_drive_root_parts(self, parts): path = self.cls(*parts) @@ -1647,6 +1649,26 @@ def test_home(self): env['HOME'] = os.path.join(BASE, 'home') self._test_home(self.cls.home()) + def test_with_segments(self): + class P(_BasePurePathSubclass, self.cls): + pass + p = P(BASE, session_id=42) + self.assertEqual(42, p.absolute().session_id) + self.assertEqual(42, p.resolve().session_id) + self.assertEqual(42, p.with_segments('~').expanduser().session_id) + self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) + self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) + if os_helper.can_symlink(): + self.assertEqual(42, (p / 'linkA').readlink().session_id) + for path in p.iterdir(): + self.assertEqual(42, path.session_id) + for path in p.glob('*'): + self.assertEqual(42, path.session_id) + for path in p.rglob('*'): + self.assertEqual(42, path.session_id) + for dirpath, dirnames, filenames in p.walk(): + self.assertEqual(42, dirpath.session_id) + def test_samefile(self): fileA_path = os.path.join(BASE, 'fileA') fileB_path = os.path.join(BASE, 'dirB', 'fileB') diff --git a/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst b/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst new file mode 100644 index 00000000000000..58db90480d2ff0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-03-22-02-35.gh-issue-100479.kNBjQm.rst @@ -0,0 +1,4 @@ +Add :meth:`pathlib.PurePath.with_segments`, which creates a path object from +arguments. This method is called whenever a derivative path is created, such +as from :attr:`pathlib.PurePath.parent`. Subclasses may override this method +to share information between path objects. From 55671fe04700ccb4e73c8db3dd1e9c031dafe700 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 May 2023 13:23:00 -0600 Subject: [PATCH 5/9] gh-99113: Share the GIL via PyInterpreterState.ceval.gil (gh-104203) In preparation for a per-interpreter GIL, we add PyInterpreterState.ceval.gil, set it to the shared GIL for each interpreter, and use that rather than using _PyRuntime.ceval.gil directly. Note that _PyRuntime.ceval.gil is still the actual GIL. --- Include/internal/pycore_ceval.h | 2 +- Include/internal/pycore_ceval_state.h | 3 + Modules/_xxsubinterpretersmodule.c | 1 + Python/ceval_gil.c | 97 ++++++++++++++++----------- Python/pystate.c | 2 +- 5 files changed, 65 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index deda070a6dea79..0bbc9efdda3ba7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -96,7 +96,7 @@ _PyEval_Vector(PyThreadState *tstate, PyObject* const* args, size_t argcount, PyObject *kwnames); -extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime); +extern int _PyEval_ThreadsInitialized(void); extern PyStatus _PyEval_InitGIL(PyThreadState *tstate); extern void _PyEval_FiniGIL(PyInterpreterState *interp); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 9ba42eb03b2676..1a00ec80270e0c 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -49,6 +49,8 @@ struct _ceval_runtime_state { the main thread of the main interpreter can handle signals: see _Py_ThreadCanHandleSignals(). */ _Py_atomic_int signals_pending; + + /* This is (only) used indirectly through PyInterpreterState.ceval.gil. */ struct _gil_runtime_state gil; }; @@ -83,6 +85,7 @@ struct _pending_calls { struct _ceval_state { int recursion_limit; + struct _gil_runtime_state *gil; /* This single variable consolidates all requests to break out of the fast path in the eval loop. */ _Py_atomic_int eval_breaker; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 95273ab278d996..433fecfda655de 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -513,6 +513,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) // Create and initialize the new interpreter. PyThreadState *save_tstate = _PyThreadState_GET(); + assert(save_tstate != NULL); const PyInterpreterConfig config = isolated ? (PyInterpreterConfig)_PyInterpreterConfig_INIT : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 4d501c89661743..ad33a58dedd820 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -229,6 +229,9 @@ static void _gil_initialize(struct _gil_runtime_state *gil) static int gil_created(struct _gil_runtime_state *gil) { + if (gil == NULL) { + return 0; + } return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0); } @@ -273,10 +276,9 @@ static void recreate_gil(struct _gil_runtime_state *gil) #endif static void -drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, - PyThreadState *tstate) +drop_gil(struct _ceval_state *ceval, PyThreadState *tstate) { - struct _gil_runtime_state *gil = &ceval->gil; + struct _gil_runtime_state *gil = ceval->gil; if (!_Py_atomic_load_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } @@ -296,7 +298,7 @@ drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, MUTEX_UNLOCK(gil->mutex); #ifdef FORCE_SWITCHING - if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request) && tstate != NULL) { + if (_Py_atomic_load_relaxed(&ceval->gil_drop_request) && tstate != NULL) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate) @@ -358,9 +360,8 @@ take_gil(PyThreadState *tstate) assert(is_tstate_valid(tstate)); PyInterpreterState *interp = tstate->interp; - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - struct _gil_runtime_state *gil = &ceval->gil; + struct _ceval_state *ceval = &interp->ceval; + struct _gil_runtime_state *gil = ceval->gil; /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); @@ -434,12 +435,12 @@ take_gil(PyThreadState *tstate) in take_gil() while the main thread called wait_for_thread_shutdown() from Py_Finalize(). */ MUTEX_UNLOCK(gil->mutex); - drop_gil(ceval, ceval2, tstate); + drop_gil(ceval, tstate); PyThread_exit_thread(); } assert(is_tstate_valid(tstate)); - if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) { + if (_Py_atomic_load_relaxed(&ceval->gil_drop_request)) { RESET_GIL_DROP_REQUEST(interp); } else { @@ -448,7 +449,7 @@ take_gil(PyThreadState *tstate) handle signals. Note: RESET_GIL_DROP_REQUEST() calls COMPUTE_EVAL_BREAKER(). */ - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); + COMPUTE_EVAL_BREAKER(interp, &_PyRuntime.ceval, ceval); } /* Don't access tstate if the thread must exit */ @@ -463,63 +464,86 @@ take_gil(PyThreadState *tstate) void _PyEval_SetSwitchInterval(unsigned long microseconds) { - struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; + /* XXX per-interpreter GIL */ + PyInterpreterState *interp = _PyInterpreterState_Main(); + struct _gil_runtime_state *gil = interp->ceval.gil; + assert(gil != NULL); gil->interval = microseconds; } unsigned long _PyEval_GetSwitchInterval(void) { - struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; + /* XXX per-interpreter GIL */ + PyInterpreterState *interp = _PyInterpreterState_Main(); + struct _gil_runtime_state *gil = interp->ceval.gil; + assert(gil != NULL); return gil->interval; } int -_PyEval_ThreadsInitialized(_PyRuntimeState *runtime) +_PyEval_ThreadsInitialized(void) { - return gil_created(&runtime->ceval.gil); + /* XXX per-interpreter GIL */ + PyInterpreterState *interp = _PyInterpreterState_Main(); + if (interp == NULL) { + return 0; + } + struct _gil_runtime_state *gil = interp->ceval.gil; + return gil_created(gil); } int PyEval_ThreadsInitialized(void) { - _PyRuntimeState *runtime = &_PyRuntime; - return _PyEval_ThreadsInitialized(runtime); + return _PyEval_ThreadsInitialized(); } PyStatus _PyEval_InitGIL(PyThreadState *tstate) { + assert(tstate->interp->ceval.gil == NULL); + + /* XXX per-interpreter GIL */ + struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; if (!_Py_IsMainInterpreter(tstate->interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ + assert(gil_created(gil)); + tstate->interp->ceval.gil = gil; return _PyStatus_OK(); } - struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; assert(!gil_created(gil)); PyThread_init_thread(); create_gil(gil); - - take_gil(tstate); - assert(gil_created(gil)); + tstate->interp->ceval.gil = gil; + take_gil(tstate); return _PyStatus_OK(); } void _PyEval_FiniGIL(PyInterpreterState *interp) { + if (interp->ceval.gil == NULL) { + /* It was already finalized (or hasn't been initialized yet). */ + return; + } + + /* XXX per-interpreter GIL */ + struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; if (!_Py_IsMainInterpreter(interp)) { /* Currently, the GIL is shared by all interpreters, and only the main interpreter is responsible to create and destroy it. */ + assert(interp->ceval.gil == gil); + interp->ceval.gil = NULL; return; } - struct _gil_runtime_state *gil = &interp->runtime->ceval.gil; if (!gil_created(gil)) { /* First Py_InitializeFromConfig() call: the GIL doesn't exist yet: do nothing. */ @@ -528,6 +552,7 @@ _PyEval_FiniGIL(PyInterpreterState *interp) destroy_gil(gil); assert(!gil_created(gil)); + interp->ceval.gil = NULL; } void @@ -555,22 +580,19 @@ PyEval_AcquireLock(void) void PyEval_ReleaseLock(void) { - _PyRuntimeState *runtime = &_PyRuntime; PyThreadState *tstate = _PyThreadState_GET(); /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + drop_gil(ceval, tstate); } void _PyEval_ReleaseLock(PyThreadState *tstate) { - struct _ceval_runtime_state *ceval = &tstate->interp->runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + drop_gil(ceval, tstate); } void @@ -595,9 +617,8 @@ PyEval_ReleaseThread(PyThreadState *tstate) if (new_tstate != tstate) { Py_FatalError("wrong thread state"); } - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + drop_gil(ceval, tstate); } #ifdef HAVE_FORK @@ -607,9 +628,9 @@ PyEval_ReleaseThread(PyThreadState *tstate) PyStatus _PyEval_ReInitThreads(PyThreadState *tstate) { - _PyRuntimeState *runtime = tstate->interp->runtime; + assert(tstate->interp == _PyInterpreterState_Main()); - struct _gil_runtime_state *gil = &runtime->ceval.gil; + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; if (!gil_created(gil)) { return _PyStatus_OK(); } @@ -644,10 +665,9 @@ PyEval_SaveThread(void) PyThreadState *tstate = _PyThreadState_Swap(runtime, NULL); _Py_EnsureTstateNotNULL(tstate); - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *ceval2 = &tstate->interp->ceval; - assert(gil_created(&ceval->gil)); - drop_gil(ceval, ceval2, tstate); + struct _ceval_state *ceval = &tstate->interp->ceval; + assert(gil_created(ceval->gil)); + drop_gil(ceval, tstate); return tstate; } @@ -906,6 +926,7 @@ Py_MakePendingCalls(void) void _PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval) { + /* XXX per-interpreter GIL */ _gil_initialize(&ceval->gil); } @@ -964,7 +985,7 @@ _Py_HandlePending(PyThreadState *tstate) if (_PyThreadState_Swap(runtime, NULL) != tstate) { Py_FatalError("tstate mix-up"); } - drop_gil(ceval, interp_ceval_state, tstate); + drop_gil(interp_ceval_state, tstate); /* Other threads may run now */ diff --git a/Python/pystate.c b/Python/pystate.c index f09d37657f9c27..75bd9f41e301a3 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2186,7 +2186,7 @@ PyGILState_Ensure(void) /* Ensure that _PyEval_InitThreads() and _PyGILState_Init() have been called by Py_Initialize() */ - assert(_PyEval_ThreadsInitialized(runtime)); + assert(_PyEval_ThreadsInitialized()); assert(gilstate_tss_initialized(runtime)); assert(runtime->gilstate.autoInterpreterState != NULL); From 1c420e138fd828895b6bd3c44ef99156e8796095 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 May 2023 14:04:55 -0600 Subject: [PATCH 6/9] gh-104108: Add the Py_mod_multiple_interpreters Module Def Slot (gh-104148) I'll be adding a value to indicate support for per-interpreter GIL in gh-99114. --- Include/moduleobject.h | 7 +- Lib/test/test_import/__init__.py | 82 ++++++++++++++----- ...-05-03-17-46-47.gh-issue-104108.GOxAYt.rst | 6 ++ Modules/_testmultiphase.c | 19 +++++ Objects/moduleobject.c | 30 +++++++ 5 files changed, 122 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 555564ec73b4a2..7ac6f6e8a4a24e 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -78,11 +78,16 @@ struct PyModuleDef_Slot { #define Py_mod_create 1 #define Py_mod_exec 2 +#define Py_mod_multiple_interpreters 3 #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 2 +#define _Py_mod_LAST_SLOT 3 #endif +/* for Py_mod_multiple_interpreters: */ +#define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0) +#define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1) + #endif /* New in 3.5 */ struct PyModuleDef { diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 41dfdaabe24664..9211639b016e7a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1652,26 +1652,44 @@ def pipe(self): os.set_blocking(r, False) return (r, w) - def import_script(self, name, fd, check_override=None): + def import_script(self, name, fd, filename=None, check_override=None): override_text = '' if check_override is not None: override_text = f''' - import _imp - _imp._override_multi_interp_extensions_check({check_override}) - ''' - return textwrap.dedent(f''' - import os, sys - {override_text} - try: - import {name} - except ImportError as exc: - text = 'ImportError: ' + str(exc) - else: - text = 'okay' - os.write({fd}, text.encode('utf-8')) - ''') + import _imp + _imp._override_multi_interp_extensions_check({check_override}) + ''' + if filename: + return textwrap.dedent(f''' + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import ExtensionFileLoader + import os, sys + {override_text} + loader = ExtensionFileLoader({name!r}, {filename!r}) + spec = spec_from_loader({name!r}, loader) + try: + module = module_from_spec(spec) + loader.exec_module(module) + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + else: + return textwrap.dedent(f''' + import os, sys + {override_text} + try: + import {name} + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') - def run_here(self, name, *, + def run_here(self, name, filename=None, *, check_singlephase_setting=False, check_singlephase_override=None, isolated=False, @@ -1700,26 +1718,30 @@ def run_here(self, name, *, ) r, w = self.pipe() - script = self.import_script(name, w, check_singlephase_override) + script = self.import_script(name, w, filename, + check_singlephase_override) ret = run_in_subinterp_with_config(script, **kwargs) self.assertEqual(ret, 0) return os.read(r, 100) - def check_compatible_here(self, name, *, strict=False, isolated=False): + def check_compatible_here(self, name, filename=None, *, + strict=False, + isolated=False, + ): # Verify that the named module may be imported in a subinterpreter. # (See run_here() for more info.) - out = self.run_here(name, + out = self.run_here(name, filename, check_singlephase_setting=strict, isolated=isolated, ) self.assertEqual(out, b'okay') - def check_incompatible_here(self, name, *, isolated=False): + def check_incompatible_here(self, name, filename=None, *, isolated=False): # Differences from check_compatible_here(): # * verify that import fails # * "strict" is always True - out = self.run_here(name, + out = self.run_here(name, filename, check_singlephase_setting=True, isolated=isolated, ) @@ -1820,6 +1842,24 @@ def test_multi_init_extension_compat(self): with self.subTest(f'{module}: strict, fresh'): self.check_compatible_fresh(module, strict=True) + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_non_isolated_compat(self): + modname = '_test_non_isolated' + filename = _testmultiphase.__file__ + loader = ExtensionFileLoader(modname, filename) + spec = importlib.util.spec_from_loader(modname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + sys.modules[modname] = module + + require_extension(module) + with self.subTest(f'{modname}: isolated'): + self.check_incompatible_here(modname, filename, isolated=True) + with self.subTest(f'{modname}: not isolated'): + self.check_incompatible_here(modname, filename, isolated=False) + with self.subTest(f'{modname}: not strict'): + self.check_compatible_here(modname, filename, strict=False) + def test_python_compat(self): module = 'threading' require_pure_python(module) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst new file mode 100644 index 00000000000000..dad843636493ae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst @@ -0,0 +1,6 @@ +Multi-phase init extension modules may now indicate whether or not they +actually support multiple interpreters. By default such modules are +expected to support use in multiple interpreters. In the uncommon case that +one does not, it may use the new ``Py_mod_multiple_interpreters`` module def +slot. A value of ``0`` means the module does not support them. ``1`` means +it does. The default is ``1``. diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index cf8990a2df0a9b..bc7d8b64a94322 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -884,3 +884,22 @@ PyInit__test_module_state_shared(void) } return module; } + + +/* multiple interpreters supports */ + +static PyModuleDef_Slot non_isolated_slots[] = { + {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL}, +}; + +static PyModuleDef non_isolated_def = TEST_MODULE_DEF("_test_non_isolated", + non_isolated_slots, + testexport_methods); + +PyMODINIT_FUNC +PyInit__test_non_isolated(void) +{ + return PyModuleDef_Init(&non_isolated_def); +} diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index ffcd90ed122e77..63f4226dd9e94a 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -245,6 +245,8 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio PyObject *(*create)(PyObject *, PyModuleDef*) = NULL; PyObject *nameobj; PyObject *m = NULL; + int has_multiple_interpreters_slot = 0; + void *multiple_interpreters = (void *)0; int has_execution_slots = 0; const char *name; int ret; @@ -287,6 +289,17 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio case Py_mod_exec: has_execution_slots = 1; break; + case Py_mod_multiple_interpreters: + if (has_multiple_interpreters_slot) { + PyErr_Format( + PyExc_SystemError, + "module %s has more than one 'multiple interpreters' slots", + name); + goto error; + } + multiple_interpreters = cur_slot->value; + has_multiple_interpreters_slot = 1; + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -297,6 +310,20 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } } + /* By default, multi-phase init modules are expected + to work under multiple interpreters. */ + if (!has_multiple_interpreters_slot) { + multiple_interpreters = Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; + } + if (multiple_interpreters == Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!_Py_IsMainInterpreter(interp) + && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) + { + goto error; + } + } + if (create) { m = create(spec, def); if (m == NULL) { @@ -421,6 +448,9 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) return -1; } break; + case Py_mod_multiple_interpreters: + /* handled in PyModule_FromDefAndSpec2 */ + break; default: PyErr_Format( PyExc_SystemError, From a9c6e0618f26270e2591b3d99ffeef55eea02a33 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 May 2023 15:11:27 -0600 Subject: [PATCH 7/9] gh-99113: Add Py_MOD_PER_INTERPRETER_GIL_SUPPORTED (gh-104205) Here we are doing no more than adding the value for Py_mod_multiple_interpreters and using it for stdlib modules. We will start checking for it in gh-104206 (once PyInterpreterState.ceval.own_gil is added in gh-104204). --- Include/moduleobject.h | 1 + Modules/_abc.c | 1 + Modules/_asynciomodule.c | 1 + Modules/_bisectmodule.c | 1 + Modules/_blake2/blake2module.c | 3 ++- Modules/_bz2module.c | 1 + Modules/_codecsmodule.c | 1 + Modules/_collectionsmodule.c | 1 + Modules/_contextvarsmodule.c | 1 + Modules/_cryptmodule.c | 1 + Modules/_csv.c | 1 + Modules/_ctypes/_ctypes_test.c | 1 + Modules/_curses_panel.c | 3 +++ Modules/_dbmmodule.c | 1 + Modules/_elementtree.c | 3 +++ Modules/_functoolsmodule.c | 1 + Modules/_gdbmmodule.c | 1 + Modules/_hashopenssl.c | 1 + Modules/_heapqmodule.c | 1 + Modules/_json.c | 1 + Modules/_localemodule.c | 1 + Modules/_lsprof.c | 3 +++ Modules/_lzmamodule.c | 1 + Modules/_multiprocessing/multiprocessing.c | 1 + Modules/_multiprocessing/posixshmem.c | 7 +++++++ Modules/_opcode.c | 8 +++++++- Modules/_operator.c | 1 + Modules/_pickle.c | 1 + Modules/_posixsubprocess.c | 1 + Modules/_queuemodule.c | 1 + Modules/_randommodule.c | 1 + Modules/_scproxy.c | 1 + Modules/_sha3/sha3module.c | 1 + Modules/_sqlite/module.c | 1 + Modules/_sre/sre.c | 1 + Modules/_ssl.c | 3 +++ Modules/_stat.c | 1 + Modules/_statisticsmodule.c | 1 + Modules/_struct.c | 1 + Modules/_testinternalcapi.c | 1 + Modules/_testmultiphase.c | 6 ++++++ Modules/_threadmodule.c | 1 + Modules/_typingmodule.c | 1 + Modules/_uuidmodule.c | 1 + Modules/_weakref.c | 1 + Modules/_winapi.c | 1 + Modules/_xxinterpchannelsmodule.c | 1 + Modules/_xxsubinterpretersmodule.c | 1 + Modules/_zoneinfo.c | 5 ++++- Modules/arraymodule.c | 1 + Modules/atexitmodule.c | 6 ++++++ Modules/audioop.c | 1 + Modules/binascii.c | 1 + Modules/cjkcodecs/cjkcodecs.h | 1 + Modules/cjkcodecs/multibytecodec.c | 1 + Modules/cmathmodule.c | 1 + Modules/errnomodule.c | 1 + Modules/faulthandler.c | 2 ++ Modules/fcntlmodule.c | 1 + Modules/gcmodule.c | 1 + Modules/grpmodule.c | 1 + Modules/itertoolsmodule.c | 1 + Modules/mathmodule.c | 1 + Modules/md5module.c | 1 + Modules/mmapmodule.c | 1 + Modules/nismodule.c | 3 +++ Modules/overlapped.c | 1 + Modules/posixmodule.c | 1 + Modules/pwdmodule.c | 1 + Modules/pyexpat.c | 3 +++ Modules/resource.c | 1 + Modules/selectmodule.c | 1 + Modules/sha1module.c | 1 + Modules/sha2module.c | 1 + Modules/signalmodule.c | 1 + Modules/socketmodule.c | 1 + Modules/spwdmodule.c | 1 + Modules/symtablemodule.c | 1 + Modules/syslogmodule.c | 1 + Modules/termios.c | 1 + Modules/timemodule.c | 1 + Modules/unicodedata.c | 1 + Modules/xxlimited.c | 1 + Modules/xxlimited_35.c | 1 + Modules/xxmodule.c | 1 + Modules/xxsubtype.c | 1 + Modules/zlibmodule.c | 1 + Objects/moduleobject.c | 3 ++- Objects/unicodeobject.c | 6 ++++++ PC/_testconsole.c | 1 + PC/msvcrtmodule.c | 1 + PC/winreg.c | 1 + PC/winsound.c | 1 + Parser/asdl_c.py | 1 + Python/Python-ast.c | 1 + Python/Python-tokenize.c | 1 + Python/_warnings.c | 1 + Python/import.c | 1 + Python/marshal.c | 1 + 99 files changed, 144 insertions(+), 4 deletions(-) diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 7ac6f6e8a4a24e..b8bdfe29d80406 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -87,6 +87,7 @@ struct PyModuleDef_Slot { /* for Py_mod_multiple_interpreters: */ #define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0) #define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1) +#define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) #endif /* New in 3.5 */ diff --git a/Modules/_abc.c b/Modules/_abc.c index 9694331339aa78..d3e405dadb664a 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -944,6 +944,7 @@ _abcmodule_free(void *module) static PyModuleDef_Slot _abcmodule_slots[] = { {Py_mod_exec, _abcmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index fef34d65523870..822d5f2a41de33 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3803,6 +3803,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 30801c2f87eee7..0773bbd191931d 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -457,6 +457,7 @@ bisect_modexec(PyObject *m) static PyModuleDef_Slot bisect_slots[] = { {Py_mod_exec, bisect_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index 44d783b40d0453..0d1d88c6603684 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -127,6 +127,7 @@ blake2_exec(PyObject *m) static PyModuleDef_Slot _blake2_slots[] = { {Py_mod_exec, blake2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; @@ -146,4 +147,4 @@ PyMODINIT_FUNC PyInit__blake2(void) { return PyModuleDef_Init(&blake2_module); -} \ No newline at end of file +} diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 8e7b8e8078af4e..97bd44b4ac9694 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -799,6 +799,7 @@ _bz2_free(void *module) static struct PyModuleDef_Slot _bz2_slots[] = { {Py_mod_exec, _bz2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index d5035d20600ae2..777c753bd7c2a9 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -1049,6 +1049,7 @@ static PyMethodDef _codecs_functions[] = { }; static PyModuleDef_Slot _codecs_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index a9b1425177c3d7..9a81531bdffb16 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2571,6 +2571,7 @@ collections_exec(PyObject *module) { static struct PyModuleDef_Slot collections_slots[] = { {Py_mod_exec, collections_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_contextvarsmodule.c b/Modules/_contextvarsmodule.c index d13b5962c13c44..f621c1de6d42d6 100644 --- a/Modules/_contextvarsmodule.c +++ b/Modules/_contextvarsmodule.c @@ -44,6 +44,7 @@ _contextvars_exec(PyObject *m) static struct PyModuleDef_Slot _contextvars_slots[] = { {Py_mod_exec, _contextvars_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_cryptmodule.c b/Modules/_cryptmodule.c index 72a4f44600d92c..75035084c9cd29 100644 --- a/Modules/_cryptmodule.c +++ b/Modules/_cryptmodule.c @@ -58,6 +58,7 @@ static PyMethodDef crypt_methods[] = { }; static PyModuleDef_Slot _crypt_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_csv.c b/Modules/_csv.c index 2217cc2ca7a775..0cde5c5a8bdc68 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1798,6 +1798,7 @@ csv_exec(PyObject *module) { static PyModuleDef_Slot csv_slots[] = { {Py_mod_exec, csv_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index a8811d03cc91a2..ce652b362d5bb3 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1054,6 +1054,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item) } static struct PyModuleDef_Slot _ctypes_test_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 2144345de01ba3..a3124ff80551e0 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -690,6 +690,9 @@ _curses_panel_exec(PyObject *mod) static PyModuleDef_Slot _curses_slots[] = { {Py_mod_exec, _curses_panel_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 54376022dcb182..9908174c94c450 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -583,6 +583,7 @@ _dbm_module_free(void *module) static PyModuleDef_Slot _dbmmodule_slots[] = { {Py_mod_exec, _dbm_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 97be89a167104f..42de3c675c2e5a 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4419,6 +4419,9 @@ module_exec(PyObject *m) static struct PyModuleDef_Slot elementtree_slots[] = { {Py_mod_exec, module_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 4032ba79374fa4..a8001d71223fdc 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1520,6 +1520,7 @@ _functools_free(void *module) static struct PyModuleDef_Slot _functools_slots[] = { {Py_mod_exec, _functools_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 4e8acdefc722b2..4dbb5741b2ede8 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -793,6 +793,7 @@ _gdbm_module_free(void *module) static PyModuleDef_Slot _gdbm_module_slots[] = { {Py_mod_exec, _gdbm_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 7476e5dc7dd61e..99d0b72819137e 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -2260,6 +2260,7 @@ static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_md_meth_names}, {Py_mod_exec, hashlib_init_constructors}, {Py_mod_exec, hashlib_exception}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 07ddc7b0851241..00285ae01f8574 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -682,6 +682,7 @@ heapq_exec(PyObject *m) static struct PyModuleDef_Slot heapq_slots[] = { {Py_mod_exec, heapq_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_json.c b/Modules/_json.c index fa8e2a936d2c33..c90de05b046b00 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1801,6 +1801,7 @@ _json_exec(PyObject *module) static PyModuleDef_Slot _json_slots[] = { {Py_mod_exec, _json_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 96675cdfb661ad..1ada7305117bb7 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -874,6 +874,7 @@ _locale_exec(PyObject *module) static struct PyModuleDef_Slot _locale_slots[] = { {Py_mod_exec, _locale_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index a7ce328cb5307a..1c84f66ee6f579 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -1001,6 +1001,9 @@ _lsprof_exec(PyObject *module) static PyModuleDef_Slot _lsprofslots[] = { {Py_mod_exec, _lsprof_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index bccab8639159e7..e34fbad230d51a 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1611,6 +1611,7 @@ static PyMethodDef lzma_methods[] = { static PyModuleDef_Slot lzma_slots[] = { {Py_mod_exec, lzma_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 2463e1e1a8bf7e..8f9daa5c3de0cc 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -276,6 +276,7 @@ multiprocessing_exec(PyObject *module) static PyModuleDef_Slot multiprocessing_slots[] = { {Py_mod_exec, multiprocessing_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index d64ded4168228f..88c93fe313785c 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -110,12 +110,19 @@ static PyMethodDef module_methods[ ] = { }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + + static struct PyModuleDef _posixshmemmodule = { PyModuleDef_HEAD_INIT, .m_name = "_posixshmem", .m_doc = "POSIX shared memory module", .m_size = 0, .m_methods = module_methods, + .m_slots = module_slots, }; /* Module init function */ diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 99be977417743e..b70d426fa29bc0 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -94,12 +94,18 @@ opcode_functions[] = { {NULL, NULL, 0, NULL} }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + static struct PyModuleDef opcodemodule = { PyModuleDef_HEAD_INIT, .m_name = "_opcode", .m_doc = "Opcode support module.", .m_size = 0, - .m_methods = opcode_functions + .m_methods = opcode_functions, + .m_slots = module_slots, }; PyMODINIT_FUNC diff --git a/Modules/_operator.c b/Modules/_operator.c index 38335b6995016c..68ccc90562d38d 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1828,6 +1828,7 @@ operator_exec(PyObject *module) static struct PyModuleDef_Slot operator_slots[] = { {Py_mod_exec, operator_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 360c7910f67187..bf7ecae0cc0e50 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -7912,6 +7912,7 @@ _pickle_exec(PyObject *m) static PyModuleDef_Slot pickle_slots[] = { {Py_mod_exec, _pickle_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index f5bce8cd7628ad..2bf83db0e228fb 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1140,6 +1140,7 @@ static PyMethodDef module_methods[] = { }; static PyModuleDef_Slot _posixsubprocess_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index af19dd6c198b67..d36a911a57c02c 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -431,6 +431,7 @@ queuemodule_exec(PyObject *module) static PyModuleDef_Slot queuemodule_slots[] = { {Py_mod_exec, queuemodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 6e22053239305a..fda5ef267fb470 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -624,6 +624,7 @@ _random_exec(PyObject *module) static PyModuleDef_Slot _random_slots[] = { {Py_mod_exec, _random_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 344b66f9aad522..e66918016b8da6 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -232,6 +232,7 @@ static PyMethodDef mod_methods[] = { }; static PyModuleDef_Slot _scproxy_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c index 633a0c0ea08d2a..93abc3b2710e55 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -641,6 +641,7 @@ _sha3_exec(PyObject *m) static PyModuleDef_Slot _sha3_slots[] = { {Py_mod_exec, _sha3_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 9c42faa232c70d..27bd42f4595e1c 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -785,6 +785,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 4b6290a5967932..f8a1a05a318889 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -3221,6 +3221,7 @@ sre_exec(PyObject *m) static PyModuleDef_Slot sre_slots[] = { {Py_mod_exec, sre_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index c9e2f24d66cc00..016a5a5cbca548 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6161,6 +6161,9 @@ static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_constants}, {Py_mod_exec, sslmodule_init_versioninfo}, {Py_mod_exec, sslmodule_init_strings}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_stat.c b/Modules/_stat.c index 546e6a5f94ca15..4218799103b59d 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -612,6 +612,7 @@ stat_exec(PyObject *module) static PyModuleDef_Slot stat_slots[] = { {Py_mod_exec, stat_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c index b9d1e4f1616036..1d5465fbe6d04e 100644 --- a/Modules/_statisticsmodule.c +++ b/Modules/_statisticsmodule.c @@ -129,6 +129,7 @@ PyDoc_STRVAR(statistics_doc, "Accelerators for the statistics module.\n"); static struct PyModuleDef_Slot _statisticsmodule_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_struct.c b/Modules/_struct.c index 3db7b991acd0a1..26434f714de5cc 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2572,6 +2572,7 @@ _structmodule_exec(PyObject *m) static PyModuleDef_Slot _structmodule_slots[] = { {Py_mod_exec, _structmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 1e38f1aa63499b..24152412107c09 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -789,6 +789,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index bc7d8b64a94322..58b064bb17cd87 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -441,6 +441,7 @@ static int execfunc(PyObject *m) static PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -745,6 +746,7 @@ PyInit__testmultiphase_create_unreported_exception(void) static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = { {Py_mod_create, createfunc_nonmodule}, {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -765,6 +767,7 @@ execfunc_err(PyObject *mod) static PyModuleDef_Slot slots_exec_err[] = { {Py_mod_exec, execfunc_err}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -786,6 +789,7 @@ execfunc_raise(PyObject *spec) static PyModuleDef_Slot slots_exec_raise[] = { {Py_mod_exec, execfunc_raise}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -807,6 +811,7 @@ execfunc_unreported_exception(PyObject *mod) static PyModuleDef_Slot slots_exec_unreported_exception[] = { {Py_mod_exec, execfunc_unreported_exception}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; @@ -845,6 +850,7 @@ meth_state_access_exec(PyObject *m) static PyModuleDef_Slot meth_state_access_slots[] = { {Py_mod_exec, meth_state_access_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index fd2fd9ab25f113..5d753b4a0ebc5e 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1710,6 +1710,7 @@ The 'threading' module provides a more convenient interface."); static PyModuleDef_Slot thread_module_slots[] = { {Py_mod_exec, thread_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 262dddb63fd5fe..64286375636aff 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -36,6 +36,7 @@ PyDoc_STRVAR(typing_doc, "Accelerators for the typing module.\n"); static struct PyModuleDef_Slot _typingmodule_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index eae38f5c98cc7f..ed3b2fedfd4d88 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -106,6 +106,7 @@ static PyMethodDef uuid_methods[] = { static PyModuleDef_Slot uuid_slots[] = { {Py_mod_exec, uuid_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_weakref.c b/Modules/_weakref.c index 157a852ae9a378..387b8fa9d0a6f1 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -174,6 +174,7 @@ weakref_exec(PyObject *module) static struct PyModuleDef_Slot weakref_slots[] = { {Py_mod_exec, weakref_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index fa380b8b798405..473bcb4736e925 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -2259,6 +2259,7 @@ static int winapi_exec(PyObject *m) static PyModuleDef_Slot winapi_slots[] = { {Py_mod_exec, winapi_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 13b005eaef9866..616dd577688116 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -2418,6 +2418,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 433fecfda655de..d7daae254638ec 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -822,6 +822,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index c215a75b804fdb..3b2d282d65cab9 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -2822,7 +2822,10 @@ zoneinfomodule_exec(PyObject *m) } static PyModuleDef_Slot zoneinfomodule_slots[] = { - {Py_mod_exec, zoneinfomodule_exec}, {0, NULL}}; + {Py_mod_exec, zoneinfomodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; static struct PyModuleDef zoneinfomodule = { .m_base = PyModuleDef_HEAD_INIT, diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 798a7629257966..f94bbec8e0bb3c 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3111,6 +3111,7 @@ array_modexec(PyObject *m) static PyModuleDef_Slot arrayslots[] = { {Py_mod_exec, array_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 47afd7f0751039..5882d405636400 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -314,12 +314,18 @@ upon normal program termination.\n\ Two public functions, register and unregister, are defined.\n\ "); +static PyModuleDef_Slot atexitmodule_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + static struct PyModuleDef atexitmodule = { PyModuleDef_HEAD_INIT, .m_name = "atexit", .m_doc = atexit__doc__, .m_size = 0, .m_methods = atexit_methods, + .m_slots = atexitmodule_slots, }; PyMODINIT_FUNC diff --git a/Modules/audioop.c b/Modules/audioop.c index 9325f82f9a17e0..604306d449265c 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -1975,6 +1975,7 @@ audioop_exec(PyObject* module) static PyModuleDef_Slot audioop_slots[] = { {Py_mod_exec, audioop_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/binascii.c b/Modules/binascii.c index 95ddb26988d6c9..4ecff4793be9a0 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -1291,6 +1291,7 @@ binascii_exec(PyObject *module) { static PyModuleDef_Slot binascii_slots[] = { {Py_mod_exec, binascii_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index e553ff3e17b898..36bc7024df9acc 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -502,6 +502,7 @@ static struct PyMethodDef _cjk_methods[] = { static PyModuleDef_Slot _cjk_slots[] = { {Py_mod_exec, _cjk_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 233fc3020fd6a8..b501e4fb923232 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -2062,6 +2062,7 @@ static struct PyMethodDef _multibytecodec_methods[] = { static PyModuleDef_Slot _multibytecodec_slots[] = { {Py_mod_exec, _multibytecodec_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index b4f7e5424b4ccf..914a697f8e173b 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -1411,6 +1411,7 @@ cmath_exec(PyObject *mod) static PyModuleDef_Slot cmath_slots[] = { {Py_mod_exec, cmath_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index df4e494ba8a973..fddde960a5fe9a 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -940,6 +940,7 @@ errno_exec(PyObject *module) static PyModuleDef_Slot errno_slots[] = { {Py_mod_exec, errno_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 9b4e4199cdc20a..428b090193f093 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1274,6 +1274,8 @@ PyExec_faulthandler(PyObject *module) { static PyModuleDef_Slot faulthandler_slots[] = { {Py_mod_exec, PyExec_faulthandler}, + // XXX gh-103092: fix isolation. + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 9a8ec8dc9858d7..6ca0b62bc5dca8 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -686,6 +686,7 @@ fcntl_exec(PyObject *module) static PyModuleDef_Slot fcntl_slots[] = { {Py_mod_exec, fcntl_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index f4d5186ff1552e..26ddcdd538a4d4 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2044,6 +2044,7 @@ gcmodule_exec(PyObject *module) static PyModuleDef_Slot gcmodule_slots[] = { {Py_mod_exec, gcmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index f6298ca0ee84c1..57cdde6064c24e 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -327,6 +327,7 @@ grpmodule_exec(PyObject *module) static PyModuleDef_Slot grpmodule_slots[] = { {Py_mod_exec, grpmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index c986e02867ca82..555eab09935e9e 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4693,6 +4693,7 @@ itertoolsmodule_exec(PyObject *mod) static struct PyModuleDef_Slot itertoolsmodule_slots[] = { {Py_mod_exec, itertoolsmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 4a2381d9611776..3737a9654575ab 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -4064,6 +4064,7 @@ static PyMethodDef math_methods[] = { static PyModuleDef_Slot math_slots[] = { {Py_mod_exec, math_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/md5module.c b/Modules/md5module.c index 4f7bc77a8836a3..86605771d9643f 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -340,6 +340,7 @@ md5_exec(PyObject *m) static PyModuleDef_Slot _md5_slots[] = { {Py_mod_exec, md5_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index fe76ca6eafaa88..a470dd3c2f3bba 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1713,6 +1713,7 @@ mmap_exec(PyObject *module) static PyModuleDef_Slot mmap_slots[] = { {Py_mod_exec, mmap_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/nismodule.c b/Modules/nismodule.c index ec7f6d8031e84b..6d094490cea731 100644 --- a/Modules/nismodule.c +++ b/Modules/nismodule.c @@ -502,6 +502,9 @@ nis_exec(PyObject *module) static PyModuleDef_Slot nis_slots[] = { {Py_mod_exec, nis_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 02c0f401be4c9e..ac637316583d2d 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -2050,6 +2050,7 @@ overlapped_exec(PyObject *module) static PyModuleDef_Slot overlapped_slots[] = { {Py_mod_exec, overlapped_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b395c265c72d0c..5022fdeb03703a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16793,6 +16793,7 @@ posixmodule_exec(PyObject *m) static PyModuleDef_Slot posixmodile_slots[] = { {Py_mod_exec, posixmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index a757380bd09f70..cc2e2a43893971 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -336,6 +336,7 @@ pwdmodule_exec(PyObject *module) static PyModuleDef_Slot pwdmodule_slots[] = { {Py_mod_exec, pwdmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 0a744998b6c514..c0fbd4d39f0096 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -2056,6 +2056,9 @@ pyexpat_free(void *module) static PyModuleDef_Slot pyexpat_slots[] = { {Py_mod_exec, pyexpat_exec}, + // XXX gh-103092: fix isolation. + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/resource.c b/Modules/resource.c index a97fb870062b82..2a8158c9be5359 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -514,6 +514,7 @@ resource_exec(PyObject *module) static struct PyModuleDef_Slot resource_slots[] = { {Py_mod_exec, resource_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 5a1e40d0b4a482..79bd5b59ab68f9 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2651,6 +2651,7 @@ _select_exec(PyObject *m) static PyModuleDef_Slot _select_slots[] = { {Py_mod_exec, _select_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index f8d4056fd34b65..bdb76c56f1a6e8 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -344,6 +344,7 @@ _sha1_exec(PyObject *module) static PyModuleDef_Slot _sha1_slots[] = { {Py_mod_exec, _sha1_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 72de20b44762d7..37d9b5c538fd0b 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -785,6 +785,7 @@ static int sha2_exec(PyObject *module) static PyModuleDef_Slot _sha2_slots[] = { {Py_mod_exec, sha2_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index fdd1450050fa1b..2350236ad46b25 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1695,6 +1695,7 @@ _signal_module_free(void *module) static PyModuleDef_Slot signal_slots[] = { {Py_mod_exec, signal_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index e5478382e11f89..60219593be61e2 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8870,6 +8870,7 @@ socket_exec(PyObject *m) static struct PyModuleDef_Slot socket_slots[] = { {Py_mod_exec, socket_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/spwdmodule.c b/Modules/spwdmodule.c index 42123c93b59365..13f1115feefa86 100644 --- a/Modules/spwdmodule.c +++ b/Modules/spwdmodule.c @@ -224,6 +224,7 @@ spwdmodule_exec(PyObject *module) static PyModuleDef_Slot spwdmodule_slots[] = { {Py_mod_exec, spwdmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 91538b4fb15cbd..1cc319cc3410d8 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -100,6 +100,7 @@ symtable_init_constants(PyObject *m) static PyModuleDef_Slot symtable_slots[] = { {Py_mod_exec, symtable_init_constants}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index f45aa5227f1cbf..6db8de9c491dd9 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -406,6 +406,7 @@ syslog_exec(PyObject *module) static PyModuleDef_Slot syslog_slots[] = { {Py_mod_exec, syslog_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/termios.c b/Modules/termios.c index fcc8f042679870..169a36fc6477d8 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -1253,6 +1253,7 @@ termios_exec(PyObject *mod) static PyModuleDef_Slot termios_slots[] = { {Py_mod_exec, termios_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index c50e689bb6986c..3607855dbd8f27 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -2107,6 +2107,7 @@ time_module_free(void *module) static struct PyModuleDef_Slot time_slots[] = { {Py_mod_exec, time_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index c108f14871f946..41dcd5f8f883f2 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -1516,6 +1516,7 @@ unicodedata_exec(PyObject *module) static PyModuleDef_Slot unicodedata_slots[] = { {Py_mod_exec, unicodedata_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 5f5297ba6337af..3935c00fc26530 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -390,6 +390,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 361c7e76d77f50..1ff3ef1cb6f296 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -293,6 +293,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index a676fdb4ec773a..1e4e0ea3743ce3 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -383,6 +383,7 @@ xx_exec(PyObject *m) static struct PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 8512baf7cd0a2d..744ba7bf5d28b6 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -286,6 +286,7 @@ xxsubtype_exec(PyObject* m) static struct PyModuleDef_Slot xxsubtype_slots[] = { {Py_mod_exec, xxsubtype_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index e2f7dbaca87a9f..b67844a67c315c 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -2109,6 +2109,7 @@ zlib_exec(PyObject *mod) static PyModuleDef_Slot zlib_slots[] = { {Py_mod_exec, zlib_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 63f4226dd9e94a..c100d018d3f0df 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -250,6 +250,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio int has_execution_slots = 0; const char *name; int ret; + PyInterpreterState *interp = _PyInterpreterState_GET(); PyModuleDef_Init(def); @@ -316,13 +317,13 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio multiple_interpreters = Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; } if (multiple_interpreters == Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED) { - PyInterpreterState *interp = _PyInterpreterState_GET(); if (!_Py_IsMainInterpreter(interp) && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) { goto error; } } + // XXX Do a similar check once we have PyInterpreterState.ceval.own_gil. if (create) { m = create(spec, def); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 6ae68cc20f7dee..1585a582f00aad 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15192,12 +15192,18 @@ static PyMethodDef _string_methods[] = { {NULL, NULL} }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + static struct PyModuleDef _string_module = { PyModuleDef_HEAD_INIT, .m_name = "_string", .m_doc = PyDoc_STR("string helper module"), .m_size = 0, .m_methods = _string_methods, + .m_slots = module_slots, }; PyMODINIT_FUNC diff --git a/PC/_testconsole.c b/PC/_testconsole.c index f14a2d45b1be26..3221b985d01ba0 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -31,6 +31,7 @@ static int execfunc(PyObject *m) PyModuleDef_Slot testconsole_slots[] = { {Py_mod_exec, execfunc}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL}, }; diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 090254befc934d..53ef26b732f615 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -661,6 +661,7 @@ exec_module(PyObject* m) static PyModuleDef_Slot msvcrt_slots[] = { {Py_mod_exec, exec_module}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/PC/winreg.c b/PC/winreg.c index 4884125c3609ad..e2d5322f458c2a 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -2184,6 +2184,7 @@ exec_module(PyObject *m) static PyModuleDef_Slot winreg_slots[] = { {Py_mod_exec, exec_module}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/PC/winsound.c b/PC/winsound.c index 17ce2ef423b1f9..68a917810f884d 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -235,6 +235,7 @@ exec_module(PyObject *module) static PyModuleDef_Slot sound_slots[] = { {Py_mod_exec, exec_module}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index b44e303ac2594b..5d5a05a70ca7ec 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1206,6 +1206,7 @@ def visitModule(self, mod): self.emit(""" static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 6c878474afb192..81ab71c0fc3b29 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -12193,6 +12193,7 @@ astmodule_exec(PyObject *m) static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 416dc5971bca3d..3394a5108cb535 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -151,6 +151,7 @@ static PyMethodDef tokenize_methods[] = { static PyModuleDef_Slot tokenizemodule_slots[] = { {Py_mod_exec, tokenizemodule_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/_warnings.c b/Python/_warnings.c index d510381c365b66..5644db9a3770cb 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1449,6 +1449,7 @@ warnings_module_exec(PyObject *module) static PyModuleDef_Slot warnings_slots[] = { {Py_mod_exec, warnings_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/import.c b/Python/import.c index 0bf107b28d3990..9e1857d5f3e4e6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3840,6 +3840,7 @@ imp_module_exec(PyObject *module) static PyModuleDef_Slot imp_slots[] = { {Py_mod_exec, imp_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Python/marshal.c b/Python/marshal.c index 2966139cec9ae9..208996b05fc484 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1870,6 +1870,7 @@ marshal_module_exec(PyObject *mod) static PyModuleDef_Slot marshalmodule_slots[] = { {Py_mod_exec, marshal_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; From 66558d2a16ee42afc0e2c02e6a90bfd62dcb67f6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 May 2023 23:35:24 +0200 Subject: [PATCH 8/9] gh-104146: Remove unused var 'parser_body_declarations' from clinic.py (#104214) --- Tools/clinic/clinic.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 5f6e67e7d65a1d..704325670bc994 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -814,13 +814,11 @@ def output_templates(self, f): # parser_body_fields remembers the fields passed in to the # previous call to parser_body. this is used for an awful hack. parser_body_fields = () - parser_body_declarations = '' def parser_body(prototype, *fields, declarations=''): - nonlocal parser_body_fields, parser_body_declarations + nonlocal parser_body_fields add, output = text_accumulator() add(prototype) parser_body_fields = fields - parser_body_declarations = declarations fields = list(fields) fields.insert(0, normalize_snippet(""" From f3e7eb48f86057919c347f56dabf417acfd55845 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 May 2023 15:59:20 -0600 Subject: [PATCH 9/9] gh-99113: Add PyInterpreterConfig.own_gil (gh-104204) We also add PyInterpreterState.ceval.own_gil to record if the interpreter actually has its own GIL. Note that for now we don't actually respect own_gil; all interpreters still share the one GIL. However, PyInterpreterState.ceval.own_gil does reflect PyInterpreterConfig.own_gil. That lie is a temporary one that we will fix when the GIL really becomes per-interpreter. --- Include/cpython/initconfig.h | 3 +++ Include/internal/pycore_ceval.h | 2 +- Include/internal/pycore_ceval_state.h | 1 + Lib/test/test_capi/test_misc.py | 36 +++++++++++++++++++-------- Lib/test/test_embed.py | 1 + Lib/test/test_import/__init__.py | 1 + Lib/test/test_threading.py | 1 + Modules/_testcapimodule.c | 12 +++++++-- Modules/_testinternalcapi.c | 7 ++++++ Python/ceval_gil.c | 23 ++++++++++++++++- Python/pylifecycle.c | 12 +++++---- 11 files changed, 80 insertions(+), 19 deletions(-) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 9c1783d272f1cd..efae2409b50069 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -252,6 +252,7 @@ typedef struct { int allow_threads; int allow_daemon_threads; int check_multi_interp_extensions; + int own_gil; } PyInterpreterConfig; #define _PyInterpreterConfig_INIT \ @@ -262,6 +263,7 @@ typedef struct { .allow_threads = 1, \ .allow_daemon_threads = 0, \ .check_multi_interp_extensions = 1, \ + .own_gil = 1, \ } #define _PyInterpreterConfig_LEGACY_INIT \ @@ -272,6 +274,7 @@ typedef struct { .allow_threads = 1, \ .allow_daemon_threads = 1, \ .check_multi_interp_extensions = 0, \ + .own_gil = 0, \ } /* --- Helper functions --------------------------------------- */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 0bbc9efdda3ba7..b7a9bf40425bc7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -97,7 +97,7 @@ _PyEval_Vector(PyThreadState *tstate, PyObject *kwnames); extern int _PyEval_ThreadsInitialized(void); -extern PyStatus _PyEval_InitGIL(PyThreadState *tstate); +extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); extern void _PyEval_ReleaseLock(PyThreadState *tstate); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 1a00ec80270e0c..4781dd5735dcf6 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -86,6 +86,7 @@ struct _pending_calls { struct _ceval_state { int recursion_limit; struct _gil_runtime_state *gil; + int own_gil; /* This single variable consolidates all requests to break out of the fast path in the eval loop. */ _Py_atomic_int eval_breaker; diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 22be3c0814278e..3fc2c07f933061 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1401,23 +1401,37 @@ def test_configured_settings(self): DAEMON_THREADS = 1<<11 FORK = 1<<15 EXEC = 1<<16 - - features = ['obmalloc', 'fork', 'exec', 'threads', 'daemon_threads', - 'extensions'] + ALL_FLAGS = (OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS + | EXTENSIONS); + + features = [ + 'obmalloc', + 'fork', + 'exec', + 'threads', + 'daemon_threads', + 'extensions', + 'own_gil', + ] kwlist = [f'allow_{n}' for n in features] kwlist[0] = 'use_main_obmalloc' - kwlist[-1] = 'check_multi_interp_extensions' + kwlist[-2] = 'check_multi_interp_extensions' + kwlist[-1] = 'own_gil' # expected to work for config, expected in { - (True, True, True, True, True, True): - OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS, - (True, False, False, False, False, False): OBMALLOC, - (False, False, False, True, False, True): THREADS | EXTENSIONS, + (True, True, True, True, True, True, True): + (ALL_FLAGS, True), + (True, False, False, False, False, False, False): + (OBMALLOC, False), + (False, False, False, True, False, True, False): + (THREADS | EXTENSIONS, False), }.items(): kwargs = dict(zip(kwlist, config)) + exp_flags, exp_gil = expected expected = { - 'feature_flags': expected, + 'feature_flags': exp_flags, + 'own_gil': exp_gil, } with self.subTest(config): r, w = os.pipe() @@ -1437,7 +1451,7 @@ def test_configured_settings(self): # expected to fail for config in [ - (False, False, False, False, False, False), + (False, False, False, False, False, False, False), ]: kwargs = dict(zip(kwlist, config)) with self.subTest(config): @@ -1473,6 +1487,7 @@ def test_overridden_setting_extensions_subinterp_check(self): 'allow_exec': True, 'allow_threads': True, 'allow_daemon_threads': True, + 'own_gil': False, } def check(enabled, override): @@ -1483,6 +1498,7 @@ def check(enabled, override): flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS settings = { 'feature_flags': flags, + 'own_gil': False, } expected = { diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index c9691bbf304915..582392ecddcb91 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1666,6 +1666,7 @@ def test_init_main_interpreter_settings(self): # All optional features should be enabled. 'feature_flags': OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS, + 'own_gil': True, } out, err = self.run_embedded_interpreter( 'test_init_main_interpreter_settings', diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 9211639b016e7a..773b7094c6b8ce 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1640,6 +1640,7 @@ class SubinterpImportTests(unittest.TestCase): ) ISOLATED = dict( use_main_obmalloc=False, + own_gil=True, ) NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()} diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index fdd74c37e26235..97165264b34bbe 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1349,6 +1349,7 @@ def func(): allow_threads={allowed}, allow_daemon_threads={daemon_allowed}, check_multi_interp_extensions=False, + own_gil=False, ) """) with test.support.SuppressCrashReport(): diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 376f04f23e324a..79ab7d3f5555c2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1488,6 +1488,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) int allow_threads = -1; int allow_daemon_threads = -1; int check_multi_interp_extensions = -1; + int own_gil = -1; int r; PyThreadState *substate, *mainstate; /* only initialise 'cflags.cf_flags' to test backwards compatibility */ @@ -1500,13 +1501,15 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) "allow_threads", "allow_daemon_threads", "check_multi_interp_extensions", + "own_gil", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$pppppp:run_in_subinterp_with_config", kwlist, + "s$ppppppp:run_in_subinterp_with_config", kwlist, &code, &use_main_obmalloc, &allow_fork, &allow_exec, &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions)) { + &check_multi_interp_extensions, + &own_gil)) { return NULL; } if (use_main_obmalloc < 0) { @@ -1525,6 +1528,10 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_ValueError, "missing allow_threads"); return NULL; } + if (own_gil < 0) { + PyErr_SetString(PyExc_ValueError, "missing own_gil"); + return NULL; + } if (allow_daemon_threads < 0) { PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); return NULL; @@ -1545,6 +1552,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) .allow_threads = allow_threads, .allow_daemon_threads = allow_daemon_threads, .check_multi_interp_extensions = check_multi_interp_extensions, + .own_gil = own_gil, }; PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); if (PyStatus_Exception(status)) { diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 24152412107c09..f35e3b48df9321 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -729,6 +729,13 @@ get_interp_settings(PyObject *self, PyObject *args) return NULL; } + /* "own GIL" */ + PyObject *own_gil = interp->ceval.own_gil ? Py_True : Py_False; + if (PyDict_SetItemString(settings, "own_gil", own_gil) != 0) { + Py_DECREF(settings); + return NULL; + } + return settings; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index ad33a58dedd820..a390bec80d556c 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -500,9 +500,18 @@ PyEval_ThreadsInitialized(void) } PyStatus -_PyEval_InitGIL(PyThreadState *tstate) +_PyEval_InitGIL(PyThreadState *tstate, int own_gil) { assert(tstate->interp->ceval.gil == NULL); + if (!own_gil) { + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(tstate->interp != main_interp); + struct _gil_runtime_state *gil = main_interp->ceval.gil; + assert(gil_created(gil)); + tstate->interp->ceval.gil = gil; + tstate->interp->ceval.own_gil = 0; + return _PyStatus_OK(); + } /* XXX per-interpreter GIL */ struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil; @@ -512,8 +521,11 @@ _PyEval_InitGIL(PyThreadState *tstate) and destroy it. */ assert(gil_created(gil)); tstate->interp->ceval.gil = gil; + // XXX For now we lie. + tstate->interp->ceval.own_gil = 1; return _PyStatus_OK(); } + assert(own_gil); assert(!gil_created(gil)); @@ -521,6 +533,7 @@ _PyEval_InitGIL(PyThreadState *tstate) create_gil(gil); assert(gil_created(gil)); tstate->interp->ceval.gil = gil; + tstate->interp->ceval.own_gil = 1; take_gil(tstate); return _PyStatus_OK(); } @@ -530,6 +543,14 @@ _PyEval_FiniGIL(PyInterpreterState *interp) { if (interp->ceval.gil == NULL) { /* It was already finalized (or hasn't been initialized yet). */ + assert(!interp->ceval.own_gil); + return; + } + else if (!interp->ceval.own_gil) { + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(interp != main_interp); + assert(interp->ceval.gil == main_interp->ceval.gil); + interp->ceval.gil = NULL; return; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 97957d3f17e6db..705708698c6a8b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -585,7 +585,7 @@ init_interp_settings(PyInterpreterState *interp, static PyStatus -init_interp_create_gil(PyThreadState *tstate) +init_interp_create_gil(PyThreadState *tstate, int own_gil) { PyStatus status; @@ -600,7 +600,7 @@ init_interp_create_gil(PyThreadState *tstate) } /* Create the GIL and take it */ - status = _PyEval_InitGIL(tstate); + status = _PyEval_InitGIL(tstate, own_gil); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -632,7 +632,9 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } - const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + // The main interpreter always has its own GIL. + config.own_gil = 1; status = init_interp_settings(interp, &config); if (_PyStatus_EXCEPTION(status)) { return status; @@ -645,7 +647,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, _PyThreadState_Bind(tstate); (void) PyThreadState_Swap(tstate); - status = init_interp_create_gil(tstate); + status = init_interp_create_gil(tstate, config.own_gil); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -2047,7 +2049,7 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) goto error; } - status = init_interp_create_gil(tstate); + status = init_interp_create_gil(tstate, config->own_gil); if (_PyStatus_EXCEPTION(status)) { goto error; }