python · ericsnowcurrently · Mar 9, 2022 · Mar 9, 2022 · Mar 9, 2022 · Mar 9, 2022
@@ -252,6 +252,7 @@ typedef struct {
     int allow_threads;
     int allow_daemon_threads;
     int check_multi_interp_extensions;
+    int own_gil;
 } PyInterpreterConfig;
 
 #define _PyInterpreterConfig_INIT \
@@ -262,6 +263,7 @@ typedef struct {
         .allow_threads = 1, \
         .allow_daemon_threads = 0, \
         .check_multi_interp_extensions = 1, \
+        .own_gil = 1, \
     }
 
 #define _PyInterpreterConfig_LEGACY_INIT \
@@ -272,6 +274,7 @@ typedef struct {
         .allow_threads = 1, \
         .allow_daemon_threads = 1, \
         .check_multi_interp_extensions = 0, \
+        .own_gil = 0, \
     }
 
 /* --- Helper functions --------------------------------------- */

diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
@@ -21,8 +21,7 @@ struct _ceval_runtime_state;
 
 
 extern void _Py_FinishPendingCalls(PyThreadState *tstate);
-extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *);
-extern void _PyEval_InitState(struct _ceval_state *, PyThread_type_lock);
+extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock);
 extern void _PyEval_FiniState(struct _ceval_state *ceval);
 PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp);
 PyAPI_FUNC(int) _PyEval_AddPendingCall(
@@ -96,11 +95,13 @@ _PyEval_Vector(PyThreadState *tstate,
             PyObject* const* args, size_t argcount,
             PyObject *kwnames);
 
-extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime);
-extern PyStatus _PyEval_InitGIL(PyThreadState *tstate);
+extern int _PyEval_ThreadsInitialized(void);
+extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil);
 extern void _PyEval_FiniGIL(PyInterpreterState *interp);
 
+extern void _PyEval_AcquireLock(PyThreadState *tstate);
 extern void _PyEval_ReleaseLock(PyThreadState *tstate);
+extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *);
 
 extern void _PyEval_DeactivateOpCache(void);
 

@@ -49,7 +49,6 @@ struct _ceval_runtime_state {
        the main thread of the main interpreter can handle signals: see
        _Py_ThreadCanHandleSignals(). */
     _Py_atomic_int signals_pending;
-    struct _gil_runtime_state gil;
 };
 
 #ifdef PY_HAVE_PERF_TRAMPOLINE
@@ -83,6 +82,8 @@ struct _pending_calls {
 
 struct _ceval_state {
     int recursion_limit;
+    struct _gil_runtime_state *gil;
+    int own_gil;
     /* This single variable consolidates all requests to break out of
        the fast path in the eval loop. */
     _Py_atomic_int eval_breaker;

@@ -178,6 +178,9 @@ struct _is {
        basis.  Also see _PyRuntimeState regarding the various mutex fields.
        */
 
+    /* The per-interpreter GIL, which might not be used. */
+    struct _gil_runtime_state _gil;
+
     /* the initial PyInterpreterState.threads.head */
     PyThreadState _initial_thread;
 };

@@ -32,8 +32,6 @@ struct _getargs_runtime_state {
     struct _PyArg_Parser *static_parsers;
 };
 
-/* ceval state */
-
 /* GIL state */
 
 struct _gilstate_runtime_state {

@@ -78,11 +78,17 @@ struct PyModuleDef_Slot {
 
 #define Py_mod_create 1
 #define Py_mod_exec 2
+#define Py_mod_multiple_interpreters 3
 
 #ifndef Py_LIMITED_API
-#define _Py_mod_LAST_SLOT 2
+#define _Py_mod_LAST_SLOT 3
 #endif
 
+/* for Py_mod_multiple_interpreters: */
+#define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED 0
+#define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED 1
+#define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED 2
+
 #endif /* New in 3.5 */
 
 struct PyModuleDef {

diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py
@@ -10,6 +10,7 @@
 import _testcapi
 from test import support
 from test.support import import_helper
+from test.support import os_helper
 from test.support import script_helper
 
 
@@ -917,7 +918,7 @@ def f():
         t = threading.Thread(target=f)
         t.start()
         """)
-        with support.temp_dir() as dirname:
+        with os_helper.temp_dir() as dirname:
             filename = script_helper.make_script(dirname, 'interp', script)
             with script_helper.spawn_python(filename) as proc:
                 retcode = proc.wait()

@@ -1247,21 +1247,25 @@ def test_configured_settings(self):
         EXEC = 1<<16
 
         features = ['obmalloc', 'fork', 'exec', 'threads', 'daemon_threads',
-                    'extensions']
+                    'extensions', 'own_gil']
         kwlist = [f'allow_{n}' for n in features]
         kwlist[0] = 'use_main_obmalloc'
-        kwlist[-1] = 'check_multi_interp_extensions'
-
+        kwlist[-2] = 'check_multi_interp_extensions'
+        kwlist[-1] = 'own_gil'
         # expected to work
         for config, expected in {
-            (True, True, True, True, True, True):
-                OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
-            (True, False, False, False, False, False): OBMALLOC,
-            (False, False, False, True, False, True): THREADS | EXTENSIONS,
+            (True, True, True, True, True, True, True):
+                (OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS, True),
+            (True, False, False, False, False, False, False):
+                (OBMALLOC, False),
+            (False, False, False, True, False, True, False):
+                (THREADS | EXTENSIONS, False),
         }.items():
             kwargs = dict(zip(kwlist, config))
+            exp_flags, exp_gil = expected
             expected = {
-                'feature_flags': expected,
+                'feature_flags': exp_flags,
+                'own_gil': exp_gil,
             }
             with self.subTest(config):
                 r, w = os.pipe()
@@ -1281,7 +1285,7 @@ def test_configured_settings(self):
 
         # expected to fail
         for config in [
-            (False, False, False, False, False, False),
+            (False, False, False, False, False, False, False),
         ]:
             kwargs = dict(zip(kwlist, config))
             with self.subTest(config):
@@ -1317,6 +1321,7 @@ def test_overridden_setting_extensions_subinterp_check(self):
             'allow_exec': True,
             'allow_threads': True,
             'allow_daemon_threads': True,
+            'own_gil': False,
         }
 
         def check(enabled, override):
@@ -1327,6 +1332,7 @@ def check(enabled, override):
             flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS
             settings = {
                 'feature_flags': flags,
+                'own_gil': False,
             }
 
             expected = {

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
@@ -1666,6 +1666,7 @@ def test_init_main_interpreter_settings(self):
             # All optional features should be enabled.
             'feature_flags':
                 OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS,
+            'own_gil': True,
         }
         out, err = self.run_embedded_interpreter(
             'test_init_main_interpreter_settings',

@@ -1639,6 +1639,7 @@ class SubinterpImportTests(unittest.TestCase):
         # Isolation-related config values aren't included here.
     )
     ISOLATED = dict(
+        own_gil=True,
         use_main_obmalloc=False,
     )
     NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()}

diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py
@@ -1349,6 +1349,7 @@ def func():
                 allow_threads={allowed},
                 allow_daemon_threads={daemon_allowed},
                 check_multi_interp_extensions=False,
+                own_gil=False,
             )
             """)
         with test.support.SuppressCrashReport():

diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst
@@ -0,0 +1,6 @@
+Multi-phase init extension modules may now indicate whether or not they
+actually support multiple interpreters.  By default such modules are
+expected to support use in multiple interpreters.  In the uncommon case that
+one does not, it may use the new ``Py_mod_multiple_interpreters`` module def
+slot.  A value of ``0`` means the module does not support them. ``1`` means
+it does.  The default is ``1``.
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
@@ -1488,6 +1488,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
     int allow_threads = -1;
     int allow_daemon_threads = -1;
     int check_multi_interp_extensions = -1;
+    int own_gil = -1;
     int r;
     PyThreadState *substate, *mainstate;
     /* only initialise 'cflags.cf_flags' to test backwards compatibility */
@@ -1500,13 +1501,19 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
                              "allow_threads",
                              "allow_daemon_threads",
                              "check_multi_interp_extensions",
+                             "own_gil",
                              NULL};
     if (!PyArg_ParseTupleAndKeywords(args, kwargs,
-                    "s$pppppp:run_in_subinterp_with_config", kwlist,
+                    "s$ppppppp:run_in_subinterp_with_config", kwlist,
                     &code, &use_main_obmalloc,
                     &allow_fork, &allow_exec,
                     &allow_threads, &allow_daemon_threads,
-                    &check_multi_interp_extensions)) {
+                    &check_multi_interp_extensions,
+                    &own_gil)) {
+        return NULL;
+    }
+    if (use_main_obmalloc < 0) {
+        PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc");
         return NULL;
     }
     if (use_main_obmalloc < 0) {
@@ -1525,6 +1532,10 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
         PyErr_SetString(PyExc_ValueError, "missing allow_threads");
         return NULL;
     }
+    if (own_gil < 0) {
+        PyErr_SetString(PyExc_ValueError, "missing own_gil");
+        return NULL;
+    }
     if (allow_daemon_threads < 0) {
         PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads");
         return NULL;
@@ -1545,6 +1556,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
         .allow_threads = allow_threads,
         .allow_daemon_threads = allow_daemon_threads,
         .check_multi_interp_extensions = check_multi_interp_extensions,
+        .own_gil = own_gil,
     };
     PyStatus status = Py_NewInterpreterFromConfig(&substate, &config);
     if (PyStatus_Exception(status)) {

diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
@@ -729,6 +729,13 @@ get_interp_settings(PyObject *self, PyObject *args)
         return NULL;
     }
 
+    /* "own GIL" */
+    PyObject *own_gil = interp->ceval.own_gil ? Py_True : Py_False;
+    if (PyDict_SetItemString(settings, "own_gil", own_gil) != 0) {
+        Py_DECREF(settings);
+        return NULL;
+    }
+
     return settings;
 }
 

diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
@@ -513,6 +513,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
 
     // Create and initialize the new interpreter.
     PyThreadState *save_tstate = _PyThreadState_GET();
+    assert(save_tstate != NULL);
     const PyInterpreterConfig config = isolated
         ? (PyInterpreterConfig)_PyInterpreterConfig_INIT
         : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT;

@@ -245,9 +245,11 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
     PyObject *(*create)(PyObject *, PyModuleDef*) = NULL;
     PyObject *nameobj;
     PyObject *m = NULL;
+    Py_ssize_t multiple_interpreters = -1;
     int has_execution_slots = 0;
     const char *name;
     int ret;
+    PyInterpreterState *interp = _PyInterpreterState_GET();
 
     PyModuleDef_Init(def);
 
@@ -287,6 +289,16 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
             case Py_mod_exec:
                 has_execution_slots = 1;
                 break;
+            case Py_mod_multiple_interpreters:
+                if (multiple_interpreters >= 0) {
+                    PyErr_Format(
+                        PyExc_SystemError,
+                        "module %s has more than one 'multiple interpreters' slots",
+                        name);
+                    goto error;
+                }
+                multiple_interpreters = (Py_ssize_t)cur_slot->value;
+                break;
             default:
                 assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT);
                 PyErr_Format(
@@ -297,6 +309,26 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
         }
     }
 
+    /* By default, multi-phase init modules are expected
+       to work under multiple interpreters. */
+    if (multiple_interpreters < 0) {
+        multiple_interpreters = Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED;
+    }
+    if (!multiple_interpreters
+        && !_Py_IsMainInterpreter(interp)
+        && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0)
+    {
+        goto error;
+    }
+    // XXX This case needs a test.
+    else if ( multiple_interpreters != Py_MOD_PER_INTERPRETER_GIL_SUPPORTED
+             && interp->ceval.own_gil
+             && !_Py_IsMainInterpreter(interp)
+             && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0)
+    {
+        goto error;
+    }
+
     if (create) {
         m = create(spec, def);
         if (m == NULL) {
@@ -421,6 +453,9 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def)
                     return -1;
                 }
                 break;
+            case Py_mod_multiple_interpreters:
+                /* handled in PyModule_FromDefAndSpec2 */
+                break;
             default:
                 PyErr_Format(
                     PyExc_SystemError,