gh-99113: A Per-Interpreter GIL!

Include/cpython/initconfig.h

@@ -252,6 +252,7 @@ typedef struct {
     int allow_threads;
     int allow_daemon_threads;
     int check_multi_interp_extensions;
+    int own_gil;
 } PyInterpreterConfig;
 
 #define _PyInterpreterConfig_INIT \
@@ -262,6 +263,7 @@ typedef struct {
         .allow_threads = 1, \
         .allow_daemon_threads = 0, \
         .check_multi_interp_extensions = 1, \
+        .own_gil = 1, \
     }
 
 #define _PyInterpreterConfig_LEGACY_INIT \
@@ -272,6 +274,7 @@ typedef struct {
         .allow_threads = 1, \
         .allow_daemon_threads = 1, \
         .check_multi_interp_extensions = 0, \
+        .own_gil = 0, \
     }
 
 /* --- Helper functions --------------------------------------- */

Include/internal/pycore_ceval.h

@@ -21,8 +21,7 @@ struct _ceval_runtime_state;
 
 
 extern void _Py_FinishPendingCalls(PyThreadState *tstate);
-extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *);
-extern void _PyEval_InitState(struct _ceval_state *, PyThread_type_lock);
+extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock);
 extern void _PyEval_FiniState(struct _ceval_state *ceval);
 PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp);
 PyAPI_FUNC(int) _PyEval_AddPendingCall(
@@ -96,11 +95,13 @@ _PyEval_Vector(PyThreadState *tstate,
             PyObject* const* args, size_t argcount,
             PyObject *kwnames);
 
-extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime);
-extern PyStatus _PyEval_InitGIL(PyThreadState *tstate);
+extern int _PyEval_ThreadsInitialized(void);
+extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil);
 extern void _PyEval_FiniGIL(PyInterpreterState *interp);
 
+extern void _PyEval_AcquireLock(PyThreadState *tstate);
 extern void _PyEval_ReleaseLock(PyThreadState *tstate);
+extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *);
 
 extern void _PyEval_DeactivateOpCache(void);
 

Include/internal/pycore_ceval_state.h

@@ -49,7 +49,6 @@ struct _ceval_runtime_state {
        the main thread of the main interpreter can handle signals: see
        _Py_ThreadCanHandleSignals(). */
     _Py_atomic_int signals_pending;
-    struct _gil_runtime_state gil;
 };
 
 #ifdef PY_HAVE_PERF_TRAMPOLINE
@@ -83,6 +82,8 @@ struct _pending_calls {
 
 struct _ceval_state {
     int recursion_limit;
+    struct _gil_runtime_state *gil;
+    int own_gil;
     /* This single variable consolidates all requests to break out of
        the fast path in the eval loop. */
     _Py_atomic_int eval_breaker;

Include/internal/pycore_interp.h

@@ -178,6 +178,9 @@ struct _is {
        basis.  Also see _PyRuntimeState regarding the various mutex fields.
        */
 
+    /* The per-interpreter GIL, which might not be used. */
+    struct _gil_runtime_state _gil;
+
     /* the initial PyInterpreterState.threads.head */
     PyThreadState _initial_thread;
 };

Include/internal/pycore_runtime.h

@@ -32,8 +32,6 @@ struct _getargs_runtime_state {
     struct _PyArg_Parser *static_parsers;
 };
 
-/* ceval state */
-
 /* GIL state */
 
 struct _gilstate_runtime_state {

Include/moduleobject.h

@@ -78,11 +78,17 @@ struct PyModuleDef_Slot {
 
 #define Py_mod_create 1
 #define Py_mod_exec 2
+#define Py_mod_multiple_interpreters 3
 
 #ifndef Py_LIMITED_API
-#define _Py_mod_LAST_SLOT 2
+#define _Py_mod_LAST_SLOT 3
 #endif
 
+/* for Py_mod_multiple_interpreters: */
+#define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0)
+#define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1)
+#define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2)
+
 #endif /* New in 3.5 */
 
 struct PyModuleDef {

Lib/test/test__xxsubinterpreters.py

@@ -10,6 +10,7 @@
 import _testcapi
 from test import support
 from test.support import import_helper
+from test.support import os_helper
 from test.support import script_helper
 
 
@@ -917,7 +918,7 @@ def f():
         t = threading.Thread(target=f)
         t.start()
         """)
-        with support.temp_dir() as dirname:
+        with os_helper.temp_dir() as dirname:
             filename = script_helper.make_script(dirname, 'interp', script)
             with script_helper.spawn_python(filename) as proc:
                 retcode = proc.wait()

Lib/test/test_capi/test_misc.py

@@ -1247,21 +1247,25 @@ def test_configured_settings(self):
         EXEC = 1<<16
 
         features = ['obmalloc', 'fork', 'exec', 'threads', 'daemon_threads',
-                    'extensions']
+                    'extensions', 'own_gil']
         kwlist = [f'allow_{n}' for n in features]
         kwlist[0] = 'use_main_obmalloc'
-        kwlist[-1] = 'check_multi_interp_extensions'
-
+        kwlist[-2] = 'check_multi_interp_extensions'
+        kwlist[-1] = 'own_gil'
         # expected to work
         for config, expected in {
-            (True, True, True, True, True, True):
-                OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
-            (True, False, False, False, False, False): OBMALLOC,
-            (False, False, False, True, False, True): THREADS | EXTENSIONS,
+            (True, True, True, True, True, True, True):
+                (OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS, True),
+            (True, False, False, False, False, False, False):
+                (OBMALLOC, False),
+            (False, False, False, True, False, True, False):
+                (THREADS | EXTENSIONS, False),
         }.items():
             kwargs = dict(zip(kwlist, config))
+            exp_flags, exp_gil = expected
             expected = {
-                'feature_flags': expected,
+                'feature_flags': exp_flags,
+                'own_gil': exp_gil,
             }
             with self.subTest(config):
                 r, w = os.pipe()
@@ -1281,7 +1285,7 @@ def test_configured_settings(self):
 
         # expected to fail
         for config in [
-            (False, False, False, False, False, False),
+            (False, False, False, False, False, False, False),
         ]:
             kwargs = dict(zip(kwlist, config))
             with self.subTest(config):
@@ -1317,6 +1321,7 @@ def test_overridden_setting_extensions_subinterp_check(self):
             'allow_exec': True,
             'allow_threads': True,
             'allow_daemon_threads': True,
+            'own_gil': False,
         }
 
         def check(enabled, override):
@@ -1327,6 +1332,7 @@ def check(enabled, override):
             flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS
             settings = {
                 'feature_flags': flags,
+                'own_gil': False,
             }
 
             expected = {

Lib/test/test_embed.py

@@ -1666,6 +1666,7 @@ def test_init_main_interpreter_settings(self):
             # All optional features should be enabled.
             'feature_flags':
                 OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS,
+            'own_gil': True,
         }
         out, err = self.run_embedded_interpreter(
             'test_init_main_interpreter_settings',

Lib/test/test_import/__init__.py

@@ -1639,6 +1639,7 @@ class SubinterpImportTests(unittest.TestCase):
         # Isolation-related config values aren't included here.
     )
     ISOLATED = dict(
+        own_gil=True,
         use_main_obmalloc=False,
     )
     NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()}

Lib/test/test_threading.py

@@ -1349,6 +1349,7 @@ def func():
                 allow_threads={allowed},
                 allow_daemon_threads={daemon_allowed},
                 check_multi_interp_extensions=False,
+                own_gil=False,
             )
             """)
         with test.support.SuppressCrashReport():

Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst

@@ -0,0 +1,6 @@
+Multi-phase init extension modules may now indicate whether or not they
+actually support multiple interpreters.  By default such modules are
+expected to support use in multiple interpreters.  In the uncommon case that
+one does not, it may use the new ``Py_mod_multiple_interpreters`` module def
+slot.  A value of ``0`` means the module does not support them. ``1`` means
+it does.  The default is ``1``.

Modules/_abc.c

@@ -944,6 +944,7 @@ _abcmodule_free(void *module)
 
 static PyModuleDef_Slot _abcmodule_slots[] = {
     {Py_mod_exec, _abcmodule_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_asynciomodule.c

@@ -3805,6 +3805,8 @@ module_exec(PyObject *mod)
 
 static struct PyModuleDef_Slot module_slots[] = {
     {Py_mod_exec, module_exec},
+    // XXX gh-103092: fix isolation.
+    //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL},
 };
 

Modules/_bisectmodule.c

@@ -457,6 +457,7 @@ bisect_modexec(PyObject *m)
 
 static PyModuleDef_Slot bisect_slots[] = {
     {Py_mod_exec, bisect_modexec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_blake2/blake2module.c

@@ -127,6 +127,7 @@ blake2_exec(PyObject *m)
 
 static PyModuleDef_Slot _blake2_slots[] = {
     {Py_mod_exec, blake2_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 
@@ -146,4 +147,4 @@ PyMODINIT_FUNC
 PyInit__blake2(void)
 {
     return PyModuleDef_Init(&blake2_module);
-}
+}

Modules/_bz2module.c

@@ -799,6 +799,7 @@ _bz2_free(void *module)
 
 static struct PyModuleDef_Slot _bz2_slots[] = {
     {Py_mod_exec, _bz2_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_codecsmodule.c

@@ -1049,6 +1049,7 @@ static PyMethodDef _codecs_functions[] = {
 };
 
 static PyModuleDef_Slot _codecs_slots[] = {
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_collectionsmodule.c

@@ -2571,6 +2571,7 @@ collections_exec(PyObject *module) {
 
 static struct PyModuleDef_Slot collections_slots[] = {
     {Py_mod_exec, collections_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_contextvarsmodule.c

@@ -44,6 +44,7 @@ _contextvars_exec(PyObject *m)
 
 static struct PyModuleDef_Slot _contextvars_slots[] = {
     {Py_mod_exec, _contextvars_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_cryptmodule.c

@@ -58,6 +58,7 @@ static PyMethodDef crypt_methods[] = {
 };
 
 static PyModuleDef_Slot _crypt_slots[] = {
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_csv.c

@@ -1798,6 +1798,7 @@ csv_exec(PyObject *module) {
 
 static PyModuleDef_Slot csv_slots[] = {
     {Py_mod_exec, csv_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_ctypes/_ctypes_test.c

@@ -1054,6 +1054,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item)
 }
 
 static struct PyModuleDef_Slot _ctypes_test_slots[] = {
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_curses_panel.c

@@ -690,6 +690,8 @@ _curses_panel_exec(PyObject *mod)
 
 static PyModuleDef_Slot _curses_slots[] = {
     {Py_mod_exec, _curses_panel_exec},
+    // XXX gh-103092: fix isolation.
+    //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_dbmmodule.c

@@ -583,6 +583,7 @@ _dbm_module_free(void *module)
 
 static PyModuleDef_Slot _dbmmodule_slots[] = {
     {Py_mod_exec, _dbm_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_elementtree.c

@@ -4419,6 +4419,8 @@ module_exec(PyObject *m)
 
 static struct PyModuleDef_Slot elementtree_slots[] = {
     {Py_mod_exec, module_exec},
+    // XXX gh-103092: fix isolation.
+    //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL},
 };
 

Modules/_functoolsmodule.c

@@ -1520,6 +1520,7 @@ _functools_free(void *module)
 
 static struct PyModuleDef_Slot _functools_slots[] = {
     {Py_mod_exec, _functools_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_gdbmmodule.c

@@ -793,6 +793,7 @@ _gdbm_module_free(void *module)
 
 static PyModuleDef_Slot _gdbm_module_slots[] = {
     {Py_mod_exec, _gdbm_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_hashopenssl.c

@@ -2260,6 +2260,7 @@ static PyModuleDef_Slot hashlib_slots[] = {
     {Py_mod_exec, hashlib_md_meth_names},
     {Py_mod_exec, hashlib_init_constructors},
     {Py_mod_exec, hashlib_exception},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_heapqmodule.c

@@ -682,6 +682,7 @@ heapq_exec(PyObject *m)
 
 static struct PyModuleDef_Slot heapq_slots[] = {
     {Py_mod_exec, heapq_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_json.c

@@ -1801,6 +1801,7 @@ _json_exec(PyObject *module)
 
 static PyModuleDef_Slot _json_slots[] = {
     {Py_mod_exec, _json_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_localemodule.c

@@ -874,6 +874,7 @@ _locale_exec(PyObject *module)
 
 static struct PyModuleDef_Slot _locale_slots[] = {
     {Py_mod_exec, _locale_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_lsprof.c

@@ -885,6 +885,8 @@ _lsprof_exec(PyObject *module)
 
 static PyModuleDef_Slot _lsprofslots[] = {
     {Py_mod_exec, _lsprof_exec},
+    // XXX gh-103092: fix isolation.
+    //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };
 

Modules/_lzmamodule.c

@@ -1611,6 +1611,7 @@ static PyMethodDef lzma_methods[] = {
 
 static PyModuleDef_Slot lzma_slots[] = {
     {Py_mod_exec, lzma_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {0, NULL}
 };