From e5503bd65820095eb074f59b017a018a5535758d Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 19 Mar 2024 14:40:20 -0400 Subject: [PATCH] [3.12] gh-113964: Don't prevent new threads until all non-daemon threads exit (GH-116677) Starting in Python 3.12, we prevented calling fork() and starting new threads during interpreter finalization (shutdown). This has led to a number of regressions and flaky tests. We should not prevent starting new threads (or `fork()`) until all non-daemon threads exit and finalization starts in earnest. This changes the checks to use `_PyInterpreterState_GetFinalizing(interp)`, which is set immediately before terminating non-daemon threads. (cherry picked from commit 60e105c1c11ecca1680d03c38aa06bcc77a28714) Co-authored-by: Sam Gross --- Lib/test/test_os.py | 17 +++++---- Lib/test/test_subprocess.py | 13 ++++--- Lib/test/test_threading.py | 38 +++++++++++++++---- ...-03-12-20-31-57.gh-issue-113964.bJppzg.rst | 2 + Modules/_posixsubprocess.c | 4 +- Modules/_threadmodule.c | 2 +- Modules/posixmodule.c | 6 +-- Objects/unicodeobject.c | 2 +- 8 files changed, 57 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 46aec219c13673..e8f80046649cd4 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4840,20 +4840,21 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - def test_fork_at_exit(self): + def test_fork_at_finalization(self): code = """if 1: import atexit import os - def exit_handler(): - pid = os.fork() - if pid != 0: - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + pid = os.fork() + if pid != 0: + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(b"", out) + self.assertEqual(b"OK\n", out) self.assertIn(b"can't fork at interpreter shutdown", err) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index d83bd9be140616..79efa42791bb2c 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -3413,14 +3413,15 @@ def test_preexec_at_exit(self): def dummy(): pass - def exit_handler(): - subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"preexec_fn not supported at interpreter shutdown", err) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 119b41f39245ca..2e4b860b975ff8 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1133,21 +1133,21 @@ def import_threading(): self.assertEqual(out, b'') self.assertEqual(err, b'') - def test_start_new_thread_at_exit(self): + def test_start_new_thread_at_finalization(self): code = """if 1: - import atexit import _thread def f(): print("shouldn't be printed") - def exit_handler(): - _thread.start_new_thread(f, ()) - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + _thread.start_new_thread(f, ()) + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"can't create new thread at interpreter shutdown", err) class ThreadJoinOnShutdown(BaseTestCase): @@ -1276,6 +1276,30 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) + def test_thread_from_thread(self): + script = """if True: + import threading + import time + + def thread2(): + time.sleep(0.05) + print("OK") + + def thread1(): + time.sleep(0.05) + t2 = threading.Thread(target=thread2) + t2.start() + + t = threading.Thread(target=thread1) + t.start() + # do not join() -- the interpreter waits for non-daemon threads to + # finish. + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(err, b"") + self.assertEqual(out.strip(), b"OK") + self.assertEqual(rc, 0) + @skip_unless_reliable_fork def test_reinit_tls_after_fork(self): # Issue #13817: fork() would deadlock in a multithreaded program with diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst new file mode 100644 index 00000000000000..ab370d4aa1baee --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst @@ -0,0 +1,2 @@ +Starting new threads and process creation through :func:`os.fork` are now +only prevented once all non-daemon threads exit. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index d75bb92757c7a4..35ea2ac306a3e6 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -946,7 +946,9 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, Py_ssize_t fds_to_keep_len = PyTuple_GET_SIZE(py_fds_to_keep); PyInterpreterState *interp = PyInterpreterState_Get(); - if ((preexec_fn != Py_None) && interp->finalizing) { + if ((preexec_fn != Py_None) && + _PyInterpreterState_GetFinalizing(interp) != NULL) + { PyErr_SetString(PyExc_RuntimeError, "preexec_fn not supported at interpreter shutdown"); return NULL; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 568fe8375d1eb4..365f4460088aab 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1190,7 +1190,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs) "thread is not supported for isolated subinterpreters"); return NULL; } - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_RuntimeError, "can't create new thread at interpreter shutdown"); return NULL; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d665d069d2c4fe..9f69c792614eec 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7663,7 +7663,7 @@ os_fork1_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_RuntimeError, "can't fork at interpreter shutdown"); return NULL; @@ -7707,7 +7707,7 @@ os_fork_impl(PyObject *module) { pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_RuntimeError, "can't fork at interpreter shutdown"); return NULL; @@ -8391,7 +8391,7 @@ os_forkpty_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_RuntimeError, "can't fork at interpreter shutdown"); return NULL; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 65d0842cb111e0..fc9379be02f4dc 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -516,7 +516,7 @@ unicode_check_encoding_errors(const char *encoding, const char *errors) /* Disable checks during Python finalization. For example, it allows to call _PyObject_Dump() during finalization for debugging purpose. */ - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { return 0; }