From 410840a806ea462c94d7c3baaad1a963138ce6ed Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 24 Aug 2023 12:57:04 -0700 Subject: [PATCH] gh-108724: Add PyMutex and _PyParkingLot APIs `PyMutex` is a one byte lock with fast, inlineable lock and unlock functions for the common uncontended case. The design is based on WebKit's `WTF::Lock`. PyMutex is built using the `_PyParkingLot` APIs, which provides a cross-platform futex-like API (based on WebKit's `WTF::ParkingLot`). This internal API will be used for building other synchronization primitives used to implement PEP 703, such as one-time initialization and events. --- Include/Python.h | 1 + Include/cpython/pyatomic.h | 3 +- Include/internal/pycore_llist.h | 107 ++++ Include/internal/pycore_lock.h | 161 ++++++ Include/internal/pycore_parking_lot.h | 129 +++++ Lib/test/test_capi/test_lock.py | 15 + Makefile.pre.in | 5 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/parts.h | 1 + Modules/_testinternalcapi.c | 3 + .../_testinternalcapi/clinic/test_lock.c.h | 74 +++ Modules/_testinternalcapi/parts.h | 1 + Modules/_testinternalcapi/test_lock.c | 353 +++++++++++++ PCbuild/_testinternalcapi.vcxproj | 1 + PCbuild/_testinternalcapi.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 5 + PCbuild/pythoncore.vcxproj.filters | 12 + Python/lock.c | 292 +++++++++++ Python/parking_lot.c | 466 ++++++++++++++++++ Python/pystate.c | 5 + Tools/lockbench/lockbench.py | 53 ++ 21 files changed, 1690 insertions(+), 2 deletions(-) create mode 100644 Include/internal/pycore_llist.h create mode 100644 Include/internal/pycore_lock.h create mode 100644 Include/internal/pycore_parking_lot.h create mode 100644 Lib/test/test_capi/test_lock.py create mode 100644 Modules/_testinternalcapi/clinic/test_lock.c.h create mode 100644 Modules/_testinternalcapi/test_lock.c create mode 100644 Python/lock.c create mode 100644 Python/parking_lot.c create mode 100644 Tools/lockbench/lockbench.py diff --git a/Include/Python.h b/Include/Python.h index 8b28200000ab56..d6429856b4b9f4 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -48,6 +48,7 @@ #include "pytypedefs.h" #include "pybuffer.h" #include "pystats.h" +#include "cpython/pyatomic.h" #include "object.h" #include "objimpl.h" #include "typeslots.h" diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 73712db847087d..066a969e0e486a 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -83,6 +83,7 @@ // # release // ... +#ifndef Py_LIMITED_API #ifndef Py_ATOMIC_H #define Py_ATOMIC_H @@ -503,4 +504,4 @@ static inline void _Py_atomic_fence_release(void); #endif #endif /* Py_ATOMIC_H */ - +#endif /* Py_LIMITED_API */ diff --git a/Include/internal/pycore_llist.h b/Include/internal/pycore_llist.h new file mode 100644 index 00000000000000..5fd261da05fa5d --- /dev/null +++ b/Include/internal/pycore_llist.h @@ -0,0 +1,107 @@ +// A doubly-linked list that can be embedded in a struct. +// +// Usage: +// struct llist_node head = LLIST_INIT(head); +// typedef struct { +// ... +// struct llist_node node; +// ... +// } MyObj; +// +// llist_insert_tail(&head, &obj->node); +// llist_remove(&obj->node); +// +// struct llist_node *node; +// llist_for_each(node, &head) { +// MyObj *obj = llist_data(node, MyObj, node); +// ... +// } +// + +#ifndef Py_INTERNAL_LLIST_H +#define Py_INTERNAL_LLIST_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "Py_BUILD_CORE must be defined to include this header" +#endif + +struct llist_node { + struct llist_node *next; + struct llist_node *prev; +}; + +// Get the struct containing a node. +#define llist_data(node, type, member) \ + (type*)((char*)node - offsetof(type, member)) + +// Iterate over a list. +#define llist_for_each(node, head) \ + for (node = (head)->next; node != (head); node = node->next) + +// Iterate over a list, but allow removal of the current node. +#define llist_for_each_safe(node, head) \ + for (struct llist_node *_next = (node = (head)->next, node->next); \ + node != (head); node = _next, _next = node->next) + +#define LLIST_INIT(head) { &head, &head } + +static inline void +llist_init(struct llist_node *head) +{ + head->next = head; + head->prev = head; +} + +// Returns 1 if the list is empty, 0 otherwise. +static inline int +llist_empty(struct llist_node *head) +{ + return head->next == head; +} + +// Appends to the tail of the list. +static inline void +llist_insert_tail(struct llist_node *head, struct llist_node *node) +{ + node->prev = head->prev; + node->next = head; + head->prev->next = node; + head->prev = node; +} + +// Remove a node from the list. +static inline void +llist_remove(struct llist_node *node) +{ + struct llist_node *prev = node->prev; + struct llist_node *next = node->next; + prev->next = next; + next->prev = prev; + node->prev = NULL; + node->next = NULL; +} + +// Append all nodes from head2 onto head1. head2 is left empty. +static inline void +llist_concat(struct llist_node *head1, struct llist_node *head2) +{ + if (!llist_empty(head2)) { + head1->prev->next = head2->next; + head2->next->prev = head1->prev; + + head1->prev = head2->prev; + head2->prev->next = head1; + llist_init(head2); + } +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_LLIST_H */ diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h new file mode 100644 index 00000000000000..36c76358f5bcae --- /dev/null +++ b/Include/internal/pycore_lock.h @@ -0,0 +1,161 @@ +// Lightweight locks and other synchronization mechanisms. +// +// These implementations are based on WebKit's WTF::Lock. See +// https://webkit.org/blog/6161/locking-in-webkit/ for a description of the +// design. +#ifndef Py_INTERNAL_LOCK_H +#define Py_INTERNAL_LOCK_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_time.h" // _PyTime_t + + +// A mutex that occupies one byte. The lock can be zero initialized. +// +// Only the two least significant bits are used. The remaining bits should be +// zero: +// 0b00: unlocked +// 0b01: locked +// 0b10: unlocked and has parked threads +// 0b11: locked and has parked threads +// +// Typical initialization: +// PyMutex m; +// memset(&m, 0, sizeof(m)); +// +// Typical usage: +// PyMutex_Lock(&m); +// ... +// PyMutex_Unlock(&m); +typedef struct _PyMutex { + uint8_t v; +} PyMutex; + +typedef enum { + _Py_UNLOCKED = 0, + _Py_LOCKED = 1, + _Py_HAS_PARKED = 2, +} _PyMutex_State; + +// (private) slow path for locking the mutex +PyAPI_FUNC(void) _PyMutex_LockSlow(PyMutex *m); + +// (private) slow path for unlocking the mutex +PyAPI_FUNC(void) _PyMutex_UnlockSlow(PyMutex *m); + +// Locks the mutex. +// +// If the mutex is currently locked, the calling thread will be parked until +// the mutex is unlocked. If the current thread holds the GIL, then the GIL +// will be released while the thread is parked. +static inline void +PyMutex_Lock(PyMutex *m) +{ + uint8_t expected = _Py_UNLOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_LOCKED)) { + _PyMutex_LockSlow(m); + } +} + +// Unlocks the mutex. +static inline void +PyMutex_Unlock(PyMutex *m) +{ + uint8_t expected = _Py_LOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->v, &expected, _Py_UNLOCKED)) { + _PyMutex_UnlockSlow(m); + } +} + +// Checks if the mutex is currently locked. +static inline int +PyMutex_IsLocked(PyMutex *m) +{ + return (_Py_atomic_load_uint8(&m->v) & _Py_LOCKED) != 0; +} + +typedef enum _PyLockFlags { + // Do not detach/release the GIL when waiting on the lock. + _Py_LOCK_DONT_DETACH = 0, + + // Detach/release the GIL when waiting on the lock. + _PY_LOCK_DETACH = 1, + + // Handle signals if interrupted while waiting on the lock. + _PY_LOCK_MAKE_PENDING_CALLS = 2, +} _PyLockFlags; + +// Lock a mutex with an optional timeout and additional options. See +// _PyLockFlags for details. +extern PyLockStatus +_PyMutex_TimedLock(PyMutex *m, _PyTime_t timeout_ns, _PyLockFlags flags); + +// Unlock a mutex, returns 0 if the mutex is not locked (used for improved +// error messages). +extern int _PyMutex_TryUnlock(PyMutex *m); + + +// PyEvent is a one-time event notification +typedef struct { + uint8_t v; +} PyEvent; + +// Set the event and notify any waiting threads. +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(void) _PyEvent_Notify(PyEvent *evt); + +// Wait for the event to be set. If the event is already set, then this returns +// immediately. +PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); + +// Wait for the event to be set, or until the timeout expires. If the event is +// already set, then this returns immediately. Returns 1 if the event was set, +// and 0 if the timeout expired or thread was interrupted. +PyAPI_FUNC(int) PyEvent_TimedWait(PyEvent *evt, _PyTime_t timeout_ns); + + +// _PyRawMutex implements a word-sized mutex that that does not depend on the +// parking lot API, and therefore can be used in the parking lot +// implementation. +// +// The mutex uses a packed representation: the least significant bit is used to +// indicate whether the mutex is locked or not. The remaining bits are either +// zero or a pointer to a `struct raw_mutex_entry` (see lock.c). +typedef struct { + uintptr_t v; +} _PyRawMutex; + +// Slow paths for lock/unlock +extern void _PyRawMutex_LockSlow(_PyRawMutex *m); +extern void _PyRawMutex_UnlockSlow(_PyRawMutex *m); + +static inline void +_PyRawMutex_Lock(_PyRawMutex *m) +{ + uintptr_t unlocked = _Py_UNLOCKED; + if (_Py_atomic_compare_exchange_uintptr(&m->v, &unlocked, _Py_LOCKED)) { + return; + } + _PyRawMutex_LockSlow(m); +} + +static inline void +_PyRawMutex_Unlock(_PyRawMutex *m) +{ + uintptr_t locked = _Py_LOCKED; + if (_Py_atomic_compare_exchange_uintptr(&m->v, &locked, _Py_UNLOCKED)) { + return; + } + _PyRawMutex_UnlockSlow(m); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_LOCK_H */ diff --git a/Include/internal/pycore_parking_lot.h b/Include/internal/pycore_parking_lot.h new file mode 100644 index 00000000000000..32a289912888f7 --- /dev/null +++ b/Include/internal/pycore_parking_lot.h @@ -0,0 +1,129 @@ +// ParkingLot is an internal API for building efficient synchronization +// primitives like mutexes and events. +// +// The API and name is inspired by WebKit's WTF::ParkingLot, which in turn +// is inspired Linux's futex API. +// See https://webkit.org/blog/6161/locking-in-webkit/. +// +// The core functionality is an atomic "compare-and-sleep" operation along with +// an atomic "wake-up" operation. + +#ifndef Py_INTERNAL_PARKING_LOT_H +#define Py_INTERNAL_PARKING_LOT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_time.h" // _PyTime_t +#include "cpython/pyatomic.h" + + +enum { + // The thread was unparked by another thread. + Py_PARK_OK = 0, + + // The value of `address` did not match `expected`. + Py_PARK_AGAIN = -1, + + // The thread was unparked due to a timeout. + Py_PARK_TIMEOUT = -2, + + // The thread was interrupted by a signal. + Py_PARK_INTR = -3, +}; + +// Checks that `*address == *expected` and puts the thread to sleep until an +// unpark operation is called on the same `address`. Otherwise, the function +// returns `Py_PARK_AGAIN`. The comparison is performed atomically +// with respect to unpark operations. +// +// The `address_size` argument is the size of the data pointed to by the +// `address` and `expected` pointers (i.e., sizeof(*address)). +// +// `arg`, which can be NULL, is passed to the unpark operation. +// +// The `timeout_ns` argument specifies the maximum amount of time to wait, with +// -1 indicating an infinite wait. +PyAPI_FUNC(int) +_PyParkingLot_Park(const void *address, const void *expected, + size_t address_size, _PyTime_t timeout_ns, + void *arg, int detach); + +struct _PyUnpark { + // The `arg` value passed to _PyParkingLot_Park(). + void *arg; + + // Are there more threads waiting on the address? May be true in cases + // where threads are waiting on a different address that maps to the same + // internal bucket. + int more_waiters; +}; + +// Unpark a single thread waiting on `address`. +// +// The `unpark` is a pointer to a `struct _PyUnpark`. +// +// Usage: +// _PyParkingLot_Unpark(address, unpark, { +// if (unpark) { +// void *arg = unpark->arg; +// int more_waiters = unpark->more_waiters; +// ... +// } +// }); +#define _PyParkingLot_Unpark(address, unpark, ...) \ + do { \ + struct _PyUnpark *(unpark); \ + unpark = _PyParkingLot_BeginUnpark((address)); \ + __VA_ARGS__ \ + _PyParkingLot_FinishUnpark((address), unpark); \ + } while (0); + +// Implements half of an unpark operation. +// Prefer using the _PyParkingLot_Unpark() macro. +PyAPI_FUNC(struct _PyUnpark *) +_PyParkingLot_BeginUnpark(const void *address); + +// Finishes the unpark operation and wakes up the thread selected by +// _PyParkingLot_BeginUnpark. +// Prefer using the _PyParkingLot_Unpark() macro. +PyAPI_FUNC(void) +_PyParkingLot_FinishUnpark(const void *address, struct _PyUnpark *unpark); + +// Unparks all threads waiting on `address`. +PyAPI_FUNC(void) _PyParkingLot_UnparkAll(const void *address); + +// Initialize/deinitialize the thread-local state used by parking lot. +void _PyParkingLot_InitThread(void); +void _PyParkingLot_DeinitThread(void); + +// Resets the parking lot state after a fork. Forgets all parked threads. +PyAPI_FUNC(void) _PyParkingLot_AfterFork(void); + + +// The _PySemaphore API a simplified cross-platform semaphore used to implement +// parking lot. It is not intended to be used directly by other modules. +typedef struct _PySemaphore _PySemaphore; + +// Puts the current thread to sleep until _PySemaphore_Wakeup() is called. +PyAPI_FUNC(int) +_PySemaphore_Wait(_PySemaphore *sema, _PyTime_t timeout_ns, int detach); + +// Wakes up a single thread waiting on sema. Note that _PySemaphore_Wakeup() +// can be called before _PySemaphore_Wait(). +PyAPI_FUNC(void) +_PySemaphore_Wakeup(_PySemaphore *sema); + +// Allocates/releases a semaphore from the thread-local pool. +PyAPI_FUNC(_PySemaphore *) _PySemaphore_Alloc(void); +PyAPI_FUNC(void) _PySemaphore_Free(_PySemaphore *sema); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PARKING_LOT_H */ diff --git a/Lib/test/test_capi/test_lock.py b/Lib/test/test_capi/test_lock.py new file mode 100644 index 00000000000000..756fd8a5ce793b --- /dev/null +++ b/Lib/test/test_capi/test_lock.py @@ -0,0 +1,15 @@ +import unittest +from test.support import import_helper + +# Skip this test if the _testcapi module isn't available. +_testinternalcapi = import_helper.import_module('_testinternalcapi') + +class PyAtomicTests(unittest.TestCase): + pass + +for name in sorted(dir(_testinternalcapi)): + if name.startswith('test_lock_'): + setattr(PyAtomicTests, name, getattr(_testinternalcapi, name)) + +if __name__ == "__main__": + unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 19a802997838a4..e634d3f30f857e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -400,12 +400,14 @@ PYTHON_OBJS= \ Python/instrumentation.o \ Python/intrinsics.o \ Python/legacy_tracing.o \ + Python/lock.o \ Python/marshal.o \ Python/modsupport.o \ Python/mysnprintf.o \ Python/mystrtoul.o \ Python/optimizer.o \ Python/optimizer_analysis.o \ + Python/parking_lot.o \ Python/pathconfig.o \ Python/preconfig.o \ Python/pyarena.o \ @@ -1779,6 +1781,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_interp.h \ $(srcdir)/Include/internal/pycore_intrinsics.h \ $(srcdir)/Include/internal/pycore_list.h \ + $(srcdir)/Include/internal/pycore_llist.h \ + $(srcdir)/Include/internal/pycore_lock.h \ $(srcdir)/Include/internal/pycore_long.h \ $(srcdir)/Include/internal/pycore_modsupport.h \ $(srcdir)/Include/internal/pycore_moduleobject.h \ @@ -1790,6 +1794,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Include/internal/pycore_opcode_utils.h \ $(srcdir)/Include/internal/pycore_optimizer.h \ + $(srcdir)/Include/internal/pycore_parking_lot.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ $(srcdir)/Include/internal/pycore_pyerrors.h \ diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 56c1badf6b44a0..7b3216a50bb284 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -158,7 +158,7 @@ @MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c -@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/pytime.c +@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index c162dbc65db81a..a21bc381d709db 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -27,6 +27,7 @@ int _PyTestCapi_Init_PyOS(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *mod); + int _PyTestCapi_Init_VectorcallLimited(PyObject *module); int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 922672d1a9f915..934e3637a9164d 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1543,6 +1543,9 @@ static PyMethodDef module_functions[] = { static int module_exec(PyObject *module) { + if (_PyTestInternalCapi_Init_Lock(module) < 0) { + return 1; + } if (_PyTestInternalCapi_Init_PyTime(module) < 0) { return 1; } diff --git a/Modules/_testinternalcapi/clinic/test_lock.c.h b/Modules/_testinternalcapi/clinic/test_lock.c.h new file mode 100644 index 00000000000000..3cbe5ef12c5fa6 --- /dev/null +++ b/Modules/_testinternalcapi/clinic/test_lock.c.h @@ -0,0 +1,74 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_abstract.h" // _PyNumber_Index() + +PyDoc_STRVAR(_testinternalcapi_benchmark_locks__doc__, +"benchmark_locks($module, num_threads, use_pymutex=True,\n" +" critical_section_length=1, time_ms=1000, /)\n" +"--\n" +"\n"); + +#define _TESTINTERNALCAPI_BENCHMARK_LOCKS_METHODDEF \ + {"benchmark_locks", _PyCFunction_CAST(_testinternalcapi_benchmark_locks), METH_FASTCALL, _testinternalcapi_benchmark_locks__doc__}, + +static PyObject * +_testinternalcapi_benchmark_locks_impl(PyObject *module, + Py_ssize_t num_threads, + int use_pymutex, + int critical_section_length, + int time_ms); + +static PyObject * +_testinternalcapi_benchmark_locks(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t num_threads; + int use_pymutex = 1; + int critical_section_length = 1; + int time_ms = 1000; + + if (!_PyArg_CheckPositional("benchmark_locks", nargs, 1, 4)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + num_threads = ival; + } + if (nargs < 2) { + goto skip_optional; + } + use_pymutex = PyObject_IsTrue(args[1]); + if (use_pymutex < 0) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + critical_section_length = PyLong_AsInt(args[2]); + if (critical_section_length == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 4) { + goto skip_optional; + } + time_ms = PyLong_AsInt(args[3]); + if (time_ms == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = _testinternalcapi_benchmark_locks_impl(module, num_threads, use_pymutex, critical_section_length, time_ms); + +exit: + return return_value; +} +/*[clinic end generated code: output=97c85dff601fed4b input=a9049054013a1b77]*/ diff --git a/Modules/_testinternalcapi/parts.h b/Modules/_testinternalcapi/parts.h index 43e7714b235156..bbb8e62ddaf7a2 100644 --- a/Modules/_testinternalcapi/parts.h +++ b/Modules/_testinternalcapi/parts.h @@ -10,6 +10,7 @@ #include "Python.h" +int _PyTestInternalCapi_Init_Lock(PyObject *module); int _PyTestInternalCapi_Init_PyTime(PyObject *module); #endif // Py_TESTINTERNALCAPI_PARTS_H diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c new file mode 100644 index 00000000000000..adfb694908c836 --- /dev/null +++ b/Modules/_testinternalcapi/test_lock.c @@ -0,0 +1,353 @@ +// C Extension module to test pycore_lock.h API + +#include "parts.h" + +#include "pycore_lock.h" +#include "clinic/test_lock.c.h" + +#ifdef _WIN32 +#include +#else +#include // usleep() +#endif + +/*[clinic input] +module _testinternalcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7bb583d8c9eb9a78]*/ + + +static void +pysleep(int ms) +{ +#ifdef _WIN32 + Sleep(ms); +#else + usleep(ms * 1000); +#endif +} + +static PyObject * +test_lock_basic(PyObject *self, PyObject *obj) +{ + PyMutex m; + memset(&m, 0, sizeof(m)); + + // uncontended lock and unlock + PyMutex_Lock(&m); + assert(m.v == 1); + PyMutex_Unlock(&m); + assert(m.v == 0); + + Py_RETURN_NONE; +} + +struct test_lock2_data { + PyMutex m; + PyEvent done; + int started; +}; + +static void +lock_thread(void *arg) +{ + struct test_lock2_data *test_data = arg; + PyMutex *m = &test_data->m; + _Py_atomic_store_int(&test_data->started, 1); + + PyMutex_Lock(m); + assert(m->v == 1); + + PyMutex_Unlock(m); + assert(m->v == 0); + + _PyEvent_Notify(&test_data->done); +} + +static PyObject * +test_lock_two_threads(PyObject *self, PyObject *obj) +{ + // lock attempt by two threads + struct test_lock2_data test_data; + memset(&test_data, 0, sizeof(test_data)); + + PyMutex_Lock(&test_data.m); + assert(test_data.m.v == 1); + + PyThread_start_new_thread(lock_thread, &test_data); + while (!_Py_atomic_load_int(&test_data.started)) { + pysleep(10); + } + pysleep(10); // allow some time for the other thread to try to lock + assert(test_data.m.v == 3); + + PyMutex_Unlock(&test_data.m); + PyEvent_Wait(&test_data.done); + assert(test_data.m.v == 0); + + Py_RETURN_NONE; +} + +#define COUNTER_THREADS 5 +#define COUNTER_ITERS 10000 + +struct test_data_counter { + PyMutex m; + Py_ssize_t counter; +}; + +struct thread_data_counter { + struct test_data_counter *test_data; + PyEvent done_event; +}; + +static void +counter_thread(void *arg) +{ + struct thread_data_counter *thread_data = arg; + struct test_data_counter *test_data = thread_data->test_data; + + for (Py_ssize_t i = 0; i < COUNTER_ITERS; i++) { + PyMutex_Lock(&test_data->m); + test_data->counter++; + PyMutex_Unlock(&test_data->m); + } + _PyEvent_Notify(&thread_data->done_event); +} + +static PyObject * +test_lock_counter(PyObject *self, PyObject *obj) +{ + // Test with rapidly locking and unlocking mutex + struct test_data_counter test_data; + memset(&test_data, 0, sizeof(test_data)); + + struct thread_data_counter thread_data[COUNTER_THREADS]; + memset(&thread_data, 0, sizeof(thread_data)); + + for (Py_ssize_t i = 0; i < COUNTER_THREADS; i++) { + thread_data[i].test_data = &test_data; + PyThread_start_new_thread(counter_thread, &thread_data[i]); + } + + for (Py_ssize_t i = 0; i < COUNTER_THREADS; i++) { + PyEvent_Wait(&thread_data[i].done_event); + } + + assert(test_data.counter == COUNTER_THREADS * COUNTER_ITERS); + Py_RETURN_NONE; +} + +#define SLOW_COUNTER_ITERS 100 + +static void +slow_counter_thread(void *arg) +{ + struct thread_data_counter *thread_data = arg; + struct test_data_counter *test_data = thread_data->test_data; + + for (Py_ssize_t i = 0; i < SLOW_COUNTER_ITERS; i++) { + PyMutex_Lock(&test_data->m); + if (i % 7 == 0) { + pysleep(2); + } + test_data->counter++; + PyMutex_Unlock(&test_data->m); + } + _PyEvent_Notify(&thread_data->done_event); +} + +static PyObject * +test_lock_counter_slow(PyObject *self, PyObject *obj) +{ + // Test lock/unlock with occasional "long" critical section, which will + // trigger handoff of the lock. + struct test_data_counter test_data; + memset(&test_data, 0, sizeof(test_data)); + + struct thread_data_counter thread_data[COUNTER_THREADS]; + memset(&thread_data, 0, sizeof(thread_data)); + + for (Py_ssize_t i = 0; i < COUNTER_THREADS; i++) { + thread_data[i].test_data = &test_data; + PyThread_start_new_thread(slow_counter_thread, &thread_data[i]); + } + + for (Py_ssize_t i = 0; i < COUNTER_THREADS; i++) { + PyEvent_Wait(&thread_data[i].done_event); + } + + assert(test_data.counter == COUNTER_THREADS * SLOW_COUNTER_ITERS); + Py_RETURN_NONE; +} + +struct bench_data_locks { + int stop; + int use_pymutex; + int critical_section_length; + char padding[200]; + PyThread_type_lock lock; + PyMutex m; + double value; + Py_ssize_t total_iters; +}; + +struct bench_thread_data { + struct bench_data_locks *bench_data; + Py_ssize_t iters; + PyEvent done; +}; + +static void +thread_benchmark_locks(void *arg) +{ + struct bench_thread_data *thread_data = arg; + struct bench_data_locks *bench_data = thread_data->bench_data; + int use_pymutex = bench_data->use_pymutex; + int critical_section_length = bench_data->critical_section_length; + + double my_value = 1.0; + Py_ssize_t iters = 0; + while (!_Py_atomic_load_int_relaxed(&bench_data->stop)) { + if (use_pymutex) { + PyMutex_Lock(&bench_data->m); + for (int i = 0; i < critical_section_length; i++) { + bench_data->value += my_value; + my_value = bench_data->value; + } + PyMutex_Unlock(&bench_data->m); + } + else { + PyThread_acquire_lock(bench_data->lock, 1); + for (int i = 0; i < critical_section_length; i++) { + bench_data->value += my_value; + my_value = bench_data->value; + } + PyThread_release_lock(bench_data->lock); + } + iters++; + } + + thread_data->iters = iters; + _Py_atomic_add_ssize(&bench_data->total_iters, iters); + _PyEvent_Notify(&thread_data->done); +} + +/*[clinic input] +_testinternalcapi.benchmark_locks + + num_threads: Py_ssize_t + use_pymutex: bool = True + critical_section_length: int = 1 + time_ms: int = 1000 + / + +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_benchmark_locks_impl(PyObject *module, + Py_ssize_t num_threads, + int use_pymutex, + int critical_section_length, + int time_ms) +/*[clinic end generated code: output=381df8d7e9a74f18 input=f3aeaf688738c121]*/ +{ + // Run from Tools/lockbench/lockbench.py + // Based on the WebKit lock benchmarks: + // https://github.com/WebKit/WebKit/blob/main/Source/WTF/benchmarks/LockSpeedTest.cpp + // See also https://webkit.org/blog/6161/locking-in-webkit/ + PyObject *thread_iters = NULL; + PyObject *res = NULL; + + struct bench_data_locks bench_data; + memset(&bench_data, 0, sizeof(bench_data)); + bench_data.use_pymutex = use_pymutex; + bench_data.critical_section_length = critical_section_length; + + bench_data.lock = PyThread_allocate_lock(); + if (bench_data.lock == NULL) { + return PyErr_NoMemory(); + } + + struct bench_thread_data *thread_data = NULL; + thread_data = PyMem_Calloc(num_threads, sizeof(*thread_data)); + if (thread_data == NULL) { + PyErr_NoMemory(); + goto exit; + } + + thread_iters = PyList_New(num_threads); + if (thread_iters == NULL) { + goto exit; + } + + _PyTime_t start = _PyTime_GetMonotonicClock(); + + for (Py_ssize_t i = 0; i < num_threads; i++) { + thread_data[i].bench_data = &bench_data; + PyThread_start_new_thread(thread_benchmark_locks, &thread_data[i]); + } + + // Let the threads run for `time_ms` milliseconds + pysleep(time_ms); + _Py_atomic_store_int(&bench_data.stop, 1); + + // Wait for the threads to finish + for (Py_ssize_t i = 0; i < num_threads; i++) { + PyEvent_Wait(&thread_data[i].done); + } + + Py_ssize_t total_iters = bench_data.total_iters; + _PyTime_t end = _PyTime_GetMonotonicClock(); + + // Return the total number of acquisitions and the number of acquisitions + // for each thread. + for (Py_ssize_t i = 0; i < num_threads; i++) { + PyObject *iter = PyLong_FromSsize_t(thread_data[i].iters); + if (iter == NULL) { + goto exit; + } + PyList_SET_ITEM(thread_iters, i, iter); + } + + double rate = total_iters * 1000000000.0 / (end - start); + res = Py_BuildValue("(dO)", rate, thread_iters); + +exit: + PyThread_free_lock(bench_data.lock); + PyMem_Free(thread_data); + Py_XDECREF(thread_iters); + return res; +} + +static PyObject * +test_lock_benchmark(PyObject *module, PyObject *obj) +{ + // Just make sure the benchmark runs without crashing + PyObject *res = _testinternalcapi_benchmark_locks_impl( + module, 1, 1, 1, 100); + if (res == NULL) { + return NULL; + } + Py_DECREF(res); + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"test_lock_basic", test_lock_basic, METH_NOARGS}, + {"test_lock_two_threads", test_lock_two_threads, METH_NOARGS}, + {"test_lock_counter", test_lock_counter, METH_NOARGS}, + {"test_lock_counter_slow", test_lock_counter_slow, METH_NOARGS}, + _TESTINTERNALCAPI_BENCHMARK_LOCKS_METHODDEF + {"test_lock_benchmark", test_lock_benchmark, METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; + +int +_PyTestInternalCapi_Init_Lock(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index 59491c644b6655..fb474f06f38fe8 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -95,6 +95,7 @@ + diff --git a/PCbuild/_testinternalcapi.vcxproj.filters b/PCbuild/_testinternalcapi.vcxproj.filters index 21a66a2aa79f76..9c8a5d793ee0f4 100644 --- a/PCbuild/_testinternalcapi.vcxproj.filters +++ b/PCbuild/_testinternalcapi.vcxproj.filters @@ -15,6 +15,9 @@ Source Files + + Source Files + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 04752a8029acc2..b35aeb58e11460 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -245,6 +245,8 @@ + + @@ -254,6 +256,7 @@ + @@ -552,12 +555,14 @@ + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4ad02778466925..c2430f00cb7e9a 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -645,6 +645,12 @@ Include\internal + + Include\internal + + + Include\internal + Include\internal @@ -1241,6 +1247,9 @@ Source Files + + Source Files + Python @@ -1259,6 +1268,9 @@ Python + + Python + Python diff --git a/Python/lock.c b/Python/lock.c new file mode 100644 index 00000000000000..fce4c4c4363f2d --- /dev/null +++ b/Python/lock.c @@ -0,0 +1,292 @@ +// Lock implementation + +#include "Python.h" + +#include "pycore_lock.h" +#include "pycore_parking_lot.h" + +#ifdef _WIN32 +#include // SwitchToThread() +#elif defined(HAVE_SCHED_H) +#include // sched_yield() +#endif + +// If a thread waits on a lock for longer than TIME_TO_BE_FAIR_NS (1 ms), then +// the unlocking thread directly hands off ownership of the lock. This avoids +// starvation. +static const _PyTime_t TIME_TO_BE_FAIR_NS = 1000*1000; + +// Spin for a bit before parking the thread. This is only enabled for +// `--disable-gil` builds because it is unlikely to be helpful if the GIL is +// enabled. +#if Py_NOGIL +static const int MAX_SPIN_COUNT = 40; +#else +static const int MAX_SPIN_COUNT = 0; +#endif + +struct mutex_entry { + // The time at which the thread should be handed off the lock. Written by + // the waiting thread. + _PyTime_t time_to_be_fair; + + // Set to 1 if the lock was handed off. Written by the unlocking thread. + int handoff; +}; + +static void +_Py_yield(void) +{ +#ifdef _WIN32 + SwitchToThread(); +#elif defined(HAVE_SCHED_H) + sched_yield(); +#endif +} + +void +_PyMutex_LockSlow(PyMutex *m) +{ + _PyMutex_TimedLock(m, -1, _PY_LOCK_DETACH); +} + +PyLockStatus +_PyMutex_TimedLock(PyMutex *m, _PyTime_t timeout, _PyLockFlags flags) +{ + uint8_t v = _Py_atomic_load_uint8_relaxed(&m->v); + if ((v & _Py_LOCKED) == _Py_UNLOCKED) { + if (_Py_atomic_compare_exchange_uint8(&m->v, &v, v|_Py_LOCKED)) { + return PY_LOCK_ACQUIRED; + } + } + else if (timeout == 0) { + return PY_LOCK_FAILURE; + } + + _PyTime_t now = _PyTime_GetMonotonicClock(); + _PyTime_t endtime = 0; + if (timeout > 0) { + endtime = _PyTime_Add(now, timeout); + } + + struct mutex_entry entry = { + .time_to_be_fair = now + TIME_TO_BE_FAIR_NS, + .handoff = 0, + }; + + Py_ssize_t spin_count = 0; + for (;;) { + if (!(v & _Py_LOCKED)) { + // The lock is unlocked. Try to grab it. + if (_Py_atomic_compare_exchange_uint8(&m->v, &v, v|_Py_LOCKED)) { + return PY_LOCK_ACQUIRED; + } + continue; + } + + if (!(v & _Py_HAS_PARKED) && spin_count < MAX_SPIN_COUNT) { + // Spin for a bit. + _Py_yield(); + spin_count++; + continue; + } + + if (timeout == 0) { + return PY_LOCK_FAILURE; + } + + uint8_t newv = v; + if (!(v & _Py_HAS_PARKED)) { + // We are the first waiter. Set the _Py_HAS_PARKED flag. + newv = v | _Py_HAS_PARKED; + if (!_Py_atomic_compare_exchange_uint8(&m->v, &v, newv)) { + continue; + } + } + + int ret = _PyParkingLot_Park(&m->v, &newv, sizeof(newv), timeout, + &entry, (flags & _PY_LOCK_DETACH) != 0); + if (ret == Py_PARK_OK) { + if (entry.handoff) { + // We own the lock now. + assert(_Py_atomic_load_uint8_relaxed(&m->v) & _Py_LOCKED); + return PY_LOCK_ACQUIRED; + } + } + else if (ret == Py_PARK_INTR && (flags & _PY_LOCK_MAKE_PENDING_CALLS)) { + if (Py_MakePendingCalls() < 0) { + return PY_LOCK_INTR; + } + } + else if (ret == Py_PARK_TIMEOUT) { + assert(timeout >= 0); + return PY_LOCK_FAILURE; + } + + if (timeout > 0) { + timeout = _PyDeadline_Get(endtime); + if (timeout <= 0) { + // Avoid negative values because those mean block forever. + timeout = 0; + } + } + + v = _Py_atomic_load_uint8_relaxed(&m->v); + } +} + +int +_PyMutex_TryUnlock(PyMutex *m) +{ + uint8_t v = _Py_atomic_load_uint8(&m->v); + for (;;) { + if ((v & _Py_LOCKED) == _Py_UNLOCKED) { + // error: the mutex is not locked + return -1; + } + else if ((v & _Py_HAS_PARKED)) { + // wake up a single thread + _PyParkingLot_Unpark(&m->v, unpark, { + v = 0; + if (unpark) { + struct mutex_entry *entry = unpark->arg; + _PyTime_t now = _PyTime_GetMonotonicClock(); + int should_be_fair = now > entry->time_to_be_fair; + + entry->handoff = should_be_fair; + if (should_be_fair) { + v |= _Py_LOCKED; + } + if (unpark->more_waiters) { + v |= _Py_HAS_PARKED; + } + } + _Py_atomic_store_uint8(&m->v, v); + }); + return 0; + } + else if (_Py_atomic_compare_exchange_uint8(&m->v, &v, _Py_UNLOCKED)) { + // fast-path: no waiters + return 0; + } + } +} + +void +_PyMutex_UnlockSlow(PyMutex *m) +{ + if (_PyMutex_TryUnlock(m) < 0) { + Py_FatalError("unlocking mutex that is not locked"); + } +} + +// _PyRawMutex stores a linked list of `struct raw_mutex_entry`, one for each +// thread waiting on the mutex, directly in the mutex itself. +struct raw_mutex_entry { + struct raw_mutex_entry *next; + _PySemaphore *sema; +}; + +void +_PyRawMutex_LockSlow(_PyRawMutex *m) +{ + struct raw_mutex_entry waiter; + waiter.sema = _PySemaphore_Alloc(); + + uintptr_t v = _Py_atomic_load_uintptr(&m->v); + for (;;) { + if ((v & _Py_LOCKED) == _Py_UNLOCKED) { + // Unlocked: try to grab it (even if it has a waiter). + if (_Py_atomic_compare_exchange_uintptr(&m->v, &v, v|_Py_LOCKED)) { + break; + } + continue; + } + + // Locked: try to add ourselves as a waiter. + waiter.next = (struct raw_mutex_entry *)(v & ~1); + uintptr_t desired = ((uintptr_t)&waiter)|_Py_LOCKED; + if (!_Py_atomic_compare_exchange_uintptr(&m->v, &v, desired)) { + continue; + } + + // Wait for us to be woken up. Note that we still have to lock the + // mutex ourselves: it is NOT handed off to us. + _PySemaphore_Wait(waiter.sema, -1, /*detach=*/0); + } + + _PySemaphore_Free(waiter.sema); +} + +void +_PyRawMutex_UnlockSlow(_PyRawMutex *m) +{ + uintptr_t v = _Py_atomic_load_uintptr(&m->v); + for (;;) { + if ((v & _Py_LOCKED) == _Py_UNLOCKED) { + Py_FatalError("unlocking mutex that is not locked"); + } + + struct raw_mutex_entry *waiter = (struct raw_mutex_entry *)(v & ~1); + if (waiter) { + uintptr_t next_waiter = (uintptr_t)waiter->next; + if (_Py_atomic_compare_exchange_uintptr(&m->v, &v, next_waiter)) { + _PySemaphore_Wakeup(waiter->sema); + return; + } + } + else { + if (_Py_atomic_compare_exchange_uintptr(&m->v, &v, _Py_UNLOCKED)) { + return; + } + } + } +} + +void +_PyEvent_Notify(PyEvent *evt) +{ + uintptr_t v = _Py_atomic_exchange_uint8(&evt->v, _Py_LOCKED); + if (v == _Py_UNLOCKED) { + // no waiters + return; + } + else if (v == _Py_LOCKED) { + // event already set + return; + } + else { + assert(v == _Py_HAS_PARKED); + _PyParkingLot_UnparkAll(&evt->v); + } +} + +void +PyEvent_Wait(PyEvent *evt) +{ + while (!PyEvent_TimedWait(evt, -1)) + ; +} + +int +PyEvent_TimedWait(PyEvent *evt, _PyTime_t timeout_ns) +{ + for (;;) { + uint8_t v = _Py_atomic_load_uint8(&evt->v); + if (v == _Py_LOCKED) { + // event already set + return 1; + } + if (v == _Py_UNLOCKED) { + if (!_Py_atomic_compare_exchange_uint8(&evt->v, &v, _Py_HAS_PARKED)) { + continue; + } + } + + uint8_t expected = _Py_HAS_PARKED; + (void) _PyParkingLot_Park(&evt->v, &expected, sizeof(evt->v), + timeout_ns, NULL, 1); + + return _Py_atomic_load_uint8(&evt->v) == _Py_LOCKED; + } +} diff --git a/Python/parking_lot.c b/Python/parking_lot.c new file mode 100644 index 00000000000000..8e6995653ce6f3 --- /dev/null +++ b/Python/parking_lot.c @@ -0,0 +1,466 @@ +#include "Python.h" + +#include "pycore_llist.h" +#include "pycore_lock.h" // _PyRawMutex +#include "pycore_parking_lot.h" +#include "pycore_pyerrors.h" // _Py_FatalErrorFormat +#include "pycore_pystate.h" // _PyThreadState_GET + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#elif (defined(_POSIX_SEMAPHORES) && !defined(HAVE_BROKEN_POSIX_SEMAPHORES) && \ + defined(HAVE_SEM_TIMEDWAIT)) +#define USE_SEMAPHORES +#include +#else +#include +#endif + +// A simple, cross-platform binary semaphore that can be used to implement +// wakeup/sleep. +struct _PySemaphore { +#if defined(_WIN32) + HANDLE platform_sem; +#elif defined(USE_SEMAPHORES) + sem_t platform_sem; +#else + PyMUTEX_T mutex; + PyCOND_T cond; + int counter; +#endif +}; + +typedef struct { + // The mutex protects the waiter queue and the num_waiters counter. + _PyRawMutex mutex; + + // Linked list of `struct wait_entry` waiters in this bucket. + struct llist_node root; + size_t num_waiters; +} Bucket; + +struct wait_entry { + struct _PyUnpark unpark; + uintptr_t addr; + _PySemaphore *sema; + struct llist_node node; +}; + +#define MAX_SEMA_DEPTH 3 + +typedef struct { + Py_ssize_t refcount; + + int depth; + _PySemaphore semas[MAX_SEMA_DEPTH]; +} ThreadData; + +#define NUM_BUCKETS 251 + +// Table of waiters (hashed by address) +static Bucket buckets[NUM_BUCKETS]; + +#ifdef HAVE_THREAD_LOCAL +_Py_thread_local ThreadData *thread_data = NULL; +#else +#error "no supported thread-local variable storage classifier" +#endif + +static void +_PySemaphore_Init(_PySemaphore *sema) +{ +#if defined(_WIN32) + sema->platform_sem = CreateSemaphore( + NULL, // attributes + 0, // initial count + 10, // maximum count + NULL // unnamed + ); + if (!sema->platform_sem) { + Py_FatalError("parking_lot: CreateSemaphore failed"); + } +#elif defined(USE_SEMAPHORES) + if (sem_init(&sema->platform_sem, /*pshared=*/0, /*value=*/0) < 0) { + Py_FatalError("parking_lot: sem_init failed"); + } +#else + if (pthread_mutex_init(&sema->mutex, NULL) != 0) { + Py_FatalError("parking_lot: pthread_mutex_init failed"); + } + if (pthread_cond_init(&sema->cond, NULL)) { + Py_FatalError("parking_lot: pthread_cond_init failed"); + } +#endif +} + +static void +_PySemaphore_Destroy(_PySemaphore *sema) +{ +#if defined(_WIN32) + CloseHandle(sema->platform_sem); +#elif defined(USE_SEMAPHORES) + sem_destroy(&sema->platform_sem); +#else + pthread_mutex_destroy(&sema->mutex); + pthread_cond_destroy(&sema->cond); +#endif +} + +static int +_PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout) +{ + int res = Py_PARK_INTR; +#if defined(_WIN32) + DWORD wait; + DWORD millis = 0; + if (timeout < 0) { + millis = INFINITE; + } + else { + millis = (DWORD) (timeout / 1000000); + } + wait = WaitForSingleObjectEx(sema->platform_sem, millis, FALSE); + if (wait == WAIT_OBJECT_0) { + res = Py_PARK_OK; + } + else if (wait == WAIT_TIMEOUT) { + res = Py_PARK_TIMEOUT; + } +#elif defined(USE_SEMAPHORES) + int err; + if (timeout >= 0) { + struct timespec ts; + + _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); + _PyTime_AsTimespec(deadline, &ts); + + err = sem_timedwait(&sema->platform_sem, &ts); + } + else { + err = sem_wait(&sema->platform_sem); + } + if (err == -1) { + err = errno; + if (err == EINTR) { + res = Py_PARK_INTR; + } + else if (err == ETIMEDOUT) { + res = Py_PARK_TIMEOUT; + } + else { + _Py_FatalErrorFormat(__func__, + "unexpected error from semaphore: %d", + err); + } + } + else { + res = Py_PARK_OK; + } +#else + pthread_mutex_lock(&sema->mutex); + if (sema->counter == 0) { + int err; + if (timeout >= 0) { + struct timespec ts; + + _PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout); + _PyTime_AsTimespec(deadline, &ts); + + err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts); + } + else { + err = pthread_cond_wait(&sema->cond, &sema->mutex); + } + if (err) { + res = Py_PARK_TIMEOUT; + } + } + if (sema->counter > 0) { + sema->counter--; + res = Py_PARK_OK; + } + pthread_mutex_unlock(&sema->mutex); +#endif + return res; +} + +int +_PySemaphore_Wait(_PySemaphore *sema, _PyTime_t timeout, int detach) +{ + PyThreadState *tstate = _PyThreadState_GET(); + int was_attached = 0; + if (tstate) { + was_attached = (tstate->_status.active); + if (was_attached && detach) { + PyEval_ReleaseThread(tstate); + } + } + + int res = _PySemaphore_PlatformWait(sema, timeout); + + if (tstate) { + if (was_attached && detach) { + PyEval_AcquireThread(tstate); + } + } + return res; +} + +void +_PySemaphore_Wakeup(_PySemaphore *sema) +{ +#if defined(_WIN32) + if (!ReleaseSemaphore(sema->platform_sem, 1, NULL)) { + Py_FatalError("parking_lot: ReleaseSemaphore failed"); + } +#elif defined(USE_SEMAPHORES) + int err = sem_post(&sema->platform_sem); + if (err != 0) { + Py_FatalError("parking_lot: sem_post failed"); + } +#else + pthread_mutex_lock(&sema->mutex); + sema->counter++; + pthread_cond_signal(&sema->cond); + pthread_mutex_unlock(&sema->mutex); +#endif +} + +_PySemaphore * +_PySemaphore_Alloc(void) +{ + // Make sure we have a valid thread_data. We need to acquire + // some locks before we have a fully initialized PyThreadState. + _PyParkingLot_InitThread(); + + ThreadData *this_thread = thread_data; + if (this_thread->depth >= MAX_SEMA_DEPTH) { + Py_FatalError("_PySemaphore_Alloc(): too many calls"); + } + return &this_thread->semas[this_thread->depth++]; +} + +void +_PySemaphore_Free(_PySemaphore *sema) +{ + ThreadData *this_thread = thread_data; + this_thread->depth--; + if (&this_thread->semas[this_thread->depth] != sema) { + Py_FatalError("_PySemaphore_Free(): mismatch wakeup"); + } + _PyParkingLot_DeinitThread(); +} + +void +_PyParkingLot_InitThread(void) +{ + if (thread_data != NULL) { + thread_data->refcount++; + return; + } + ThreadData *this_thread = PyMem_RawMalloc(sizeof(ThreadData)); + if (this_thread == NULL) { + Py_FatalError("_PyParkingLot_InitThread: unable to allocate thread data"); + } + memset(this_thread, 0, sizeof(*this_thread)); + this_thread->refcount = 1; + this_thread->depth = 0; + for (int i = 0; i < MAX_SEMA_DEPTH; i++) { + _PySemaphore_Init(&this_thread->semas[i]); + } + thread_data = this_thread; +} + +void +_PyParkingLot_DeinitThread(void) +{ + ThreadData *td = thread_data; + if (td == NULL) { + return; + } + + if (--td->refcount != 0) { + assert(td->refcount > 0); + return; + } + + thread_data = NULL; + for (int i = 0; i < MAX_SEMA_DEPTH; i++) { + _PySemaphore_Destroy(&td->semas[i]); + } + + PyMem_RawFree(td); +} + +static void +enqueue(Bucket *bucket, const void *address, struct wait_entry *wait) +{ + if (!bucket->root.next) { + // initialize bucket + llist_init(&bucket->root); + } + llist_insert_tail(&bucket->root, &wait->node); + ++bucket->num_waiters; +} + +static struct wait_entry * +dequeue(Bucket *bucket, const void *address) +{ + struct llist_node *root = &bucket->root; + if (!root->next) { + // bucket was not yet initialized + return NULL; + } + + // find the first waiter that is waiting on `address` + struct llist_node *node; + llist_for_each(node, root) { + struct wait_entry *wait = llist_data(node, struct wait_entry, node); + if (wait->addr == (uintptr_t)address) { + llist_remove(node); + --bucket->num_waiters; + return wait; + } + } + return NULL; +} + +static void +dequeue_all(Bucket *bucket, const void *address, struct llist_node *dst) +{ + struct llist_node *root = &bucket->root; + if (!root->next) { + // bucket was not yet initialized + return; + } + + // remove and append all matching waiters to dst + struct llist_node *node; + llist_for_each_safe(node, root) { + struct wait_entry *wait = llist_data(node, struct wait_entry, node); + if (wait->addr == (uintptr_t)address) { + llist_remove(node); + llist_insert_tail(dst, node); + --bucket->num_waiters; + } + } +} + +// Checks that `*addr == *expected` +static int +validate_addr(const void *addr, const void *expected, size_t addr_size) +{ + switch (addr_size) { + case 1: return _Py_atomic_load_uint8(addr) == *(const uint8_t *)expected; + case 2: return _Py_atomic_load_uint16(addr) == *(const uint16_t *)expected; + case 4: return _Py_atomic_load_uint32(addr) == *(const uint32_t *)expected; + case 8: return _Py_atomic_load_uint64(addr) == *(const uint64_t *)expected; + default: Py_UNREACHABLE(); + } +} + +int +_PyParkingLot_Park(const void *addr, const void *expected, size_t size, + _PyTime_t timeout_ns, void *arg, int detach) +{ + struct wait_entry wait; + wait.unpark.arg = arg; + wait.addr = (uintptr_t)addr; + + Bucket *bucket = &buckets[((uintptr_t)addr) % NUM_BUCKETS]; + + _PyRawMutex_Lock(&bucket->mutex); + if (!validate_addr(addr, expected, size)) { + _PyRawMutex_Unlock(&bucket->mutex); + return Py_PARK_AGAIN; + } + wait.sema = _PySemaphore_Alloc(); + enqueue(bucket, addr, &wait); + _PyRawMutex_Unlock(&bucket->mutex); + + int res = _PySemaphore_Wait(wait.sema, timeout_ns, detach); + if (res == Py_PARK_OK) { + goto done; + } + + // timeout or interrupt + _PyRawMutex_Lock(&bucket->mutex); + if (wait.node.next == NULL) { + _PyRawMutex_Unlock(&bucket->mutex); + // We've been removed the waiter queue. Wait until we process the + // wakeup signal. + do { + res = _PySemaphore_Wait(wait.sema, -1, detach); + } while (res != Py_PARK_OK); + goto done; + } + else { + llist_remove(&wait.node); + --bucket->num_waiters; + } + _PyRawMutex_Unlock(&bucket->mutex); + +done: + _PySemaphore_Free(wait.sema); + return res; + +} + +struct _PyUnpark * +_PyParkingLot_BeginUnpark(const void *addr) +{ + Bucket *bucket = &buckets[((uintptr_t)addr) % NUM_BUCKETS]; + _PyRawMutex_Lock(&bucket->mutex); + + struct wait_entry *waiter = dequeue(bucket, addr); + if (!waiter) { + return NULL; + } + + waiter->unpark.more_waiters = (bucket->num_waiters > 0); + return &waiter->unpark; +} + +#define container_of(ptr, type, member) \ + ((type *)((char *)(ptr) - offsetof(type, member))) + +void +_PyParkingLot_FinishUnpark(const void *addr, struct _PyUnpark *unpark) +{ + Bucket *bucket = &buckets[((uintptr_t)addr) % NUM_BUCKETS]; + _PyRawMutex_Unlock(&bucket->mutex); + + if (unpark) { + struct wait_entry *waiter; + waiter = container_of(unpark, struct wait_entry, unpark); + + _PySemaphore_Wakeup(waiter->sema); + } +} + +void +_PyParkingLot_UnparkAll(const void *addr) +{ + struct llist_node head = LLIST_INIT(head); + Bucket *bucket = &buckets[((uintptr_t)addr) % NUM_BUCKETS]; + + _PyRawMutex_Lock(&bucket->mutex); + dequeue_all(bucket, addr, &head); + _PyRawMutex_Unlock(&bucket->mutex); + + struct llist_node *node; + llist_for_each_safe(node, &head) { + struct wait_entry *waiter = llist_data(node, struct wait_entry, node); + llist_remove(node); + _PySemaphore_Wakeup(waiter->sema); + } +} + +void +_PyParkingLot_AfterFork(void) +{ + // After a fork only one thread remains. That thread cannot be blocked + // so all entries in the parking lot are for dead threads. + memset(buckets, 0, sizeof(buckets)); +} diff --git a/Python/pystate.c b/Python/pystate.c index b5c4fd7fb50616..b7e8429b1ca403 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -8,6 +8,7 @@ #include "pycore_frame.h" #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_object.h" // _PyType_InitCache() +#include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_pyerrors.h" // _PyErr_Clear() #include "pycore_pylifecycle.h" // _PyAST_Fini() #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() @@ -549,6 +550,10 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + // Clears the parking lot. Any waiting threads are dead. This must be + // called before releasing any locks that use the parking lot. + _PyParkingLot_AfterFork(); + /* bpo-42540: id_mutex is freed by _PyInterpreterState_Delete, which does * not force the default allocator. */ reinit_err += _PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex); diff --git a/Tools/lockbench/lockbench.py b/Tools/lockbench/lockbench.py new file mode 100644 index 00000000000000..9833d703e00cbb --- /dev/null +++ b/Tools/lockbench/lockbench.py @@ -0,0 +1,53 @@ +# Measure the performance of PyMutex and PyThread_type_lock locks +# with short critical sections. +# +# Usage: python Tools/lockbench/lockbench.py [CRITICAL_SECTION_LENGTH] +# +# How to interpret the results: +# +# Acquisitions (kHz): Reports the total number of lock acquisitions in +# thousands of acquisitions per second. This is the most important metric, +# particularly for the 1 thread case because even in multithreaded programs, +# most locks acquisitions are not contended. Values for 2+ threads are +# only meaningful for `--disable-gil` builds, because the GIL prevents most +# situations where there is lock contention with short critical sections. +# +# Fairness: A measure of how evenly the lock acquisitions are distributed. +# A fairness of 1.0 means that all threads acquired the lock the same number +# of times. A fairness of 1/N means that only one thread ever acquired the +# lock. +# See https://en.wikipedia.org/wiki/Fairness_measure#Jain's_fairness_index + +from _testinternalcapi import benchmark_locks +import sys + +# Max number of threads to test +MAX_THREADS = 10 + +# How much "work" to do while holding the lock +CRITICAL_SECTION_LENGTH = 1 + + +def jains_fairness(values): + # Jain's fairness index + # See https://en.wikipedia.org/wiki/Fairness_measure + return (sum(values) ** 2) / (len(values) * sum(x ** 2 for x in values)) + +def main(): + print("Lock Type Threads Acquisitions (kHz) Fairness") + for lock_type in ["PyMutex", "PyThread_type_lock"]: + use_pymutex = (lock_type == "PyMutex") + for num_threads in range(1, MAX_THREADS + 1): + acquisitions, thread_iters = benchmark_locks( + num_threads, use_pymutex, CRITICAL_SECTION_LENGTH) + + acquisitions /= 1000 # report in kHz for readability + fairness = jains_fairness(thread_iters) + + print(f"{lock_type: <20}{num_threads: <18}{acquisitions: >5.0f}{fairness: >20.2f}") + + +if __name__ == "__main__": + if len(sys.argv) > 1: + CRITICAL_SECTION_LENGTH = int(sys.argv[1]) + main()