From b84bb003f8b4fb24300a7f93dfddf379ac770085 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 10 Jul 2023 15:29:07 +0100 Subject: [PATCH 1/3] gh-106597: Add debugging struct with offsets for out-of-process tools --- Include/internal/pycore_runtime.h | 44 +++++++++++++++++++ Include/internal/pycore_runtime_init.h | 33 +++++++++++++- ...-07-10-15-30-45.gh-issue-106597.WAZ14y.rst | 5 +++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-10-15-30-45.gh-issue-106597.WAZ14y.rst diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 5ed97e9715b2b0..33980e24d2833e 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -53,12 +53,56 @@ typedef struct _Py_AuditHookEntry { void *userData; } _Py_AuditHookEntry; +typedef struct _Py_DebugOffsets { + // Runtime state offset; + off_t rs_finalizing; + off_t rs_interpreters_head; + + // Interpreter state offset; + off_t is_next; + off_t is_threads_head; + off_t is_gc; + off_t is_imports_modules; + off_t is_sysdict; + off_t is_builtins; + off_t is_ceval_gil; + + // Thread state offset; + off_t ts_prev; + off_t ts_next; + off_t ts_interp; + off_t ts_cframe; + off_t ts_thread_id; + + // Frame object offset; + off_t fo_previous; + off_t fo_executable; + off_t fo_prev_instr; + off_t fo_localsplus; + off_t fo_owner; + + // Code object offset; + off_t co_filename; + off_t co_name; + off_t co_linetable; + off_t co_firstlineno; + off_t co_argcount; + off_t co_localsplusnames; + off_t co_co_code_adaptive; +} _Py_DebugOffsets; + /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. That data is exposed in the internal API as a static variable (_PyRuntime). */ typedef struct pyruntimestate { + /* This field must be first to facilitate locating it by out of process + * debuggers. Out of process debuggers will use the offsets contained in this + * field to be able to locate other fields in several interpreter structures + * in a way that doesn't require them to know the exact layout of those + * structures */ + _Py_DebugOffsets debug_offsets; /* Has been initialized to a safe state. In order to be effective, this must be set to 0 during or right diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b507de0437d9aa..f251f62c00c50b 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -21,9 +21,40 @@ extern PyTypeObject _PyExc_MemoryError; /* The static initializers defined here should only be used in the runtime init code (in pystate.c and pylifecycle.c). */ - #define _PyRuntimeState_INIT(runtime) \ { \ + .debug_offsets = { \ + .rs_finalizing = offsetof(_PyRuntimeState, _finalizing), \ + .rs_interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ + \ + .is_next = offsetof(PyInterpreterState, next), \ + .is_threads_head = offsetof(PyInterpreterState, threads.head), \ + .is_gc = offsetof(PyInterpreterState, gc), \ + .is_imports_modules = offsetof(PyInterpreterState, imports.modules), \ + .is_sysdict = offsetof(PyInterpreterState, sysdict), \ + .is_builtins = offsetof(PyInterpreterState, builtins), \ + .is_ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ + \ + .ts_prev = offsetof(PyThreadState, prev), \ + .ts_next = offsetof(PyThreadState, next), \ + .ts_interp = offsetof(PyThreadState, interp), \ + .ts_cframe = offsetof(PyThreadState, cframe), \ + .ts_thread_id = offsetof(PyThreadState, thread_id), \ + \ + .fo_previous = offsetof(_PyInterpreterFrame, previous), \ + .fo_executable = offsetof(_PyInterpreterFrame, f_executable), \ + .fo_prev_instr = offsetof(_PyInterpreterFrame, prev_instr), \ + .fo_localsplus = offsetof(_PyInterpreterFrame, localsplus), \ + .fo_owner = offsetof(_PyInterpreterFrame, owner), \ + \ + .co_filename = offsetof(PyCodeObject, co_filename), \ + .co_name = offsetof(PyCodeObject, co_name), \ + .co_linetable = offsetof(PyCodeObject, co_linetable), \ + .co_firstlineno = offsetof(PyCodeObject, co_firstlineno), \ + .co_argcount = offsetof(PyCodeObject, co_argcount), \ + .co_localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ + .co_co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + }, \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ .debug = _pymem_allocators_debug_INIT, \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-10-15-30-45.gh-issue-106597.WAZ14y.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-10-15-30-45.gh-issue-106597.WAZ14y.rst new file mode 100644 index 00000000000000..bbe455d652f50e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-10-15-30-45.gh-issue-106597.WAZ14y.rst @@ -0,0 +1,5 @@ +A new debug structure of offsets has been added to the ``_PyRuntimeState`` +that will help out-of-process debuggers and profilers to obtain the offsets +to relevant interpreter structures in a way that is agnostic of how Python +was compiled and that doesn't require copying the headers. Patch by Pablo +Galindo From ff5d47eddffb094b3090e588cfb99e5efef36c18 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 10 Jul 2023 16:35:03 +0100 Subject: [PATCH 2/3] Use smaller structs --- Include/internal/pycore_runtime.h | 62 ++++++++++++++---------- Include/internal/pycore_runtime_init.h | 66 ++++++++++++++------------ 2 files changed, 72 insertions(+), 56 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 33980e24d2833e..e059d6a659dd66 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -55,40 +55,50 @@ typedef struct _Py_AuditHookEntry { typedef struct _Py_DebugOffsets { // Runtime state offset; - off_t rs_finalizing; - off_t rs_interpreters_head; + struct _runtime_state { + off_t finalizing; + off_t interpreters_head; + } runtime_state; // Interpreter state offset; - off_t is_next; - off_t is_threads_head; - off_t is_gc; - off_t is_imports_modules; - off_t is_sysdict; - off_t is_builtins; - off_t is_ceval_gil; + struct _interpreter_state { + off_t next; + off_t threads_head; + off_t gc; + off_t imports_modules; + off_t sysdict; + off_t builtins; + off_t ceval_gil; + } interpreter_state; // Thread state offset; - off_t ts_prev; - off_t ts_next; - off_t ts_interp; - off_t ts_cframe; - off_t ts_thread_id; + struct _thread_state{ + off_t prev; + off_t next; + off_t interp; + off_t cframe; + off_t thread_id; + } thread_state; // Frame object offset; - off_t fo_previous; - off_t fo_executable; - off_t fo_prev_instr; - off_t fo_localsplus; - off_t fo_owner; + struct _frame_object { + off_t previous; + off_t executable; + off_t prev_instr; + off_t localsplus; + off_t owner; + } frame_object; // Code object offset; - off_t co_filename; - off_t co_name; - off_t co_linetable; - off_t co_firstlineno; - off_t co_argcount; - off_t co_localsplusnames; - off_t co_co_code_adaptive; + struct _code_object { + off_t filename; + off_t name; + off_t linetable; + off_t firstlineno; + off_t argcount; + off_t localsplusnames; + off_t co_code_adaptive; + } code_object; } _Py_DebugOffsets; /* Full Python runtime state */ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index f251f62c00c50b..421c8bad08e421 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -24,36 +24,42 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyRuntimeState_INIT(runtime) \ { \ .debug_offsets = { \ - .rs_finalizing = offsetof(_PyRuntimeState, _finalizing), \ - .rs_interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ - \ - .is_next = offsetof(PyInterpreterState, next), \ - .is_threads_head = offsetof(PyInterpreterState, threads.head), \ - .is_gc = offsetof(PyInterpreterState, gc), \ - .is_imports_modules = offsetof(PyInterpreterState, imports.modules), \ - .is_sysdict = offsetof(PyInterpreterState, sysdict), \ - .is_builtins = offsetof(PyInterpreterState, builtins), \ - .is_ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ - \ - .ts_prev = offsetof(PyThreadState, prev), \ - .ts_next = offsetof(PyThreadState, next), \ - .ts_interp = offsetof(PyThreadState, interp), \ - .ts_cframe = offsetof(PyThreadState, cframe), \ - .ts_thread_id = offsetof(PyThreadState, thread_id), \ - \ - .fo_previous = offsetof(_PyInterpreterFrame, previous), \ - .fo_executable = offsetof(_PyInterpreterFrame, f_executable), \ - .fo_prev_instr = offsetof(_PyInterpreterFrame, prev_instr), \ - .fo_localsplus = offsetof(_PyInterpreterFrame, localsplus), \ - .fo_owner = offsetof(_PyInterpreterFrame, owner), \ - \ - .co_filename = offsetof(PyCodeObject, co_filename), \ - .co_name = offsetof(PyCodeObject, co_name), \ - .co_linetable = offsetof(PyCodeObject, co_linetable), \ - .co_firstlineno = offsetof(PyCodeObject, co_firstlineno), \ - .co_argcount = offsetof(PyCodeObject, co_argcount), \ - .co_localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ - .co_co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + .runtime_state = { \ + .finalizing = offsetof(_PyRuntimeState, _finalizing), \ + .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ + }, \ + .interpreter_state = { \ + .next = offsetof(PyInterpreterState, next), \ + .threads_head = offsetof(PyInterpreterState, threads.head), \ + .gc = offsetof(PyInterpreterState, gc), \ + .imports_modules = offsetof(PyInterpreterState, imports.modules), \ + .sysdict = offsetof(PyInterpreterState, sysdict), \ + .builtins = offsetof(PyInterpreterState, builtins), \ + .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ + }, \ + .thread_state = { \ + .prev = offsetof(PyThreadState, prev), \ + .next = offsetof(PyThreadState, next), \ + .interp = offsetof(PyThreadState, interp), \ + .cframe = offsetof(PyThreadState, cframe), \ + .thread_id = offsetof(PyThreadState, thread_id), \ + }, \ + .frame_object = { \ + .previous = offsetof(_PyInterpreterFrame, previous), \ + .executable = offsetof(_PyInterpreterFrame, f_executable), \ + .prev_instr = offsetof(_PyInterpreterFrame, prev_instr), \ + .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ + .owner = offsetof(_PyInterpreterFrame, owner), \ + }, \ + .code_object = { \ + .filename = offsetof(PyCodeObject, co_filename), \ + .name = offsetof(PyCodeObject, co_name), \ + .linetable = offsetof(PyCodeObject, co_linetable), \ + .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ + .argcount = offsetof(PyCodeObject, co_argcount), \ + .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ + .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + }, \ }, \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ From de2154a9e0b1b61b5ecc214a26600bc26702314d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 11 Jul 2023 14:57:26 +0100 Subject: [PATCH 3/3] Address feedback Signed-off-by: Pablo Galindo --- Include/internal/pycore_runtime.h | 43 +++++++++++++++++++++++--- Include/internal/pycore_runtime_init.h | 21 ++++++++++++- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index e059d6a659dd66..a16d4202b616db 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -54,6 +54,8 @@ typedef struct _Py_AuditHookEntry { } _Py_AuditHookEntry; typedef struct _Py_DebugOffsets { + char cookie[8]; + uint64_t version; // Runtime state offset; struct _runtime_state { off_t finalizing; @@ -69,6 +71,8 @@ typedef struct _Py_DebugOffsets { off_t sysdict; off_t builtins; off_t ceval_gil; + off_t gil_runtime_state_locked; + off_t gil_runtime_state_holder; } interpreter_state; // Thread state offset; @@ -78,16 +82,23 @@ typedef struct _Py_DebugOffsets { off_t interp; off_t cframe; off_t thread_id; + off_t native_thread_id; } thread_state; - // Frame object offset; - struct _frame_object { + // InterpreterFrame offset; + struct _interpreter_frame { off_t previous; off_t executable; off_t prev_instr; off_t localsplus; off_t owner; - } frame_object; + } interpreter_frame; + + // CFrame offset; + struct _cframe { + off_t current_frame; + off_t previous; + } cframe; // Code object offset; struct _code_object { @@ -97,8 +108,24 @@ typedef struct _Py_DebugOffsets { off_t firstlineno; off_t argcount; off_t localsplusnames; + off_t localspluskinds; off_t co_code_adaptive; } code_object; + + // PyObject offset; + struct _pyobject { + off_t ob_type; + } pyobject; + + // PyTypeObject object offset; + struct _type_object { + off_t tp_name; + } type_object; + + // PyTuple object offset; + struct _tuple_object { + off_t ob_item; + } tuple_object; } _Py_DebugOffsets; /* Full Python runtime state */ @@ -111,8 +138,16 @@ typedef struct pyruntimestate { * debuggers. Out of process debuggers will use the offsets contained in this * field to be able to locate other fields in several interpreter structures * in a way that doesn't require them to know the exact layout of those - * structures */ + * structures. + * + * IMPORTANT: + * This struct is **NOT** backwards compatible between minor version of the + * interpreter and the members, order of members and size can change between + * minor versions. This struct is only guaranteed to be stable between patch + * versions for a given minor version of the interpreter. + */ _Py_DebugOffsets debug_offsets; + /* Has been initialized to a safe state. In order to be effective, this must be set to 0 during or right diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 421c8bad08e421..e72e7422c7207e 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -24,6 +24,8 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyRuntimeState_INIT(runtime) \ { \ .debug_offsets = { \ + .cookie = "xdebugpy", \ + .version = PY_VERSION_HEX, \ .runtime_state = { \ .finalizing = offsetof(_PyRuntimeState, _finalizing), \ .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \ @@ -36,6 +38,8 @@ extern PyTypeObject _PyExc_MemoryError; .sysdict = offsetof(PyInterpreterState, sysdict), \ .builtins = offsetof(PyInterpreterState, builtins), \ .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \ + .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ + .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ }, \ .thread_state = { \ .prev = offsetof(PyThreadState, prev), \ @@ -43,14 +47,19 @@ extern PyTypeObject _PyExc_MemoryError; .interp = offsetof(PyThreadState, interp), \ .cframe = offsetof(PyThreadState, cframe), \ .thread_id = offsetof(PyThreadState, thread_id), \ + .native_thread_id = offsetof(PyThreadState, native_thread_id), \ }, \ - .frame_object = { \ + .interpreter_frame = { \ .previous = offsetof(_PyInterpreterFrame, previous), \ .executable = offsetof(_PyInterpreterFrame, f_executable), \ .prev_instr = offsetof(_PyInterpreterFrame, prev_instr), \ .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ }, \ + .cframe = { \ + .current_frame = offsetof(_PyCFrame, current_frame), \ + .previous = offsetof(_PyCFrame, previous), \ + }, \ .code_object = { \ .filename = offsetof(PyCodeObject, co_filename), \ .name = offsetof(PyCodeObject, co_name), \ @@ -58,8 +67,18 @@ extern PyTypeObject _PyExc_MemoryError; .firstlineno = offsetof(PyCodeObject, co_firstlineno), \ .argcount = offsetof(PyCodeObject, co_argcount), \ .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ + .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ }, \ + .pyobject = { \ + .ob_type = offsetof(PyObject, ob_type), \ + }, \ + .type_object = { \ + .tp_name = offsetof(PyTypeObject, tp_name), \ + }, \ + .tuple_object = { \ + .ob_item = offsetof(PyTupleObject, ob_item), \ + }, \ }, \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \