From 9a268e3e33a3e047f8f1c7e24477d290282f96a2 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 5 May 2026 01:29:55 +0100 Subject: [PATCH 01/10] gh-98894: Restore function entry/exit DTrace probes (#142397) The function__entry and function__return probes stopped working in Python 3.11 when the interpreter was restructured around the new bytecode system. This change restores these probes by adding DTRACE_FUNCTION_ENTRY() at the start_frame label in bytecodes.c and DTRACE_FUNCTION_RETURN() in the RETURN_VALUE and YIELD_VALUE instructions. The helper functions are defined in ceval.c and extract the filename, function name, and line number from the frame before firing the probe. This builds on the approach from https://github.com/python/cpython/pull/125019 but avoids modifying the JIT template since the JIT does not currently support DTrace. The macros are conditionally compiled with WITH_DTRACE and are no-ops otherwise. The tests have been updated to use modern opcode names (CALL, CALL_KW, CALL_FUNCTION_EX) and a new bpftrace backend was added for Linux CI alongside the existing SystemTap tests. Line probe tests were removed since that probe was never restored after 3.11. --- Lib/test/dtracedata/call_stack.py | 6 +- Lib/test/dtracedata/call_stack.stp.expected | 6 +- Lib/test/dtracedata/line.d | 7 - Lib/test/dtracedata/line.d.expected | 20 -- Lib/test/dtracedata/line.py | 17 -- Lib/test/test_dtrace.py | 204 +++++++++++++++++- ...12-08-00-25-35.gh-issue-98894.hKWyfqNx.rst | 2 + Modules/_testinternalcapi/test_cases.c.h | 27 ++- Modules/_testinternalcapi/test_targets.h | 1 + Python/bytecodes.c | 13 +- Python/ceval.c | 32 +++ Python/ceval_macros.h | 13 ++ Python/executor_cases.c.h | 3 + Python/generated_cases.c.h | 27 ++- Python/opcode_targets.h | 1 + 15 files changed, 319 insertions(+), 60 deletions(-) delete mode 100644 Lib/test/dtracedata/line.d delete mode 100644 Lib/test/dtracedata/line.d.expected delete mode 100644 Lib/test/dtracedata/line.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-08-00-25-35.gh-issue-98894.hKWyfqNx.rst diff --git a/Lib/test/dtracedata/call_stack.py b/Lib/test/dtracedata/call_stack.py index ee9f3ae8d6c9f4..11c0369d4baa04 100644 --- a/Lib/test/dtracedata/call_stack.py +++ b/Lib/test/dtracedata/call_stack.py @@ -5,16 +5,16 @@ def function_1(): def function_2(): function_1() -# CALL_FUNCTION_VAR +# CALL with positional args def function_3(dummy, dummy2): pass -# CALL_FUNCTION_KW +# CALL_KW (keyword arguments) def function_4(**dummy): return 1 return 2 # unreachable -# CALL_FUNCTION_VAR_KW +# CALL_FUNCTION_EX (unpacking) def function_5(dummy, dummy2, **dummy3): if False: return 7 diff --git a/Lib/test/dtracedata/call_stack.stp.expected b/Lib/test/dtracedata/call_stack.stp.expected index 32cf396f820629..044eae62018f57 100644 --- a/Lib/test/dtracedata/call_stack.stp.expected +++ b/Lib/test/dtracedata/call_stack.stp.expected @@ -1,8 +1,11 @@ -function__entry:call_stack.py:start:23 function__entry:call_stack.py:function_1:1 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 function__return:call_stack.py:function_1:2 function__entry:call_stack.py:function_2:5 function__entry:call_stack.py:function_1:1 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 function__return:call_stack.py:function_1:2 function__return:call_stack.py:function_2:6 function__entry:call_stack.py:function_3:9 @@ -11,4 +14,3 @@ function__entry:call_stack.py:function_4:13 function__return:call_stack.py:function_4:14 function__entry:call_stack.py:function_5:18 function__return:call_stack.py:function_5:21 -function__return:call_stack.py:start:28 diff --git a/Lib/test/dtracedata/line.d b/Lib/test/dtracedata/line.d deleted file mode 100644 index 03f22db6fcc1a0..00000000000000 --- a/Lib/test/dtracedata/line.d +++ /dev/null @@ -1,7 +0,0 @@ -python$target:::line -/(copyinstr(arg1)=="test_line")/ -{ - printf("%d\t%s:%s:%s:%d\n", timestamp, - probename, basename(copyinstr(arg0)), - copyinstr(arg1), arg2); -} diff --git a/Lib/test/dtracedata/line.d.expected b/Lib/test/dtracedata/line.d.expected deleted file mode 100644 index 9b16ce76ee60a4..00000000000000 --- a/Lib/test/dtracedata/line.d.expected +++ /dev/null @@ -1,20 +0,0 @@ -line:line.py:test_line:2 -line:line.py:test_line:3 -line:line.py:test_line:4 -line:line.py:test_line:5 -line:line.py:test_line:6 -line:line.py:test_line:7 -line:line.py:test_line:8 -line:line.py:test_line:9 -line:line.py:test_line:10 -line:line.py:test_line:11 -line:line.py:test_line:4 -line:line.py:test_line:5 -line:line.py:test_line:6 -line:line.py:test_line:7 -line:line.py:test_line:8 -line:line.py:test_line:10 -line:line.py:test_line:11 -line:line.py:test_line:4 -line:line.py:test_line:12 -line:line.py:test_line:13 diff --git a/Lib/test/dtracedata/line.py b/Lib/test/dtracedata/line.py deleted file mode 100644 index 0930ff391f7a05..00000000000000 --- a/Lib/test/dtracedata/line.py +++ /dev/null @@ -1,17 +0,0 @@ -def test_line(): - a = 1 - print('# Preamble', a) - for i in range(2): - a = i - b = i+2 - c = i+3 - if c < 4: - a = c - d = a + b +c - print('#', a, b, c, d) - a = 1 - print('# Epilogue', a) - - -if __name__ == '__main__': - test_line() diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index ba2fa99707cd46..61320a472f3e02 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -33,11 +33,17 @@ def normalize_trace_output(output): result = [ row.split("\t") for row in output.splitlines() - if row and not row.startswith('#') + if row and not row.startswith('#') and not row.startswith('@') ] result.sort(key=lambda row: int(row[0])) result = [row[1] for row in result] - return "\n".join(result) + # Normalize paths to basenames (bpftrace outputs full paths) + normalized = [] + for line in result: + # Replace full paths with just the filename + line = re.sub(r'/[^:]+/([^/:]+\.py)', r'\1', line) + normalized.append(line) + return "\n".join(normalized) except (IndexError, ValueError): raise AssertionError( "tracer produced unparsable output:\n{}".format(output) @@ -96,6 +102,8 @@ def assert_usable(self): class DTraceBackend(TraceBackend): EXTENSION = ".d" COMMAND = ["dtrace", "-q", "-s"] + if sys.platform == "sunos5": + COMMAND.insert(2, "-Z") class SystemTapBackend(TraceBackend): @@ -103,6 +111,177 @@ class SystemTapBackend(TraceBackend): COMMAND = ["stap", "-g"] +class BPFTraceBackend(TraceBackend): + EXTENSION = ".bt" + COMMAND = ["bpftrace"] + + # Inline bpftrace programs for each test case + PROGRAMS = { + "call_stack": """ + usdt:{python}:python:function__entry {{ + printf("%lld\\tfunction__entry:%s:%s:%d\\n", + nsecs, str(arg0), str(arg1), arg2); + }} + usdt:{python}:python:function__return {{ + printf("%lld\\tfunction__return:%s:%s:%d\\n", + nsecs, str(arg0), str(arg1), arg2); + }} + """, + "gc": """ + usdt:{python}:python:function__entry {{ + if (str(arg1) == "start") {{ @tracing = 1; }} + }} + usdt:{python}:python:function__return {{ + if (str(arg1) == "start") {{ @tracing = 0; }} + }} + usdt:{python}:python:gc__start {{ + if (@tracing) {{ + printf("%lld\\tgc__start:%d\\n", nsecs, arg0); + }} + }} + usdt:{python}:python:gc__done {{ + if (@tracing) {{ + printf("%lld\\tgc__done:%lld\\n", nsecs, arg0); + }} + }} + END {{ clear(@tracing); }} + """, + } + + # Which test scripts to filter by filename (None = use @tracing flag) + FILTER_BY_FILENAME = {"call_stack": "call_stack.py"} + + @staticmethod + def _filter_probe_rows(output): + return "\n".join( + line for line in output.splitlines() + if line.partition("\t")[0].isdigit() + ) + + # Expected outputs for each test case + # Note: bpftrace captures entry/return and may have slight timing + # differences compared to SystemTap due to probe firing order + EXPECTED = { + "call_stack": """function__entry:call_stack.py::0 +function__entry:call_stack.py:start:23 +function__entry:call_stack.py:function_1:1 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 +function__return:call_stack.py:function_1:2 +function__entry:call_stack.py:function_2:5 +function__entry:call_stack.py:function_1:1 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 +function__return:call_stack.py:function_1:2 +function__return:call_stack.py:function_2:6 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 +function__entry:call_stack.py:function_4:13 +function__return:call_stack.py:function_4:14 +function__entry:call_stack.py:function_5:18 +function__return:call_stack.py:function_5:21 +function__return:call_stack.py:start:28 +function__return:call_stack.py::30""", + "gc": """gc__start:0 +gc__done:0 +gc__start:1 +gc__done:0 +gc__start:2 +gc__done:0 +gc__start:2 +gc__done:1""", + } + + def run_case(self, name, optimize_python=None): + if name not in self.PROGRAMS: + raise unittest.SkipTest(f"No bpftrace program for {name}") + + python_file = abspath(name + ".py") + python_flags = [] + if optimize_python: + python_flags.extend(["-O"] * optimize_python) + + subcommand = [sys.executable] + python_flags + [python_file] + program = self.PROGRAMS[name].format(python=sys.executable) + + try: + proc = subprocess.Popen( + ["bpftrace", "-e", program, "-c", " ".join(subcommand)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = proc.communicate(timeout=60) + except subprocess.TimeoutExpired: + proc.kill() + raise AssertionError("bpftrace timed out") + except (FileNotFoundError, PermissionError) as e: + raise unittest.SkipTest(f"bpftrace not available: {e}") + + if proc.returncode != 0: + raise AssertionError( + f"bpftrace failed with code {proc.returncode}:\n{stderr}" + ) + + stdout = self._filter_probe_rows(stdout) + + # Filter output by filename if specified (bpftrace captures everything) + if name in self.FILTER_BY_FILENAME: + filter_filename = self.FILTER_BY_FILENAME[name] + filtered_lines = [ + line for line in stdout.splitlines() + if filter_filename in line + ] + stdout = "\n".join(filtered_lines) + + actual_output = normalize_trace_output(stdout) + expected_output = self.EXPECTED[name].strip() + + return (expected_output, actual_output) + + def assert_usable(self): + # Check if bpftrace is available and can attach to USDT probes + program = f'usdt:{sys.executable}:python:function__entry {{ printf("probe: success\\n"); exit(); }}' + try: + proc = subprocess.Popen( + ["bpftrace", "-e", program, "-c", f"{sys.executable} -c pass"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = proc.communicate(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() # Clean up + raise unittest.SkipTest("bpftrace timed out during usability check") + except OSError as e: + raise unittest.SkipTest(f"bpftrace not available: {e}") + + # Check for permission errors (bpftrace usually requires root) + if proc.returncode != 0: + raise unittest.SkipTest( + f"bpftrace(1) failed with code {proc.returncode}: {stderr}" + ) + + if "probe: success" not in stdout: + raise unittest.SkipTest( + f"bpftrace(1) failed: stdout={stdout!r} stderr={stderr!r}" + ) + + +class BPFTraceOutputTests(unittest.TestCase): + def test_filter_probe_rows_ignores_warnings(self): + output = """stdin:1-19: WARNING: found external warnings +HINT: include/vmlinux.h:1439:3: warning: declaration does not declare anything +4623214882928\tgc__start:0 +4623214885575\tgc__done:0 +""" + self.assertEqual( + BPFTraceBackend._filter_probe_rows(output), + "4623214882928\tgc__start:0\n4623214885575\tgc__done:0", + ) + + @unittest.skipIf(MS_WINDOWS, "Tests not compliant with trace on Windows.") class TraceTests: # unittest.TestCase options @@ -127,7 +306,8 @@ def test_function_entry_return(self): def test_verify_call_opcodes(self): """Ensure our call stack test hits all function call opcodes""" - opcodes = set(["CALL_FUNCTION", "CALL_FUNCTION_EX", "CALL_FUNCTION_KW"]) + # Modern Python uses CALL, CALL_KW, and CALL_FUNCTION_EX + opcodes = set(["CALL", "CALL_FUNCTION_EX", "CALL_KW"]) with open(abspath("call_stack.py")) as f: code_string = f.read() @@ -152,9 +332,6 @@ def get_function_instructions(funcname): def test_gc(self): self.run_case("gc") - def test_line(self): - self.run_case("line") - class DTraceNormalTests(TraceTests, unittest.TestCase): backend = DTraceBackend() @@ -175,6 +352,17 @@ class SystemTapOptimizedTests(TraceTests, unittest.TestCase): backend = SystemTapBackend() optimize_python = 2 + +class BPFTraceNormalTests(TraceTests, unittest.TestCase): + backend = BPFTraceBackend() + optimize_python = 0 + + +class BPFTraceOptimizedTests(TraceTests, unittest.TestCase): + backend = BPFTraceBackend() + optimize_python = 2 + + class CheckDtraceProbes(unittest.TestCase): @classmethod def setUpClass(cls): @@ -235,6 +423,8 @@ def test_check_probes(self): "Name: audit", "Name: gc__start", "Name: gc__done", + "Name: function__entry", + "Name: function__return", ] for probe_name in available_probe_names: @@ -247,8 +437,6 @@ def test_missing_probes(self): # Missing probes will be added in the future. missing_probe_names = [ - "Name: function__entry", - "Name: function__return", "Name: line", ] diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-08-00-25-35.gh-issue-98894.hKWyfqNx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-08-00-25-35.gh-issue-98894.hKWyfqNx.rst new file mode 100644 index 00000000000000..09ccf198a90583 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-08-00-25-35.gh-issue-98894.hKWyfqNx.rst @@ -0,0 +1,2 @@ +Restore ``function__entry`` and ``function__return`` DTrace/SystemTap probes +that were broken since Python 3.11. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index c65767e42da2e9..46627c33d2a7c9 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -779,6 +779,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2005,6 +2006,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2148,6 +2150,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2276,6 +2279,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2871,6 +2875,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -3465,6 +3470,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -3652,6 +3658,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -4489,6 +4496,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -4589,6 +4597,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -6174,6 +6183,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -7865,6 +7875,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; @@ -7927,6 +7938,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -8533,6 +8545,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -9069,6 +9082,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -11050,6 +11064,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; @@ -11247,6 +11262,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -12933,6 +12949,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -13079,6 +13096,13 @@ JUMP_TO_LABEL(error); } LABEL(exit_unwind) + { + assert(_PyErr_Occurred(tstate)); + DTRACE_FUNCTION_RETURN(); + JUMP_TO_LABEL(exit_unwind_notrace); + } + + LABEL(exit_unwind_notrace) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); @@ -13111,8 +13135,9 @@ JUMP_TO_LABEL(error); { int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { - JUMP_TO_LABEL(exit_unwind); + JUMP_TO_LABEL(exit_unwind_notrace); } + DTRACE_FUNCTION_ENTRY(); next_instr = frame->instr_ptr; #ifdef Py_DEBUG int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index d99b618c8393fc..43d4656a4b7f7e 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -527,6 +527,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index daa989eb32ca1f..8b411829214b52 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1568,6 +1568,7 @@ dummy_func( DEAD(retval); SAVE_STACK(); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -1760,6 +1761,7 @@ dummy_func( _PyStackRef temp = retval; DEAD(retval); SAVE_STACK(); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -4569,6 +4571,7 @@ dummy_func( tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -6469,6 +6472,12 @@ dummy_func( } spilled label(exit_unwind) { + assert(_PyErr_Occurred(tstate)); + DTRACE_FUNCTION_RETURN(); + goto exit_unwind_notrace; + } + + spilled label(exit_unwind_notrace) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); @@ -6501,8 +6510,9 @@ dummy_func( spilled label(start_frame) { int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { - goto exit_unwind; + goto exit_unwind_notrace; } + DTRACE_FUNCTION_ENTRY(); next_instr = frame->instr_ptr; #ifdef Py_DEBUG int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); @@ -6601,6 +6611,7 @@ dummy_func( error: exception_unwind: exit_unwind: + exit_unwind_notrace: handle_eval_breaker: resume_frame: start_frame: diff --git a/Python/ceval.c b/Python/ceval.c index 28087ba58d4855..060e948e6b01c9 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1174,6 +1174,38 @@ _Py_ReachedRecursionLimit(PyThreadState *tstate) { #define DONT_SLP_VECTORIZE #endif +#ifdef WITH_DTRACE +static void +dtrace_function_entry(_PyInterpreterFrame *frame) +{ + const char *filename; + const char *funcname; + int lineno; + + PyCodeObject *code = _PyFrame_GetCode(frame); + filename = PyUnicode_AsUTF8(code->co_filename); + funcname = PyUnicode_AsUTF8(code->co_name); + lineno = PyUnstable_InterpreterFrame_GetLine(frame); + + PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); +} + +static void +dtrace_function_return(_PyInterpreterFrame *frame) +{ + const char *filename; + const char *funcname; + int lineno; + + PyCodeObject *code = _PyFrame_GetCode(frame); + filename = PyUnicode_AsUTF8(code->co_filename); + funcname = PyUnicode_AsUTF8(code->co_name); + lineno = PyUnstable_InterpreterFrame_GetLine(frame); + + PyDTrace_FUNCTION_RETURN(filename, funcname, lineno); +} +#endif + PyObject* _Py_HOT_FUNCTION DONT_SLP_VECTORIZE _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a4e9980589e4a3..be3d5fb0a69ad4 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -328,11 +328,24 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define CONSTS() _PyFrame_GetCode(frame)->co_consts #define NAMES() _PyFrame_GetCode(frame)->co_names +#if defined(WITH_DTRACE) && !defined(Py_BUILD_CORE_MODULE) +static void dtrace_function_entry(_PyInterpreterFrame *); +static void dtrace_function_return(_PyInterpreterFrame *); + #define DTRACE_FUNCTION_ENTRY() \ if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ dtrace_function_entry(frame); \ } +#define DTRACE_FUNCTION_RETURN() \ + if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ + dtrace_function_return(frame); \ + } +#else +#define DTRACE_FUNCTION_ENTRY() ((void)0) +#define DTRACE_FUNCTION_RETURN() ((void)0) +#endif + /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 29c901b2bae723..76caff06ca61f7 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8623,6 +8623,7 @@ _PyStackRef temp = retval; _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; @@ -8830,6 +8831,7 @@ assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; _PyFrame_SetStackPointer(frame, stack_pointer); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -16840,6 +16842,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); SET_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7fea3ddfc6f559..0419991ca7761a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -779,6 +779,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2005,6 +2006,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2148,6 +2150,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2276,6 +2279,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2871,6 +2875,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -3465,6 +3470,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -3652,6 +3658,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -4489,6 +4496,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -4589,6 +4597,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -6174,6 +6183,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -7864,6 +7874,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; @@ -7926,6 +7937,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -8532,6 +8544,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -9068,6 +9081,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -11047,6 +11061,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; @@ -11244,6 +11259,7 @@ tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -12930,6 +12946,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -13076,6 +13093,13 @@ JUMP_TO_LABEL(error); } LABEL(exit_unwind) + { + assert(_PyErr_Occurred(tstate)); + DTRACE_FUNCTION_RETURN(); + JUMP_TO_LABEL(exit_unwind_notrace); + } + + LABEL(exit_unwind_notrace) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); @@ -13108,8 +13132,9 @@ JUMP_TO_LABEL(error); { int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { - JUMP_TO_LABEL(exit_unwind); + JUMP_TO_LABEL(exit_unwind_notrace); } + DTRACE_FUNCTION_ENTRY(); next_instr = frame->instr_ptr; #ifdef Py_DEBUG int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index d99b618c8393fc..43d4656a4b7f7e 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -527,6 +527,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); From 9dca1ff759190ec775cc3f62b6b153ffb1e7a360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Tue, 5 May 2026 02:31:53 +0200 Subject: [PATCH 02/10] gh-149300: `_remote_debugging`: clean up magic and duplicate consts in the binary format helper (#149301) --- Modules/_remote_debugging/binary_io.h | 27 +++++++++++++++++- Modules/_remote_debugging/binary_io_reader.c | 20 ++++++-------- Modules/_remote_debugging/binary_io_writer.c | 29 +++++++++----------- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/Modules/_remote_debugging/binary_io.h b/Modules/_remote_debugging/binary_io.h index 18f989f672e103..87a54371c774f1 100644 --- a/Modules/_remote_debugging/binary_io.h +++ b/Modules/_remote_debugging/binary_io.h @@ -61,11 +61,36 @@ extern "C" { #define HDR_SIZE_COMPRESSION 4 #define FILE_HEADER_SIZE (HDR_OFF_COMPRESSION + HDR_SIZE_COMPRESSION) #define FILE_HEADER_PLACEHOLDER_SIZE 64 -#define SAMPLE_HEADER_FIXED_SIZE (sizeof(uint64_t) + sizeof(uint32_t) + 1) static_assert(FILE_HEADER_SIZE <= FILE_HEADER_PLACEHOLDER_SIZE, "FILE_HEADER_SIZE exceeds FILE_HEADER_PLACEHOLDER_SIZE"); +/* Sample header field offsets and sizes */ +#define SMP_OFF_THREAD_ID 0 +#define SMP_SIZE_THREAD_ID sizeof(uint64_t) +#define SMP_OFF_INTERPRETER_ID (SMP_OFF_THREAD_ID + SMP_SIZE_THREAD_ID) +#define SMP_SIZE_INTERPRETER_ID sizeof(uint32_t) +#define SMP_OFF_ENCODING (SMP_OFF_INTERPRETER_ID + SMP_SIZE_INTERPRETER_ID) +#define SMP_SIZE_ENCODING sizeof(uint8_t) +#define SAMPLE_HEADER_FIXED_SIZE (SMP_OFF_ENCODING + SMP_SIZE_ENCODING) + +static_assert(SAMPLE_HEADER_FIXED_SIZE == 13, + "SAMPLE_HEADER_FIXED_SIZE must remain 13"); + +/* Footer field offsets and sizes */ +#define FTR_OFF_STRINGS 0 +#define FTR_SIZE_STRINGS sizeof(uint32_t) +#define FTR_OFF_FRAMES (FTR_OFF_STRINGS + FTR_SIZE_STRINGS) +#define FTR_SIZE_FRAMES sizeof(uint32_t) +#define FTR_OFF_FILE_SIZE (FTR_OFF_FRAMES + FTR_SIZE_FRAMES) +#define FTR_SIZE_FILE_SIZE sizeof(uint64_t) +#define FTR_OFF_CHECKSUM (FTR_OFF_FILE_SIZE + FTR_SIZE_FILE_SIZE) +#define FTR_SIZE_CHECKSUM (2 * sizeof(uint64_t)) +#define FILE_FOOTER_SIZE (FTR_OFF_CHECKSUM + FTR_SIZE_CHECKSUM) + +static_assert(FILE_FOOTER_SIZE == 32, + "FILE_FOOTER_SIZE must remain 32"); + /* Buffer sizes: 512KB balances syscall amortization against memory use, * and aligns well with filesystem block sizes and zstd dictionary windows */ #define WRITE_BUFFER_SIZE (512 * 1024) diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 6c32ef70ac3f65..3ec4e0c77964c8 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -23,15 +23,11 @@ * ============================================================================ */ /* File structure sizes */ -#define FILE_FOOTER_SIZE 32 #define MIN_DECOMPRESS_BUFFER_SIZE (64 * 1024) /* Minimum decompression buffer */ /* Progress callback frequency */ #define PROGRESS_CALLBACK_INTERVAL 1000 -/* Maximum decompression size limit (1GB) */ -#define MAX_DECOMPRESS_SIZE (1ULL << 30) - /* ============================================================================ * BINARY READER IMPLEMENTATION * ============================================================================ */ @@ -47,8 +43,8 @@ reader_parse_header(BinaryReader *reader, const uint8_t *data, size_t file_size) /* Use memcpy to avoid strict aliasing violations and unaligned access */ uint32_t magic; uint32_t version; - memcpy(&magic, &data[0], sizeof(magic)); - memcpy(&version, &data[4], sizeof(version)); + memcpy(&magic, &data[HDR_OFF_MAGIC], HDR_SIZE_MAGIC); + memcpy(&version, &data[HDR_OFF_VERSION], HDR_SIZE_VERSION); /* Detect endianness from magic number */ if (magic == BINARY_FORMAT_MAGIC) { @@ -119,8 +115,8 @@ reader_parse_footer(BinaryReader *reader, const uint8_t *data, size_t file_size) const uint8_t *footer = data + file_size - FILE_FOOTER_SIZE; /* Use memcpy to avoid strict aliasing violations */ uint32_t strings_count, frames_count; - memcpy(&strings_count, &footer[0], sizeof(strings_count)); - memcpy(&frames_count, &footer[4], sizeof(frames_count)); + memcpy(&strings_count, &footer[FTR_OFF_STRINGS], FTR_SIZE_STRINGS); + memcpy(&frames_count, &footer[FTR_OFF_FRAMES], FTR_SIZE_FRAMES); reader->strings_count = SWAP32_IF(reader->needs_swap, strings_count); reader->frames_count = SWAP32_IF(reader->needs_swap, frames_count); @@ -984,11 +980,11 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre /* Use memcpy to avoid strict aliasing violations, then byte-swap if needed */ uint64_t thread_id_raw; uint32_t interpreter_id_raw; - memcpy(&thread_id_raw, &reader->sample_data[offset], sizeof(thread_id_raw)); - offset += 8; + memcpy(&thread_id_raw, &reader->sample_data[offset], SMP_SIZE_THREAD_ID); + offset += SMP_SIZE_THREAD_ID; - memcpy(&interpreter_id_raw, &reader->sample_data[offset], sizeof(interpreter_id_raw)); - offset += 4; + memcpy(&interpreter_id_raw, &reader->sample_data[offset], SMP_SIZE_INTERPRETER_ID); + offset += SMP_SIZE_INTERPRETER_ID; uint64_t thread_id = SWAP64_IF(reader->needs_swap, thread_id_raw); uint32_t interpreter_id = SWAP32_IF(reader->needs_swap, interpreter_id_raw); diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index 0ac6c88d0373a7..4e29c3142e2d4c 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -29,9 +29,6 @@ /* Frame buffer: depth varint (max 2 bytes for 256) + 256 frames * 5 bytes/varint + margin */ #define MAX_FRAME_BUFFER_SIZE ((MAX_STACK_DEPTH * MAX_VARINT_SIZE_U32) + MAX_VARINT_SIZE_U32 + 16) -/* File structure sizes */ -#define FILE_FOOTER_SIZE 32 - /* Helper macro: convert PyLong to int32, using default_val if conversion fails */ #define PYLONG_TO_INT32_OR_DEFAULT(obj, var, default_val) \ do { \ @@ -588,9 +585,9 @@ static inline int write_sample_header(BinaryWriter *writer, ThreadEntry *entry, uint8_t encoding) { uint8_t header[SAMPLE_HEADER_FIXED_SIZE]; - memcpy(header, &entry->thread_id, 8); - memcpy(header + 8, &entry->interpreter_id, 4); - header[12] = encoding; + memcpy(header + SMP_OFF_THREAD_ID, &entry->thread_id, SMP_SIZE_THREAD_ID); + memcpy(header + SMP_OFF_INTERPRETER_ID, &entry->interpreter_id, SMP_SIZE_INTERPRETER_ID); + header[SMP_OFF_ENCODING] = encoding; return writer_write_bytes(writer, header, SAMPLE_HEADER_FIXED_SIZE); } @@ -649,9 +646,9 @@ write_sample_with_encoding(BinaryWriter *writer, ThreadEntry *entry, { /* Header: thread_id(8) + interpreter_id(4) + encoding(1) + delta(varint) + status(1) */ uint8_t header_buf[SAMPLE_HEADER_MAX_SIZE]; - memcpy(header_buf, &entry->thread_id, 8); - memcpy(header_buf + 8, &entry->interpreter_id, 4); - header_buf[12] = (uint8_t)encoding_type; + memcpy(header_buf + SMP_OFF_THREAD_ID, &entry->thread_id, SMP_SIZE_THREAD_ID); + memcpy(header_buf + SMP_OFF_INTERPRETER_ID, &entry->interpreter_id, SMP_SIZE_INTERPRETER_ID); + header_buf[SMP_OFF_ENCODING] = (uint8_t)encoding_type; size_t varint_len = encode_varint_u64( header_buf + SAMPLE_HEADER_FIXED_SIZE, timestamp_delta); @@ -1145,17 +1142,17 @@ binary_writer_finalize(BinaryWriter *writer) PyErr_SetFromErrno(PyExc_IOError); return -1; } - uint64_t file_size = (uint64_t)footer_offset + 32; - uint8_t footer[32] = {0}; + uint64_t file_size = (uint64_t)footer_offset + FILE_FOOTER_SIZE; + uint8_t footer[FILE_FOOTER_SIZE] = {0}; /* Cast size_t to uint32_t before memcpy to ensure correct bytes are copied * on both little-endian and big-endian systems (size_t is 8 bytes on 64-bit) */ uint32_t string_count_u32 = (uint32_t)writer->string_count; uint32_t frame_count_u32 = (uint32_t)writer->frame_count; - memcpy(footer + 0, &string_count_u32, 4); - memcpy(footer + 4, &frame_count_u32, 4); - memcpy(footer + 8, &file_size, 8); - /* bytes 16-31: checksum placeholder (zeros) */ - if (fwrite_checked_allow_threads(footer, 32, writer->fp) < 0) { + memcpy(footer + FTR_OFF_STRINGS, &string_count_u32, FTR_SIZE_STRINGS); + memcpy(footer + FTR_OFF_FRAMES, &frame_count_u32, FTR_SIZE_FRAMES); + memcpy(footer + FTR_OFF_FILE_SIZE, &file_size, FTR_SIZE_FILE_SIZE); + /* checksum (FTR_OFF_CHECKSUM..FILE_FOOTER_SIZE-1): placeholder zeros */ + if (fwrite_checked_allow_threads(footer, FILE_FOOTER_SIZE, writer->fp) < 0) { return -1; } From 6f8c964dc09c4a062b9f06b4c418c2538b774975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Tue, 5 May 2026 02:32:06 +0200 Subject: [PATCH 03/10] gh-149342: `_remote_debugging`: Fix binary profile corruption when sampling a (temporarily) empty stack (#149343) --- .../test_binary_format.py | 134 +++++++++++++++++- ...-05-04-04-06-36.gh-issue-149342.d3CK-y.rst | 6 + Modules/_remote_debugging/binary_io_writer.c | 16 ++- Modules/_remote_debugging/module.c | 45 ++++-- 4 files changed, 181 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-04-04-06-36.gh-issue-149342.d3CK-y.rst diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index 29f83c843561cd..7e6cb724c407e3 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -148,6 +148,11 @@ def tearDown(self): def create_binary_file(self, samples, interval=1000, compression="none"): """Create a test binary file and track it for cleanup.""" + filename, _ = self.write_binary_file(samples, interval, compression) + return filename + + def write_binary_file(self, samples, interval=1000, compression="none"): + """Like create_binary_file but also returns the writer collector.""" with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: filename = f.name self.temp_files.append(filename) @@ -158,7 +163,7 @@ def create_binary_file(self, samples, interval=1000, compression="none"): for sample in samples: collector.collect(sample) collector.export(None) - return filename + return filename, collector def roundtrip(self, samples, interval=1000, compression="none"): """Write samples to binary and read back.""" @@ -805,6 +810,133 @@ def test_invalid_file_path(self): with BinaryReader("/nonexistent/path/file.bin") as reader: reader.replay_samples(RawCollector()) + def test_writer_handles_empty_stack_first_sample(self): + """BinaryWriter.write_sample tolerates an empty stack on a fresh thread. + + Regression test for the C-level RLE bug in process_thread_sample: a + freshly-created ThreadEntry has prev_stack_depth == 0, so an empty + curr_stack compares as STACK_REPEAT against the zero-initialized + previous stack. Before the fix, this fell through the + `&& !is_new_thread` guard into write_sample_with_encoding, which had + no handler for STACK_REPEAT and raised + RuntimeError("Invalid stack encoding type"). Goes through + BinaryWriter.write_sample directly so the test cannot be masked by + any Python-level filtering. + """ + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + writer = _remote_debugging.BinaryWriter(filename, 1000, 0, compression=0) + empty_sample = [ + make_interpreter( + 0, [make_thread(99, [], status=THREAD_STATUS_UNKNOWN)] + ) + ] + # First sample for a fresh thread has empty frame_info — the exact + # scenario that exposes the bug. + writer.write_sample(empty_sample, 1000) + writer.write_sample(empty_sample, 2000) + # Mix in a real sample to exercise the transition out of the + # empty-stack RLE buffer. + real_sample = [ + make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])]) + ] + writer.write_sample(real_sample, 3000) + writer.finalize() + + reader_collector = RawCollector() + with BinaryReader(filename) as reader: + count = reader.replay_samples(reader_collector) + # Empty-stack samples are recorded as STACK_REPEAT records with + # depth-0 stacks; the file must replay all three samples. + self.assertEqual(count, 3) + + def test_writer_handles_mixed_empty_and_real_first_sample(self): + """First sample with one empty + one real thread roundtrips through C.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + writer = _remote_debugging.BinaryWriter(filename, 1000, 0, compression=0) + sample = [ + make_interpreter( + 0, + [ + make_thread(1, [make_frame("a.py", 1, "f")]), + make_thread(99, [], status=THREAD_STATUS_UNKNOWN), + ], + ) + ] + # Two samples so RLE state is exercised. + writer.write_sample(sample, 1000) + writer.write_sample(sample, 2000) + writer.finalize() + + # Replay must succeed without raising RuntimeError, and the real + # thread's frames must round-trip. + reader_collector = RawCollector() + with BinaryReader(filename) as reader: + reader.replay_samples(reader_collector) + self.assertIn((0, 1), reader_collector.by_thread) + self.assertEqual(len(reader_collector.by_thread[(0, 1)]), 2) + + def test_writer_total_samples_after_finalize_matches_reader(self): + """BinaryWriter.total_samples after finalize() matches the reader's count.""" + # Five IDENTICAL samples force every sample beyond the first into the + # per-thread RLE buffer. Regression for the cached_total_samples + # ordering bug: capturing the cache BEFORE binary_writer_finalize() + # missed the buffered samples that flush_pending_rle() counts. Keep + # the samples identical to preserve coverage. See gh-149342. + samples = [ + [make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])])] + ] * 5 + filename, writer_collector = self.write_binary_file(samples) + reader_collector = RawCollector() + with BinaryReader(filename) as reader: + replayed = reader.replay_samples(reader_collector) + self.assertEqual(writer_collector.total_samples, len(samples)) + self.assertEqual(writer_collector.total_samples, replayed) + + def test_writer_total_samples_after_context_manager_matches_reader(self): + """total_samples after `with BinaryWriter(...)` matches the reader's count. + + Regression for the asymmetry between finalize() and __exit__ in + module.c: __exit__ also calls binary_writer_finalize and must + preserve cached_total_samples like finalize() does, otherwise the + getter returns 0 once self->writer is NULL. + """ + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + sample = [ + make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])]) + ] + with _remote_debugging.BinaryWriter(filename, 1000, 0, compression=0) as w: + for i in range(5): + w.write_sample(sample, i * 1000) + self.assertEqual(w.total_samples, 5) + + reader_collector = RawCollector() + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(reader_collector), 5) + + def test_writer_total_samples_after_close_returns_zero(self): + """close() discards data; total_samples reflects no cached count.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + w = _remote_debugging.BinaryWriter(filename, 1000, 0, compression=0) + sample = [ + make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])]) + ] + for i in range(5): + w.write_sample(sample, i * 1000) + w.close() + self.assertEqual(w.total_samples, 0) + class TestBinaryEncodings(BinaryFormatTestBase): """Tests specifically targeting different stack encodings.""" diff --git a/Misc/NEWS.d/next/Library/2026-05-04-04-06-36.gh-issue-149342.d3CK-y.rst b/Misc/NEWS.d/next/Library/2026-05-04-04-06-36.gh-issue-149342.d3CK-y.rst new file mode 100644 index 00000000000000..660a28ba52e679 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-04-04-06-36.gh-issue-149342.d3CK-y.rst @@ -0,0 +1,6 @@ +Fix :mod:`!_remote_debugging` binary writing so that sampling a thread +whose Python frame stack is empty (for example while it is in a C call or +mid-syscall) no longer raises ``RuntimeError("Invalid stack encoding +type")``, and so that ``BinaryWriter.total_samples`` after :meth:`!finalize` +or context-manager exit includes samples flushed from the RLE buffer. +Patch by Maurycy Pawłowski-Wieroński. diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index 4e29c3142e2d4c..4cfed7300ac5ab 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -484,7 +484,7 @@ writer_get_or_create_thread_entry(BinaryWriter *writer, uint64_t thread_id, entry->prev_stack_capacity = MAX_STACK_DEPTH; entry->pending_rle_capacity = INITIAL_RLE_CAPACITY; - entry->prev_stack = PyMem_Malloc(entry->prev_stack_capacity * sizeof(uint32_t)); + entry->prev_stack = PyMem_Calloc(entry->prev_stack_capacity, sizeof(uint32_t)); if (!entry->prev_stack) { PyErr_NoMemory(); return NULL; @@ -938,9 +938,8 @@ process_thread_sample(BinaryWriter *writer, PyObject *thread_info, } uint8_t status = (uint8_t)status_long; - int is_new_thread = 0; ThreadEntry *entry = writer_get_or_create_thread_entry( - writer, thread_id, interpreter_id, &is_new_thread); + writer, thread_id, interpreter_id, NULL); if (!entry) { return -1; } @@ -963,8 +962,15 @@ process_thread_sample(BinaryWriter *writer, PyObject *thread_info, curr_stack, curr_depth, &shared_count, &pop_count, &push_count); - if (encoding == STACK_REPEAT && !is_new_thread) { - /* Buffer this sample for RLE */ + if (encoding == STACK_REPEAT) { + /* Buffer this sample for RLE. + * + * STACK_REPEAT also covers the "first sample for a fresh thread, + * empty stack" case: a new ThreadEntry has prev_stack_depth == 0 + * and a zero-initialized prev_stack, so compare_stacks() returns + * STACK_REPEAT against an empty curr_stack (depth 0). Buffering + * it here is correct; the RLE flush path emits it as a normal + * STACK_REPEAT record. */ if (GROW_ARRAY(entry->pending_rle, entry->pending_rle_count, entry->pending_rle_capacity, PendingRLESample) < 0) { return -1; diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index c694e587e7cccb..172f8711a2a2a0 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -1544,6 +1544,24 @@ _remote_debugging_BinaryWriter_write_sample_impl(BinaryWriterObject *self, Py_RETURN_NONE; } +/* Finalize the writer, cache total_samples, and destroy it. + * + * The cache assignment must happen AFTER binary_writer_finalize(): finalize + * flushes pending RLE samples via flush_pending_rle(), which increments + * writer->total_samples for each one. Caching before finalize would lose + * those trailing samples. */ +static int +binary_writer_finalize_and_cache(BinaryWriterObject *self) +{ + if (binary_writer_finalize(self->writer) < 0) { + return -1; + } + self->cached_total_samples = self->writer->total_samples; + binary_writer_destroy(self->writer); + self->writer = NULL; + return 0; +} + /*[clinic input] _remote_debugging.BinaryWriter.finalize @@ -1561,16 +1579,10 @@ _remote_debugging_BinaryWriter_finalize_impl(BinaryWriterObject *self) return NULL; } - /* Save total_samples before finalizing */ - self->cached_total_samples = self->writer->total_samples; - - if (binary_writer_finalize(self->writer) < 0) { + if (binary_writer_finalize_and_cache(self) < 0) { return NULL; } - binary_writer_destroy(self->writer); - self->writer = NULL; - Py_RETURN_NONE; } @@ -1624,14 +1636,18 @@ _remote_debugging_BinaryWriter___exit___impl(BinaryWriterObject *self, if (self->writer) { /* Only finalize on normal exit (no exception) */ if (exc_type == Py_None) { - if (binary_writer_finalize(self->writer) < 0) { - binary_writer_destroy(self->writer); - self->writer = NULL; + if (binary_writer_finalize_and_cache(self) < 0) { + if (self->writer) { + binary_writer_destroy(self->writer); + self->writer = NULL; + } return NULL; } } - binary_writer_destroy(self->writer); - self->writer = NULL; + else { + binary_writer_destroy(self->writer); + self->writer = NULL; + } } Py_RETURN_FALSE; } @@ -1658,8 +1674,9 @@ _remote_debugging_BinaryWriter_get_stats_impl(BinaryWriterObject *self) } static PyObject * -BinaryWriter_get_total_samples(BinaryWriterObject *self, void *closure) +BinaryWriter_get_total_samples(PyObject *op, void *closure) { + BinaryWriterObject *self = BinaryWriter_CAST(op); if (!self->writer) { /* Use cached value after finalize/close */ return PyLong_FromUnsignedLong(self->cached_total_samples); @@ -1668,7 +1685,7 @@ BinaryWriter_get_total_samples(BinaryWriterObject *self, void *closure) } static PyGetSetDef BinaryWriter_getset[] = { - {"total_samples", (getter)BinaryWriter_get_total_samples, NULL, "Total samples written", NULL}, + {"total_samples", BinaryWriter_get_total_samples, NULL, "Total samples written", NULL}, {NULL} }; From f025dba62e4deee9cb740cb94dcdf0a9b0a229cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Tue, 5 May 2026 02:33:56 +0200 Subject: [PATCH 04/10] gh-149230: `_remote_debugging`: Fix async-aware for tasks in non-main threads (#149235) --- Lib/test/test_external_inspection.py | 154 +++++++++++++++++++++++++++ Modules/_remote_debugging/threads.c | 4 +- 2 files changed, 156 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 401136de8de666..a29e6cdbbf6c78 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -1437,6 +1437,160 @@ def matches_awaited_by_pattern(task): finally: _cleanup_sockets(client_socket, server_socket) + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_async_global_awaited_by_from_non_main_thread(self): + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import asyncio + import socket + import threading + import time + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + async def worker_main(): + task = asyncio.create_task( + asyncio.sleep(10_000), + name="worker task", + ) + await asyncio.sleep(0) + sock.sendall(f"ready:{{threading.get_native_id()}}\\n".encode()) + await task + + def run_worker_loop(): + asyncio.run(worker_main()) + + threading.Thread( + target=run_worker_loop, + name="async-worker", + daemon=True, + ).start() + time.sleep(10_000) + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + response = _wait_for_signal(client_socket, b"ready:") + worker_thread_id = int( + response.split(b"ready:", 1)[1].splitlines()[0] + ) + + for _ in busy_retry(SHORT_TIMEOUT): + all_awaited_by = get_all_awaited_by(p.pid) + if any( + task.task_name == "worker task" + for info in all_awaited_by + if info.thread_id == worker_thread_id + for task in info.awaited_by + ): + break + else: + self.fail( + "get_all_awaited_by() did not report " + "the asyncio task from the non-main thread" + ) + finally: + _cleanup_sockets(client_socket, server_socket) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_async_remote_stack_trace_from_non_main_thread(self): + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import asyncio + import socket + import threading + import time + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def blocking_call(): + sock.sendall(f"ready:{{threading.get_native_id()}}\\n".encode()) + time.sleep(10_000) + + async def worker_task(): + await asyncio.sleep(0) + blocking_call() + + async def worker_main(): + task = asyncio.create_task( + worker_task(), + name="worker task", + ) + await task + + def run_worker_loop(): + asyncio.run(worker_main()) + + threading.Thread( + target=run_worker_loop, + name="async-worker", + daemon=True, + ).start() + time.sleep(10_000) + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + response = _wait_for_signal(client_socket, b"ready:") + worker_thread_id = int( + response.split(b"ready:", 1)[1].splitlines()[0] + ) + + for _ in busy_retry(SHORT_TIMEOUT): + stack_trace = get_async_stack_trace(p.pid) + if any( + task.task_name == "worker task" + for info in stack_trace + if info.thread_id == worker_thread_id + for task in info.awaited_by + ): + break + else: + self.fail( + "get_async_stack_trace() did not report " + "the running asyncio task from the non-main thread" + ) + finally: + _cleanup_sockets(client_socket, server_socket) + @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index e303c667ea013a..d775234b8d78d7 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -34,11 +34,11 @@ iterate_threads( if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - unwinder->interpreter_addr + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_main, + unwinder->interpreter_addr + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_head, sizeof(void*), &thread_state_addr)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read main thread state"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read threads head"); return -1; } From 04ce31852260b3d39e35286c1b6a134a3c475b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Tue, 5 May 2026 02:44:37 +0200 Subject: [PATCH 05/10] gh-146256: Add `--jsonl` collector to the `profiling.sampling` (#146257) --- Lib/profiling/sampling/__init__.py | 11 +- Lib/profiling/sampling/binary_reader.py | 3 + Lib/profiling/sampling/cli.py | 26 +- Lib/profiling/sampling/collector.py | 10 +- Lib/profiling/sampling/constants.py | 8 + Lib/profiling/sampling/jsonl_collector.py | 266 +++++++++ .../test_sampling_profiler/helpers.py | 26 + .../test_binary_format.py | 70 ++- .../test_sampling_profiler/test_cli.py | 77 ++- .../test_sampling_profiler/test_collectors.py | 544 +++++++++++++++++- ...-03-31-17-33-10.gh-issue-146256.Nm_Ke_.rst | 4 + Modules/_remote_debugging/binary_io_reader.c | 6 +- 12 files changed, 1037 insertions(+), 14 deletions(-) create mode 100644 Lib/profiling/sampling/jsonl_collector.py create mode 100644 Misc/NEWS.d/next/Library/2026-03-31-17-33-10.gh-issue-146256.Nm_Ke_.rst diff --git a/Lib/profiling/sampling/__init__.py b/Lib/profiling/sampling/__init__.py index 6a0bb5e5c2f387..71579a3903253e 100644 --- a/Lib/profiling/sampling/__init__.py +++ b/Lib/profiling/sampling/__init__.py @@ -9,6 +9,15 @@ from .stack_collector import CollapsedStackCollector from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector +from .jsonl_collector import JsonlCollector from .string_table import StringTable -__all__ = ("Collector", "PstatsCollector", "CollapsedStackCollector", "HeatmapCollector", "GeckoCollector", "StringTable") +__all__ = ( + "Collector", + "PstatsCollector", + "CollapsedStackCollector", + "HeatmapCollector", + "GeckoCollector", + "JsonlCollector", + "StringTable", +) diff --git a/Lib/profiling/sampling/binary_reader.py b/Lib/profiling/sampling/binary_reader.py index a11be3652597a6..a29dad91ae339d 100644 --- a/Lib/profiling/sampling/binary_reader.py +++ b/Lib/profiling/sampling/binary_reader.py @@ -4,6 +4,7 @@ from .gecko_collector import GeckoCollector from .stack_collector import FlamegraphCollector, CollapsedStackCollector +from .jsonl_collector import JsonlCollector from .pstats_collector import PstatsCollector @@ -117,6 +118,8 @@ def convert_binary_to_format(input_file, output_file, output_format, collector = PstatsCollector(interval) elif output_format == 'gecko': collector = GeckoCollector(interval) + elif output_format == "jsonl": + collector = JsonlCollector(interval) else: raise ValueError(f"Unknown output format: {output_format}") diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index 9900415ae8a927..0648713edc52af 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -20,6 +20,7 @@ from .stack_collector import CollapsedStackCollector, FlamegraphCollector, DiffFlamegraphCollector from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector +from .jsonl_collector import JsonlCollector from .binary_collector import BinaryCollector from .binary_reader import BinaryReader from .constants import ( @@ -101,6 +102,7 @@ def __call__(self, parser, namespace, values, option_string=None): "diff_flamegraph": "html", "gecko": "json", "heatmap": "html", + "jsonl": "jsonl", "binary": "bin", } @@ -111,6 +113,7 @@ def __call__(self, parser, namespace, values, option_string=None): "diff_flamegraph": DiffFlamegraphCollector, "gecko": GeckoCollector, "heatmap": HeatmapCollector, + "jsonl": JsonlCollector, "binary": BinaryCollector, } @@ -488,6 +491,13 @@ def _add_format_options(parser, include_compression=True, include_binary=True): action=DiffFlamegraphAction, help="Generate differential flamegraph comparing current profile to `BASELINE` binary file", ) + format_group.add_argument( + "--jsonl", + action="store_const", + const="jsonl", + dest="format", + help="Generate newline-delimited JSON (JSONL) for programmatic consumers", + ) if include_binary: format_group.add_argument( "--binary", @@ -611,15 +621,18 @@ def _sort_to_mode(sort_choice): return sort_map.get(sort_choice, SORT_MODE_NSAMPLES) def _create_collector(format_type, sample_interval_usec, skip_idle, opcodes=False, - output_file=None, compression='auto', diff_baseline=None): + mode=None, output_file=None, compression='auto', + diff_baseline=None): """Create the appropriate collector based on format type. Args: - format_type: The output format ('pstats', 'collapsed', 'flamegraph', 'gecko', 'heatmap', 'binary', 'diff_flamegraph') + format_type: The output format ('pstats', 'collapsed', 'flamegraph', + 'gecko', 'heatmap', 'jsonl', 'binary', 'diff_flamegraph') sample_interval_usec: Sampling interval in microseconds skip_idle: Whether to skip idle samples opcodes: Whether to collect opcode information (only used by gecko format for creating interval markers in Firefox Profiler) + mode: Profiling mode for collectors that expose it in metadata output_file: Output file path (required for binary format) compression: Compression type for binary format ('auto', 'zstd', 'none') diff_baseline: Path to baseline binary file for differential flamegraph @@ -655,6 +668,11 @@ def _create_collector(format_type, sample_interval_usec, skip_idle, opcodes=Fals skip_idle = False return collector_class(sample_interval_usec, skip_idle=skip_idle, opcodes=opcodes) + if format_type == "jsonl": + return collector_class( + sample_interval_usec, skip_idle=skip_idle, mode=mode + ) + return collector_class(sample_interval_usec, skip_idle=skip_idle) @@ -1142,7 +1160,7 @@ def _handle_attach(args): # Create the appropriate collector collector = _create_collector( - args.format, args.sample_interval_usec, skip_idle, args.opcodes, + args.format, args.sample_interval_usec, skip_idle, args.opcodes, mode, output_file=output_file, compression=getattr(args, 'compression', 'auto'), diff_baseline=args.diff_baseline @@ -1249,7 +1267,7 @@ def _handle_run(args): # Create the appropriate collector collector = _create_collector( - args.format, args.sample_interval_usec, skip_idle, args.opcodes, + args.format, args.sample_interval_usec, skip_idle, args.opcodes, mode, output_file=output_file, compression=getattr(args, 'compression', 'auto'), diff_baseline=args.diff_baseline diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index 08759b611696b7..81ec6344ebdea4 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -20,13 +20,16 @@ def normalize_location(location): """Normalize location to a 4-tuple format. Args: - location: tuple (lineno, end_lineno, col_offset, end_col_offset) or None + location: tuple (lineno, end_lineno, col_offset, end_col_offset), + an integer line number, or None Returns: tuple: (lineno, end_lineno, col_offset, end_col_offset) """ if location is None: return DEFAULT_LOCATION + if isinstance(location, int): + return (location, location, -1, -1) return location @@ -34,13 +37,16 @@ def extract_lineno(location): """Extract lineno from location. Args: - location: tuple (lineno, end_lineno, col_offset, end_col_offset) or None + location: tuple (lineno, end_lineno, col_offset, end_col_offset), + an integer line number, or None Returns: int: The line number (0 for synthetic frames) """ if location is None: return 0 + if isinstance(location, int): + return location return location[0] def _is_internal_frame(frame): diff --git a/Lib/profiling/sampling/constants.py b/Lib/profiling/sampling/constants.py index a364d0b8fde1e0..d7c710f943b1b7 100644 --- a/Lib/profiling/sampling/constants.py +++ b/Lib/profiling/sampling/constants.py @@ -11,6 +11,14 @@ PROFILING_MODE_ALL = 3 # Combines GIL + CPU checks PROFILING_MODE_EXCEPTION = 4 # Only samples when thread has an active exception +PROFILING_MODE_NAMES = { + PROFILING_MODE_WALL: "wall", + PROFILING_MODE_CPU: "cpu", + PROFILING_MODE_GIL: "gil", + PROFILING_MODE_ALL: "all", + PROFILING_MODE_EXCEPTION: "exception", +} + # Sort mode constants SORT_MODE_NSAMPLES = 0 SORT_MODE_TOTTIME = 1 diff --git a/Lib/profiling/sampling/jsonl_collector.py b/Lib/profiling/sampling/jsonl_collector.py new file mode 100644 index 00000000000000..7d26129b80de86 --- /dev/null +++ b/Lib/profiling/sampling/jsonl_collector.py @@ -0,0 +1,266 @@ +"""JSON Lines (JSONL) collector for the sampling profiler. + +Emits a normalized newline-delimited JSON record stream suitable for +programmatic consumption by external tools, scripts, and agents. Each line +is one JSON object; consumers can parse the file incrementally line by +line, but the producer writes the whole file at the end of the run (it is +not a live/streaming producer). + +Record schema +============= + +Every record is a JSON object with at least ``"type"``, ``"v"`` (record +schema version), and ``"run_id"`` (UUID4 hex tagging the run; allows +demultiplexing concatenated streams). Records appear in this fixed order: + +1. ``meta`` (exactly one, first line):: + + {"type":"meta","v":0,"run_id":"", + "sample_interval_usec":,"mode":"wall|cpu|gil|all|exception"} + + ``mode`` is omitted when not provided. + +2. ``string_table`` (zero or more):: + + {"type":"string_table","v":0,"run_id":"", + "strings":[{"str_id":,"value":""}, ...]} + + Strings (filenames, function names) are interned to keep repeated values + compact. IDs are zero-based. Each chunk holds up to ``_CHUNK_SIZE`` + entries, and each entry carries its explicit ``str_id`` so consumers do + not need to infer offsets across chunks. + +3. ``frame_table`` (zero or more):: + + {"type":"frame_table","v":0,"run_id":"", + "frames":[{"frame_id":,"path_str_id":,"func_str_id":, + "line":,"end_line":,"col":, + "end_col":}, ...]} + + ``end_line``/``col``/``end_col`` are *omitted* when source location data + is unavailable (a missing key means "not available", not zero or null). + ``line`` is ``0`` for synthetic frames (for example, internal marker + frames whose source location is None). Frame IDs are zero-based. + +4. ``agg`` (zero or more):: + + {"type":"agg","v":0,"run_id":"","kind":"frame","scope":"final", + "samples_total":, + "entries":[{"frame_id":,"self":,"cumulative":}, ...]} + + ``self`` counts samples where the frame was the leaf (currently + executing); ``cumulative`` counts samples where the frame appeared + anywhere in the stack (deduped per sample so recursion does not + double-count). ``samples_total`` is the run-wide total, repeated on + each chunk so a streaming consumer always knows the denominator. + +5. ``end`` (exactly one, last line):: + + {"type":"end","v":0,"run_id":"","samples_total":} + + Presence of ``end`` is the consumer's signal that the file is complete. + +Forward compatibility +===================== + +Consumers MUST ignore unknown record ``"type"`` values and unknown object +fields. New fields will be added by adding optional keys; an incompatible +schema change will bump the per-record ``"v"``. +""" + +from collections import Counter +import json +import uuid +from itertools import batched + +from .constants import PROFILING_MODE_NAMES +from .collector import normalize_location +from .stack_collector import StackTraceCollector + + +_CHUNK_SIZE = 256 +_SCHEMA_VERSION = 0 + + +class JsonlCollector(StackTraceCollector): + """Collector that exports finalized profiling data as JSONL. + + See the module docstring for the full record schema. The collector + accumulates samples in memory and writes the complete file at + ``export()`` time. + """ + + def __init__(self, sample_interval_usec, *, skip_idle=False, mode=None): + super().__init__(sample_interval_usec, skip_idle=skip_idle) + self.run_id = uuid.uuid4().hex + + self._string_to_id = {} + self._strings = [] + + self._frame_to_id = {} + self._frames = [] + + self._frame_self = Counter() + self._frame_cumulative = Counter() + self._samples_total = 0 + self._seen_frame_ids = set() + + self._mode = mode + + def process_frames(self, frames, _thread_id, weight=1): + self._samples_total += weight + self._seen_frame_ids.clear() + + for i, (filename, location, funcname, _opcode) in enumerate(frames): + frame_id = self._get_or_create_frame_id( + filename, location, funcname + ) + is_leaf = i == 0 + count_cumulative = frame_id not in self._seen_frame_ids + + if count_cumulative: + self._seen_frame_ids.add(frame_id) + + if is_leaf: + self._frame_self[frame_id] += weight + + if count_cumulative: + self._frame_cumulative[frame_id] += weight + + def export(self, filename): + with open(filename, "w", encoding="utf-8") as output: + self._write_message(output, self._build_meta_record()) + self._write_chunked_records( + output, + { + "type": "string_table", + "v": _SCHEMA_VERSION, + "run_id": self.run_id, + }, + "strings", + self._strings, + ) + self._write_chunked_records( + output, + { + "type": "frame_table", + "v": _SCHEMA_VERSION, + "run_id": self.run_id, + }, + "frames", + self._frames, + ) + self._write_chunked_records( + output, + { + "type": "agg", + "v": _SCHEMA_VERSION, + "run_id": self.run_id, + "kind": "frame", + "scope": "final", + "samples_total": self._samples_total, + }, + "entries", + self._iter_final_agg_entries(), + ) + self._write_message(output, self._build_end_record()) + + def _build_meta_record(self): + record = { + "type": "meta", + "v": _SCHEMA_VERSION, + "run_id": self.run_id, + "sample_interval_usec": self.sample_interval_usec, + } + + if self._mode is not None: + record["mode"] = PROFILING_MODE_NAMES.get( + self._mode, str(self._mode) + ) + + return record + + def _build_end_record(self): + record = { + "type": "end", + "v": _SCHEMA_VERSION, + "run_id": self.run_id, + "samples_total": self._samples_total, + } + + return record + + def _iter_final_agg_entries(self): + for frame_record in self._frames: + frame_id = frame_record["frame_id"] + yield { + "frame_id": frame_id, + "self": self._frame_self[frame_id], + "cumulative": self._frame_cumulative[frame_id], + } + + def _get_or_create_frame_id(self, filename, location, funcname): + location_fields = self._location_to_export_fields(location) + func_str_id = self._intern_string(funcname) + path_str_id = self._intern_string(filename) + + frame_key = ( + path_str_id, + func_str_id, + location_fields["line"], + location_fields.get("end_line"), + location_fields.get("col"), + location_fields.get("end_col"), + ) + + if (frame_id := self._frame_to_id.get(frame_key)) is not None: + return frame_id + + frame_id = len(self._frames) + frame_record = { + "frame_id": frame_id, + "path_str_id": path_str_id, + "func_str_id": func_str_id, + **location_fields, + } + + self._frame_to_id[frame_key] = frame_id + self._frames.append(frame_record) + return frame_id + + def _intern_string(self, value): + value = str(value) + + if (string_id := self._string_to_id.get(value)) is not None: + return string_id + + string_id = len(self._strings) + self._string_to_id[value] = string_id + self._strings.append({"str_id": string_id, "value": value}) + return string_id + + @staticmethod + def _location_to_export_fields(location): + lineno, end_lineno, col_offset, end_col_offset = normalize_location( + location + ) + + fields = {"line": lineno} + if end_lineno > 0: + fields["end_line"] = end_lineno + if col_offset >= 0: + fields["col"] = col_offset + if end_col_offset >= 0: + fields["end_col"] = end_col_offset + return fields + + def _write_chunked_records( + self, output, base_record, chunk_field, entries + ): + for chunk in batched(entries, _CHUNK_SIZE): + self._write_message(output, {**base_record, chunk_field: chunk}) + + @staticmethod + def _write_message(output, record): + output.write(json.dumps(record, separators=(",", ":"))) + output.write("\n") diff --git a/Lib/test/test_profiling/test_sampling_profiler/helpers.py b/Lib/test/test_profiling/test_sampling_profiler/helpers.py index 0e32d8dd9eabef..b07776d415bb29 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/helpers.py +++ b/Lib/test/test_profiling/test_sampling_profiler/helpers.py @@ -174,3 +174,29 @@ def close_and_unlink(file): """Close a file and unlink it from the filesystem.""" file.close() unlink(file.name) + + +def jsonl_tables(records): + """Extract the canonical sections of a parsed JSONL profile. + + Returns ``(meta, str_defs, frame_defs, agg, end)`` where ``str_defs`` is a + ``{str_id: value}`` dict, ``frame_defs`` is a flat list of all frame + definitions across chunks, and ``agg`` is the first agg record (sufficient + for tests that only emit one chunk). + """ + meta = next(record for record in records if record["type"] == "meta") + end = next(record for record in records if record["type"] == "end") + agg = next(record for record in records if record["type"] == "agg") + str_defs = { + item["str_id"]: item["value"] + for record in records + if record["type"] == "string_table" + for item in record["strings"] + } + frame_defs = [ + item + for record in records + if record["type"] == "frame_table" + for item in record["frames"] + ] + return meta, str_defs, frame_defs, agg, end diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index 7e6cb724c407e3..ca6cb6befaed24 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -1,5 +1,6 @@ """Tests for binary format round-trip functionality.""" +import json import os import random import tempfile @@ -21,7 +22,7 @@ THREAD_STATUS_MAIN_THREAD, ) from profiling.sampling.binary_collector import BinaryCollector - from profiling.sampling.binary_reader import BinaryReader + from profiling.sampling.binary_reader import BinaryReader, convert_binary_to_format from profiling.sampling.gecko_collector import GeckoCollector ZSTD_AVAILABLE = _remote_debugging.zstd_available() @@ -30,6 +31,8 @@ "Test only runs when _remote_debugging is available" ) +from .helpers import jsonl_tables + def make_frame(filename, lineno, funcname, end_lineno=None, column=None, end_column=None, opcode=None): @@ -1343,5 +1346,70 @@ def test_timestamp_preservation_with_rle(self): self.assertEqual(ts_collector.all_timestamps, expected_timestamps) +class TestBinaryReplayToJsonl(BinaryFormatTestBase): + """Tests for binary -> JSONL replay via convert_binary_to_format.""" + + def _replay_to_jsonl(self, samples, interval=1000): + bin_path = self.create_binary_file(samples, interval=interval) + with tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False) as f: + jsonl_path = f.name + self.temp_files.append(jsonl_path) + + convert_binary_to_format(bin_path, jsonl_path, "jsonl") + + with open(jsonl_path, "r", encoding="utf-8") as f: + return [json.loads(line) for line in f] + + def test_binary_replay_to_jsonl_basic(self): + """Replay a small .bin to JSONL: meta/end shape, samples_total, run_id.""" + frame = make_frame("hot.py", 99, "hot_func") + samples = [ + [make_interpreter(0, [make_thread(1, [frame])])] + for _ in range(5) + ] + records = self._replay_to_jsonl(samples, interval=2000) + meta, _, frame_defs, _, end = jsonl_tables(records) + + self.assertEqual(meta["sample_interval_usec"], 2000) + self.assertEqual(end["samples_total"], 5) + + run_ids = {r["run_id"] for r in records} + self.assertEqual(len(run_ids), 1) + self.assertRegex(next(iter(run_ids)), r"^[0-9a-f]{32}$") + + self.assertEqual(len(frame_defs), 1) + self.assertEqual(frame_defs[0]["line"], 99) + + def test_binary_replay_to_jsonl_rle_weight_propagation(self): + """RLE-batched identical samples land as a single agg entry with the right total.""" + frame = make_frame("rle.py", 42, "rle_func") + samples = [ + [make_interpreter(0, [make_thread(1, [frame])])] + for _ in range(50) + ] + records = self._replay_to_jsonl(samples) + _, _, _, agg, end = jsonl_tables(records) + + self.assertEqual(end["samples_total"], 50) + self.assertEqual(agg["entries"], [ + {"frame_id": 0, "self": 50, "cumulative": 50}, + ]) + + def test_binary_replay_to_jsonl_omits_unavailable_columns(self): + """Columns the binary recorder did not capture are omitted, not 0.""" + # make_frame defaults column/end_column to 0; pass column=-1 / end_column=-1 + # so the binary side records LOCATION_NOT_AVAILABLE. + frame = make_frame("nocol.py", 7, "no_col", column=-1, end_column=-1) + samples = [[make_interpreter(0, [make_thread(1, [frame])])]] + records = self._replay_to_jsonl(samples) + _, _, frame_defs, _, _ = jsonl_tables(records) + + self.assertEqual(len(frame_defs), 1) + fd = frame_defs[0] + self.assertEqual(fd["line"], 7) + self.assertNotIn("col", fd) + self.assertNotIn("end_col", fd) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py index c522c50d1fd5fa..9c0734ac804e1b 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py @@ -1,6 +1,7 @@ """Tests for sampling profiler CLI argument parsing and functionality.""" import io +import json import os import subprocess import sys @@ -21,9 +22,19 @@ requires_remote_subprocess_debugging, ) -from profiling.sampling.cli import main -from profiling.sampling.constants import PROFILING_MODE_ALL, PROFILING_MODE_WALL +from profiling.sampling.cli import ( + FORMAT_EXTENSIONS, + _create_collector, + _generate_output_filename, + main, +) +from profiling.sampling.constants import ( + PROFILING_MODE_ALL, + PROFILING_MODE_CPU, + PROFILING_MODE_WALL, +) from profiling.sampling.errors import SamplingScriptNotFoundError, SamplingModuleNotFoundError, SamplingUnknownProcessError +from profiling.sampling.jsonl_collector import JsonlCollector class TestSampleProfilerCLI(unittest.TestCase): def _setup_sync_mocks(self, mock_socket, mock_popen): @@ -912,3 +923,65 @@ def test_cli_replay_reader_errors_exit_cleanly(self): str(cm.exception), "Error: Unsupported format version 2", ) + + def test_cli_jsonl_format_mutually_exclusive_with_pstats(self): + """--jsonl and --pstats cannot be combined (mutually exclusive group).""" + with ( + mock.patch( + "sys.argv", + [ + "profiling.sampling.cli", + "attach", + "12345", + "--jsonl", + "--pstats", + ], + ), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + main() + + def test_cli_jsonl_extension_in_format_extensions(self): + """FORMAT_EXTENSIONS maps 'jsonl' -> 'jsonl' so default filenames work.""" + self.assertEqual(FORMAT_EXTENSIONS["jsonl"], "jsonl") + self.assertEqual(_generate_output_filename("jsonl", 12345), "jsonl_12345.jsonl") + + def test_cli_jsonl_create_collector_propagates_mode(self): + """_create_collector('jsonl', ..., mode=X) lands X in the meta record.""" + collector = _create_collector( + "jsonl", + sample_interval_usec=1000, + skip_idle=False, + mode=PROFILING_MODE_CPU, + ) + self.assertIsInstance(collector, JsonlCollector) + + with tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False) as f: + jsonl_path = f.name + self.addCleanup(os.unlink, jsonl_path) + collector.export(jsonl_path) + with open(jsonl_path, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + meta = next(r for r in records if r["type"] == "meta") + self.assertEqual(meta["mode"], "cpu") + + def test_cli_jsonl_rejects_opcodes_combination(self): + """--opcodes is incompatible with --jsonl per opcodes_compatible_formats.""" + test_args = [ + "profiling.sampling.cli", + "attach", + "12345", + "--jsonl", + "--opcodes", + ] + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + mock.patch("profiling.sampling.cli.sample"), + self.assertRaises(SystemExit) as cm, + ): + main() + + self.assertEqual(cm.exception.code, 2) + self.assertIn("--opcodes", mock_stderr.getvalue()) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 240ec8a195c43b..b42e7aa579f40c 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -16,6 +16,7 @@ CollapsedStackCollector, FlamegraphCollector, ) + from profiling.sampling.jsonl_collector import JsonlCollector from profiling.sampling.gecko_collector import GeckoCollector from profiling.sampling.collector import extract_lineno, normalize_location from profiling.sampling.opcode_utils import get_opcode_info, format_opcode @@ -38,7 +39,7 @@ from test.support import captured_stdout, captured_stderr from .mocks import MockFrameInfo, MockThreadInfo, MockInterpreterInfo, LocationInfo, make_diff_collector_with_mock_baseline -from .helpers import close_and_unlink +from .helpers import close_and_unlink, jsonl_tables def resolve_name(node, strings): @@ -1669,6 +1670,393 @@ def test_diff_flamegraph_load_baseline(self): self.assertAlmostEqual(cold_node["diff"], -1.0) self.assertAlmostEqual(cold_node["diff_pct"], -50.0) + def test_jsonl_collector_export_exact_output(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + collector.run_id = "run-123" + + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [MockFrameInfo("other.py", 5, "other_func")] + ) + ], + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + content = f.read() + + self.assertEqual( + content, + ( + '{"type":"meta","v":0,"run_id":"run-123","sample_interval_usec":1000}\n' + '{"type":"string_table","v":0,"run_id":"run-123","strings":[{"str_id":0,"value":"func1"},{"str_id":1,"value":"file.py"},{"str_id":2,"value":"func2"},{"str_id":3,"value":"other_func"},{"str_id":4,"value":"other.py"}]}\n' + '{"type":"frame_table","v":0,"run_id":"run-123","frames":[{"frame_id":0,"path_str_id":1,"func_str_id":0,"line":10,"end_line":10},{"frame_id":1,"path_str_id":1,"func_str_id":2,"line":20,"end_line":20},{"frame_id":2,"path_str_id":4,"func_str_id":3,"line":5,"end_line":5}]}\n' + '{"type":"agg","v":0,"run_id":"run-123","kind":"frame","scope":"final","samples_total":3,"entries":[{"frame_id":0,"self":2,"cumulative":2},{"frame_id":1,"self":0,"cumulative":2},{"frame_id":2,"self":1,"cumulative":1}]}\n' + '{"type":"end","v":0,"run_id":"run-123","samples_total":3}\n' + ), + ) + + def test_jsonl_collector_export_includes_mode_in_meta(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000, mode=PROFILING_MODE_CPU) + collector.collect( + [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [MockFrameInfo("file.py", 10, "func")] + ) + ], + ) + ] + ) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + meta_record = next( + record for record in records if record["type"] == "meta" + ) + self.assertEqual(meta_record["mode"], "cpu") + + def test_jsonl_collector_export_empty_profile(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + collector.run_id = "run-123" + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + self.assertEqual( + [record["type"] for record in records], ["meta", "end"] + ) + self.assertEqual(records[0]["sample_interval_usec"], 1000) + self.assertEqual(records[0]["run_id"], "run-123") + self.assertEqual(records[1]["samples_total"], 0) + self.assertEqual(records[1]["run_id"], "run-123") + + def test_jsonl_collector_recursive_frames_counted_once_per_sample(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + collector.collect( + [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo( + "recursive.py", 10, "recursive_func" + ), + MockFrameInfo( + "recursive.py", 10, "recursive_func" + ), + MockFrameInfo( + "recursive.py", 10, "recursive_func" + ), + ], + ) + ], + ) + ] + ) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, _, frame_defs, agg_record, end_record = jsonl_tables(records) + self.assertEqual(len(frame_defs), 1) + self.assertEqual( + agg_record["entries"], + [ + { + "frame_id": frame_defs[0]["frame_id"], + "self": 1, + "cumulative": 1, + } + ], + ) + self.assertEqual(agg_record["samples_total"], 1) + self.assertEqual(end_record["samples_total"], 1) + + def test_jsonl_collector_skip_idle_filters_threads(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + active_status = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU + frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("active1.py", 10, "active_func1")], + status=active_status, + ), + MockThreadInfo( + 2, + [MockFrameInfo("idle.py", 20, "idle_func")], + status=0, + ), + MockThreadInfo( + 3, + [MockFrameInfo("active2.py", 30, "active_func2")], + status=active_status, + ), + ], + ) + ] + + def export_summary(skip_idle): + collector = JsonlCollector(1000, skip_idle=skip_idle) + collector.collect(frames) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, str_defs, frame_defs, agg_record, _ = jsonl_tables(records) + paths = {str_defs[item["path_str_id"]] for item in frame_defs} + funcs = {str_defs[item["func_str_id"]] for item in frame_defs} + return paths, funcs, agg_record["samples_total"] + + paths, funcs, samples_total = export_summary(skip_idle=True) + self.assertEqual(paths, {"active1.py", "active2.py"}) + self.assertEqual(funcs, {"active_func1", "active_func2"}) + self.assertEqual(samples_total, 2) + + paths, funcs, samples_total = export_summary(skip_idle=False) + self.assertEqual(paths, {"active1.py", "idle.py", "active2.py"}) + self.assertEqual(funcs, {"active_func1", "idle_func", "active_func2"}) + self.assertEqual(samples_total, 3) + + def test_jsonl_collector_splits_large_exports_into_chunks(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + + for i in range(257): + collector.collect( + [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo( + f"file{i}.py", i + 1, f"func{i}" + ) + ], + ) + ], + ) + ] + ) + + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + run_ids = {record["run_id"] for record in records} + self.assertEqual(len(run_ids), 1) + self.assertRegex(next(iter(run_ids)), r"^[0-9a-f]{32}$") + + _, str_defs, frame_defs, agg_record, end_record = jsonl_tables( + records + ) + str_chunks = [ + record for record in records if record["type"] == "string_table" + ] + frame_chunks = [ + record for record in records if record["type"] == "frame_table" + ] + agg_chunks = [record for record in records if record["type"] == "agg"] + + self.assertEqual( + [len(record["strings"]) for record in str_chunks], + [256, 256, 2], + ) + self.assertEqual( + [len(record["frames"]) for record in frame_chunks], [256, 1] + ) + self.assertEqual( + [len(record["entries"]) for record in agg_chunks], [256, 1] + ) + self.assertEqual(len(str_defs), 514) + self.assertEqual(len(frame_defs), 257) + self.assertEqual(agg_record["samples_total"], 257) + self.assertEqual(end_record["samples_total"], 257) + + def test_jsonl_collector_respects_weight_for_rle_batched_samples(self): + """weight>1 (from binary replay RLE) is honored in self/cumulative.""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + leaf = MockFrameInfo("file.py", 10, "leaf") + non_leaf = MockFrameInfo("file.py", 20, "non_leaf") + + collector.process_frames([leaf, non_leaf], _thread_id=1, weight=5) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, str_defs, frame_defs, agg, end = jsonl_tables(records) + self.assertEqual(end["samples_total"], 5) + self.assertEqual(agg["samples_total"], 5) + self.assertEqual( + {str_defs[fd["func_str_id"]]: fd["frame_id"] for fd in frame_defs}, + {"leaf": 0, "non_leaf": 1}, + ) + self.assertEqual(agg["entries"], [ + {"frame_id": 0, "self": 5, "cumulative": 5}, + {"frame_id": 1, "self": 0, "cumulative": 5}, + ]) + + def test_jsonl_collector_recursion_with_weight(self): + """Recursion dedup respects weight, not occurrence count.""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + recursive = MockFrameInfo("rec.py", 10, "f") + + collector.process_frames([recursive] * 3, _thread_id=1, weight=3) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, _, frame_defs, agg, _ = jsonl_tables(records) + self.assertEqual(len(frame_defs), 1) + self.assertEqual(agg["entries"], [ + {"frame_id": 0, "self": 3, "cumulative": 3}, + ]) + + def test_jsonl_collector_emits_col_and_end_col_when_present(self): + """All four location fields are emitted when col/end_col are >= 0.""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + frame = MockFrameInfo("test.py", 0, "f") + frame.location = LocationInfo(42, 45, 4, 12) + frames = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [frame], status=THREAD_STATUS_HAS_GIL)] + ) + ] + collector.collect(frames) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, str_defs, frame_defs, _, _ = jsonl_tables(records) + self.assertEqual(frame_defs, [ + { + "frame_id": 0, + "path_str_id": 1, + "func_str_id": 0, + "line": 42, + "end_line": 45, + "col": 4, + "end_col": 12, + }, + ]) + self.assertEqual(str_defs, {0: "f", 1: "test.py"}) + + def test_jsonl_collector_partial_location_elision(self): + """Negative col/end_col/end_line fields are individually elided.""" + # _get_or_create_frame_id interns funcname before filename, so + # func_str_id=0 ("f") and path_str_id=1 ("test.py"). + common = {"frame_id": 0, "path_str_id": 1, "func_str_id": 0} + cases = [ + (LocationInfo(42, 45, -1, 12), + {**common, "line": 42, "end_line": 45, "end_col": 12}), + (LocationInfo(42, 45, 4, -1), + {**common, "line": 42, "end_line": 45, "col": 4}), + (LocationInfo(42, 0, 4, 8), + {**common, "line": 42, "col": 4, "end_col": 8}), + ] + for loc, expected_frame_def in cases: + with self.subTest(location=loc): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + frame = MockFrameInfo("test.py", 0, "f") + frame.location = loc + frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(1, [frame], status=THREAD_STATUS_HAS_GIL)], + ) + ] + collector.collect(frames) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, _, frame_defs, _, _ = jsonl_tables(records) + self.assertEqual(frame_defs, [expected_frame_def]) + class TestRecursiveFunctionHandling(unittest.TestCase): """Tests for correct handling of recursive functions in cumulative stats.""" @@ -1878,6 +2266,20 @@ def test_extract_lineno_from_none(self): """Test extracting lineno from None (synthetic frames).""" self.assertEqual(extract_lineno(None), 0) + def test_extract_lineno_from_int(self): + """Test extracting lineno from a bare integer line number. + + Mirrors normalize_location's int contract so callers like the + collapsed/flamegraph collectors do not crash on a bare-int location. + """ + self.assertEqual(extract_lineno(42), 42) + self.assertEqual(extract_lineno(0), 0) + + def test_normalize_location_with_int(self): + """Test normalize_location expands a legacy integer line number.""" + result = normalize_location(42) + self.assertEqual(result, (42, 42, -1, -1)) + def test_normalize_location_with_location_info(self): """Test normalize_location passes through LocationInfo.""" loc = LocationInfo(10, 15, 0, 5) @@ -2068,6 +2470,85 @@ def test_gecko_collector_with_location_info(self): # Verify function name is in string table self.assertIn("handle_request", string_array) + def test_jsonl_collector_with_location_info(self): + """Test JsonlCollector handles LocationInfo properly.""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(sample_interval_usec=1000) + + # Frame with LocationInfo + frame = MockFrameInfo("test.py", 42, "my_function") + frames = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [frame], status=THREAD_STATUS_HAS_GIL)] + ) + ] + collector.collect(frames) + + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + meta, str_defs, frame_defs, agg, end = jsonl_tables(records) + self.assertEqual(meta["sample_interval_usec"], 1000) + self.assertEqual(agg["samples_total"], 1) + self.assertEqual(end["samples_total"], 1) + self.assertEqual(len(frame_defs), 1) + self.assertEqual(str_defs[frame_defs[0]["path_str_id"]], "test.py") + self.assertEqual(str_defs[frame_defs[0]["func_str_id"]], "my_function") + self.assertEqual( + frame_defs[0], + { + "frame_id": 0, + "path_str_id": frame_defs[0]["path_str_id"], + "func_str_id": frame_defs[0]["func_str_id"], + "line": 42, + "end_line": 42, + }, + ) + + def test_jsonl_collector_with_none_location(self): + """Test JsonlCollector handles None location (synthetic frames).""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(sample_interval_usec=1000) + + # Create frame with None location (like GC frame) + frame = MockFrameInfo("~", 0, "") + frame.location = None # Synthetic frame has no location + frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(1, [frame], status=THREAD_STATUS_HAS_GIL)] + ) + ] + collector.collect(frames) + + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + meta, str_defs, frame_defs, agg, end = jsonl_tables(records) + self.assertEqual(meta["sample_interval_usec"], 1000) + self.assertEqual(agg["samples_total"], 1) + self.assertEqual(end["samples_total"], 1) + self.assertEqual(len(frame_defs), 1) + self.assertEqual(str_defs[frame_defs[0]["path_str_id"]], "~") + self.assertEqual(str_defs[frame_defs[0]["func_str_id"]], "") + self.assertEqual( + frame_defs[0], + { + "frame_id": 0, + "path_str_id": frame_defs[0]["path_str_id"], + "func_str_id": frame_defs[0]["func_str_id"], + "line": 0, + }, + ) + class TestOpcodeHandling(unittest.TestCase): """Tests for opcode field handling in collectors.""" @@ -2288,6 +2769,28 @@ def test_gecko_collector_frame_format(self): # Should have recorded 3 functions self.assertEqual(thread["funcTable"]["length"], 3) + def test_jsonl_collector_frame_format(self): + """Test JsonlCollector with 4-element frame format.""" + collector = JsonlCollector(sample_interval_usec=1000) + collector.collect(self._make_sample_frames()) + + with tempfile.NamedTemporaryFile(delete=False) as f: + self.addClassCleanup(close_and_unlink, f) + collector.export(f.name) + + with open(f.name, "r", encoding="utf-8") as fp: + records = [json.loads(line) for line in fp] + + _, str_defs, frame_defs, _, _ = jsonl_tables(records) + + self.assertEqual(len(frame_defs), 3) + + paths = {str_defs[item["path_str_id"]] for item in frame_defs} + funcs = {str_defs[item["func_str_id"]] for item in frame_defs} + + self.assertEqual(paths, {"app.py", "utils.py", "lib.py"}) + self.assertEqual(funcs, {"main", "helper", "process"}) + class TestInternalFrameFiltering(unittest.TestCase): """Tests for filtering internal profiler frames from output.""" @@ -2415,3 +2918,42 @@ def test_collapsed_stack_collector_filters_internal_frames(self): for (call_tree, _), _ in collector.stack_counter.items(): for filename, _, _ in call_tree: self.assertNotIn("_sync_coordinator", filename) + + def test_jsonl_collector_filters_internal_frames(self): + """Test that JsonlCollector filters out internal frames.""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("app.py", 50, "run"), + MockFrameInfo("/lib/_sync_coordinator.py", 100, "main"), + MockFrameInfo("", 87, "_run_code"), + ], + status=THREAD_STATUS_HAS_GIL, + ) + ], + ) + ] + + collector = JsonlCollector(sample_interval_usec=1000) + collector.collect(frames) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, str_defs, frame_defs, _, _ = jsonl_tables(records) + + paths = {str_defs[item["path_str_id"]] for item in frame_defs} + + self.assertIn("app.py", paths) + self.assertIn("", paths) + + for path in paths: + self.assertNotIn("_sync_coordinator", path) diff --git a/Misc/NEWS.d/next/Library/2026-03-31-17-33-10.gh-issue-146256.Nm_Ke_.rst b/Misc/NEWS.d/next/Library/2026-03-31-17-33-10.gh-issue-146256.Nm_Ke_.rst new file mode 100644 index 00000000000000..636f45ae8d6c70 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-31-17-33-10.gh-issue-146256.Nm_Ke_.rst @@ -0,0 +1,4 @@ +The ``profiling.sampling`` module now supports JSONL output format via +``--jsonl``. Each run emits a newline-delimited JSON file that is +sequentially parseable by external tools, scripts, and programmatic +consumers. Patch by Maurycy Pawłowski-Wieroński. diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 3ec4e0c77964c8..da3e7d55309c27 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -781,9 +781,9 @@ build_frame_list(RemoteDebuggingState *state, BinaryReader *reader, if (frame->lineno != LOCATION_NOT_AVAILABLE) { location = Py_BuildValue("(iiii)", frame->lineno, - frame->end_lineno != LOCATION_NOT_AVAILABLE ? frame->end_lineno : frame->lineno, - frame->column != LOCATION_NOT_AVAILABLE ? frame->column : 0, - frame->end_column != LOCATION_NOT_AVAILABLE ? frame->end_column : 0); + frame->end_lineno, + frame->column, + frame->end_column); if (!location) { Py_DECREF(frame_info); goto error; From c266f0c375c2e60ea46046254fa7cd5fa2fe1ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Tue, 5 May 2026 02:50:06 +0200 Subject: [PATCH 06/10] gh-149009: Validate `thread_count` in `profiling.sampling` binary reader (#149147) --- .../test_binary_format.py | 30 +++++++++++++++++++ ...-04-29-13-08-46.gh-issue-149009.rek3Tw.rst | 3 ++ Modules/_remote_debugging/binary_io_reader.c | 8 +++++ 3 files changed, 41 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-04-29-13-08-46.gh-issue-149009.rek3Tw.rst diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index ca6cb6befaed24..9cf706aa2dafee 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -3,6 +3,7 @@ import json import os import random +import struct import tempfile import unittest from collections import defaultdict @@ -941,6 +942,35 @@ def test_writer_total_samples_after_close_returns_zero(self): self.assertEqual(w.total_samples, 0) +class TestBinaryFormatValidation(BinaryFormatTestBase): + """Tests for malformed binary files.""" + + HDR_OFF_THREADS = 32 + + def test_replay_rejects_more_threads_than_declared(self): + """Replay rejects files with more unique threads than the header declares.""" + threads = [ + make_thread(1, [make_frame("t1.py", 10, "t1")]), + make_thread(2, [make_frame("t2.py", 20, "t2")]), + ] + samples = [[make_interpreter(0, threads)]] + filename = self.create_binary_file(samples, compression="none") + + with open(filename, "r+b") as raw: + raw.seek(self.HDR_OFF_THREADS) + raw.write(struct.pack("=I", 1)) + + with BinaryReader(filename) as reader: + self.assertEqual(reader.get_info()["thread_count"], 1) + with self.assertRaises(ValueError) as cm: + reader.replay_samples(RawCollector()) + self.assertEqual( + str(cm.exception), + "Invalid thread count: sample data contains more unique " + "threads than declared in header (declared 1, found at least 2)", + ) + + class TestBinaryEncodings(BinaryFormatTestBase): """Tests specifically targeting different stack encodings.""" diff --git a/Misc/NEWS.d/next/Library/2026-04-29-13-08-46.gh-issue-149009.rek3Tw.rst b/Misc/NEWS.d/next/Library/2026-04-29-13-08-46.gh-issue-149009.rek3Tw.rst new file mode 100644 index 00000000000000..e2f078742760a5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-29-13-08-46.gh-issue-149009.rek3Tw.rst @@ -0,0 +1,3 @@ +Validate that :mod:`profiling.sampling` binary profiles do not contain more +unique (thread, interpreter) pairs than declared in the header. Patch by +Maurycy Pawłowski-Wieroński. diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index da3e7d55309c27..551530b519952c 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -559,6 +559,14 @@ reader_get_or_create_thread_state(BinaryReader *reader, uint64_t thread_id, } } + if (reader->thread_state_count >= reader->thread_count) { + PyErr_Format(PyExc_ValueError, + "Invalid thread count: sample data contains more unique threads than declared in header " + "(declared %u, found at least %zu)", + reader->thread_count, reader->thread_state_count + 1); + return NULL; + } + if (!reader->thread_states) { reader->thread_state_capacity = 16; reader->thread_states = PyMem_Calloc(reader->thread_state_capacity, sizeof(ReaderThreadState)); From 88844d213593b0abdd7a22b2f619bf59b74f2dcf Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 5 May 2026 01:51:28 +0100 Subject: [PATCH 07/10] gh-149202: Highlight PEP 831 in What's New for Python 3.15 (#149390) --- Doc/whatsnew/3.15.rst | 46 ++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 7f12cc04a460d4..e98c483baec823 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -75,6 +75,8 @@ Summary -- Release highlights profiling tools ` * :pep:`799`: :ref:`Tachyon: High frequency statistical sampling profiler ` +* :pep:`831`: :ref:`Frame pointers are enabled by default for improved + system-level observability ` * :pep:`798`: :ref:`Unpacking in comprehensions ` * :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding @@ -86,7 +88,6 @@ Summary -- Release highlights * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object ` * :pep:`803`: :ref:`Stable ABI for Free-Threaded Builds ` -* :pep:`831`: :ref:`Frame pointers everywhere ` * :ref:`The JIT compiler has been significantly upgraded ` * :ref:`Improved error messages ` * :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter @@ -376,6 +377,39 @@ available output formats, profiling modes, and configuration options. (Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953` and :gh:`138122`.) +.. _whatsnew315-pep831: + +:pep:`831`: Frame pointers enabled by default +--------------------------------------------- + +CPython is now built with frame pointers by default on platforms that support +them. This uses the compiler flags ``-fno-omit-frame-pointer`` and +``-mno-omit-leaf-frame-pointer``, making native stack unwinding faster and +more reliable for system profilers, debuggers, crash analysis tools, and +eBPF-based observability tools. + +The flags are exposed through :mod:`sysconfig`, so extension modules built by +tools that consume Python's build configuration inherit frame pointers by +default. This propagation is intentional: mixed Python/native profiling needs +an unbroken frame-pointer chain through the interpreter, extension modules, +embedding applications, and native libraries. + +.. important:: + + Third-party build backends and native build systems should preserve these + flags when they consume Python's :mod:`sysconfig` values. Build systems + that compile C, C++, Rust, or other native code without inheriting Python's + compiler flags should enable equivalent frame-pointer flags themselves. A + single native component built without frame pointers can break stack + unwinding for the whole Python process. + +.. seealso:: :pep:`831` for further details. + +(Contributed by Pablo Galindo Salgado and Savannah Ostrowski in +:gh:`149201`; PEP 831 written by Pablo Galindo Salgado, Ken Jin, and +Savannah Ostrowski.) + + .. _whatsnew315-unpacking-in-comprehensions: :pep:`798`: Unpacking in Comprehensions @@ -2378,16 +2412,6 @@ Build changes and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. (Contributed by Donghee Na in :gh:`141770`.) -.. _whatsnew315-frame-pointers: - -* CPython is now built with frame pointers enabled by default - (:pep:`831`). Pass :option:`--without-frame-pointers` to opt out. - Authors of C extensions and native libraries built with custom build - systems should add ``-fno-omit-frame-pointer`` and - ``-mno-omit-leaf-frame-pointer`` to their own ``CFLAGS`` to keep the - unwind chain intact. - (Contributed by Pablo Galindo Salgado and Savannah Ostrowski in :gh:`149201`.) - .. _whatsnew315-windows-tail-calling-interpreter: * 64-bit builds using Visual Studio 2026 (MSVC 18) may now use the new From 2995d4565978365233e01ad8b472c5512f3991be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ux=C3=ADo=20Garc=C3=ADa=20Andrade?= Date: Tue, 5 May 2026 02:51:39 +0200 Subject: [PATCH 08/10] gh-137293: Ignore Exceptions when searching ELF File in Remote Debug (#137309) --- .../2025-08-01-20-31-30.gh-issue-137293.4x3JbV.rst | 1 + Python/remote_debug.h | 1 + 2 files changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-08-01-20-31-30.gh-issue-137293.4x3JbV.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-01-20-31-30.gh-issue-137293.4x3JbV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-01-20-31-30.gh-issue-137293.4x3JbV.rst new file mode 100644 index 00000000000000..83289d4d9bc875 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-01-20-31-30.gh-issue-137293.4x3JbV.rst @@ -0,0 +1 @@ +Fix :exc:`SystemError` when searching ELF Files in :func:`sys.remote_exec`. diff --git a/Python/remote_debug.h b/Python/remote_debug.h index 7628fb04ba5bae..6c089a834dcd40 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -781,6 +781,7 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c } if (strstr(filename, substr)) { + PyErr_Clear(); retval = search_elf_file_for_section(handle, secname, start, path); if (retval && (validator == NULL || validator(handle, retval))) From ed99680487b347997061ebd0138d49e601b20de8 Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Tue, 5 May 2026 03:36:43 +0200 Subject: [PATCH 09/10] gh-130472: Use fancycompleter in import completions (#148188) --- Lib/_pyrepl/_module_completer.py | 81 ++++++++++++++----- Lib/_pyrepl/fancycompleter.py | 76 +++++++++-------- Lib/_pyrepl/readline.py | 26 ++++-- Lib/test/test_pyrepl/test_fancycompleter.py | 17 +++- Lib/test/test_pyrepl/test_pyrepl.py | 51 +++++++++++- ...-04-08-21-39-01.gh-issue-130472.4Bk6qH.rst | 1 + 6 files changed, 190 insertions(+), 62 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-04-08-21-39-01.gh-issue-130472.4Bk6qH.rst diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index a22b0297b24ea0..17bf5cdc819542 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from itertools import chain from tokenize import TokenInfo +from .fancycompleter import safe_getattr TYPE_CHECKING = False @@ -71,41 +72,69 @@ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self._curr_sys_path: list[str] = sys.path[:] self._stdlib_path = os.path.dirname(importlib.__path__[0]) - def get_completions(self, line: str) -> tuple[list[str], CompletionAction | None] | None: + def get_completions( + self, line: str, *, include_values: bool = True + ) -> tuple[list[str], list[Any], CompletionAction | None] | None: """Return the next possible import completions for 'line'. For attributes completion, if the module to complete from is not imported, also return an action (prompt + callback to run if the user press TAB again) to import the module. + + If *include_values* is false, the returned values list is empty and + attribute values are not resolved. """ result = ImportParser(line).parse() if not result: return None try: - return self.complete(*result) + return self.complete(*result, include_values=include_values) except Exception: # Some unexpected error occurred, make it look like # no completions are available - return [], None - - def complete(self, from_name: str | None, name: str | None) -> tuple[list[str], CompletionAction | None]: + return [], [], None + + def complete( + self, + from_name: str | None, + name: str | None, + *, + include_values: bool = True, + ) -> tuple[list[str], list[Any], CompletionAction | None]: if from_name is None: # import x.y.z assert name is not None path, prefix = self.get_path_and_prefix(name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules], None + names = [self.format_completion(path, module) for module in modules] + # These are always modules, use dummy values to get the right color + values = [sys] * len(names) if include_values else [] + return names, values, None if name is None: # from x.y.z path, prefix = self.get_path_and_prefix(from_name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules], None + names = [self.format_completion(path, module) for module in modules] + # These are always modules, use dummy values to get the right color + values = [sys] * len(names) if include_values else [] + return names, values, None # from x.y import z submodules = self.find_modules(from_name, name) - attributes, action = self.find_attributes(from_name, name) - return sorted({*submodules, *attributes}), action + attr_names, attr_module, action = self._find_attributes(from_name, name) + all_names = sorted({*submodules, *attr_names}) + if not include_values: + return all_names, [], action + + # Build values list matching the sorted order: + # submodules use `sys` as a dummy value so they get the 'module' color, + # attributes use their actual value. + attr_map = {} + if attr_module is not None: + attr_map = {n: safe_getattr(attr_module, n) for n in attr_names} + all_values = [attr_map[n] if n in attr_map else sys for n in all_names] + return all_names, all_values, action def find_modules(self, path: str, prefix: str) -> list[str]: """Find all modules under 'path' that start with 'prefix'.""" @@ -166,31 +195,43 @@ def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: return (isinstance(module_info.module_finder, FileFinder) and module_info.module_finder.path == self._stdlib_path) - def find_attributes(self, path: str, prefix: str) -> tuple[list[str], CompletionAction | None]: + def find_attributes( + self, path: str, prefix: str + ) -> tuple[list[str], list[Any], CompletionAction | None]: """Find all attributes of module 'path' that start with 'prefix'.""" - attributes, action = self._find_attributes(path, prefix) - # Filter out invalid attribute names - # (for example those containing dashes that cannot be imported with 'import') - return [attr for attr in attributes if attr.isidentifier()], action + attributes, module, action = self._find_attributes(path, prefix) + if module is not None: + values = [safe_getattr(module, attr) for attr in attributes] + else: + values = [] + return attributes, values, action - def _find_attributes(self, path: str, prefix: str) -> tuple[list[str], CompletionAction | None]: + def _find_attributes( + self, path: str, prefix: str + ) -> tuple[list[str], ModuleType | None, CompletionAction | None]: path = self._resolve_relative_path(path) # type: ignore[assignment] if path is None: - return [], None + return [], None, None imported_module = sys.modules.get(path) if not imported_module: if path in self._failed_imports: # Do not propose to import again - return [], None + return [], None, None imported_module = self._maybe_import_module(path) if not imported_module: - return [], self._get_import_completion_action(path) + return [], None, self._get_import_completion_action(path) try: module_attributes = dir(imported_module) except Exception: module_attributes = [] - return [attr_name for attr_name in module_attributes - if self.is_suggestion_match(attr_name, prefix)], None + # Filter out invalid attribute names, such as dashes that cannot be + # imported with 'import'. + names = [ + attr_name for attr_name in module_attributes + if (self.is_suggestion_match(attr_name, prefix) + and attr_name.isidentifier()) + ] + return names, imported_module, None def is_suggestion_match(self, module_name: str, prefix: str) -> bool: if prefix: diff --git a/Lib/_pyrepl/fancycompleter.py b/Lib/_pyrepl/fancycompleter.py index 7a639afd74ef3c..ac4f0afdbc721c 100644 --- a/Lib/_pyrepl/fancycompleter.py +++ b/Lib/_pyrepl/fancycompleter.py @@ -3,11 +3,53 @@ # # All Rights Reserved """Colorful tab completion for Python prompt""" +from __future__ import annotations + from _colorize import ANSIColors, get_colors, get_theme import rlcompleter import keyword import types +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + from _colorize import Theme + + +def safe_getattr(obj, name): + # Mirror rlcompleter's safeguards so completion does not + # call properties or reify lazy module attributes. + if isinstance(getattr(type(obj), name, None), property): + return None + if (isinstance(obj, types.ModuleType) + and isinstance(obj.__dict__.get(name), types.LazyImportType) + ): + return obj.__dict__.get(name) + return getattr(obj, name, None) + + +def colorize_matches(names: list[str], values: list[Any], theme: Theme) -> list[str]: + return [ + _color_for_obj(name, obj, theme) + for name, obj in zip(names, values) + ] + +def _color_for_obj(name: str, value: Any, theme: Theme) -> str: + t = type(value) + color = _color_by_type(t, theme) + return f"{color}{name}{ANSIColors.RESET}" + + +def _color_by_type(t, theme): + typename = t.__name__ + # this is needed e.g. to turn method-wrapper into method_wrapper, + # because if we want _colorize.FancyCompleter to be "dataclassable" + # our keys need to be valid identifiers. + typename = typename.replace('-', '_').replace('.', '_') + return getattr(theme.fancycompleter, typename, ANSIColors.RESET) + + class Completer(rlcompleter.Completer): """ When doing something like a.b., keep the full a.b.attr completion @@ -143,21 +185,7 @@ def _attr_matches(self, text): word[:n] == attr and not (noprefix and word[:n+1] == noprefix) ): - # Mirror rlcompleter's safeguards so completion does not - # call properties or reify lazy module attributes. - if isinstance(getattr(type(thisobject), word, None), property): - value = None - elif ( - isinstance(thisobject, types.ModuleType) - and isinstance( - thisobject.__dict__.get(word), - types.LazyImportType, - ) - ): - value = thisobject.__dict__.get(word) - else: - value = getattr(thisobject, word, None) - + value = safe_getattr(thisobject, word) names.append(word) values.append(value) if names or not noprefix: @@ -170,23 +198,7 @@ def _attr_matches(self, text): return expr, attr, names, values def colorize_matches(self, names, values): - return [ - self._color_for_obj(name, obj) - for name, obj in zip(names, values) - ] - - def _color_for_obj(self, name, value): - t = type(value) - color = self._color_by_type(t) - return f"{color}{name}{ANSIColors.RESET}" - - def _color_by_type(self, t): - typename = t.__name__ - # this is needed e.g. to turn method-wrapper into method_wrapper, - # because if we want _colorize.FancyCompleter to be "dataclassable" - # our keys need to be valid identifiers. - typename = typename.replace('-', '_').replace('.', '_') - return getattr(self.theme.fancycompleter, typename, ANSIColors.RESET) + return colorize_matches(names, values, self.theme) def commonprefix(names): diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index f8f1727d2a1d1f..e4370b0d1462ea 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -40,7 +40,7 @@ from .completing_reader import CompletingReader, stripcolor from .console import Console as ConsoleType from ._module_completer import ModuleCompleter, make_default_module_completer -from .fancycompleter import Completer as FancyCompleter +from .fancycompleter import Completer as FancyCompleter, colorize_matches Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] @@ -104,6 +104,7 @@ class ReadlineConfig: readline_completer: Completer | None = None completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") module_completer: ModuleCompleter = field(default_factory=make_default_module_completer) + colorize_completions: Callable[[list[str], list[Any]], list[str]] | None = None @dataclass(kw_only=True) class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): @@ -169,8 +170,17 @@ def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None return result, None def get_module_completions(self) -> tuple[list[str], CompletionAction | None] | None: - line = self.get_line() - return self.config.module_completer.get_completions(line) + line = stripcolor(self.get_line()) + colorize_completions = self.config.colorize_completions + result = self.config.module_completer.get_completions( + line, include_values=bool(colorize_completions) + ) + if result is None: + return None + names, values, action = result + if colorize_completions: + names = colorize_completions(names, values) + return names, action def get_trimmed_history(self, maxlength: int) -> list[str]: if maxlength >= 0: @@ -616,13 +626,19 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) - _wrapper.config.module_completer = ModuleCompleter(namespace) use_basic_completer = ( not sys.flags.ignore_environment and os.getenv("PYTHON_BASIC_COMPLETER") ) completer_cls = RLCompleter if use_basic_completer else FancyCompleter - _wrapper.config.readline_completer = completer_cls(namespace).complete + completer = completer_cls(namespace) + _wrapper.config.readline_completer = completer.complete + if isinstance(completer, FancyCompleter) and completer.use_colors: + theme = completer.theme + def _colorize(names: list[str], values: list[object]) -> list[str]: + return colorize_matches(names, values, theme) + _wrapper.config.colorize_completions = _colorize + _wrapper.config.module_completer = ModuleCompleter(namespace) # this is not really what readline.c does. Better than nothing I guess import builtins diff --git a/Lib/test/test_pyrepl/test_fancycompleter.py b/Lib/test/test_pyrepl/test_fancycompleter.py index d2646cd3050428..0ffc1ed97b557a 100644 --- a/Lib/test/test_pyrepl/test_fancycompleter.py +++ b/Lib/test/test_pyrepl/test_fancycompleter.py @@ -1,11 +1,17 @@ import importlib +import inspect import os import types import unittest from _colorize import ANSIColors, get_theme from _pyrepl.completing_reader import stripcolor -from _pyrepl.fancycompleter import Completer, commonprefix +from _pyrepl.fancycompleter import ( + Completer, + colorize_matches, + commonprefix, + _color_for_obj, +) from test.support.import_helper import ready_to_import class MockPatch: @@ -36,6 +42,11 @@ def test_commonprefix(self): self.assertEqual(commonprefix(['isalpha', 'isdigit']), 'is') self.assertEqual(commonprefix([]), '') + def test_colorize_matches_signature(self): + signature = inspect.signature(colorize_matches) + + self.assertEqual(list(signature.parameters), ["names", "values", "theme"]) + def test_complete_attribute(self): compl = Completer({'a': None}, use_colors=False) self.assertEqual(compl.attr_matches('a.'), ['a.__']) @@ -168,8 +179,8 @@ def test_complete_global_colored(self): self.assertEqual(compl.global_matches('nothing'), []) def test_colorized_match_is_stripped(self): - compl = Completer({'a': 42}, use_colors=True) - match = compl._color_for_obj('spam', 1) + theme = get_theme() + match = _color_for_obj('spam', 1, theme) self.assertEqual(stripcolor(match), 'spam') def test_complete_with_indexer(self): diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 9d0a4ed5316a3f..4240a3c3174959 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -36,6 +36,7 @@ multiline_input, code_to_events, ) +from _colorize import ANSIColors, get_theme from _pyrepl.console import Event from _pyrepl.completing_reader import stripcolor from _pyrepl._module_completer import ( @@ -43,7 +44,7 @@ ModuleCompleter, HARDCODED_SUBMODULES, ) -from _pyrepl.fancycompleter import Completer as FancyCompleter +from _pyrepl.fancycompleter import Completer as FancyCompleter, colorize_matches import _pyrepl.readline as pyrepl_readline from _pyrepl.readline import ( ReadlineAlikeReader, @@ -1102,6 +1103,8 @@ def test_setup_ignores_basic_completer_env_when_env_is_disabled(self): class FakeFancyCompleter: def __init__(self, namespace): self.namespace = namespace + self.use_colors = Mock() + self.theme = Mock() def complete(self, text, state): return None @@ -1704,7 +1707,7 @@ def test_suggestions_and_messages(self) -> None: result = completer.get_completions(code) self.assertEqual(result is None, expected is None) if result: - compl, act = result + compl, _values, act = result self.assertEqual(compl, expected[0]) self.assertEqual(act is None, expected[1] is None) if act: @@ -1716,6 +1719,50 @@ def test_suggestions_and_messages(self) -> None: new_imports = sys.modules.keys() - _imported self.assertSetEqual(new_imports, expected_imports) + def test_colorize_import_completions(self) -> None: + theme = get_theme() + type_color = theme.fancycompleter.type + module_color = theme.fancycompleter.module + R = ANSIColors.RESET + + colorize = lambda names, values: colorize_matches(names, values, theme) + config = ReadlineConfig(colorize_completions=colorize) + reader = ReadlineAlikeReader( + console=FakeConsole(events=[]), + config=config, + ) + + # "from collections import de" -> defaultdict (type) and deque (type) + reader.buffer = list("from collections import de") + reader.pos = len(reader.buffer) + names, action = reader.get_module_completions() + self.assertEqual(names, [ + f"{type_color}defaultdict{R}", + f"{type_color}deque{R}", + ]) + self.assertIsNone(action) + + # "from importlib.m" has submodule completions colored as modules + reader.buffer = list("from importlib.m") + reader.pos = len(reader.buffer) + names, action = reader.get_module_completions() + self.assertEqual(names, [ + f"{module_color}importlib.machinery{R}", + f"{module_color}importlib.metadata{R}", + ]) + self.assertIsNone(action) + + # Make sure attributes take precedence over submodules when both exist + # Here we're using `unittest.main` which happens to be both a module and an attribute + reader.buffer = list("from unittest import m") + reader.pos = len(reader.buffer) + names, action = reader.get_module_completions() + self.assertEqual(names, [ + f"{type_color}main{R}", # Ensure that `main` is colored as an attribute (class in this case) + f"{module_color}mock{R}", + ]) + self.assertIsNone(action) + # Audit hook used to check for stdlib modules import side-effects # Defined globally to avoid adding one hook per test run (refleak) diff --git a/Misc/NEWS.d/next/Library/2026-04-08-21-39-01.gh-issue-130472.4Bk6qH.rst b/Misc/NEWS.d/next/Library/2026-04-08-21-39-01.gh-issue-130472.4Bk6qH.rst new file mode 100644 index 00000000000000..9384843b7c253b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-08-21-39-01.gh-issue-130472.4Bk6qH.rst @@ -0,0 +1 @@ +Integrate fancycompleter with import completions. From 5dd21617164cf69e848a70e3d7e32faf0bc3f279 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 4 May 2026 20:30:03 -0700 Subject: [PATCH 10/10] gh-137840: Document PEP 728 (#149207) --- Doc/library/typing.rst | 65 ++++++++++++++++++++++++++++++++++++++---- Doc/whatsnew/3.15.rst | 11 ++++++- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index f45a22addbb56a..1e544de74a040f 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2792,6 +2792,37 @@ types. y: int z: int + By default, a ``TypedDict`` is open, meaning that it may contain additional keys + at runtime beyond those defined in the class body. The *closed* class argument can + be used to control this; if ``closed=True``, the ``TypedDict`` cannot contain additional keys. + + :: + + class ClosedPoint(TypedDict, closed=True): + x: int + y: int + + class ClosedPoint3D(ClosedPoint): # type checker error: cannot add keys to a closed TypedDict + z: int + + Setting ``closed=False`` explicitly requests the default open behavior. If the argument is not + passed, this state is inherited from the parent class. + + In addition to being open or closed, a ``TypedDict`` can also be configured to have extra items. + If the *extra_items* class argument is set to a type, the ``TypedDict`` can contain arbitrary + additional keys, but the values of those keys must be of the specified type. + + :: + + class ExtraItemsPoint(TypedDict, extra_items=int): + x: int + y: int + + point: ExtraItemsPoint = {'x': 1, 'y': 2, 'anything': 3} # OK + + The *extra_items* argument is also inherited through subclassing. It is unset + by default, and it may not be used together with the *closed* argument. + A ``TypedDict`` cannot inherit from a non-\ ``TypedDict`` class, except for :class:`Generic`. For example:: @@ -2825,8 +2856,8 @@ types. group: list[T] A ``TypedDict`` can be introspected via annotations dicts - (see :ref:`annotations-howto` for more information on annotations best practices), - :attr:`__total__`, :attr:`__required_keys__`, and :attr:`__optional_keys__`. + (see :ref:`annotations-howto` for more information on annotations best practices) + and the following attributes: .. attribute:: __total__ @@ -2896,8 +2927,6 @@ types. ``__required_keys__`` and ``__optional_keys__`` rely on may not work properly, and the values of the attributes may be incorrect. - Support for :data:`ReadOnly` is reflected in the following attributes: - .. attribute:: __readonly_keys__ A :class:`frozenset` containing the names of all read-only keys. Keys @@ -2912,6 +2941,14 @@ types. .. versionadded:: 3.13 + .. attribute:: __closed__ + + The value of the *closed* class argument. It can be ``True``, ``False``, or :data:`None`. + + .. attribute:: __extra_items__ + + The value of the *extra_items* class argument. It can be a valid type or :data:`NoExtraItems`. + See the `TypedDict `_ section in the typing documentation for more examples and detailed rules. .. versionadded:: 3.8 @@ -2931,7 +2968,10 @@ types. Removed support for the keyword-argument method of creating ``TypedDict``\ s. .. versionchanged:: 3.13 - Support for the :data:`ReadOnly` qualifier was added. + Support for the :data:`ReadOnly` qualifier was added. See :pep:`705`. + + .. versionchanged:: next + Support for the *closed* and *extra_items* class arguments was added. See :pep:`728`. Protocols @@ -3679,6 +3719,21 @@ Introspection helpers .. versionadded:: 3.13 +.. data:: NoExtraItems + + A :class:`sentinel` object used to indicate that a :class:`TypedDict` + does not have the *extra_items* class argument. + + .. doctest:: + + >>> from typing import TypedDict, NoExtraItems + >>> class Point(TypedDict): + ... x: int + ... y: int + ... + >>> Point.__extra_items__ is NoExtraItems + True + Constant -------- diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index e98c483baec823..7c4ff0d8775168 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -81,7 +81,7 @@ Summary -- Release highlights ` * :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding ` -* :pep:`728`: ``TypedDict`` with typed extra items +* :pep:`728`: :ref:`TypedDict with typed extra items ` * :pep:`747`: :ref:`Annotating type forms with TypeForm ` * :pep:`800`: Disjoint bases in the type system @@ -1521,6 +1521,15 @@ typing (Contributed by Jelle Zijlstra in :gh:`145033`.) +.. _whatsnew315-typeddict: + +* :pep:`728`: Add support in :class:`~typing.TypedDict` for the *closed* + and *extra_items* class arguments. A closed :class:`~typing.TypedDict` + does not allow extra keys beyond those specified in the class body, while + a :class:`~typing.TypedDict` with ``extra_items`` allows arbitrary extra + items where the values are of the specified type. (Contributed by Angela + Liss in :gh:`137840`.) + * Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. (Contributed by Nikita Sobolev in :gh:`137191`.)