From 91f6979c906d4546b2c6dee96f9543a769aafed1 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 25 Mar 2026 21:13:04 +0100 Subject: [PATCH 001/165] Moved kernprof pre-import code to another module - `line_profiler/curated_profiling.py` New module for setting up profiling in a curated environment - `ClassifiedPreimportTargets.from_targets()` Method for creating a `ClassifiedPreimportTargets` instance, facilitating writing pre-import modules in a replicable and portable manner - `ClassifiedPreimportTargets.write_preimport_module()` Method for writing a pre-import module based on an instance; also fixed bug where the body of the written module was intercepted without appearing in the debug output - `kernprof.py` - `_gather_preimport_targets()` Migrated to `line_profiler.curated_profiling` - `_write_preimports()` Now using the new `ClassifiedPreimportTargets` class, moving esp. the logic to the `write_preimport_module()` method --- kernprof.py | 84 ++--------------- line_profiler/curated_profiling.py | 140 +++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 76 deletions(-) create mode 100644 line_profiler/curated_profiling.py diff --git a/kernprof.py b/kernprof.py index 8a7c4d6a..0a198acb 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1043,72 +1043,16 @@ def _write_tempfile(source, content, options): ) -def _gather_preimport_targets(options, exclude): - """ - Used in _write_preimports - """ - from line_profiler.autoprofile.util_static import modpath_to_modname - from line_profiler.autoprofile.eager_preimports import is_dotted_path - - filtered_targets = [] - recurse_targets = [] - invalid_targets = [] - for target in options.prof_mod: - if is_dotted_path(target): - modname = target - else: - # Paths already normalized by - # `_normalize_profiling_targets()` - if not os.path.exists(target): - invalid_targets.append(target) - continue - if any(os.path.samefile(target, excluded) for excluded in exclude): - # Ignore the script to be run in eager importing - # (`line_profiler.autoprofile.autoprofile.run()` will - # handle it) - continue - modname = modpath_to_modname(target, hide_init=False) - if modname is None: # Not import-able - invalid_targets.append(target) - continue - if modname.endswith('.__init__'): - modname = modname.rpartition('.')[0] - filtered_targets.append(modname) - else: - recurse_targets.append(modname) - if invalid_targets: - invalid_targets = sorted(set(invalid_targets)) - msg = ( - '{} profile-on-import target{} cannot be converted to ' - 'dotted-path form: {!r}'.format( - len(invalid_targets), - '' if len(invalid_targets) == 1 else 's', - invalid_targets, - ) - ) - warnings.warn(msg) - diagnostics.log.warning(msg) - - return filtered_targets, recurse_targets - - def _write_preimports(prof, options, exclude): """ Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. """ - from line_profiler.autoprofile.eager_preimports import ( - write_eager_import_module, - ) from line_profiler.autoprofile.autoprofile import ( _extend_line_profiler_for_profiling_imports as upgrade_profiler, ) + from line_profiler.curated_profiling import ClassifiedPreimportTargets - filtered_targets, recurse_targets = _gather_preimport_targets( - options, exclude - ) - if not (filtered_targets or recurse_targets): - return # We could've done everything in-memory with `io.StringIO` and `exec()`, # but that results in indecipherable tracebacks should anything goes wrong; # so we write to a tempfile and `execfile()` it @@ -1116,26 +1060,14 @@ def _write_preimports(prof, options, exclude): temp_mod_path = _touch_tempfile( dir=options.tmpdir, prefix='kernprof-eager-preimports-', suffix='.py' ) - write_module_kwargs = { - 'dotted_paths': filtered_targets, - 'recurse': recurse_targets, - 'static': options.static, - } - temp_file = open(temp_mod_path, mode='w') - if options.debug: - with StringIO() as sio: - write_eager_import_module(stream=sio, **write_module_kwargs) - code = sio.getvalue() - with temp_file as fobj: - print(code, file=fobj) - diagnostics.log.debug( - 'Wrote temporary module for pre-imports to ' - f'{short_string_path(temp_mod_path)!r}' + with open(temp_mod_path, mode='w') as fobj: + preimports = ClassifiedPreimportTargets.from_targets( + options.prof_mod, exclude, ) - else: - with temp_file as fobj: - write_eager_import_module(stream=fobj, **write_module_kwargs) - if not options.dryrun: + preimports.write_preimport_module( + fobj, debug=options.debug, static=options.static, + ) + if preimports and not options.dryrun: ns = {} # Use a fresh namespace execfile(temp_mod_path, ns, ns) # Delete the tempfile ASAP if its execution succeeded diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py new file mode 100644 index 00000000..d5bf5b3c --- /dev/null +++ b/line_profiler/curated_profiling.py @@ -0,0 +1,140 @@ +""" +Tools for setting up profiling in a curated environment (e.g. with +the use of :py:mod:`kernprof`). +""" +from __future__ import annotations + +import dataclasses +import os +import warnings +from collections.abc import Collection +from io import StringIO +from textwrap import indent +from typing import TextIO +from typing_extensions import Self + +from . import _diagnostics as diagnostics +from .autoprofile.util_static import modpath_to_modname +from .autoprofile.eager_preimports import ( + is_dotted_path, write_eager_import_module, +) +from .cli_utils import short_string_path + + +__all__ = ('ClassifiedPreimportTargets',) + + +@dataclasses.dataclass +class ClassifiedPreimportTargets: + """ + Pre-import targets classified into three bins: ``regular`` targets, + targets to ``recurse`` into, and ``invalid`` targets + """ + regular: list[str] = dataclasses.field(default_factory=list) + recurse: list[str] = dataclasses.field(default_factory=list) + invalid: list[str] = dataclasses.field(default_factory=list) + + def __bool__(self) -> bool: + return bool(self.regular or self.recurse) + + def write_preimport_module( + self, fobj: TextIO, *, debug: bool | None = None, **kwargs + ) -> None: + """ + Convenience interface with + :py:func:`~.write_eager_import_module`, writing a module which + when imported sets up profiling of the targets. + + Args: + fobj (TextIO): + File object to write said module to. + debug (Optional[bool]): + Whether to generate debugging outputs. + kwargs: + Passed to :py:func:`~.write_eager_import_module`. + """ + if self.invalid: + invalid_targets = sorted(set(self.invalid)) + msg = ( + '{} profile-on-import target{} cannot be converted to ' + 'dotted-path form: {!r}'.format( + len(invalid_targets), + '' if len(invalid_targets) == 1 else 's', + invalid_targets, + ) + ) + warnings.warn(msg) + diagnostics.log.warning(msg) + + if not self: + return None + write_module_kwargs = { + 'dotted_paths': self.regular, + 'recurse': self.recurse, + **kwargs, + } + if diagnostics.DEBUG if debug is None else debug: + with StringIO() as sio: + write_eager_import_module(stream=sio, **write_module_kwargs) + code = sio.getvalue() + print(code, file=fobj) + if hasattr(fobj, 'name'): + fobj_repr = repr(short_string_path(str(fobj.name))) + else: + fobj_repr = repr(fobj) # Fall back + diagnostics.log.debug( + f'Wrote temporary module for pre-imports to {fobj_repr}:\n' + + indent(code, ' ') + ) + else: + write_eager_import_module(stream=fobj, **write_module_kwargs) + + @classmethod + def from_targets( + cls, + targets: Collection[str], + exclude: Collection[os.PathLike[str] | str] = (), + ) -> Self: + """ + Create an instance based on a collection of targets + (like what is supplied to :cmd:`kernprof --prof-mod=...`). + + Args: + targets (Collection[str]) + Collection of dotted paths and filenames to profile. + exclude (Collection[str]) + Collections of filenames which are explicitly excluded + from being profiled. + + Return: + New instance. + """ + filtered_targets = [] + recurse_targets = [] + invalid_targets = [] + for target in targets: + if is_dotted_path(target): + modname = target + else: + # Paths already normalized by + # `_normalize_profiling_targets()` + if not os.path.exists(target): + invalid_targets.append(target) + continue + if any( + os.path.samefile(target, excluded) for excluded in exclude + ): + # Ignore the script to be run in eager importing + # (`line_profiler.autoprofile.autoprofile.run()` + # will handle it) + continue + modname = modpath_to_modname(target, hide_init=False) + if modname is None: # Not import-able + invalid_targets.append(target) + continue + if modname.endswith('.__init__'): + modname = modname.rpartition('.')[0] + filtered_targets.append(modname) + else: + recurse_targets.append(modname) + return cls(filtered_targets, recurse_targets, invalid_targets) From 4e8cc67efe8a8fc891d6b3e94393d00587c3a8d1 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 26 Mar 2026 00:11:02 +0100 Subject: [PATCH 002/165] Removed redundant check in kernprof --- kernprof.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernprof.py b/kernprof.py index 0a198acb..bd4ef5b1 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1363,8 +1363,7 @@ def _post_profile(options, prof): for _ in range(prof.enable_count): prof.disable_by_count() # Restore the state of the global `@line_profiler.profile` - if options.global_profiler: - options.install_profiler(None) + options.install_profiler(None) if __name__ == '__main__': From 9a5a0bc0f700009133e9e9fa80c76d8a0163af1d Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 26 Mar 2026 03:34:04 +0100 Subject: [PATCH 003/165] Move code between kernprof & `~.curated_profiling` - `kernprof.py::_manage_profiler` `line_profiler/curated_profiling.py::CuratedProfilerContext` New context-manager classes for handling profiler setup and teardown - `kernprof.py::_pre_profile()` Refactored into the above context managers and other private functions (`_prepare_profiler()`, `_prepare_exec_script()`) --- kernprof.py | 65 ++++++++++++++--------- line_profiler/curated_profiling.py | 83 ++++++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 29 deletions(-) diff --git a/kernprof.py b/kernprof.py index bd4ef5b1..5a873571 100755 --- a/kernprof.py +++ b/kernprof.py @@ -187,7 +187,6 @@ def main(): """ # noqa: E501 import atexit -import builtins import functools import os import sys @@ -198,9 +197,7 @@ def main(): import shutil import tempfile import time -import warnings from argparse import ArgumentParser -from io import StringIO from operator import methodcaller from runpy import run_module from pathlib import Path @@ -1163,13 +1160,42 @@ def _call_with_diagnostics(options, func, *args, **kwargs): return func(*args, **kwargs) -def _pre_profile(options, module, exit_on_error): +class _manage_profiler: """ Prepare the environment to execute profiling with requested options. Note: modifies ``options`` with extra attributes. """ + def __init__(self, options, module, exit_on_error): + self.options = options + self.module = module + self.exit_on_error = exit_on_error + + def __enter__(self): + from line_profiler.curated_profiling import CuratedProfilerContext + + self.prof = _prepare_profiler( + self.options, self.module, self.exit_on_error, + ) + self._ctx = CuratedProfilerContext( + self.prof, insert_builtin=self.options.builtin, + ) + self._ctx.install() + script_file = _prepare_exec_script( + self.options, self.module, self.prof, self.exit_on_error, + ) + return self.prof, script_file + + def __exit__(self, *_, **__): + _post_profile(self.options, self.prof) + self._ctx.uninstall() + + +def _prepare_profiler(options, module, exit_on_error): + """ + Set up the appropriate profiler instance. + """ if not options.outfile: extension = 'lprof' if options.line_by_line else 'prof' options.outfile = f'{os.path.basename(options.script)}.{extension}' @@ -1199,24 +1225,21 @@ def _pre_profile(options, module, exit_on_error): execfile(setup_file, ns, ns) if options.line_by_line: - prof = line_profiler.LineProfiler() options.builtin = True + return line_profiler.LineProfiler() elif Profile.__module__ == 'profile': raise RuntimeError( 'non-line-by-line profiling depends on cProfile, ' 'which is not available on this platform' ) else: - prof = ContextualProfile() + return ContextualProfile() - # Overwrite the explicit decorator - global_profiler = line_profiler.profile - install_profiler = global_profiler._kernprof_overwrite - install_profiler(prof) - - if options.builtin: - builtins.__dict__['profile'] = prof +def _prepare_exec_script(options, module, prof, exit_on_error): + """ + Set up the script to be executed among other things. + """ if module: script_file = find_module_script( options.script, static=options.static, exit_on_error=exit_on_error @@ -1245,8 +1268,6 @@ def _pre_profile(options, module, exit_on_error): exclude = set() if module else {script_file} _write_preimports(prof, options, exclude) - options.global_profiler = global_profiler - options.install_profiler = install_profiler if options.output_interval and not options.dryrun: options.rt = RepeatedTimer( max(options.output_interval, 1), prof.dump_stats, options.outfile @@ -1254,7 +1275,7 @@ def _pre_profile(options, module, exit_on_error): else: options.rt = None options.original_stdout = sys.stdout - return script_file, prof + return script_file def _main_profile(options, module=False, exit_on_error=True): @@ -1262,9 +1283,10 @@ def _main_profile(options, module=False, exit_on_error=True): Called by :py:func:`main()` for the actual execution and profiling of code after initial parsing of options; not to be invoked on its own. """ - script_file, prof = _pre_profile(options, module, exit_on_error) call = functools.partial(_call_with_diagnostics, options) - try: + with _manage_profiler( + options, module, exit_on_error, + ) as (prof, script_file): rmod = functools.partial( run_module, run_name='__main__', alter_sys=True ) @@ -1315,8 +1337,6 @@ def _main_profile(options, module=False, exit_on_error=True): module_ns, module_ns, ) - finally: - _post_profile(options, prof) def _post_profile(options, prof): @@ -1359,11 +1379,6 @@ def _post_profile(options, prof): f'{quote(py_exe)} -m {show_mod} ' f'{quote(short_outfile)}' ) - # Fully disable the profiler - for _ in range(prof.enable_count): - prof.disable_by_count() - # Restore the state of the global `@line_profiler.profile` - options.install_profiler(None) if __name__ == '__main__': diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index d5bf5b3c..ecad97a9 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -4,24 +4,27 @@ """ from __future__ import annotations +import builtins import dataclasses +import functools import os import warnings -from collections.abc import Collection +from collections.abc import Callable, Collection from io import StringIO from textwrap import indent -from typing import TextIO +from typing import Any, TextIO from typing_extensions import Self -from . import _diagnostics as diagnostics +from . import _diagnostics as diagnostics, profile as _global_profiler from .autoprofile.util_static import modpath_to_modname from .autoprofile.eager_preimports import ( is_dotted_path, write_eager_import_module, ) from .cli_utils import short_string_path +from .profiler_mixin import ByCountProfilerMixin -__all__ = ('ClassifiedPreimportTargets',) +__all__ = ('ClassifiedPreimportTargets', 'CuratedProfilerContext') @dataclasses.dataclass @@ -138,3 +141,75 @@ def from_targets( else: recurse_targets.append(modname) return cls(filtered_targets, recurse_targets, invalid_targets) + + +class CuratedProfilerContext: + """ + Context manager for handling various bookkeeping tasks when setting + up and tearing down profiling: + + - Slipping ``prof`` into the builtin namespace (if + ``insert_builtin`` is true) and :py::deco:`~.profile` + - At exit, clearing the ``enable_count`` of ``prof``, properly + disabling it + + Note: + The attributes on this object are to be considered + implementation details, but not its methods and their + signatures. + """ + def __init__( + self, + prof: ByCountProfilerMixin, + insert_builtin: bool = False, + builtin_loc: str = 'profile', + ) -> None: + self.prof = prof + self.insert_builtin = insert_builtin + self.builtin_loc = builtin_loc + self._installed = False + self._global_install = _global_profiler._kernprof_overwrite + + def install(self) -> None: + def del_builtin_profile() -> None: + delattr(builtins, self.builtin_loc) + + def set_builtin_profile(old: Any) -> None: + setattr(builtins, self.builtin_loc, old) + + if self._installed: + return + # Overwrite the explicit profiler (`@line_profiler.profile`) + self._global_install(self.prof) # type: ignore[arg-type] + # Set up hooks to deal with inserting `.prof` as a builtin name + if self.insert_builtin: + try: + old = getattr(builtins, self.builtin_loc) + except AttributeError: + self._restore: Callable[[], None] = del_builtin_profile + else: + self._restore = functools.partial(set_builtin_profile, old) + set_builtin_profile(self.prof) + self._installed = True + + def uninstall(self) -> None: + if not self._installed: + return + # Restore the `builtins` namespace + if ( + self.insert_builtin + and getattr(builtins, self.builtin_loc, None) is self.prof + ): + self._restore() + # Fully disable the profiler + for _i in range(getattr(self.prof, 'enable_count', 0)): + self.prof.disable_by_count() + # Restore the state of the global `@line_profiler.profile` + self._global_install(None) # type: ignore[arg-type] + self._installed = False + + def __enter__(self) -> None: + self.install() + + def __exit__(self, *_, **__) -> None: + self.uninstall() From 607d7a340928fe9f96e82869213a283e63b2663f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 28 Mar 2026 02:55:21 +0100 Subject: [PATCH 004/165] New subpackage for profiling child processes line_profiler/_child_process_profiling/cache.py::LineProfilingCache New class for passing info onto child processes so that profiling can resume there line_profiler/pth_hook.py New submodule for the .pth-file-based solution to propagating profiling into child processes: write_pth_hook() In the main process, write the temporary .pth file to be loaded in child processes load_pth_hook() Called by the .pth in child process, loading the cache and setting up profiling based thereon --- .../_child_process_profiling/__init__.py | 8 + .../_child_process_profiling/cache.py | 142 ++++++++++++ .../_child_process_profiling/pth_hook.py | 206 ++++++++++++++++++ 3 files changed, 356 insertions(+) create mode 100644 line_profiler/_child_process_profiling/__init__.py create mode 100644 line_profiler/_child_process_profiling/cache.py create mode 100644 line_profiler/_child_process_profiling/pth_hook.py diff --git a/line_profiler/_child_process_profiling/__init__.py b/line_profiler/_child_process_profiling/__init__.py new file mode 100644 index 00000000..4ff89c0a --- /dev/null +++ b/line_profiler/_child_process_profiling/__init__.py @@ -0,0 +1,8 @@ +""" +Tooling for profiling child Python processes and gathering their +profiling results. + +Note: + All contents of this subpackage is to be considered implementation + details. +""" diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py new file mode 100644 index 00000000..74a1cf4a --- /dev/null +++ b/line_profiler/_child_process_profiling/cache.py @@ -0,0 +1,142 @@ +""" +A cache object to be used by for propagating profiling down to child +processes. +""" +from __future__ import annotations + +import dataclasses +import os +try: + import _pickle as pickle +except ImportError: + import pickle # type: ignore[assignment,no-redef] +from collections.abc import Collection, Callable +from functools import partial +from pickle import HIGHEST_PROTOCOL +from typing import Any +from typing_extensions import Self, ParamSpec + +# Note: this should have been defined here in this file, but we moved it +# over to `~._child_process_hook` because that module contains the .pth +# hook, which must run with minimal overhead when a Python process isn't +# associated with a profiled process +from .pth_hook import INHERITED_PID_ENV_VARNAME + + +__all__ = ('LineProfilingCache',) + + +PS = ParamSpec('PS') + +INHERITED_CACHE_ENV_VARNAME_PREFIX = ( + 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_DIR' +) +CACHE_FILENAME = 'line_profiler_cache.pkl' + + +@dataclasses.dataclass +class LineProfilingCache: + cache_dir: os.PathLike[str] | str + profiling_targets: Collection[str] = dataclasses.field( + default_factory=list, + ) + preimports_module: os.PathLike[str] | str | None = None + main_pid: int = dataclasses.field(default_factory=os.getpid) + insert_builtin: bool = True + _cleanup_stack: list[Callable[[], Any]] = dataclasses.field( + default_factory=list, init=False, + ) + + def cleanup(self) -> None: + """ + Pop all the cleanup callbacks from the internal stack added via + :py:meth:`~.add_cleanup` and call them in order. + """ + callbacks = self._cleanup_stack + while callbacks: + callback = callbacks.pop() + try: + callback() + except Exception: + pass + + def add_cleanup( + self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, + ) -> None: + """ + Add a cleanup callback to the internal stack; they can be later + called by :py:meth:`~.cleanup`. + """ + if args or kwargs: + callback = partial(callback, *args, **kwargs) + self._cleanup_stack.append(callback) + + def copy( + self, *, inherit_cleanups: bool = False, **replacements + ) -> Self: + """ + Make a copy with optionally replaced fields; + if ``inherit_cleanups`` is set to true, the copy also makes a + (shallow) copy of the clean-callback stack. + """ + init_args: dict[str, Any] = {} + for field, value in self._get_init_args().items(): + init_args[field] = replacements.get(field, value) + copy = type(self)(**init_args) + if inherit_cleanups: + copy._cleanup_stack[:] = self._cleanup_stack + return copy + + @classmethod + def load(cls) -> Self: + """ + Reconstruct the instance from the environment variables + :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID` and + :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_DIR_`. + These should have been set from an ancestral Python process. + """ + pid = os.environ[INHERITED_PID_ENV_VARNAME] + cache_dir = os.environ[f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}'] + with open(cls._get_filename(cache_dir), mode='rb') as fobj: + return cls(**pickle.load(fobj)) + + def dump(self) -> None: + """ + Serialize the cache instance and dump into the default location + as indicated by :py:attr:`~.cache_dir`, so that they can be + :py:meth:`~.load`-ed by child processes. + + Note: + Cleanup callbacks are not serialized. + """ + with open(self.filename, mode='wb') as fobj: + pickle.dump( + self._get_init_args(), fobj, protocol=HIGHEST_PROTOCOL, + ) + + def _get_init_args(self) -> dict[str, Any]: + init_fields = [ + field_obj.name for field_obj in dataclasses.fields(self) + if field_obj.init + ] + return {name: getattr(self, name) for name in init_fields} + + @staticmethod + def _get_filename(cache_dir: os.PathLike[str] | str) -> str: + return os.path.join(cache_dir, CACHE_FILENAME) + + @property + def environ(self) -> dict[str, str]: + """ + Environment variables to be injected into and inherited by child + processes. + """ + cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{self.main_pid}' + return { + INHERITED_PID_ENV_VARNAME: str(self.main_pid), + cache_varname: str(self.cache_dir), + } + + @property + def filename(self) -> str: + return self._get_filename(self.cache_dir) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py new file mode 100644 index 00000000..82261512 --- /dev/null +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -0,0 +1,206 @@ +""" +Hooks to set up shop in a child Python process and extend profiling +to therein. + +Note: + - The current implementation writes temporary .pth files to the + site-packages directory, which are executed for all Python + processes referring to the same `lib/`. However, only processes + originating from a parent which set the requisite environment + variables will execute to the profiling code. + - Said .pth file always import this module; hence, this file is kept + intentionally lean to reduce overhead: + - Imports in this file are deferred to being as late as possible. + - Type annotations are replaced with type comments. + - Non-essential functionalities are split into small separate + submodules (e.g. :py:mod:`~.cache`). + - Inspired by similar code in `coverage.control` and + `pytest_autoprofile.startup_hook`. +""" +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .cache import LineProfilingCache # noqa: F401 + from pathlib import Path # noqa: F401 + + +__all__ = ( + 'write_pth_hook', 'load_pth_hook', '_setup_in_child_process' +) + +INHERITED_PID_ENV_VARNAME = ( + 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID' +) + + +def write_pth_hook(cache): # type: (LineProfilingCache) -> Path + """ + Write a .pth file which allows for setting up profiling in child + Python processes. + + Args: + cache (:py:class:`~.LineProfilingCache`): + Cache object + + Returns: + fpath (Path): + Path to the written .pth file + + Note: + - To be called in the main process. + - The ``cache`` is responsible for deleting the written .pth + file via the registered cleanup callback. + """ + import os + from pathlib import Path # noqa: F811 + from sysconfig import get_path + from tempfile import mkstemp + + if not os.path.exists(cache.filename): + cache.dump() + assert os.path.exists(cache.filename) + + handle, fname = mkstemp( + prefix='_line_profiler_profiling_hook_', + suffix='.pth', + dir=get_path('purelib'), + ) + fpath = Path(fname) + try: + pth_content = 'import {0}; {0}.load_pth_hook({1})'.format( + (lambda: None).__module__, cache.main_pid, + ) + fpath.write_text(pth_content) + cache.add_cleanup(fpath.unlink, missing_ok=True) + except Exception: + os.remove(fpath) + raise + finally: # Not closing the handle causes issues on Windows + os.close(handle) + return fpath + + +def load_pth_hook(ppid): # type: (int) -> None + """ + Function imported and called by the written .pth file; to reduce + overhead, we immediately return if ``ppid`` doesn't match + :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID`. + """ + from os import environ + + try: + env_ppid = int(environ[INHERITED_PID_ENV_VARNAME]) + except (KeyError, ValueError): + return + if env_ppid != ppid: + return + + # If we're here, we're most probably in a descendent process of a + # profiled Python process, so we can be more liberal with the + # imports without worrying about overhead + import warnings + from .._diagnostics import DEBUG, log + from .cache import LineProfilingCache # noqa: F811 + + # Note: .pth files may be double-loaded in a virtual environment + # (see https://stackoverflow.com/questions/58807569), so work around + # that; + # also see similar check in `coverage.control.process_startup()` + if getattr(load_pth_hook, 'called', False): + return + try: + _setup_in_child_process(LineProfilingCache.load()) + except Exception as e: + if DEBUG: + msg = f'{type(e)}: {e}' + warnings.warn(msg) + log.warning(msg) + finally: + load_pth_hook.called = True # type: ignore[attr-defined] + + +def _wrap_os_fork(cache): # type: (LineProfilingCache) -> None + """ + Create a wrapper around :py:func:`os.fork` which handles profiling. + + Args: + cache (:py:class:`~.LineProfilingCache`): + Cache object + + Side effects: + - :py:func:`os.fork` (if available) replaced with the wrapper + - Cleanup callback registered at ``cache`` undoing that + """ + import os + from functools import wraps + + try: + fork = os.fork + except AttributeError: # Can't fork on this platform + return + + @wraps(fork) + def wrapper(): + result = fork() + if not result: # In the fork + _setup_in_child_process(cache.copy(), False) + return result + + os.fork = wrapper + cache.add_cleanup(setattr, os, 'fork', fork) + + +def _setup_in_child_process(cache, wrap_os_fork=True): + # type: (LineProfilingCache, bool) -> None + """ + Set up shop in a forked/spawned child process so that + (line-)profiling can extend therein. + + Args: + cache (:py:class:`~.LineProfilingCache`): + Cache object + wrap_os_fork (bool): + Whether to wrap :py:func:`os.fork` which handles profiling; + already-forked child processes should set this to false + """ + import os + from atexit import register + from tempfile import mkstemp + from ..curated_profiling import CuratedProfilerContext + from ..line_profiler import LineProfiler + + # Create a profiler instance and manage it with + # `CuratedProfilerContext` + prof = LineProfiler() + ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) + ctx.install() + cache.add_cleanup(ctx.uninstall) + + # Do the preimports at `cache.preimports_module` where appropriate + if cache.preimports_module: + with open(cache.preimports_module, mode='rb') as fobj: + code = compile(fobj.read(), cache.preimports_module, 'exec') + exec(code, {}) # Use a fresh, empty namespace + + # Set up the importer for rewriting `__main__` (TODO) + ... + + # Occupy a tempfile slot in `cache.cache_dir` and set the profiler + # up to write thereto when the process terminates + handle, prof_outfile = mkstemp( + prefix='line-profiler-child-prof-output-', + suffix='.lprof', + dir=cache.cache_dir, + ) + try: + cache.add_cleanup(prof.dump_stats, prof_outfile) + finally: + os.close(handle) + + # Set up `os.fork()` wrapping if needed (i.e. in a spawned process) + if wrap_os_fork: + _wrap_os_fork(cache) + + # Set `cache.cleanup()` as an atexit hook to handle everything when + # the child process is about to terminate + register(cache.cleanup) From 6cf8dee571d0aafd7b4a9f1a02f1333822a8f76c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 28 Mar 2026 05:00:32 +0100 Subject: [PATCH 005/165] WIP: set up child-proc profiling via import sys line_profiler/_child_process_profiling/cache.py::LineProfilingCache Added new `.profile_imports` attribute to correspond to `kernprof`'s `--prof-imports` flag line_profiler/_child_process_profiling/meta_path_finder.py New submodule defining the `RewritingFinder` class, a meta path finder which rewrites a single module on import line_profiler/_child_process_profiling/pth_hook.py write_pth_hook() Now also handling the `os.fork()` patching/wrapping _setup_in_child_process() Now creating a `RewritingFinder` to mirror what `~.autoprofile.autoprofile.run()` does in the main process . --- .../_child_process_profiling/cache.py | 3 + .../meta_path_finder.py | 221 ++++++++++++++++++ .../_child_process_profiling/pth_hook.py | 30 ++- 3 files changed, 247 insertions(+), 7 deletions(-) create mode 100644 line_profiler/_child_process_profiling/meta_path_finder.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 74a1cf4a..bd55e65f 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -40,8 +40,11 @@ class LineProfilingCache: profiling_targets: Collection[str] = dataclasses.field( default_factory=list, ) + profile_imports: bool = False preimports_module: os.PathLike[str] | str | None = None main_pid: int = dataclasses.field(default_factory=os.getpid) + # Note: if we're using the line profiler, `kernprof` always set + # `builtin` to true insert_builtin: bool = True _cleanup_stack: list[Callable[[], Any]] = dataclasses.field( default_factory=list, init=False, diff --git a/line_profiler/_child_process_profiling/meta_path_finder.py b/line_profiler/_child_process_profiling/meta_path_finder.py new file mode 100644 index 00000000..1436fc06 --- /dev/null +++ b/line_profiler/_child_process_profiling/meta_path_finder.py @@ -0,0 +1,221 @@ +""" +A meta path finder object which rewrites a specific module. + +Note: + Based on the implementation in + :py:mod:`pytest_autoprofile.importers`. +""" +from __future__ import annotations + +import ast +import os +import sys +from collections.abc import Callable +from importlib.abc import MetaPathFinder, SourceLoader +from importlib.machinery import ModuleSpec +from pathlib import Path +from types import CodeType, ModuleType +from typing import TYPE_CHECKING + +from .. import _diagnostics as diagnostics +from ..autoprofile.run_module import AstTreeModuleProfiler +from ..line_profiler import LineProfiler +from .cache import LineProfilingCache + + +__all__ = ('RewritingFinder',) + + +class RewritingFinder(MetaPathFinder, SourceLoader): + """ + Meta path finder to be set up in child processes, so that the + ``module_to_rewrite`` is rewritten for profiling as + :py:func:`line_profiler.autoprofile.autoprofile.run` does. + """ + _cached_code_obj: CodeType + + def __init__( + self, + prof: LineProfiler, + lp_cache: LineProfilingCache, + module_to_rewrite: str = '__main__', + ) -> None: + self.prof = prof + self.lp_cache = lp_cache + self.module_to_rewrite = module_to_rewrite + + def install(self, *, index: int = 0) -> None: + """ + Install the importer into :py:data:`sys.meta_path` at the + specified ``index``; + if it's already there, it's first removed and re-inserted at + the requested position. + """ + self.uninstall(invalidate_caches=False) + sys.meta_path.insert(index, self) + + def uninstall(self, *, invalidate_caches: bool = True) -> None: + """ + Uninstall the importer from :py:data:`sys.meta_path`, and + optionally also invalidate the caches. + """ + try: + sys.meta_path.remove(self) + except ValueError: # Not in the list + return + if invalidate_caches: + self.invalidate_caches() + + @classmethod + def find_spec_by_path(cls, *args, **kwargs) -> ModuleSpec | None: + """ + Implementation of + :py:meth:`MetaPathFinder.find_spec` which looks + for module specs with the other meta-path finders. + + Returns: + maybe_spec (ModuleSpec | None) + Module spec if found, :py:const:`None` otherwise + """ + Implementation = Callable[..., ModuleSpec | None] + impls: list[Implementation] = [ + finder.find_spec for finder in sys.meta_path + if callable(getattr(finder, 'find_spec', None)) + if not isinstance(finder, cls) + ] + for impl in impls: + try: + spec = impl(*args, **kwargs) + except Exception: + continue + if spec is not None: + return spec + return None + + def write_code(self, spec: ModuleSpec) -> CodeType: + """ + Rewrite the module code that ``spec`` points to with + :py:class:`ast.autoprofile.run_module.AstTreeModuleProfiler`. + + Args: + spec (ModuleSpec) + Module spec + + Returns: + code (CodeType) + Code object + """ + assert spec.origin + fname = str(Path(spec.origin)) + module = AstTreeModuleProfiler( + spec.origin, + list(self.lp_cache.profiling_targets), + self.lp_cache.profile_imports, + ).profile() + # Slip in a helper node so as to ensure the availability of + # `@profile` + # Note: `@profile` is slipped into the local namespace by + # `RewritingFinder.exec_module()`, but that may not be + # enough for applications directly using the code objects (e.g. + # `runpy`. Hence, we provide an out by falling back to + # `@line_profiler.profile`. + ensure_profile_node, = ast.parse( + 'if profile not in globals():\n' + ' from line_profiler import profile' + ).body + module.body.insert(0, ensure_profile_node) + return compile(module, fname, 'exec') + + # Methods as dictated by the interface + + def invalidate_caches(self) -> None: + try: + del self._cached_code_obj + except AttributeError: + pass + super().invalidate_caches() + + def exec_module(self, module: ModuleType) -> None: + namespace = module.__dict__ + namespace['profile'] = prof = self.prof + spec: ModuleSpec | None = module.__spec__ + if spec is None: + raise RuntimeError(f'module = {module!r}: empty `.__spec__`') + if TYPE_CHECKING: # Appease `mypy` + assert hasattr(prof, 'object_count') + count = prof.object_count + exec(self.get_code(spec), namespace, namespace) + if prof.object_count > count: + msg = ( + 'main PID = {} / child PID = {}: ' + 'profiled {} code object{} in the `{}` module' + ).format( + self.lp_cache.main_pid, + os.getpid(), + prof.object_count, + '' if prof.object_count == 1 else 's', + self.module_to_rewrite, + ) + diagnostics.log.debug(msg) + + def find_spec(self, *args, **kwargs) -> ModuleSpec | None: + spec = self.find_spec_by_path(*args, **kwargs) + if (spec is not None and spec.name == self.module_to_rewrite): + return spec + return None + + @staticmethod + def get_data(path: os.PathLike[str] | str) -> bytes: + return Path(path).read_bytes() + + @classmethod + def get_filename(cls, name: str) -> str: + spec = cls.find_spec_by_path(name) + if spec is None: + raise ImportError(name) + origin = spec.origin + if origin is None: + raise ImportError(name) + if origin == 'frozen' or not os.path.exists(origin): + raise ImportError(name) + return origin + + def get_code(self, name_or_spec: str | ModuleSpec) -> CodeType: + if isinstance(name_or_spec, str): + spec = self.find_spec_by_path(name_or_spec) + if spec is None: + raise ImportError(name_or_spec) + else: + spec = name_or_spec + try: + try: + return self._cached_code_obj + except AttributeError: + self._cached_code_obj = code = self.write_code(spec) + return code + except Exception as e: + raise ImportError(name_or_spec) from e + + if TYPE_CHECKING: + def source_to_code( # type: ignore[override] + self, *args, **kwargs + ) -> CodeType: + """ + Notes + ----- + :py:mod:`mypy` reports that + :py:meth:`SourceLoader.source_to_code`, an instance method, + clashes with + :py:meth:`importlib.abc.InspectLoader.source_to_code`, a + static method. + + Since: + - The method is functionally only used in the superclasses + as an instance method, and + - :py:class:`importlib.abc.InspectLoader`` is merely + included as a base class because :py:class:`SourceLoader` + inherits from it, + just explicitly override the method here so that we can + catch and suppress the :py:mod:`mypy` error. + """ + return super().source_to_code(*args, **kwargs) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 82261512..f19a8c33 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -5,17 +5,17 @@ Note: - The current implementation writes temporary .pth files to the site-packages directory, which are executed for all Python - processes referring to the same `lib/`. However, only processes - originating from a parent which set the requisite environment - variables will execute to the profiling code. + processes referring to the same :path:`lib/`. However, only + processes originating from a parent which set the requisite + environment variables will execute to the profiling code. - Said .pth file always import this module; hence, this file is kept intentionally lean to reduce overhead: - Imports in this file are deferred to being as late as possible. - Type annotations are replaced with type comments. - Non-essential functionalities are split into small separate submodules (e.g. :py:mod:`~.cache`). - - Inspired by similar code in `coverage.control` and - `pytest_autoprofile.startup_hook`. + - Inspired by similar code in :py:mod:`coverage.control` and + :py:mod:`pytest_autoprofile.startup_hook`. """ from typing import TYPE_CHECKING @@ -50,6 +50,8 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path - To be called in the main process. - The ``cache`` is responsible for deleting the written .pth file via the registered cleanup callback. + - For convenience, we also wrap :py:func:`os.fork` when this + function is called. """ import os from pathlib import Path # noqa: F811 @@ -77,6 +79,9 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path raise finally: # Not closing the handle causes issues on Windows os.close(handle) + + _wrap_os_fork(cache) + return fpath @@ -166,12 +171,21 @@ def _setup_in_child_process(cache, wrap_os_fork=True): import os from atexit import register from tempfile import mkstemp + from ..autoprofile.autoprofile import ( + # Note: we need this to equip the profiler with the + # `.add_imported_function_or_module()` pseudo-method + # (see `kernprof.py::_write_preimports()`), which is required + # for the preimports to work + _extend_line_profiler_for_profiling_imports as upgrade_profiler, + ) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler + from .meta_path_finder import RewritingFinder # Create a profiler instance and manage it with # `CuratedProfilerContext` prof = LineProfiler() + upgrade_profiler(prof) ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) ctx.install() cache.add_cleanup(ctx.uninstall) @@ -182,8 +196,10 @@ def _setup_in_child_process(cache, wrap_os_fork=True): code = compile(fobj.read(), cache.preimports_module, 'exec') exec(code, {}) # Use a fresh, empty namespace - # Set up the importer for rewriting `__main__` (TODO) - ... + # Set up the importer for rewriting `__main__` + finder = RewritingFinder(prof, cache) + finder.install() + cache.add_cleanup(finder.uninstall) # Occupy a tempfile slot in `cache.cache_dir` and set the profiler # up to write thereto when the process terminates From 7fa12c0246174f4e0b69d0887211915765da0764 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 28 Mar 2026 19:07:08 +0100 Subject: [PATCH 006/165] WIP: `multiprocessing` patches line_profiler/_child_process_profiling/cache::LineProfilingCache Refactored `.load()` line_profiler/_child_process_profiling/multiprocessing_patches.py New submodule for applying patches to the `multiprocessing` package, so that profiling is automatically set up in child processes created by it --- .../_child_process_profiling/cache.py | 8 +- .../multiprocessing_patches.py | 188 ++++++++++++++++++ 2 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index bd55e65f..3a4f22eb 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -100,8 +100,7 @@ def load(cls) -> Self: """ pid = os.environ[INHERITED_PID_ENV_VARNAME] cache_dir = os.environ[f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}'] - with open(cls._get_filename(cache_dir), mode='rb') as fobj: - return cls(**pickle.load(fobj)) + return cls._from_path(cls._get_filename(cache_dir)) def dump(self) -> None: """ @@ -117,6 +116,11 @@ def dump(self) -> None: self._get_init_args(), fobj, protocol=HIGHEST_PROTOCOL, ) + @classmethod + def _from_path(cls, fname: os.PathLike[str] | str) -> Self: + with open(cls._get_filename(fname), mode='rb') as fobj: + return cls(**pickle.load(fobj)) + def _get_init_args(self) -> dict[str, Any]: init_fields = [ field_obj.name for field_obj in dataclasses.fields(self) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py new file mode 100644 index 00000000..39cabe2a --- /dev/null +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -0,0 +1,188 @@ +""" +Patch :py:mod:`multiprocessing` so that profiling extends into processes +it creates. + +Notes +----- +- Based on the implementations in :py:mod:`coverage.multiproc` and + :py:mod:`pytest_autoprofile._multiprocessing`. +- Results may vary if the process pool is not properly + :py:meth:`multiprocessing.pool.Pool.close`-d and + :py:meth:`multiprocessing.pool.Pool.join`-ed; + see `this caveat `__. +""" +from __future__ import annotations + +import multiprocessing.process +from collections.abc import Callable +from functools import partial, partialmethod +from typing import Any, TypedDict, TypeVar +from typing_extensions import Concatenate, ParamSpec + +from .cache import LineProfilingCache +from .pth_hook import _setup_in_child_process + + +__all__ = ('apply',) + + +T = TypeVar('T') +PS = ParamSpec('PS') + + +class _HookState(TypedDict): + cache_path: str # Cache to be loaded from here + + +class PickleHook: + """ + Object which, when unpickled, sets up profiling in the + :py:mod:`multiprocessing`-created process. + + See also: + :py:class:`coverage.multiproc.Stowaway` + """ + def __init__(self, cache_path: str) -> None: + self.cache_path = cache_path + + def __getstate__(self) -> _HookState: + return {'cache_path': self.cache_path} + + def __setstate__(self, state: _HookState) -> None: + self.cache_path = path = state['cache_path'] + apply(path) + + +def bootstrap( + self: multiprocessing.process.BaseProcess, + vanilla_impl: Callable[ + Concatenate[multiprocessing.process.BaseProcess, PS], T + ], + lp_cache: LineProfilingCache, + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> T: + """ + Wrap around + :py:meth:`multiprocessing.process.BaseProcess._bootstrap`, + writing the profiling results after it is run. + + Args: + self (multiprocessing.process.BaseProcess) + :py:class:`~.BaseProcess` + vanilla_impl (Callable) + Vanilla :py:meth:`~.BaseProcess._bootstrap` + lp_cache (LineProfilingCache) + Cache recovered by :py:meth:`~.LineProfilingCache.load` + *args + **kwargs + Passed to :py:meth:`~.BaseProcess._bootstrap` + + Returns: + Return value of ``vanilla_impl(*args, **kwargs)`` + + Side effects: + Profiling results are written + """ + try: + return vanilla_impl(self, *args, **kwargs) + finally: # Write profiling results + lp_cache.cleanup() + + +def get_preparation_data( + vanilla_impl: Callable[PS, dict[str, Any]], + cache_path: str, + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> dict[str, Any]: + """ + Wrap around :py:func:`multiprocessing.spawn.get_preparation_data`, + slipping a :py:class:`PickleHook` into the returned dictionary so + that profiling is + triggered upon unpickling. + + Args: + vanilla_impl + Vanilla + :py:func:`multiprocessing.spawn.get_preparation_data` + cache_path + File from which the :py:class:`LineProfilingCache` should be + loaded + *args + **kwargs + Passed to + :py:func:`multiprocessing.spawn.get_preparation_data` + + Returns + Dictionary returned by + ``get_preparation_data(*args, **kwargs)`` with an extra key + """ + key = 'line_profiler_pickle_hook' # Doesn't matter + data = vanilla_impl(*args, **kwargs) + assert key not in data + data[key] = PickleHook(cache_path) + return data + + +def apply( + cache_path: str, *, lp_cache: LineProfilingCache | None = None, +) -> None: + """ + Set up profiling in :py:mod:`multiprocessing` child processes by + applying patches to the module. + + Args: + cache_path + Path to the file whence a :py:class:`LineProfilingCache` + object can be loaded + lp_cache + Optional :py:class:`LineProfilingCache` instance; + if not provided, it is loaded from `cache_path`, and + profiling is set up therefrom in the (sub-)process + + Side effects: + - :py:mod:`multiprocessing` marked as having been set up + - :py:meth:`multiprocessing.process.BaseProcess._bootstrap` + patched + - :py:func:`multiprocessing.spawn.get_preparation_data` patched + - Cleanup callbacks registered via `lp_cache.add_cleanup()` + """ + patched_marker = '_line_profiler_patched_multiprocessing' + if getattr(multiprocessing, patched_marker, False): + return + if lp_cache is None: + lp_cache = LineProfilingCache._from_path(cache_path) + _setup_in_child_process(lp_cache) + + vanilla: Callable[..., Any] | None + + # Patch `multiprocessing.process.BaseProcess._bootstrap()` + Proc = multiprocessing.process.BaseProcess + vanilla = Proc._bootstrap # type: ignore[attr-defined] + Proc._bootstrap = ( # type: ignore[attr-defined] + partialmethod(bootstrap, vanilla, lp_cache) + ) + lp_cache.add_cleanup(setattr, Proc, '_bootstrap', vanilla) + + # Patch `multiprocessing.spawn.get_preparation_data()` + try: + from multiprocessing import spawn + except ImportError: # Incompatible platforms + pass + else: + vanilla = getattr(spawn, 'get_preparation_data', None) + if vanilla: + spawn.get_preparation_data = partial( + get_preparation_data, vanilla, cache_path, + ) + lp_cache.add_cleanup( + setattr, spawn, 'get_preparation_data', vanilla, + ) + + # Mark `multiprocessing` as having been patched + setattr(multiprocessing, patched_marker, True) + lp_cache.add_cleanup(vars(multiprocessing).pop, patched_marker, None) From e3675d90255fbe67938f4c490ad6741fbdefb74a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 28 Mar 2026 20:54:35 +0100 Subject: [PATCH 007/165] Child-process stat gathering; cache refactoring line_profiler/_child_process_profiling/cache.py::LineProfilingCache Added debug logging to various methods gather_stats() New method for gathering profiling stats from child processes inject_env_vars() New method for injecting `.environ` into `os.environ` line_profiler/line_profiler.py::LineStats get_empty_instance() New convenience method for creating an empty instance from_files() Added new argument `on_defective` to allow for processing a group of files that cannot all be correctly read --- .../_child_process_profiling/cache.py | 84 +++++++++++++++++-- line_profiler/line_profiler.py | 54 +++++++++++- 2 files changed, 126 insertions(+), 12 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 3a4f22eb..1c611fc5 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -11,11 +11,15 @@ except ImportError: import pickle # type: ignore[assignment,no-redef] from collections.abc import Collection, Callable -from functools import partial +from functools import partial, cached_property +from operator import setitem +from pathlib import Path from pickle import HIGHEST_PROTOCOL -from typing import Any +from typing import Any, cast from typing_extensions import Self, ParamSpec +from .. import _diagnostics as diagnostics +from ..line_profiler import LineStats # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth # hook, which must run with minimal overhead when a Python process isn't @@ -60,8 +64,11 @@ def cleanup(self) -> None: callback = callbacks.pop() try: callback() - except Exception: - pass + except Exception as e: + msg = f'Cleanup failed: {callback}: {type(e).__name__}: {e}' + else: + msg = f'Cleanup succeeded: {callback}' + self._debug(msg) def add_cleanup( self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, @@ -73,6 +80,7 @@ def add_cleanup( if args or kwargs: callback = partial(callback, *args, **kwargs) self._cleanup_stack.append(callback) + self._debug(f'Cleanup callback added: {callback}') def copy( self, *, inherit_cleanups: bool = False, **replacements @@ -99,7 +107,13 @@ def load(cls) -> Self: These should have been set from an ancestral Python process. """ pid = os.environ[INHERITED_PID_ENV_VARNAME] - cache_dir = os.environ[f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}'] + cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}' + cache_dir = os.environ[cache_varname] + msg = ( + f'PID {os.getpid()} (from {pid}): ' + f'Loading instance from ${{{cache_varname}}} = {cache_dir}' + ) + diagnostics.log.debug(msg) return cls._from_path(cls._get_filename(cache_dir)) def dump(self) -> None: @@ -111,14 +125,59 @@ def dump(self) -> None: Note: Cleanup callbacks are not serialized. """ + content = self._get_init_args() + msg = f'Dumping instance data to {self.filename}: {content!r}' + self._debug(msg) with open(self.filename, mode='wb') as fobj: - pickle.dump( - self._get_init_args(), fobj, protocol=HIGHEST_PROTOCOL, - ) + pickle.dump(content, fobj, protocol=HIGHEST_PROTOCOL) + + def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: + """ + Gather the profiling output files matching ``glob_pattern`` from + :py:attr:`~.cache_dir`, consolidating them into a single + :py:class:`LineStats` object. + """ + fnames = list(Path(self.cache_dir).glob(glob_pattern)) + self._debug( + 'Loading results from {} child profiling file(s): {!r}' + .format(len(fnames), fnames) + ) + if not fnames: + return LineStats.get_empty_instance() + return LineStats.from_files(*fnames, on_defective='ignore') + + def inject_env_vars( + self, env: dict[str, str] | None = None, + ) -> None: + """ + Inject the :py:attr:`~.environ` variables into ``env`` and add + cleanup callbacks to reverse them. + + Args: + env (dict[str, str] | None): + Dictionary in the format of :py:data:`os.environ`; + default is to use that + """ + if env is None: + env = cast(dict[str, str], os.environ) + for name, value in self.environ.items(): + try: + old = env[name] + except KeyError: + self.add_cleanup(env.pop, name, None) + change = f'{value!r} (new)' + else: + self.add_cleanup(setitem, env, name, old) + change = f'{old!r} -> {value!r}' + self._debug(f'Injecting env var ${{{name}}}: {change}') + env[name] = value + + def _debug(self, msg: str) -> None: + diagnostics.log.debug(f'{self._debug_message_header}: {msg}') @classmethod def _from_path(cls, fname: os.PathLike[str] | str) -> Self: - with open(cls._get_filename(fname), mode='rb') as fobj: + with open(fname, mode='rb') as fobj: return cls(**pickle.load(fobj)) def _get_init_args(self) -> dict[str, Any]: @@ -147,3 +206,10 @@ def environ(self) -> dict[str, str]: @property def filename(self) -> str: return self._get_filename(self.cache_dir) + + @cached_property + def _debug_message_header(self) -> str: + pid = os.getpid() + if self.main_pid == pid: + return f'PID {pid} (main process)' + return f'PID {pid} (from {self.main_pid})' diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index bede6f6c..03899261 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -18,6 +18,7 @@ import tempfile import types import tokenize +import warnings from argparse import ArgumentParser from datetime import datetime from os import PathLike @@ -367,17 +368,64 @@ def to_file(self, filename: PathLike[str] | str) -> None: with open(filename, 'wb') as f: pickle.dump(self, f, pickle.HIGHEST_PROTOCOL) + @classmethod + def get_empty_instance(cls) -> Self: + """ + Returns: + instance (LineStats): + New instance without any profiling data. + """ + unit = LineProfiler().timer_unit # type: ignore[attr-defined] + return cls({}, unit) + @classmethod def from_files( - cls, file: PathLike[str] | str, /, *files: PathLike[str] | str + cls, + file: PathLike[str] | str, + /, + *files: PathLike[str] | str, + on_defective: Literal['ignore', 'warn', 'error'] = 'error', ) -> Self: """ Utility function to load an instance from the given filenames. + + Args: + file (PathLike[str] | str): + File to load profiling data from + *files (PathLike[str] | str): + Ditto above + on_defective (Literal['ignore', 'warn', 'error']): + What to do if some files fail to load: ``'ignore'`` + those files, skip them but with a ``'warn'``-ing, or + raise the ``'error'`` as soon as one is encountered + + Returns: + instance (LineStats): + New instance """ stats_objs = [] - for file in [file, *files]: + failures: dict[str, str] = {} + all_files = [file, *files] + for file in all_files: with open(file, 'rb') as f: - stats_objs.append(pickle.load(f)) + try: + stats_objs.append(pickle.load(f)) + except Exception as e: + if on_defective == 'error': + raise + failures[str(file)] = f'{type(e).__name__}: {e}' + if failures: + msg = ( + '{} file(s) out of {} failed to load and are skipped: {!r}' + .format(len(failures), len(all_files), failures) + ) + if on_defective == 'warn': + warnings.warn(msg) + diagnostics.log.warning(msg) + else: # 'ignore' + diagnostics.log.debug(msg) + if not stats_objs: + return cls.get_empty_instance() return cls.from_stats_objects(*stats_objs) @classmethod From 1c7681abbf5717579a2dee4e63150dba079f2d8d Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 29 Mar 2026 03:32:42 +0200 Subject: [PATCH 008/165] WIP: kernprof now sets up child-proc profiling line_profiler/rc/line_profiler.toml::[tool.line_profiler.kernprof] Added new key-value pair `prof-child-procs` for the default value of `kernprof --prof-child-procs` kernprof.py - New boolean flags `[--prof-child-procs[=...] | --no-prof-child-procs]` for controlling whether to set up profiling in child processes - Fixed bug in `_manage_profiler.__exit__()` where `CuratedProfilerContext.uninstall()` can be skipped if the preceding code raises an error --- kernprof.py | 158 +++++++++++++++++++++++----- line_profiler/rc/line_profiler.toml | 3 + 2 files changed, 136 insertions(+), 25 deletions(-) diff --git a/kernprof.py b/kernprof.py index 5a873571..37cc14a0 100755 --- a/kernprof.py +++ b/kernprof.py @@ -79,6 +79,7 @@ def main(): [-s SETUP] [-p {path/to/script | object.dotted.path}[,...]] [--preimports [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] [--prof-imports [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] + [--prof-child-procs [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] [-o OUTFILE] [-v] [-q] [--rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] [-u UNIT] @@ -151,7 +152,7 @@ def main(): -q, --quiet Decrease verbosity level (default: 0). At level -1, disable helpful messages (e.g. "Wrote profile results to <...>"); at level -2, silence the stdout; at level -3, silence the stderr. - --rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] +--rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] Use rich formatting if viewing output. (Default: False; short form: -r) -u, --unit UNIT Output unit (in seconds) in which the timing info is @@ -226,6 +227,7 @@ def main(): short_string_path, ) from line_profiler.profiler_mixin import ByCountProfilerMixin +from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._logger import Logger from line_profiler import _diagnostics as diagnostics @@ -327,6 +329,7 @@ def resolve_module_path(mod_name): # type: (str) -> str | None fname = mod_spec.origin # type: str | None if fname and os.path.exists(fname): return fname + return None get_module_path = modname_to_modpath if static else resolve_module_path @@ -678,6 +681,14 @@ def _add_core_parser_arguments(parser): 'Only works with line profiling (`-l`/`--line-by-line`). ' f'(Default: {default.conf_dict["prof_imports"]})', ) + add_argument( + prof_opts, + '--prof-child-procs', + action='store_true', + help='Extend profiling into child Python processes. ' + 'Only works with line profiling (`-l`/`--line-by-line`). ' + f'(Default: {default.conf_dict["prof_child_procs"]})', + ) out_opts = parser.add_argument_group('output options') if default.conf_dict['outfile']: def_outfile = repr(default.conf_dict['outfile']) @@ -800,8 +811,8 @@ def _build_parsers(args=None): # We've already consumed the `-m `, so we need a dummy # parser for generating the help text; # but the real parser should not consume the `options.script` - # positional arg, and it it got the `--help` option, it should - # hand off the the dummy parser + # positional arg, and if it got the `--help` option, it should + # hand off to the dummy parser real_parser = ArgumentParser(add_help=False, **parser_kwargs) real_parser.add_argument('-h', '--help', action='store_true') help_parser = ArgumentParser(**parser_kwargs) @@ -1040,7 +1051,7 @@ def _write_tempfile(source, content, options): ) -def _write_preimports(prof, options, exclude): +def _write_preimports(prof, options, exclude, keep=False): """ Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. @@ -1068,8 +1079,9 @@ def _write_preimports(prof, options, exclude): ns = {} # Use a fresh namespace execfile(temp_mod_path, ns, ns) # Delete the tempfile ASAP if its execution succeeded - if not diagnostics.KEEP_TEMPDIRS: + if not (keep or diagnostics.KEEP_TEMPDIRS): _remove(temp_mod_path) + return temp_mod_path def _remove(path, *, recursive=False, missing_ok=False): @@ -1083,9 +1095,8 @@ def _remove(path, *, recursive=False, missing_ok=False): path.unlink(missing_ok=missing_ok) -def _dump_filtered_stats(tmpdir, prof, filename): +def _dump_filtered_stats(tmpdir, prof, filename, extra_line_stats=None): import os - import pickle # Build list of known temp file paths tempfile_paths = [ @@ -1103,21 +1114,25 @@ def _dump_filtered_stats(tmpdir, prof, filename): prof.dump_stats(filename) return + line_stats = prof.get_stats() + if extra_line_stats is not None: + line_stats += extra_line_stats + _dump_filtered_line_stats(line_stats, tempfile_paths, filename) + + +def _dump_filtered_line_stats(stats, exclude, filename): # Filter the filenames to remove data from tempfiles, which will # have been deleted by the time the results are viewed in a # separate process - stats = prof.get_stats() timings = stats.timings for key in set(timings): fname = key[0] try: - if any(os.path.samefile(fname, tmp) for tmp in tempfile_paths): + if any(os.path.samefile(fname, tmp) for tmp in exclude): del timings[key] except OSError: del timings[key] - - with open(filename, 'wb') as f: - pickle.dump(stats, f, protocol=pickle.HIGHEST_PROTOCOL) + stats.to_file(filename) def _format_call_message(func, *args, **kwargs): @@ -1167,6 +1182,8 @@ class _manage_profiler: Note: modifies ``options`` with extra attributes. """ + cache: LineProfilingCache + def __init__(self, options, module, exit_on_error): self.options = options self.module = module @@ -1182,14 +1199,36 @@ def __enter__(self): self.prof, insert_builtin=self.options.builtin, ) self._ctx.install() - script_file = _prepare_exec_script( - self.options, self.module, self.prof, self.exit_on_error, + # Keep the generated pre-imports file to be reused in child + # processes + script_file, preimports_file = _prepare_exec_script( + self.options, self.module, self.prof, + exit_on_error=self.exit_on_error, + keep_preimports_file=self.set_up_child_profiling, ) + if self.set_up_child_profiling: + self.cache = _prepare_child_profiling_cache( + self.options, self.prof, preimports_file, + ) return self.prof, script_file def __exit__(self, *_, **__): - _post_profile(self.options, self.prof) - self._ctx.uninstall() + try: + extra_stats = None + if self.set_up_child_profiling: + try: + extra_stats = self.cache.gather_stats() + finally: + self.cache.cleanup() + _post_profile(self.options, self.prof, extra_stats) + finally: + self._ctx.uninstall() + + @property + def set_up_child_profiling(self): + return bool( + self.options.line_by_line and self.options.prof_child_procs + ) def _prepare_profiler(options, module, exit_on_error): @@ -1236,7 +1275,12 @@ def _prepare_profiler(options, module, exit_on_error): return ContextualProfile() -def _prepare_exec_script(options, module, prof, exit_on_error): +def _prepare_exec_script( + options, module, prof, + *, + exit_on_error=False, + keep_preimports_file=False, +): """ Set up the script to be executed among other things. """ @@ -1259,6 +1303,8 @@ def _prepare_exec_script(options, module, prof, exit_on_error): options.prof_mod = _normalize_profiling_targets(options.prof_mod) if not options.prof_mod: options.preimports = False + + preimports_file = None if options.line_by_line and options.preimports: # We assume most items in `.prof_mod` to be import-able without # significant side effects, but the same cannot be said if it @@ -1266,7 +1312,9 @@ def _prepare_exec_script(options, module, prof, exit_on_error): # even have a `if __name__ == '__main__': ...` guard. So don't # eager-import it. exclude = set() if module else {script_file} - _write_preimports(prof, options, exclude) + preimports_file = _write_preimports( + prof, options, exclude, keep=keep_preimports_file, + ) if options.output_interval and not options.dryrun: options.rt = RepeatedTimer( @@ -1275,7 +1323,60 @@ def _prepare_exec_script(options, module, prof, exit_on_error): else: options.rt = None options.original_stdout = sys.stdout - return script_file + return script_file, preimports_file + + +def _prepare_child_profiling_cache(options, prof, preimports_file): + """ + Handle the (line-)profiling of spawned/forked child Python + processes. + """ + from line_profiler._child_process_profiling import ( + pth_hook, multiprocessing_patches, + ) + + # Create the cache dir and cache file here; the cache instance will + # be responsible for managing their lifetimes, while derivative + # instances in child processes will merely inherit and use them + cache = LineProfilingCache( + tempfile.mkdtemp(), + options.prof_mod, + options.prof_imports, + preimports_file, + insert_builtin=options.builtin, + ) + cache.add_cleanup( + _remove, cache.cache_dir, recursive=True, missing_ok=True, + ) + cache.dump() + cache.add_cleanup(_remove, cache.filename, missing_ok=True) + + # This file is handed to us at the end of + # `_manage_profiler.__enter__()`; + # normally it is deleted before `.__enter__()` returns, but when + # child-process profiling is used, it is to persist for the lifetime + # of the cache (so that child processes can do the same preimports) + if preimports_file is not None: + cache.add_cleanup(_remove, preimports_file, missing_ok=True) + + # Note: the following functions/methods all clean up after + # themselves, so there is no need to explicitly call + # `cache.add_cleanup()` + + # Inject environment variables so that child Python processes can + # inherit them and resume profiling + cache.inject_env_vars() + + # Create the .pth file which allows for setting up profiling in all + # the child Python processes (e.g. those created by `os.system()` or + # `subprocess.run()` + pth_hook.write_pth_hook(cache) + + # Patch `multiprocessing` so that child processes are properly + # handled across all "start methods" + multiprocessing_patches.apply(cache.filename, lp_cache=cache) + + return cache def _main_profile(options, module=False, exit_on_error=True): @@ -1339,14 +1440,16 @@ def _main_profile(options, module=False, exit_on_error=True): ) -def _post_profile(options, prof): +def _post_profile(options, prof, extra_line_stats=None): """ Cleanup setup after executing a main profile """ if options.rt is not None: options.rt.stop() if not options.dryrun: - _dump_filtered_stats(options.tmpdir, prof, options.outfile) + _dump_filtered_stats( + options.tmpdir, prof, options.outfile, extra_line_stats, + ) short_outfile = short_string_path(options.outfile) diagnostics.log.info( ( @@ -1357,9 +1460,15 @@ def _post_profile(options, prof): + f'to {short_outfile!r}' ) if options.verbose > 0 and not options.dryrun: - kwargs = {} - if not isinstance(prof, ContextualProfile): - kwargs.update( + if isinstance(prof, ContextualProfile): + _call_with_diagnostics(options, prof.print_stats) + else: + stats = prof.get_stats() + if extra_line_stats is not None: + stats += extra_line_stats + _call_with_diagnostics( + options, + stats.print, output_unit=options.unit, stripzeros=options.skip_zero, summarize=options.summarize, @@ -1367,7 +1476,6 @@ def _post_profile(options, prof): stream=options.original_stdout, config=options.config, ) - _call_with_diagnostics(options, prof.print_stats, **kwargs) else: py_exe = _python_command() if isinstance(prof, ContextualProfile): diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 6680c06a..a5b7740c 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -94,6 +94,9 @@ preimports = true # - `prof-imports` (bool): # `--prof-imports` (true) or `--no-prof-imports` (false) prof-imports = false +# - `prof-child-procs` (bool): +# `--prof-child-procs` (true) or `--no-prof-child-procs` (false) +prof-child-procs = false # - Misc flags # - `verbose` (count): From 41eb3b3bdef041bd4d5b3d69c0419906c6a9b523 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 1 Apr 2026 22:31:13 +0200 Subject: [PATCH 009/165] Debugging support in `kernprof` kernprof.py::_prepare_child_profiling_cache() - Now respecting ${LINE_PROFILER_KEEP_TEMPDIRS} - Now setting `LineProfilingCache.debug` line_profiler/_child_process_profiling/cache.py::LineProfilingCache - Added new attributes `.debug` and `._debug_log` - Now diverting debug messages to log files in `.cache_dir` --- kernprof.py | 13 ++++---- .../_child_process_profiling/cache.py | 31 ++++++++++++++----- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/kernprof.py b/kernprof.py index 37cc14a0..4f323ecb 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1344,20 +1344,21 @@ def _prepare_child_profiling_cache(options, prof, preimports_file): options.prof_imports, preimports_file, insert_builtin=options.builtin, + debug=options.debug, ) - cache.add_cleanup( - _remove, cache.cache_dir, recursive=True, missing_ok=True, - ) + clean_up = functools.partial(cache.add_cleanup, _remove) + if not diagnostics.KEEP_TEMPDIRS: + clean_up(cache.cache_dir, recursive=True, missing_ok=True) cache.dump() - cache.add_cleanup(_remove, cache.filename, missing_ok=True) + clean_up(cache.filename, missing_ok=True) # This file is handed to us at the end of # `_manage_profiler.__enter__()`; # normally it is deleted before `.__enter__()` returns, but when # child-process profiling is used, it is to persist for the lifetime # of the cache (so that child processes can do the same preimports) - if preimports_file is not None: - cache.add_cleanup(_remove, preimports_file, missing_ok=True) + if not (preimports_file is None or diagnostics.KEEP_TEMPDIRS): + clean_up(preimports_file, missing_ok=True) # Note: the following functions/methods all clean up after # themselves, so there is no need to explicitly call diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 1c611fc5..b8fb1818 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -50,6 +50,7 @@ class LineProfilingCache: # Note: if we're using the line profiler, `kernprof` always set # `builtin` to true insert_builtin: bool = True + debug: bool = diagnostics.DEBUG _cleanup_stack: list[Callable[[], Any]] = dataclasses.field( default_factory=list, init=False, ) @@ -68,7 +69,7 @@ def cleanup(self) -> None: msg = f'Cleanup failed: {callback}: {type(e).__name__}: {e}' else: msg = f'Cleanup succeeded: {callback}' - self._debug(msg) + self._debug_output(msg) def add_cleanup( self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, @@ -80,7 +81,7 @@ def add_cleanup( if args or kwargs: callback = partial(callback, *args, **kwargs) self._cleanup_stack.append(callback) - self._debug(f'Cleanup callback added: {callback}') + self._debug_output(f'Cleanup callback added: {callback}') def copy( self, *, inherit_cleanups: bool = False, **replacements @@ -127,7 +128,7 @@ def dump(self) -> None: """ content = self._get_init_args() msg = f'Dumping instance data to {self.filename}: {content!r}' - self._debug(msg) + self._debug_output(msg) with open(self.filename, mode='wb') as fobj: pickle.dump(content, fobj, protocol=HIGHEST_PROTOCOL) @@ -138,7 +139,7 @@ def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: :py:class:`LineStats` object. """ fnames = list(Path(self.cache_dir).glob(glob_pattern)) - self._debug( + self._debug_output( 'Loading results from {} child profiling file(s): {!r}' .format(len(fnames), fnames) ) @@ -169,11 +170,20 @@ def inject_env_vars( else: self.add_cleanup(setitem, env, name, old) change = f'{old!r} -> {value!r}' - self._debug(f'Injecting env var ${{{name}}}: {change}') + self._debug_output(f'Injecting env var ${{{name}}}: {change}') env[name] = value - def _debug(self, msg: str) -> None: - diagnostics.log.debug(f'{self._debug_message_header}: {msg}') + def _debug_output(self, msg: str) -> None: + msg = f'{self._debug_message_header}: {msg}' + diagnostics.log.debug(msg) + try: + with self._debug_log.open(mode='a') as fobj: + print(msg, file=fobj) + except ( + AttributeError, # `._debug_log` is None in non-debug mode + OSError, # Cache dir may have been removed during cleanup + ): + pass @classmethod def _from_path(cls, fname: os.PathLike[str] | str) -> Self: @@ -207,6 +217,13 @@ def environ(self) -> dict[str, str]: def filename(self) -> str: return self._get_filename(self.cache_dir) + @property + def _debug_log(self) -> Path | None: + if not self.debug: + return None + fname = f'debug_log_{self.main_pid}_{os.getpid()}.log' + return Path(self.cache_dir) / fname + @cached_property def _debug_message_header(self) -> str: pid = os.getpid() From 9bedb2dc6805c796093553bb884a70d76c0314ce Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 6 Apr 2026 22:43:24 +0200 Subject: [PATCH 010/165] Cleanup prioritization and more debug messages line_profiler/_child_process_profiling/cache.py::LineProfilingCache add_cleanup() Now deferring to a `._add_cleanup()` method which allows for cleanup-function prioritization _debug_output() Fixed type-checking line_profiler/_child_process_profiling/multiprocessing_patches.py ::apply() Added debug output before `_setup_in_child_process()` is called to help with tracing line_profiler/_child_process_profiling/pth_hook.py load_pth_hook() _wrap_os_fork() Added debug output before `_setup_in_child_process()` is called to help with tracing _setup_in_child_process() - `wrap_os_fork` now defaults to false - `prof.dump_stats()` now has increased priority over other callbacks (doesn't seem to help with the malformed prof files though...) - Child-process profiling output now written to a less randomized filename to facilitate debugging --- .../_child_process_profiling/cache.py | 50 ++++++++++++------- .../multiprocessing_patches.py | 1 + .../_child_process_profiling/pth_hook.py | 17 ++++--- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index b8fb1818..76eb62aa 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -51,8 +51,8 @@ class LineProfilingCache: # `builtin` to true insert_builtin: bool = True debug: bool = diagnostics.DEBUG - _cleanup_stack: list[Callable[[], Any]] = dataclasses.field( - default_factory=list, init=False, + _cleanup_stacks: dict[float, list[Callable[[], Any]]] = dataclasses.field( + default_factory=dict, init=False, ) def cleanup(self) -> None: @@ -60,16 +60,17 @@ def cleanup(self) -> None: Pop all the cleanup callbacks from the internal stack added via :py:meth:`~.add_cleanup` and call them in order. """ - callbacks = self._cleanup_stack - while callbacks: - callback = callbacks.pop() - try: - callback() - except Exception as e: - msg = f'Cleanup failed: {callback}: {type(e).__name__}: {e}' - else: - msg = f'Cleanup succeeded: {callback}' - self._debug_output(msg) + for priority in sorted(self._cleanup_stacks): + callbacks = self._cleanup_stacks.pop(priority) + while callbacks: + callback = callbacks.pop() + try: + callback() + except Exception as e: + msg = f'failed: {callback}: {type(e).__name__}: {e}' + else: + msg = f'succeeded: {callback}' + self._debug_output('Cleanup ' + msg) def add_cleanup( self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, @@ -78,10 +79,19 @@ def add_cleanup( Add a cleanup callback to the internal stack; they can be later called by :py:meth:`~.cleanup`. """ + self._add_cleanup(callback, 0, *args, **kwargs) + + def _add_cleanup( + self, callback: Callable[PS, Any], priority: float, + *args: PS.args, **kwargs: PS.kwargs, + ) -> None: if args or kwargs: callback = partial(callback, *args, **kwargs) - self._cleanup_stack.append(callback) - self._debug_output(f'Cleanup callback added: {callback}') + self._cleanup_stacks.setdefault(priority, []).append(callback) + header = 'Cleanup callback added' + if priority: + header = f'{header} (priority: {priority})' + self._debug_output(f'{header}: {callback}') def copy( self, *, inherit_cleanups: bool = False, **replacements @@ -96,7 +106,10 @@ def copy( init_args[field] = replacements.get(field, value) copy = type(self)(**init_args) if inherit_cleanups: - copy._cleanup_stack[:] = self._cleanup_stack + copy._cleanup_stacks = { + priority: list(callbacks) + for priority, callbacks in self._cleanup_stacks.items() + } return copy @classmethod @@ -176,13 +189,12 @@ def inject_env_vars( def _debug_output(self, msg: str) -> None: msg = f'{self._debug_message_header}: {msg}' diagnostics.log.debug(msg) + if not self._debug_log: + return try: with self._debug_log.open(mode='a') as fobj: print(msg, file=fobj) - except ( - AttributeError, # `._debug_log` is None in non-debug mode - OSError, # Cache dir may have been removed during cleanup - ): + except OSError: # Cache dir may have been rm-ed during cleanup pass @classmethod diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 39cabe2a..a617cb3e 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -156,6 +156,7 @@ def apply( return if lp_cache is None: lp_cache = LineProfilingCache._from_path(cache_path) + lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') _setup_in_child_process(lp_cache) vanilla: Callable[..., Any] | None diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index f19a8c33..e073d820 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -114,7 +114,10 @@ def load_pth_hook(ppid): # type: (int) -> None if getattr(load_pth_hook, 'called', False): return try: - _setup_in_child_process(LineProfilingCache.load()) + cache = LineProfilingCache.load() + cache._debug_output(f'cache {id(cache):#x} setting up (pth)...') + _setup_in_child_process(cache, True) + # _setup_in_child_process(LineProfilingCache.load()) except Exception as e: if DEBUG: msg = f'{type(e)}: {e}' @@ -148,14 +151,15 @@ def _wrap_os_fork(cache): # type: (LineProfilingCache) -> None def wrapper(): result = fork() if not result: # In the fork - _setup_in_child_process(cache.copy(), False) + cache._debug_output(f'cache {id(cache):#x} setting up (fork)...') + _setup_in_child_process(cache.copy()) return result os.fork = wrapper cache.add_cleanup(setattr, os, 'fork', fork) -def _setup_in_child_process(cache, wrap_os_fork=True): +def _setup_in_child_process(cache, wrap_os_fork=False): # type: (LineProfilingCache, bool) -> None """ Set up shop in a forked/spawned child process so that @@ -204,12 +208,13 @@ def _setup_in_child_process(cache, wrap_os_fork=True): # Occupy a tempfile slot in `cache.cache_dir` and set the profiler # up to write thereto when the process terminates handle, prof_outfile = mkstemp( - prefix='line-profiler-child-prof-output-', + prefix='child-prof-output-{}-{}-{:#x}-' + .format(cache.main_pid, os.getpid(), id(prof)), suffix='.lprof', dir=cache.cache_dir, ) - try: - cache.add_cleanup(prof.dump_stats, prof_outfile) + try: # Whatever else we do, write the profiling stats first + cache._add_cleanup(prof.dump_stats, -1, prof_outfile) finally: os.close(handle) From 2d5ffa5e4df0199cba0f0b2e242717391fb145d8 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 7 Apr 2026 02:20:54 +0200 Subject: [PATCH 011/165] Try to keep one cache per process line_profiler/_child_process_profiling/cache.py::LineProfilingCache profiler New attribute for the profiler instance copy(..., inherit_profiler=...) New argument for inheriting the `.profiler` load() Now keeping track of the loaded instance and returning it in subsequent calls line_profiler/_child_process_profiling/multiprocessing_patches.py ::apply(..., lp_cache=None) - If the `LineProfilingCache.load()`-ed instance is consistent with that loaded from `cache_path`, the former is used - Added more debugging output line_profiler/_child_process_profiling/pth_hook.py load_pth_hook() Added more debugging output _wrap_os_fork() Updated debugging output _setup_in_child_process() - Now returning a boolean (whether setup has been newly done) - Now setting `.profiler` of the cache instance - Added moew debugging output --- .../_child_process_profiling/cache.py | 58 ++++++++++++++----- .../multiprocessing_patches.py | 12 +++- .../_child_process_profiling/pth_hook.py | 32 ++++++++-- 3 files changed, 80 insertions(+), 22 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 76eb62aa..589e54ab 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -15,11 +15,11 @@ from operator import setitem from pathlib import Path from pickle import HIGHEST_PROTOCOL -from typing import Any, cast +from typing import Any, ClassVar, cast from typing_extensions import Self, ParamSpec from .. import _diagnostics as diagnostics -from ..line_profiler import LineStats +from ..line_profiler import LineProfiler, LineStats # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth # hook, which must run with minimal overhead when a Python process isn't @@ -51,9 +51,12 @@ class LineProfilingCache: # `builtin` to true insert_builtin: bool = True debug: bool = diagnostics.DEBUG + + profiler: LineProfiler | None = dataclasses.field(default=None, init=False) _cleanup_stacks: dict[float, list[Callable[[], Any]]] = dataclasses.field( default_factory=dict, init=False, ) + _loaded_instance: ClassVar[Self | None] = None def cleanup(self) -> None: """ @@ -94,12 +97,27 @@ def _add_cleanup( self._debug_output(f'{header}: {callback}') def copy( - self, *, inherit_cleanups: bool = False, **replacements + self, *, + inherit_cleanups: bool = False, + inherit_profiler: bool = False, + **replacements ) -> Self: """ - Make a copy with optionally replaced fields; - if ``inherit_cleanups`` is set to true, the copy also makes a - (shallow) copy of the clean-callback stack. + Make a copy with optionally replaced fields. + + Args: + inherit_cleanups (bool): + If true, the copy also makes a (shallow) copy of the + cleanup-callback stack. + inherit_profiler (bool): + If true, the copy also gets a reference to + :py:attr:`~.profiler` + **replacements (Any): + Optional fields to replace + + Return: + inst (LineProfilingCache): + New instance """ init_args: dict[str, Any] = {} for field, value in self._get_init_args().items(): @@ -110,6 +128,8 @@ def copy( priority: list(callbacks) for priority, callbacks in self._cleanup_stacks.items() } + if inherit_profiler: + copy.profiler = self.profiler return copy @classmethod @@ -119,16 +139,24 @@ def load(cls) -> Self: :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID` and :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_DIR_`. These should have been set from an ancestral Python process. + + Note: + If a previously :py:meth:`.~.load`-ed instance exists, it is + returned instead of a new instance. """ - pid = os.environ[INHERITED_PID_ENV_VARNAME] - cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}' - cache_dir = os.environ[cache_varname] - msg = ( - f'PID {os.getpid()} (from {pid}): ' - f'Loading instance from ${{{cache_varname}}} = {cache_dir}' - ) - diagnostics.log.debug(msg) - return cls._from_path(cls._get_filename(cache_dir)) + instance = cls._loaded_instance + if instance is None: + pid = os.environ[INHERITED_PID_ENV_VARNAME] + cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}' + cache_dir = os.environ[cache_varname] + msg = ( + f'PID {os.getpid()} (from {pid}): ' + f'Loading instance from ${{{cache_varname}}} = {cache_dir}' + ) + diagnostics.log.debug(msg) + instance = cls._from_path(cls._get_filename(cache_dir)) + cls._loaded_instance = instance + return instance def dump(self) -> None: """ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index a617cb3e..8fa88db3 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -156,8 +156,18 @@ def apply( return if lp_cache is None: lp_cache = LineProfilingCache._from_path(cache_path) + # Hack to retrieve the `.load()`-ed instance if one exists + loaded_cache = LineProfilingCache._loaded_instance + if ( + loaded_cache is not None + and loaded_cache._get_init_args() == lp_cache._get_init_args() + ): + lp_cache = loaded_cache lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') - _setup_in_child_process(lp_cache) + has_set_up = _setup_in_child_process(lp_cache) + lp_cache._debug_output('cache {:#x} setup {}'.format( + id(lp_cache), 'done' if has_set_up else 'aborted', + )) vanilla: Callable[..., Any] | None diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index e073d820..75a74ed9 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -116,7 +116,10 @@ def load_pth_hook(ppid): # type: (int) -> None try: cache = LineProfilingCache.load() cache._debug_output(f'cache {id(cache):#x} setting up (pth)...') - _setup_in_child_process(cache, True) + has_set_up = _setup_in_child_process(cache, True) + cache._debug_output('cache {:#x} setup {}'.format( + id(cache), 'done' if has_set_up else 'aborted', + )) # _setup_in_child_process(LineProfilingCache.load()) except Exception as e: if DEBUG: @@ -150,9 +153,15 @@ def _wrap_os_fork(cache): # type: (LineProfilingCache) -> None @wraps(fork) def wrapper(): result = fork() - if not result: # In the fork - cache._debug_output(f'cache {id(cache):#x} setting up (fork)...') - _setup_in_child_process(cache.copy()) + if result: + return result + # If we're here, we are in the fork + forked = cache.copy() + forked._debug_output(f'cache {id(forked):#x} setting up (fork)...') + has_set_up = _setup_in_child_process(forked) + forked._debug_output('cache {:#x} setup {}'.format( + id(forked), 'done' if has_set_up else 'aborted', + )) return result os.fork = wrapper @@ -160,7 +169,7 @@ def wrapper(): def _setup_in_child_process(cache, wrap_os_fork=False): - # type: (LineProfilingCache, bool) -> None + # type: (LineProfilingCache, bool) -> bool """ Set up shop in a forked/spawned child process so that (line-)profiling can extend therein. @@ -171,7 +180,15 @@ def _setup_in_child_process(cache, wrap_os_fork=False): wrap_os_fork (bool): Whether to wrap :py:func:`os.fork` which handles profiling; already-forked child processes should set this to false + + Returns: + has_set_up (bool): + False is ``cache`` has already been set up prior to calling + this function, true otherwise """ + if cache.profiler is not None: # Already set up + return False + import os from atexit import register from tempfile import mkstemp @@ -188,7 +205,7 @@ def _setup_in_child_process(cache, wrap_os_fork=False): # Create a profiler instance and manage it with # `CuratedProfilerContext` - prof = LineProfiler() + cache.profiler = prof = LineProfiler() upgrade_profiler(prof) ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) ctx.install() @@ -196,6 +213,7 @@ def _setup_in_child_process(cache, wrap_os_fork=False): # Do the preimports at `cache.preimports_module` where appropriate if cache.preimports_module: + cache._debug_output(f'cache {id(cache):#x} loading preimports...') with open(cache.preimports_module, mode='rb') as fobj: code = compile(fobj.read(), cache.preimports_module, 'exec') exec(code, {}) # Use a fresh, empty namespace @@ -225,3 +243,5 @@ def _setup_in_child_process(cache, wrap_os_fork=False): # Set `cache.cleanup()` as an atexit hook to handle everything when # the child process is about to terminate register(cache.cleanup) + + return True From f3f14182e234e1a5cbc9dd0fac9a8c2b4b66c833 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 7 Apr 2026 04:17:54 +0200 Subject: [PATCH 012/165] Renamed submodule --- .../{meta_path_finder.py => import_machinery.py} | 0 line_profiler/_child_process_profiling/pth_hook.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename line_profiler/_child_process_profiling/{meta_path_finder.py => import_machinery.py} (100%) diff --git a/line_profiler/_child_process_profiling/meta_path_finder.py b/line_profiler/_child_process_profiling/import_machinery.py similarity index 100% rename from line_profiler/_child_process_profiling/meta_path_finder.py rename to line_profiler/_child_process_profiling/import_machinery.py diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 75a74ed9..0c6a8b19 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -201,7 +201,7 @@ def _setup_in_child_process(cache, wrap_os_fork=False): ) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler - from .meta_path_finder import RewritingFinder + from .import_machinery import RewritingFinder # Create a profiler instance and manage it with # `CuratedProfilerContext` From cf00675b550643461ca3d6df25503e12ee549cb8 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 00:10:02 +0200 Subject: [PATCH 013/165] WIP: pass along more data kernprof.py::_manage_profiler.__enter__() Updated so that the created `LineProfilingCache` instance carries a `.rewrite_module` line_profiler/_child_process_profiling/cache.py::LineProfilingCache Added an optional `.rewrite_module` attribute line_profiler/_child_process_profiling/import_machinery.py ::RewritingFinder.find_spec() Now looking at `.lp_cache.rewrite_module` (where available) to check for specs to return --- kernprof.py | 13 +++--- .../_child_process_profiling/cache.py | 1 + .../import_machinery.py | 45 ++++++++++++++----- .../multiprocessing_patches.py | 3 +- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/kernprof.py b/kernprof.py index 4f323ecb..cc7ebe5c 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1208,7 +1208,7 @@ def __enter__(self): ) if self.set_up_child_profiling: self.cache = _prepare_child_profiling_cache( - self.options, self.prof, preimports_file, + self.options, self.prof, preimports_file, script_file, ) return self.prof, script_file @@ -1326,7 +1326,7 @@ def _prepare_exec_script( return script_file, preimports_file -def _prepare_child_profiling_cache(options, prof, preimports_file): +def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): """ Handle the (line-)profiling of spawned/forked child Python processes. @@ -1339,10 +1339,11 @@ def _prepare_child_profiling_cache(options, prof, preimports_file): # be responsible for managing their lifetimes, while derivative # instances in child processes will merely inherit and use them cache = LineProfilingCache( - tempfile.mkdtemp(), - options.prof_mod, - options.prof_imports, - preimports_file, + cache_dir=tempfile.mkdtemp(), + profiling_targets=options.prof_mod, + rewrite_module=script_file, + profile_imports=options.prof_imports, + preimports_module=preimports_file, insert_builtin=options.builtin, debug=options.debug, ) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 589e54ab..784cdf5c 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -44,6 +44,7 @@ class LineProfilingCache: profiling_targets: Collection[str] = dataclasses.field( default_factory=list, ) + rewrite_module: os.PathLike[str] | str | None = None profile_imports: bool = False preimports_module: os.PathLike[str] | str | None = None main_pid: int = dataclasses.field(default_factory=os.getpid) diff --git a/line_profiler/_child_process_profiling/import_machinery.py b/line_profiler/_child_process_profiling/import_machinery.py index 1436fc06..d4ddc1a0 100644 --- a/line_profiler/_child_process_profiling/import_machinery.py +++ b/line_profiler/_child_process_profiling/import_machinery.py @@ -11,13 +11,13 @@ import os import sys from collections.abc import Callable +from functools import partial from importlib.abc import MetaPathFinder, SourceLoader from importlib.machinery import ModuleSpec from pathlib import Path from types import CodeType, ModuleType from typing import TYPE_CHECKING -from .. import _diagnostics as diagnostics from ..autoprofile.run_module import AstTreeModuleProfiler from ..line_profiler import LineProfiler from .cache import LineProfilingCache @@ -26,6 +26,18 @@ __all__ = ('RewritingFinder',) +def _check_module_name(name: str, spec: ModuleSpec) -> bool: + return spec.name == name + + +def _check_module_origin( + path: os.PathLike[str] | str, spec: ModuleSpec, +) -> bool: + if spec.origin is None: + return False + return os.path.samefile(path, spec.origin) + + class RewritingFinder(MetaPathFinder, SourceLoader): """ Meta path finder to be set up in child processes, so that the @@ -33,6 +45,7 @@ class RewritingFinder(MetaPathFinder, SourceLoader): :py:func:`line_profiler.autoprofile.autoprofile.run` does. """ _cached_code_obj: CodeType + _checks: list[Callable[[ModuleSpec], bool]] def __init__( self, @@ -42,7 +55,12 @@ def __init__( ) -> None: self.prof = prof self.lp_cache = lp_cache - self.module_to_rewrite = module_to_rewrite + self._checks = [] + self._checks.append(partial(_check_module_name, module_to_rewrite)) + if lp_cache.rewrite_module: + self._checks.append( + partial(_check_module_origin, lp_cache.rewrite_module) + ) def install(self, *, index: int = 0) -> None: """ @@ -146,22 +164,25 @@ def exec_module(self, module: ModuleType) -> None: count = prof.object_count exec(self.get_code(spec), namespace, namespace) if prof.object_count > count: - msg = ( - 'main PID = {} / child PID = {}: ' - 'profiled {} code object{} in the `{}` module' - ).format( - self.lp_cache.main_pid, - os.getpid(), + msg = '{}: profiled {} code object{} in the `{}` module'.format( + type(self).__name__, prof.object_count, '' if prof.object_count == 1 else 's', - self.module_to_rewrite, + spec.name, ) - diagnostics.log.debug(msg) + self.lp_cache._debug_output(msg) def find_spec(self, *args, **kwargs) -> ModuleSpec | None: spec = self.find_spec_by_path(*args, **kwargs) - if (spec is not None and spec.name == self.module_to_rewrite): - return spec + if spec is None: + return None + for check in self._checks: + try: + if check(spec): + spec.loader = self + return spec + except Exception: + pass return None @staticmethod diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 8fa88db3..3abe0a82 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -102,8 +102,7 @@ def get_preparation_data( """ Wrap around :py:func:`multiprocessing.spawn.get_preparation_data`, slipping a :py:class:`PickleHook` into the returned dictionary so - that profiling is - triggered upon unpickling. + that profiling is triggered upon unpickling. Args: vanilla_impl From 3119f0a38fd4e2fca41ab1fea19d75047e0360c1 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 02:07:47 +0200 Subject: [PATCH 014/165] WIP: fix profiling for forked processes line_profiler/_child_process_profiling/ cache.py::LineProfilingCache _replace_loaded_instance() New convenience method for an instance in a fork to replace the instance to be `.load()`-ed _consistent_with_loaded_instance New attribute for checking whether the instance is consistent with what would have been `.load()`-ed multiprocessing_patches.py bootstrap(..., lp_cache=...) Can now be `None`, which defers the `.load()`-ing of the cache instance apply() - Streamlined logic for retrieving the loaded instance - Now using the above deferred loading whenever appropriate, so that cleanup and profiling is preserved in forked processes pth_hook.py::_wrap_os_fork() Now using `._replace_loaded_instance()`, so that future calls to `.load()` in the forked process retrieves the newly-created instance --- .../_child_process_profiling/cache.py | 10 +++++++ .../multiprocessing_patches.py | 26 ++++++++++--------- .../_child_process_profiling/pth_hook.py | 5 ++++ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 784cdf5c..58801457 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -226,6 +226,12 @@ def _debug_output(self, msg: str) -> None: except OSError: # Cache dir may have been rm-ed during cleanup pass + def _replace_loaded_instance(self) -> bool: + if self._consistent_with_loaded_instance: + type(self)._loaded_instance = self + return True + return False + @classmethod def _from_path(cls, fname: os.PathLike[str] | str) -> Self: with open(fname, mode='rb') as fobj: @@ -271,3 +277,7 @@ def _debug_message_header(self) -> str: if self.main_pid == pid: return f'PID {pid} (main process)' return f'PID {pid} (from {self.main_pid})' + + @cached_property + def _consistent_with_loaded_instance(self) -> bool: + return type(self).load()._get_init_args() == self._get_init_args() diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 3abe0a82..ca55c6e7 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -59,7 +59,7 @@ def bootstrap( vanilla_impl: Callable[ Concatenate[multiprocessing.process.BaseProcess, PS], T ], - lp_cache: LineProfilingCache, + lp_cache: LineProfilingCache | None, /, *args: PS.args, **kwargs: PS.kwargs @@ -74,8 +74,10 @@ def bootstrap( :py:class:`~.BaseProcess` vanilla_impl (Callable) Vanilla :py:meth:`~.BaseProcess._bootstrap` - lp_cache (LineProfilingCache) - Cache recovered by :py:meth:`~.LineProfilingCache.load` + lp_cache (LineProfilingCache | None) + Cache recovered by :py:meth:`~.LineProfilingCache.load`; + if :py:const:`None`, the ``.load()`` is deferred to after + ``vanilla_impl()`` is run *args **kwargs Passed to :py:meth:`~.BaseProcess._bootstrap` @@ -89,6 +91,8 @@ def bootstrap( try: return vanilla_impl(self, *args, **kwargs) finally: # Write profiling results + if lp_cache is None: + lp_cache = LineProfilingCache.load() lp_cache.cleanup() @@ -155,13 +159,8 @@ def apply( return if lp_cache is None: lp_cache = LineProfilingCache._from_path(cache_path) - # Hack to retrieve the `.load()`-ed instance if one exists - loaded_cache = LineProfilingCache._loaded_instance - if ( - loaded_cache is not None - and loaded_cache._get_init_args() == lp_cache._get_init_args() - ): - lp_cache = loaded_cache + if lp_cache._consistent_with_loaded_instance: + lp_cache = LineProfilingCache.load() lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') has_set_up = _setup_in_child_process(lp_cache) lp_cache._debug_output('cache {:#x} setup {}'.format( @@ -173,8 +172,11 @@ def apply( # Patch `multiprocessing.process.BaseProcess._bootstrap()` Proc = multiprocessing.process.BaseProcess vanilla = Proc._bootstrap # type: ignore[attr-defined] - Proc._bootstrap = ( # type: ignore[attr-defined] - partialmethod(bootstrap, vanilla, lp_cache) + Proc._bootstrap = partialmethod( # type: ignore[attr-defined] + bootstrap, vanilla, + # Always defer to the `.load()`-ed instance where possible + # (helps with keeping forked processes consistent) + None if lp_cache._consistent_with_loaded_instance else lp_cache, ) lp_cache.add_cleanup(setattr, Proc, '_bootstrap', vanilla) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 0c6a8b19..29ac7822 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -157,6 +157,11 @@ def wrapper(): return result # If we're here, we are in the fork forked = cache.copy() + if forked._replace_loaded_instance(): + forked._debug_output( + f'cache {id(forked):#x} in fork ' + 'superseded cached `.load()`-ed instance' + ) forked._debug_output(f'cache {id(forked):#x} setting up (fork)...') has_set_up = _setup_in_child_process(forked) forked._debug_output('cache {:#x} setup {}'.format( From 12a2ae06e7b759cead93a5d6cebc83f1229c35fc Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 02:59:59 +0200 Subject: [PATCH 015/165] Cleanup: `[...].multiprocessing_patches` kernprof.py::_prepare_child_profiling_cache() - Updated call to `[...].multiprocessing_patches.apply()` - Now always setting up the created instance as the one returned by further calls to `.load()` line_profiler/_child_process_profiling/multiprocessing_patches.py PickleHook - Refactored to contain no instance variables - Now always using `LineProfilingCache.load()` to retrieve the appropriate cache instance bootstrap() Removed argument `lp_cache` get_preparation_data() Removed arguemnt `cache_path` apply() - Removed argument `cache_path` - Argument `lp_cache` now required - Simplified implementation --- kernprof.py | 5 +- .../multiprocessing_patches.py | 100 +++++++----------- 2 files changed, 44 insertions(+), 61 deletions(-) diff --git a/kernprof.py b/kernprof.py index cc7ebe5c..55c0d8d9 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1376,7 +1376,10 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): # Patch `multiprocessing` so that child processes are properly # handled across all "start methods" - multiprocessing_patches.apply(cache.filename, lp_cache=cache) + multiprocessing_patches.apply(cache) + + # Set up the cache instance to be the default one `.load()`-ed + cache._replace_loaded_instance() return cache diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index ca55c6e7..3210ef39 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -17,7 +17,7 @@ import multiprocessing.process from collections.abc import Callable from functools import partial, partialmethod -from typing import Any, TypedDict, TypeVar +from typing import Any, TypeVar from typing_extensions import Concatenate, ParamSpec from .cache import LineProfilingCache @@ -30,9 +30,7 @@ T = TypeVar('T') PS = ParamSpec('PS') - -class _HookState(TypedDict): - cache_path: str # Cache to be loaded from here +_PATCHED_MARKER = '_line_profiler_patched_multiprocessing' class PickleHook: @@ -43,15 +41,24 @@ class PickleHook: See also: :py:class:`coverage.multiproc.Stowaway` """ - def __init__(self, cache_path: str) -> None: - self.cache_path = cache_path - - def __getstate__(self) -> _HookState: - return {'cache_path': self.cache_path} - - def __setstate__(self, state: _HookState) -> None: - self.cache_path = path = state['cache_path'] - apply(path) + def __getstate__(_) -> int: + # Cannot return `None`, or nothing will be pickled and + # `.__getstate__()` will not be invoked in the child + return 1 + + def __setstate__(*_) -> None: + # We're in a child process created by `multiprocessing`, so set + # up shop here... + lp_cache = LineProfilingCache.load() + lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') + has_set_up = _setup_in_child_process(lp_cache) + lp_cache._debug_output('cache {:#x} setup {}'.format( + id(lp_cache), 'done' if has_set_up else 'aborted', + )) + # ... and we don't care about polluting the `multiprocessing` + # namespace either, so don't bother with cleanup + if not getattr(multiprocessing, _PATCHED_MARKER, False): + _apply_mp_patches(lp_cache, _no_op) def bootstrap( @@ -59,7 +66,6 @@ def bootstrap( vanilla_impl: Callable[ Concatenate[multiprocessing.process.BaseProcess, PS], T ], - lp_cache: LineProfilingCache | None, /, *args: PS.args, **kwargs: PS.kwargs @@ -74,10 +80,6 @@ def bootstrap( :py:class:`~.BaseProcess` vanilla_impl (Callable) Vanilla :py:meth:`~.BaseProcess._bootstrap` - lp_cache (LineProfilingCache | None) - Cache recovered by :py:meth:`~.LineProfilingCache.load`; - if :py:const:`None`, the ``.load()`` is deferred to after - ``vanilla_impl()`` is run *args **kwargs Passed to :py:meth:`~.BaseProcess._bootstrap` @@ -91,14 +93,11 @@ def bootstrap( try: return vanilla_impl(self, *args, **kwargs) finally: # Write profiling results - if lp_cache is None: - lp_cache = LineProfilingCache.load() - lp_cache.cleanup() + LineProfilingCache.load().cleanup() def get_preparation_data( vanilla_impl: Callable[PS, dict[str, Any]], - cache_path: str, /, *args: PS.args, **kwargs: PS.kwargs @@ -112,9 +111,6 @@ def get_preparation_data( vanilla_impl Vanilla :py:func:`multiprocessing.spawn.get_preparation_data` - cache_path - File from which the :py:class:`LineProfilingCache` should be - loaded *args **kwargs Passed to @@ -127,25 +123,18 @@ def get_preparation_data( key = 'line_profiler_pickle_hook' # Doesn't matter data = vanilla_impl(*args, **kwargs) assert key not in data - data[key] = PickleHook(cache_path) + data[key] = PickleHook() return data -def apply( - cache_path: str, *, lp_cache: LineProfilingCache | None = None, -) -> None: +def apply(lp_cache: LineProfilingCache) -> None: """ Set up profiling in :py:mod:`multiprocessing` child processes by applying patches to the module. Args: - cache_path - Path to the file whence a :py:class:`LineProfilingCache` - object can be loaded - lp_cache - Optional :py:class:`LineProfilingCache` instance; - if not provided, it is loaded from `cache_path`, and - profiling is set up therefrom in the (sub-)process + lp_cache (LineProfilingCache) + Cache instance governing the profiling run Side effects: - :py:mod:`multiprocessing` marked as having been set up @@ -154,19 +143,13 @@ def apply( - :py:func:`multiprocessing.spawn.get_preparation_data` patched - Cleanup callbacks registered via `lp_cache.add_cleanup()` """ - patched_marker = '_line_profiler_patched_multiprocessing' - if getattr(multiprocessing, patched_marker, False): - return - if lp_cache is None: - lp_cache = LineProfilingCache._from_path(cache_path) - if lp_cache._consistent_with_loaded_instance: - lp_cache = LineProfilingCache.load() - lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') - has_set_up = _setup_in_child_process(lp_cache) - lp_cache._debug_output('cache {:#x} setup {}'.format( - id(lp_cache), 'done' if has_set_up else 'aborted', - )) + if not getattr(multiprocessing, _PATCHED_MARKER, False): + _apply_mp_patches(lp_cache, lp_cache.add_cleanup) + +def _apply_mp_patches( + lp_cache: LineProfilingCache, add_cleanup: Callable[..., None], +) -> None: vanilla: Callable[..., Any] | None # Patch `multiprocessing.process.BaseProcess._bootstrap()` @@ -174,11 +157,8 @@ def apply( vanilla = Proc._bootstrap # type: ignore[attr-defined] Proc._bootstrap = partialmethod( # type: ignore[attr-defined] bootstrap, vanilla, - # Always defer to the `.load()`-ed instance where possible - # (helps with keeping forked processes consistent) - None if lp_cache._consistent_with_loaded_instance else lp_cache, ) - lp_cache.add_cleanup(setattr, Proc, '_bootstrap', vanilla) + add_cleanup(setattr, Proc, '_bootstrap', vanilla) # Patch `multiprocessing.spawn.get_preparation_data()` try: @@ -188,13 +168,13 @@ def apply( else: vanilla = getattr(spawn, 'get_preparation_data', None) if vanilla: - spawn.get_preparation_data = partial( - get_preparation_data, vanilla, cache_path, - ) - lp_cache.add_cleanup( - setattr, spawn, 'get_preparation_data', vanilla, - ) + spawn.get_preparation_data = partial(get_preparation_data, vanilla) + add_cleanup(setattr, spawn, 'get_preparation_data', vanilla) # Mark `multiprocessing` as having been patched - setattr(multiprocessing, patched_marker, True) - lp_cache.add_cleanup(vars(multiprocessing).pop, patched_marker, None) + setattr(multiprocessing, _PATCHED_MARKER, True) + add_cleanup(vars(multiprocessing).pop, _PATCHED_MARKER, None) + + +def _no_op(*_, **__) -> None: + pass From 10d208bebe0c304454754456c68dd50b589ed19f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 05:00:15 +0200 Subject: [PATCH 016/165] Removed `import_machinery` line_profiler/_child_process_profiling/import_machinery.py Removed line_profiler/_child_process_profiling/pth_hook.py ::_setup_in_child_process() No longer set up the `RewritingFinder` because messing with the import system doesn't help with propagating autoprofiling rewrites to child processes... --- .../import_machinery.py | 242 ------------------ .../_child_process_profiling/pth_hook.py | 6 - 2 files changed, 248 deletions(-) delete mode 100644 line_profiler/_child_process_profiling/import_machinery.py diff --git a/line_profiler/_child_process_profiling/import_machinery.py b/line_profiler/_child_process_profiling/import_machinery.py deleted file mode 100644 index d4ddc1a0..00000000 --- a/line_profiler/_child_process_profiling/import_machinery.py +++ /dev/null @@ -1,242 +0,0 @@ -""" -A meta path finder object which rewrites a specific module. - -Note: - Based on the implementation in - :py:mod:`pytest_autoprofile.importers`. -""" -from __future__ import annotations - -import ast -import os -import sys -from collections.abc import Callable -from functools import partial -from importlib.abc import MetaPathFinder, SourceLoader -from importlib.machinery import ModuleSpec -from pathlib import Path -from types import CodeType, ModuleType -from typing import TYPE_CHECKING - -from ..autoprofile.run_module import AstTreeModuleProfiler -from ..line_profiler import LineProfiler -from .cache import LineProfilingCache - - -__all__ = ('RewritingFinder',) - - -def _check_module_name(name: str, spec: ModuleSpec) -> bool: - return spec.name == name - - -def _check_module_origin( - path: os.PathLike[str] | str, spec: ModuleSpec, -) -> bool: - if spec.origin is None: - return False - return os.path.samefile(path, spec.origin) - - -class RewritingFinder(MetaPathFinder, SourceLoader): - """ - Meta path finder to be set up in child processes, so that the - ``module_to_rewrite`` is rewritten for profiling as - :py:func:`line_profiler.autoprofile.autoprofile.run` does. - """ - _cached_code_obj: CodeType - _checks: list[Callable[[ModuleSpec], bool]] - - def __init__( - self, - prof: LineProfiler, - lp_cache: LineProfilingCache, - module_to_rewrite: str = '__main__', - ) -> None: - self.prof = prof - self.lp_cache = lp_cache - self._checks = [] - self._checks.append(partial(_check_module_name, module_to_rewrite)) - if lp_cache.rewrite_module: - self._checks.append( - partial(_check_module_origin, lp_cache.rewrite_module) - ) - - def install(self, *, index: int = 0) -> None: - """ - Install the importer into :py:data:`sys.meta_path` at the - specified ``index``; - if it's already there, it's first removed and re-inserted at - the requested position. - """ - self.uninstall(invalidate_caches=False) - sys.meta_path.insert(index, self) - - def uninstall(self, *, invalidate_caches: bool = True) -> None: - """ - Uninstall the importer from :py:data:`sys.meta_path`, and - optionally also invalidate the caches. - """ - try: - sys.meta_path.remove(self) - except ValueError: # Not in the list - return - if invalidate_caches: - self.invalidate_caches() - - @classmethod - def find_spec_by_path(cls, *args, **kwargs) -> ModuleSpec | None: - """ - Implementation of - :py:meth:`MetaPathFinder.find_spec` which looks - for module specs with the other meta-path finders. - - Returns: - maybe_spec (ModuleSpec | None) - Module spec if found, :py:const:`None` otherwise - """ - Implementation = Callable[..., ModuleSpec | None] - impls: list[Implementation] = [ - finder.find_spec for finder in sys.meta_path - if callable(getattr(finder, 'find_spec', None)) - if not isinstance(finder, cls) - ] - for impl in impls: - try: - spec = impl(*args, **kwargs) - except Exception: - continue - if spec is not None: - return spec - return None - - def write_code(self, spec: ModuleSpec) -> CodeType: - """ - Rewrite the module code that ``spec`` points to with - :py:class:`ast.autoprofile.run_module.AstTreeModuleProfiler`. - - Args: - spec (ModuleSpec) - Module spec - - Returns: - code (CodeType) - Code object - """ - assert spec.origin - fname = str(Path(spec.origin)) - module = AstTreeModuleProfiler( - spec.origin, - list(self.lp_cache.profiling_targets), - self.lp_cache.profile_imports, - ).profile() - # Slip in a helper node so as to ensure the availability of - # `@profile` - # Note: `@profile` is slipped into the local namespace by - # `RewritingFinder.exec_module()`, but that may not be - # enough for applications directly using the code objects (e.g. - # `runpy`. Hence, we provide an out by falling back to - # `@line_profiler.profile`. - ensure_profile_node, = ast.parse( - 'if profile not in globals():\n' - ' from line_profiler import profile' - ).body - module.body.insert(0, ensure_profile_node) - return compile(module, fname, 'exec') - - # Methods as dictated by the interface - - def invalidate_caches(self) -> None: - try: - del self._cached_code_obj - except AttributeError: - pass - super().invalidate_caches() - - def exec_module(self, module: ModuleType) -> None: - namespace = module.__dict__ - namespace['profile'] = prof = self.prof - spec: ModuleSpec | None = module.__spec__ - if spec is None: - raise RuntimeError(f'module = {module!r}: empty `.__spec__`') - if TYPE_CHECKING: # Appease `mypy` - assert hasattr(prof, 'object_count') - count = prof.object_count - exec(self.get_code(spec), namespace, namespace) - if prof.object_count > count: - msg = '{}: profiled {} code object{} in the `{}` module'.format( - type(self).__name__, - prof.object_count, - '' if prof.object_count == 1 else 's', - spec.name, - ) - self.lp_cache._debug_output(msg) - - def find_spec(self, *args, **kwargs) -> ModuleSpec | None: - spec = self.find_spec_by_path(*args, **kwargs) - if spec is None: - return None - for check in self._checks: - try: - if check(spec): - spec.loader = self - return spec - except Exception: - pass - return None - - @staticmethod - def get_data(path: os.PathLike[str] | str) -> bytes: - return Path(path).read_bytes() - - @classmethod - def get_filename(cls, name: str) -> str: - spec = cls.find_spec_by_path(name) - if spec is None: - raise ImportError(name) - origin = spec.origin - if origin is None: - raise ImportError(name) - if origin == 'frozen' or not os.path.exists(origin): - raise ImportError(name) - return origin - - def get_code(self, name_or_spec: str | ModuleSpec) -> CodeType: - if isinstance(name_or_spec, str): - spec = self.find_spec_by_path(name_or_spec) - if spec is None: - raise ImportError(name_or_spec) - else: - spec = name_or_spec - try: - try: - return self._cached_code_obj - except AttributeError: - self._cached_code_obj = code = self.write_code(spec) - return code - except Exception as e: - raise ImportError(name_or_spec) from e - - if TYPE_CHECKING: - def source_to_code( # type: ignore[override] - self, *args, **kwargs - ) -> CodeType: - """ - Notes - ----- - :py:mod:`mypy` reports that - :py:meth:`SourceLoader.source_to_code`, an instance method, - clashes with - :py:meth:`importlib.abc.InspectLoader.source_to_code`, a - static method. - - Since: - - The method is functionally only used in the superclasses - as an instance method, and - - :py:class:`importlib.abc.InspectLoader`` is merely - included as a base class because :py:class:`SourceLoader` - inherits from it, - just explicitly override the method here so that we can - catch and suppress the :py:mod:`mypy` error. - """ - return super().source_to_code(*args, **kwargs) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 29ac7822..a46b4dac 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -206,7 +206,6 @@ def _setup_in_child_process(cache, wrap_os_fork=False): ) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler - from .import_machinery import RewritingFinder # Create a profiler instance and manage it with # `CuratedProfilerContext` @@ -223,11 +222,6 @@ def _setup_in_child_process(cache, wrap_os_fork=False): code = compile(fobj.read(), cache.preimports_module, 'exec') exec(code, {}) # Use a fresh, empty namespace - # Set up the importer for rewriting `__main__` - finder = RewritingFinder(prof, cache) - finder.install() - cache.add_cleanup(finder.uninstall) - # Occupy a tempfile slot in `cache.cache_dir` and set the profiler # up to write thereto when the process terminates handle, prof_outfile = mkstemp( From 4b6bb3806e7e420a01ae6cdfa9600c8ad03c0cd8 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 07:54:14 +0200 Subject: [PATCH 017/165] Fixed stat-aggregation bug kernprof.py _dump_filtered_stats() Fixed bug where if no tempfile remains, the `extra_line_stats` are not merged into the dumped stats _prepare_child_profiling_cache() Now setting the `.profiler` of the returned cache object --- kernprof.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/kernprof.py b/kernprof.py index 55c0d8d9..50097fde 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1098,25 +1098,30 @@ def _remove(path, *, recursive=False, missing_ok=False): def _dump_filtered_stats(tmpdir, prof, filename, extra_line_stats=None): import os + if isinstance(prof, ContextualProfile): + # - Not using `line_profiler` + # -> doesn't matter if the source lines can't be retrieved + # -> no need to filter anything + prof.dump_stats(filename) + return + + # Remember to incorporate extra stats where available + line_stats = prof.get_stats() + if extra_line_stats is not None: + line_stats += extra_line_stats + # Build list of known temp file paths tempfile_paths = [ os.path.join(dirpath, fname) for dirpath, _, fnames in os.walk(tmpdir) for fname in fnames ] - - if not tempfile_paths or isinstance(prof, ContextualProfile): + if not tempfile_paths: # - No tempfiles written -> no function lives in tempfiles # -> no need to filter anything - # - Not using `line_profiler` - # -> doesn't matter if the source lines can't be retrieved - # -> no need to filter anything - prof.dump_stats(filename) + line_stats.to_file(filename) return - line_stats = prof.get_stats() - if extra_line_stats is not None: - line_stats += extra_line_stats _dump_filtered_line_stats(line_stats, tempfile_paths, filename) @@ -1378,8 +1383,11 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): # handled across all "start methods" multiprocessing_patches.apply(cache) - # Set up the cache instance to be the default one `.load()`-ed + # Misc. setup: + # - Set up the cache instance to be the default one `.load()`-ed cache._replace_loaded_instance() + # - Set `cache.profiler` + cache.profiler = prof return cache From ab1f9215ce838fd57e7a2c44e2114f2250bd4e56 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 08:20:07 +0200 Subject: [PATCH 018/165] More forked-process fixes line_profiler/_child_process_profiling/multiprocessing_patches.py ::_apply_mp_patches() - Added debugging output for the patches - Now patching the copy of `runpy` imported by `multiprocessing.runpy` line_profiler/_child_process_profiling/pth_hook.py _wrap_os_fork() No longer creating a new `LineProfiler` instance (helps with handling forked processes) _setup_in_child_process(..., prof=...) New argument for avoiding instantiating a new profiler when not necessary (e.g. in a forked process) line_profiler/_child_process_profiling/runpy_patches.py New submodule for the aforementioned patching of `runpy` --- .../multiprocessing_patches.py | 45 ++++-- .../_child_process_profiling/pth_hook.py | 22 ++- .../_child_process_profiling/runpy_patches.py | 149 ++++++++++++++++++ 3 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 line_profiler/_child_process_profiling/runpy_patches.py diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 3210ef39..28ce9acc 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -22,6 +22,7 @@ from .cache import LineProfilingCache from .pth_hook import _setup_in_child_process +from .runpy_patches import create_runpy_wrapper __all__ = ('apply',) @@ -150,26 +151,50 @@ def apply(lp_cache: LineProfilingCache) -> None: def _apply_mp_patches( lp_cache: LineProfilingCache, add_cleanup: Callable[..., None], ) -> None: - vanilla: Callable[..., Any] | None + def replace( + obj: Any, attr: str, value: Any, obj_name: str | None = None, + ) -> None: + try: + old = getattr(obj, attr) + except AttributeError: + add_cleanup(setattr, obj, attr, old) + else: + add_cleanup(delattr, obj, attr) + setattr(obj, attr, value) + if obj_name is None: + obj_name = repr(obj) + lp_cache._debug_output('cache {:#x}: patched `{}.{}` -> `{}`'.format( + id(lp_cache), obj_name, attr, value, + )) # Patch `multiprocessing.process.BaseProcess._bootstrap()` Proc = multiprocessing.process.BaseProcess - vanilla = Proc._bootstrap # type: ignore[attr-defined] - Proc._bootstrap = partialmethod( # type: ignore[attr-defined] - bootstrap, vanilla, + bootstrap_wrapper = partialmethod( + bootstrap, + Proc._bootstrap, # type: ignore[attr-defined] + ) + replace( + Proc, '_bootstrap', bootstrap_wrapper, + f'{Proc.__module__}.{Proc.__qualname__}', ) - add_cleanup(setattr, Proc, '_bootstrap', vanilla) - # Patch `multiprocessing.spawn.get_preparation_data()` + # Patch `multiprocessing.spawn` try: from multiprocessing import spawn except ImportError: # Incompatible platforms pass else: - vanilla = getattr(spawn, 'get_preparation_data', None) - if vanilla: - spawn.get_preparation_data = partial(get_preparation_data, vanilla) - add_cleanup(setattr, spawn, 'get_preparation_data', vanilla) + # Patch `get_preparation_data()` + gpd_wrapper = partial( # type: ignore[call-arg] + get_preparation_data, spawn.get_preparation_data, + ) + replace(spawn, 'get_preparation_data', gpd_wrapper, spawn.__name__) + # Patch `runpy` (do it locally instead of tempering with the + # global `runpy` mmodule) + if hasattr(spawn, 'runpy'): + replace( + spawn, 'runpy', create_runpy_wrapper(lp_cache), spawn.__name__, + ) # Mark `multiprocessing` as having been patched setattr(multiprocessing, _PATCHED_MARKER, True) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index a46b4dac..e8dafaae 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -20,8 +20,11 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from .cache import LineProfilingCache # noqa: F401 + from collections.abc import Callable # noqa: F401 from pathlib import Path # noqa: F401 + from typing import Any # noqa: F401 + from ..line_profiler import LineProfiler # noqa: F401 + from .cache import LineProfilingCache # noqa: F401 __all__ = ( @@ -151,7 +154,7 @@ def _wrap_os_fork(cache): # type: (LineProfilingCache) -> None return @wraps(fork) - def wrapper(): + def wrapper(): # type: () -> int result = fork() if result: return result @@ -162,8 +165,11 @@ def wrapper(): f'cache {id(forked):#x} in fork ' 'superseded cached `.load()`-ed instance' ) + # Note: we can reuse the profiler instance in the fork, but it + # needs to go through setup so that the separate profiling + # results are dumped into another output file forked._debug_output(f'cache {id(forked):#x} setting up (fork)...') - has_set_up = _setup_in_child_process(forked) + has_set_up = _setup_in_child_process(forked, prof=cache.profiler) forked._debug_output('cache {:#x} setup {}'.format( id(forked), 'done' if has_set_up else 'aborted', )) @@ -173,8 +179,8 @@ def wrapper(): cache.add_cleanup(setattr, os, 'fork', fork) -def _setup_in_child_process(cache, wrap_os_fork=False): - # type: (LineProfilingCache, bool) -> bool +def _setup_in_child_process(cache, wrap_os_fork=False, prof=None): + # type: (LineProfilingCache, bool, LineProfiler | None) -> bool """ Set up shop in a forked/spawned child process so that (line-)profiling can extend therein. @@ -205,11 +211,13 @@ def _setup_in_child_process(cache, wrap_os_fork=False): _extend_line_profiler_for_profiling_imports as upgrade_profiler, ) from ..curated_profiling import CuratedProfilerContext - from ..line_profiler import LineProfiler + from ..line_profiler import LineProfiler # noqa: F811 # Create a profiler instance and manage it with # `CuratedProfilerContext` - cache.profiler = prof = LineProfiler() + if prof is None: + prof = LineProfiler() + cache.profiler = prof upgrade_profiler(prof) ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) ctx.install() diff --git a/line_profiler/_child_process_profiling/runpy_patches.py b/line_profiler/_child_process_profiling/runpy_patches.py new file mode 100644 index 00000000..69d38da4 --- /dev/null +++ b/line_profiler/_child_process_profiling/runpy_patches.py @@ -0,0 +1,149 @@ +""" +Patches for :py:mod:`runpy` to be patched into the namespace of +:py:mod:`multiprocessing.spawn`, so that the rewriting of ``__main__`` +can be continued into child processes. +""" +from __future__ import annotations + +import os +from collections.abc import Callable +from functools import partial +from importlib.util import find_spec +from types import ModuleType +from typing import cast, TypeVar +from typing_extensions import Concatenate, ParamSpec + +from ..autoprofile.ast_tree_profiler import AstTreeProfiler +from ..autoprofile.run_module import AstTreeModuleProfiler +from ..autoprofile.util_static import modname_to_modpath +from .cache import LineProfilingCache + + +__all__ = ('create_runpy_wrapper',) + + +PS = ParamSpec('PS') +T = TypeVar('T') + + +THIS_MODULE = (lambda: None).__module__ + + +def _copy_module(name: str) -> ModuleType: + """ + Returns: + module (ModuleType): + Module object, which is a fresh copy of the module named + ``name`` + """ + spec = find_spec(name) + if spec is None: + raise ModuleNotFoundError(name) + assert spec.loader + assert callable(getattr(spec.loader, 'exec_module', None)) + module = ModuleType(spec.name) + for attr, value in { + '__spec__': spec, + '__name__': spec.name, + '__file__': spec.origin, + '__path__': spec.submodule_search_locations, + }.items(): + if value is not None: + setattr(module, attr, value) + spec.loader.exec_module(module) + return module + + +def _exec( + cache: LineProfilingCache, + CodeWriter: type[AstTreeProfiler], + _code, # This represents the first pos arg to `exec()` (ignored) + /, + *args, **kwargs, +) -> None: + """ + To be monkey-patched into :py:mod:`runpy`'s namespace as `exec()` + so that rewritten and autoprofiled code at ``cache.rewrite_module`` + is always executed. + """ + assert cache.rewrite_module + cache._debug_output('Calling via {}: `exec({})`'.format( + THIS_MODULE, + ', '.join( + [repr(a) for a in (_code, *args)] + + [f'{k}={v!r}' for k, v in kwargs.items()] + ), + )) + fname = str(cache.rewrite_module) + code_writer = CodeWriter( + fname, + list(cache.profiling_targets), + cache.profile_imports, + ) + code = compile(code_writer.profile(), fname, 'exec') + exec(code, *args, **kwargs) + + +def _run( + cache: LineProfilingCache, + runpy: ModuleType, + func: Callable[Concatenate[str, PS], T], + resolve_target_to_path: Callable[[str], str], + CodeWriter: type[AstTreeProfiler], + target: str, + /, + *args: PS.args, **kwargs: PS.kwargs +) -> T: + cache._debug_output('Calling via {}: `runpy.{}({})`'.format( + THIS_MODULE, + func.__name__, + ', '.join( + [repr(a) for a in (target, *args)] + + [f'{k}={v!r}' for k, v in kwargs.items()] + ), + )) + if cache.rewrite_module: + try: + filename = resolve_target_to_path(target) + profile = os.path.samefile(filename, cache.rewrite_module) + except Exception as e: + cache._debug_output( + f'{THIS_MODULE}: failed to check whether to ' + f'rewrite code loaded in `runpy.{func.__name__}(...)` ' + f'({type(e).__name__}: {e})' + ) + profile = False + else: + profile = False + # If we are about to run the code to be autoprofiled, monkey-patch + # `exec()` into the `runpy` namespace which just rewrites + # `cache.rewrite_module` and executes it + if profile: + runpy.exec = ( # type: ignore[attr-defined] + partial(_exec, cache, CodeWriter) + ) + try: + return func(target, *args, **kwargs) + finally: + try: + del runpy.exec + except AttributeError: + pass + + +def create_runpy_wrapper(cache: LineProfilingCache) -> ModuleType: + """ + Create a copy of :py:mod:`runpy` which does code rewriting similar + to :py:func:`line_profiler.autoprofile.autoprofile.run` for the + appropriate file as indicated by ``cache``. + """ + runpy = _copy_module('runpy') + for func, resolver, CodeWriter in [ + ('run_path', str, AstTreeProfiler), + ('run_module', modname_to_modpath, AstTreeModuleProfiler), + ]: + impl = getattr(runpy, func) + res = cast(Callable[[str], str], resolver) # Help `mypy` out + wrapper = partial(_run, cache, runpy, impl, res, CodeWriter) + setattr(runpy, func, wrapper) + return runpy From ee3f439402ef6b58c36206682718438b272a34b7 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 22:53:33 +0200 Subject: [PATCH 019/165] Streamlined debug output in `_child_process_profiling` --- .../_child_process_profiling/cache.py | 8 +++-- .../multiprocessing_patches.py | 13 +++---- .../_child_process_profiling/pth_hook.py | 34 ++++++++++--------- .../_child_process_profiling/runpy_patches.py | 4 +-- 4 files changed, 29 insertions(+), 30 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 58801457..7ef9d0f0 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -274,9 +274,11 @@ def _debug_log(self) -> Path | None: @cached_property def _debug_message_header(self) -> str: pid = os.getpid() - if self.main_pid == pid: - return f'PID {pid} (main process)' - return f'PID {pid} (from {self.main_pid})' + return 'PID {} ({}): Cache {:#x}'.format( + pid, + 'main process' if self.main_pid == pid else self.main_pid, + id(self), + ) @cached_property def _consistent_with_loaded_instance(self) -> bool: diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 28ce9acc..9bfc5abf 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -51,11 +51,7 @@ def __setstate__(*_) -> None: # We're in a child process created by `multiprocessing`, so set # up shop here... lp_cache = LineProfilingCache.load() - lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') - has_set_up = _setup_in_child_process(lp_cache) - lp_cache._debug_output('cache {:#x} setup {}'.format( - id(lp_cache), 'done' if has_set_up else 'aborted', - )) + _setup_in_child_process(lp_cache, False, 'multiprocessing') # ... and we don't care about polluting the `multiprocessing` # namespace either, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): @@ -163,15 +159,14 @@ def replace( setattr(obj, attr, value) if obj_name is None: obj_name = repr(obj) - lp_cache._debug_output('cache {:#x}: patched `{}.{}` -> `{}`'.format( - id(lp_cache), obj_name, attr, value, + lp_cache._debug_output('Patched `{}.{}` -> `{}`'.format( + obj_name, attr, value, )) # Patch `multiprocessing.process.BaseProcess._bootstrap()` Proc = multiprocessing.process.BaseProcess bootstrap_wrapper = partialmethod( - bootstrap, - Proc._bootstrap, # type: ignore[attr-defined] + bootstrap, Proc._bootstrap, # type: ignore[attr-defined] ) replace( Proc, '_bootstrap', bootstrap_wrapper, diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index e8dafaae..db6daea4 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -118,11 +118,7 @@ def load_pth_hook(ppid): # type: (int) -> None return try: cache = LineProfilingCache.load() - cache._debug_output(f'cache {id(cache):#x} setting up (pth)...') - has_set_up = _setup_in_child_process(cache, True) - cache._debug_output('cache {:#x} setup {}'.format( - id(cache), 'done' if has_set_up else 'aborted', - )) + _setup_in_child_process(cache, True, 'pth') # _setup_in_child_process(LineProfilingCache.load()) except Exception as e: if DEBUG: @@ -162,42 +158,47 @@ def wrapper(): # type: () -> int forked = cache.copy() if forked._replace_loaded_instance(): forked._debug_output( - f'cache {id(forked):#x} in fork ' - 'superseded cached `.load()`-ed instance' + 'Superseded cached `.load()`-ed instance in forked process' ) # Note: we can reuse the profiler instance in the fork, but it # needs to go through setup so that the separate profiling # results are dumped into another output file - forked._debug_output(f'cache {id(forked):#x} setting up (fork)...') - has_set_up = _setup_in_child_process(forked, prof=cache.profiler) - forked._debug_output('cache {:#x} setup {}'.format( - id(forked), 'done' if has_set_up else 'aborted', - )) + _setup_in_child_process(forked, False, 'fork', cache.profiler) return result os.fork = wrapper cache.add_cleanup(setattr, os, 'fork', fork) -def _setup_in_child_process(cache, wrap_os_fork=False, prof=None): - # type: (LineProfilingCache, bool, LineProfiler | None) -> bool +def _setup_in_child_process(cache, wrap_os_fork=False, context='', prof=None): + # type: (LineProfilingCache, bool, str, LineProfiler | None) -> bool """ Set up shop in a forked/spawned child process so that (line-)profiling can extend therein. Args: - cache (:py:class:`~.LineProfilingCache`): + cache (LineProfilingCache): Cache object wrap_os_fork (bool): Whether to wrap :py:func:`os.fork` which handles profiling; already-forked child processes should set this to false + context (str): + Optional context from which the function is called, to be + used in log messages + prof (LineProfiler | None): + Optional profiler instance to associate with the cache; + if not provided, an instance is created Returns: has_set_up (bool): False is ``cache`` has already been set up prior to calling this function, true otherwise """ + if not context: + context = '...' + cache._debug_output(f'Setting up ({context})...') if cache.profiler is not None: # Already set up + cache._debug_output(f'Setup aborted ({context})') return False import os @@ -225,7 +226,7 @@ def _setup_in_child_process(cache, wrap_os_fork=False, prof=None): # Do the preimports at `cache.preimports_module` where appropriate if cache.preimports_module: - cache._debug_output(f'cache {id(cache):#x} loading preimports...') + cache._debug_output('Loading preimports...') with open(cache.preimports_module, mode='rb') as fobj: code = compile(fobj.read(), cache.preimports_module, 'exec') exec(code, {}) # Use a fresh, empty namespace @@ -251,4 +252,5 @@ def _setup_in_child_process(cache, wrap_os_fork=False, prof=None): # the child process is about to terminate register(cache.cleanup) + cache._debug_output(f'Setup successful ({context})') return True diff --git a/line_profiler/_child_process_profiling/runpy_patches.py b/line_profiler/_child_process_profiling/runpy_patches.py index 69d38da4..6ebd3496 100644 --- a/line_profiler/_child_process_profiling/runpy_patches.py +++ b/line_profiler/_child_process_profiling/runpy_patches.py @@ -108,8 +108,8 @@ def _run( profile = os.path.samefile(filename, cache.rewrite_module) except Exception as e: cache._debug_output( - f'{THIS_MODULE}: failed to check whether to ' - f'rewrite code loaded in `runpy.{func.__name__}(...)` ' + f'{THIS_MODULE}: Failed to check whether code loaded by ' + f'`runpy.{func.__name__}(...)` is to be rewritten ' f'({type(e).__name__}: {e})' ) profile = False From 78c153373b9f175c6b68aae05bae3ac626e0a849 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 9 Apr 2026 03:56:57 +0200 Subject: [PATCH 020/165] WIP: refactoring `tests/test_child_procs.py` tests/test_child_procs.py test_running_multiproc_literal_code() New test paralleling `test_running_multiproc_{script,module}` to test `kernprof -c ...` test_multiproc_script_sanity_check() - Refactored parameters for better `pytest` output - Added testing for running the code with `python -c ...` - Added CLI argument `--local` to the profiled module to toggle between a locally-defined summing function and an imported one - Refactored how the test modules are injected - Added debugging output to `subprocess.run()` calls - Added provisional support for examining the profiling data --- tests/test_child_procs.py | 375 +++++++++++++++++++++++++++++--------- 1 file changed, 292 insertions(+), 83 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 10be3f78..7d48feab 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1,33 +1,61 @@ from __future__ import annotations import os +import shlex import subprocess import sys -from collections.abc import Callable, Generator, Mapping +from collections.abc import ( + Callable, Collection, Generator, Mapping, Sequence, +) +from dataclasses import dataclass +from functools import partial from pathlib import Path from tempfile import TemporaryDirectory from textwrap import dedent, indent import pytest import ubelt as ub +from _pytest.fixtures import FixtureFunctionDefinition + +from line_profiler.line_profiler import LineStats NUM_NUMBERS = 100 NUM_PROCS = 4 + +EXTERNAL_MODULE_BODY = dedent(""" +from __future__ import annotations + + +def my_external_sum(x: list[int]) -> int: + result: int = 0 # GREP_MARKER[EXT-INVOCATION] + for item in x: + result += item # GREP_MARKER[EXT-LOOP] + return result +""").strip('\n') + TEST_MODULE_BODY = dedent(f""" from __future__ import annotations + from argparse import ArgumentParser +from collections.abc import Callable from multiprocessing import Pool +from my_ext_module import my_external_sum -def my_sum(x: list[int]) -> int: - result: int = 0 - for item in x: - result += item + +def my_local_sum(x: list[int]) -> int: + result: int = 0 # GREP_MARKER[LOCAL-INVOCATION] + # The reversing is to prevent bytecode aliasing with + # `my_external_sum()` (see issue #424, PR #425) + for item in reversed(x): + result += item # GREP_MARKER[LOCAL-LOOP] return result -def sum_in_child_procs(length: int, n: int) -> int: +def sum_in_child_procs( + length: int, n: int, my_sum: Callable[[list[int]], int], +) -> int: my_list: list[int] = list(range(1, length + 1)) sublists: list[list[int]] = [] subsums: list[int] @@ -48,8 +76,15 @@ def main(args: list[str] | None = None) -> None: parser = ArgumentParser() parser.add_argument('-l', '--length', type=int, default={NUM_NUMBERS}) parser.add_argument('-n', type=int, default={NUM_PROCS}) + parser.add_argument( + '--local', + action='store_const', + dest='my_sum', + default=my_external_sum, + const=my_local_sum, + ) options = parser.parse_args(args) - print(sum_in_child_procs(options.length, options.n)) + print(sum_in_child_procs(options.length, options.n, options.my_sum)) if __name__ == '__main__': @@ -57,37 +92,79 @@ def main(args: list[str] | None = None) -> None: """).strip('\n') -@pytest.fixture(scope='session') -def test_module() -> Generator[Path, None, None]: - with TemporaryDirectory() as mydir_str: - my_dir = Path(mydir_str) - my_dir.mkdir(exist_ok=True) - my_module = my_dir / 'my_test_module.py' - with my_module.open('w') as fobj: - fobj.write(TEST_MODULE_BODY + '\n') - yield my_module +@dataclass +class _ModuleFixture: + path: Path + monkeypatch: pytest.MonkeyPatch + dependencies: Collection[_ModuleFixture] = () + + def install( + self, *, + local: bool = False, children: bool = False, deps_only: bool = False, + ) -> None: + for dep in self.dependencies: + dep.install(local=local, children=children) + if deps_only: + return + path = str(self.path.parent) + if local: + self.monkeypatch.syspath_prepend(path) + if children: + self.monkeypatch.setenv('PYTHONPATH', path, prepend=os.pathsep) + + @property + def name(self) -> str: + return self.path.stem + + +def _module_path_fixture( + name: str, body: str, **kwargs, +) -> FixtureFunctionDefinition: + def my_fixture_func() -> Generator[Path, None, None]: + with TemporaryDirectory() as mydir_str: + my_dir = Path(mydir_str) + my_dir.mkdir(exist_ok=True) + my_module = my_dir / f'{name}.py' + my_module.write_text(body) + yield my_module + + my_fixture_func.__name__ = name + return pytest.fixture(my_fixture_func, **kwargs) + + +# Only write the files once per test session +_ext_module = _module_path_fixture( + 'my_ext_module', EXTERNAL_MODULE_BODY, scope='session', +) +_test_module = _module_path_fixture( + 'my_test_module', TEST_MODULE_BODY, scope='session', +) -@pytest.mark.parametrize('as_module', [True, False]) -@pytest.mark.parametrize( - ('nnums', 'nprocs'), [(None, None), (None, 3), (200, None)], -) -def test_multiproc_script_sanity_check( - test_module: Path, - tmp_path_factory: pytest.TempPathFactory, - nnums: int, - nprocs: int, - as_module: bool, -) -> None: +@pytest.fixture +def ext_module( + _ext_module: Path, monkeypatch: pytest.MonkeyPatch, +) -> Generator[_ModuleFixture, None, None]: """ - Sanity check that the test module functions as expected when run - with vanilla Python. + Yields: + :py:class:`_ModuleFixture` helper object containing the code at + :py:data:`EXTERNAL_MODULE_BODY` """ - _run_test_module( - _run_as_module if as_module else _run_as_script, - test_module, tmp_path_factory, [sys.executable], None, False, - nnums=nnums, nprocs=nprocs, - ) + yield _ModuleFixture(_ext_module, monkeypatch) + + +@pytest.fixture +def test_module( + _test_module: Path, + ext_module: _ModuleFixture, + monkeypatch: pytest.MonkeyPatch, +) -> Generator[_ModuleFixture, None, None]: + """ + Yields: + :py:class:`_ModuleFixture` helper object containing the code at + :py:data:`TEST_MODULE_BODY` + """ + yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) # Note: @@ -99,20 +176,20 @@ def test_multiproc_script_sanity_check( fuzz_invocations = pytest.mark.parametrize( ('runner', 'outfile', 'profile', 'label'), # Dummy argument to make `pytest` output more legible - [ - (['kernprof', '-q'], 'out.prof', False, 'cProfile'), - # Run with `line_profiler` with and w/o profiling targets - (['kernprof', '-q', '-l'], 'out.lprof', False, - 'line_profiler-inactive'), - (['kernprof', '-q', '-l'], 'out.lprof', True, - 'line_profiler-active'), - ], + # This is essentially a no-op since it doesn't actually do + # line-profiling, but we check that code path for completeness + [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] + # Run line profiling with and w/o profiling targets + + [(['kernprof', '-q', '-l'], 'out.lprof', False, + 'line_profiler-inactive'), + (['kernprof', '-q', '-l'], 'out.lprof', True, + 'line_profiler-active')], ) @fuzz_invocations def test_running_multiproc_script( - test_module: Path, + test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, runner: str | list[str], outfile: str | None, @@ -123,15 +200,12 @@ def test_running_multiproc_script( Check that `kernprof` can run the test module as a script (`kernprof [...] `). """ - _run_test_module( - _run_as_script, - test_module, tmp_path_factory, runner, outfile, profile, - ) + run_script(test_module, tmp_path_factory, runner, outfile, profile) @fuzz_invocations def test_running_multiproc_module( - test_module: Path, + test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, runner: str | list[str], outfile: str | None, @@ -142,62 +216,150 @@ def test_running_multiproc_module( Check that `kernprof` can run the test module as a module (`kernprof [...] -m `). """ - _run_test_module( - _run_as_module, - test_module, tmp_path_factory, runner, outfile, profile, - ) + run_module(test_module, tmp_path_factory, runner, outfile, profile) + + +@fuzz_invocations +def test_running_multiproc_literal_code( + test_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + runner: str | list[str], + outfile: str | None, + profile: bool, + label: str, +) -> None: + """ + Check that `kernprof` can run the test module as literal code + (`kernprof [...] -c "code"`). + """ + run_literal_code(test_module, tmp_path_factory, runner, outfile, profile) def _run_as_script( - runner_args: list[str], test_args: list[str], test_module: Path, **kwargs + runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, + **kwargs ) -> subprocess.CompletedProcess: - cmd = runner_args + [str(test_module)] + test_args - return subprocess.run(cmd, **kwargs) + cmd = runner_args + [str(test_module.path)] + test_args + test_module.install(children=True, deps_only=True) + return _run_subproc(cmd, **kwargs) def _run_as_module( - runner_args: list[str], - test_args: list[str], - test_module: Path, - *, + runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, + **kwargs +) -> subprocess.CompletedProcess: + cmd = runner_args + ['-m', test_module.name] + test_args + test_module.install(children=True) + return _run_subproc(cmd, **kwargs) + + +def _run_as_literal_code( + runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, + **kwargs +) -> subprocess.CompletedProcess: + cmd = runner_args + ['-c', test_module.path.read_text()] + test_args + test_module.install(children=True, deps_only=True) + return _run_subproc(cmd, **kwargs) + + +def _run_subproc( + cmd: Sequence[str] | str, + /, + *args, env: Mapping[str, str] | None = None, **kwargs ) -> subprocess.CompletedProcess: - cmd = runner_args + ['-m', test_module.stem] + test_args - env_dict = {**os.environ, **(env or {})} - python_path = env_dict.pop('PYTHONPATH', '') - if python_path: - env_dict['PYTHONPATH'] = '{}:{}'.format( - test_module.parent, python_path, + """ + Wrapper around :py:func:`subprocess.run` which writes debugging + output. + """ + if isinstance(cmd, str): + cmd_str = cmd + else: + cmd_str = shlex.join(cmd) + print('Command:', cmd_str) + if env is not None: + diff: list[str] = [] + for key in set(os.environ).union(env): + old = os.environ.get(key) + new = env.get(key) + if old is not None is new: + item = f'{old!r} -> (deleted)' + elif old is None is not new: + item = f'{new!r} (added)' + else: + if old == new: + continue + item = f'{old!r} -> {new!r}' + diff.append(f'${{{key}}}: {item}') + if diff: + print('Env:', indent('\n'.join(diff), ' '), sep='\n') + print('-- Process start --') + # Note: somehow `mypy` doesn't agree with simply unpacking the + # `*args` into `subprocess.run()`... + status: int | str = '???' + try: + proc = subprocess.run( # type: ignore[call-overload] + cmd, *args, env=env, **kwargs, ) + except Exception: + status = 'error' + raise else: - env_dict['PYTHONPATH'] = str(test_module.parent) - return subprocess.run(cmd, env=env_dict, **kwargs) + status = proc.returncode + return proc + finally: + print(f'-- Process end (return status: {status})--') def _run_test_module( run_helper: Callable[..., subprocess.CompletedProcess], - test_module: Path, + test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, - runner: str | list[str], - outfile: str | None, - profile: bool, + runner: str | list[str] = 'kernprof', + outfile: str | None = None, + profile: bool = True, *, + profiled_code_is_tempfile: bool = False, nnums: int | None = None, nprocs: int | None = None, check: bool = True, -) -> tuple[subprocess.CompletedProcess, Path | None]: + nhits: Mapping[str, int] | None = None, +) -> tuple[subprocess.CompletedProcess, LineStats | None]: """ - Return - ------ - `(process_running_the_test_module, path_to_profiling_output | None)` + Returns: + process_running_the_test_module (subprocess.CompletedProcess): + Process object + profliing_stats (LineStats | None): + Line-profiling stats (where available) """ + def check_output(output: str, tag: str, nhits: int) -> None: + # The line should be preixed with 5 numbers: + # lineno, nhits, time, time-per-hit, % time + actual_nhits = 0 + for line in output.splitlines(): + if line.endswith(f'# GREP_MARKER[{tag}]'): + try: + _, n, _, _, _, *_ = line.split() + actual_nhits += int(n) + except Exception: + pass + assert actual_nhits == nhits + if isinstance(runner, str): runner_args: list[str] = [runner] else: runner_args = list(runner) - if profile: - runner_args.extend(['--prof-mod', str(test_module)]) + + if not profile: + nhits = None + + if profile and not profiled_code_is_tempfile: + runner_args.extend(['--prof-mod', str(test_module.path)]) + if nhits is not None: + # We need `kernprof` to write the profliing results immediately + # to preserve data from tempfiles (see note below) + runner_args.append('--view') test_args: list[str] = [] if nnums is None: @@ -221,13 +383,60 @@ def _run_test_module( print(f'stdout:\n{indent(proc.stdout, " ")}') print(f'stderr:\n{indent(proc.stderr, " ")}', file=sys.stderr) - assert proc.stdout == f'{nnums * (nnums + 1) // 2}\n' - - prof_result: Path | None = None + # Checks: + # - The result is correctly calculated + assert proc.stdout.splitlines()[0] == str(nnums * (nnums + 1) // 2) + # - Profiling results are written to the specified file + prof_result: LineStats | None = None if outfile is None: assert not list(Path.cwd().iterdir()) else: - prof_result = Path(outfile).resolve() - assert prof_result.exists() - assert prof_result.stat().st_size + assert os.path.exists(outfile) + assert os.stat(outfile).st_size + if profile: + prof_result = LineStats.from_files(outfile) + # - If we're keeping track, the function is called the expected + # number of times and has run the expected # of loops + # (Note: we do it by parsing the output of `kernprof -v` + # instead of reading the `--outfile`, because if the profiled + # code is in a tempfile the profiling data will be dropped in + # the written outfile) + for tag, num in (nhits or {}).items(): + check_output(proc.stdout, tag, num) return proc, prof_result + + +run_module = partial(_run_test_module, _run_as_module) +run_script = partial(_run_test_module, _run_as_script) +run_literal_code = partial( + _run_test_module, _run_as_literal_code, profiled_code_is_tempfile=True, +) + + +@pytest.mark.parametrize( + ('run_func', + 'label'), # Dummy argument to make `pytest` output more legible + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')], +) +@pytest.mark.parametrize( + ('nnums', 'nprocs'), [(None, None), (None, 3), (200, None)], +) +def test_multiproc_script_sanity_check( + run_func: Callable[..., subprocess.CompletedProcess], + test_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + nnums: int, + nprocs: int, + label: str, +) -> None: + """ + Sanity check that the test module functions as expected when run + with vanilla Python. + """ + run_func( + test_module, tmp_path_factory, + runner=sys.executable, profile=False, + nnums=nnums, nprocs=nprocs, + ) From 73ac303bcc412b54d98bdad93ffe27fadc2d07ec Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 9 Apr 2026 06:28:56 +0200 Subject: [PATCH 021/165] Test local funcs in the profiled script tests/test_child_procs.py::test_multiproc_script_sanity_check() Now parametrized to test passing the function defined in the test module itself to `multiprocessing` --- tests/test_child_procs.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 7d48feab..26bb14eb 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -321,6 +321,7 @@ def _run_test_module( profile: bool = True, *, profiled_code_is_tempfile: bool = False, + use_local_func: bool = False, nnums: int | None = None, nprocs: int | None = None, check: bool = True, @@ -362,6 +363,8 @@ def check_output(output: str, tag: str, nhits: int) -> None: runner_args.append('--view') test_args: list[str] = [] + if use_local_func: + test_args.append('--local') if nnums is None: nnums = NUM_NUMBERS else: @@ -414,11 +417,13 @@ def check_output(output: str, tag: str, nhits: int) -> None: @pytest.mark.parametrize( - ('run_func', + ('run_func', 'use_local_func', 'label'), # Dummy argument to make `pytest` output more legible - [(run_module, 'module'), - (run_script, 'script'), - (run_literal_code, 'literal-code')], + [(run_module, True, 'module-local'), (run_module, False, 'module-ext'), + (run_script, True, 'script-local'), (run_script, False, 'script-ext')] + # Python can't pickle things unless they resided in a retrievable + # location (so not the script supplied by `python -c`) + + [(run_literal_code, False, 'literal-code-ext')], ) @pytest.mark.parametrize( ('nnums', 'nprocs'), [(None, None), (None, 3), (200, None)], @@ -427,6 +432,7 @@ def test_multiproc_script_sanity_check( run_func: Callable[..., subprocess.CompletedProcess], test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, + use_local_func: bool, nnums: int, nprocs: int, label: str, @@ -438,5 +444,6 @@ def test_multiproc_script_sanity_check( run_func( test_module, tmp_path_factory, runner=sys.executable, profile=False, + use_local_func=use_local_func, nnums=nnums, nprocs=nprocs, ) From 44c4eb04d6d860d5fd713d6aade1d9cd0a1bc9d9 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 9 Apr 2026 06:41:56 +0200 Subject: [PATCH 022/165] Consolidated tests tests/test_child_procs.py test_running_multiproc_{module,literal_code}() Integrated into `test_running_multiproc_script()` test_running_multiproc_script() Extended parametrization --- tests/test_child_procs.py | 112 +++++++++++++++----------------------- 1 file changed, 44 insertions(+), 68 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 26bb14eb..94915fc0 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -167,74 +167,6 @@ def test_module( yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) -# Note: -# Currently code execution in child processes is not properly profiled; -# these tests are just for checking that `kernprof` doesn't impair the -# proper execution of `multiprocessing` code - - -fuzz_invocations = pytest.mark.parametrize( - ('runner', 'outfile', 'profile', - 'label'), # Dummy argument to make `pytest` output more legible - # This is essentially a no-op since it doesn't actually do - # line-profiling, but we check that code path for completeness - [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] - # Run line profiling with and w/o profiling targets - + [(['kernprof', '-q', '-l'], 'out.lprof', False, - 'line_profiler-inactive'), - (['kernprof', '-q', '-l'], 'out.lprof', True, - 'line_profiler-active')], -) - - -@fuzz_invocations -def test_running_multiproc_script( - test_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - runner: str | list[str], - outfile: str | None, - profile: bool, - label: str, -) -> None: - """ - Check that `kernprof` can run the test module as a script - (`kernprof [...] `). - """ - run_script(test_module, tmp_path_factory, runner, outfile, profile) - - -@fuzz_invocations -def test_running_multiproc_module( - test_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - runner: str | list[str], - outfile: str | None, - profile: bool, - label: str, -) -> None: - """ - Check that `kernprof` can run the test module as a module - (`kernprof [...] -m `). - """ - run_module(test_module, tmp_path_factory, runner, outfile, profile) - - -@fuzz_invocations -def test_running_multiproc_literal_code( - test_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - runner: str | list[str], - outfile: str | None, - profile: bool, - label: str, -) -> None: - """ - Check that `kernprof` can run the test module as literal code - (`kernprof [...] -c "code"`). - """ - run_literal_code(test_module, tmp_path_factory, runner, outfile, profile) - - def _run_as_script( runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, **kwargs @@ -447,3 +379,47 @@ def test_multiproc_script_sanity_check( use_local_func=use_local_func, nnums=nnums, nprocs=nprocs, ) + + +@pytest.mark.parametrize( + ('run_func', + 'label2'), # Dummy argument to make `pytest` output more legible + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')] +) +@pytest.mark.parametrize( + ('runner', 'outfile', 'profile', + 'label'), # Dummy argument to make `pytest` output more legible + # This is essentially a no-op since it doesn't actually do + # line-profiling, but we check that code path for completeness + [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] + # Run line profiling with and w/o profiling targets + + [(['kernprof', '-q', '-l'], 'out.lprof', False, + 'line_profiler-inactive'), + (['kernprof', '-q', '-l'], 'out.lprof', True, + 'line_profiler-active')], +) +def test_running_multiproc_script( + run_func: Callable[..., subprocess.CompletedProcess], + test_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + runner: str | list[str], + outfile: str | None, + profile: bool, + label: str, + label2: str, +) -> None: + """ + Check that `kernprof` can RUN the test module in various contexts + (`kernprof [...] `, `kernprof [...] -m `, and + `kernprof [...] -c "code"`). + + Notes: + - See issue #422 for the original motivation. + + - This test does not test the actual profiling, just the + execution of the code and presence of profiling data + thereafter. + """ + run_func(test_module, tmp_path_factory, runner, outfile, profile) From 5c5556a6b8a50f1ee18d97ba3e571fb26edae87a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 12 Apr 2026 00:25:38 +0200 Subject: [PATCH 023/165] Added tests for our `multiprocessing` patches tests/test_child_procs.py test_profiling_multiproc_script() Test parallel to `test_running_multiproc_script()`, checking whether we are correctly profiling the child processes - Added more docs - Updated dummy parameter names _ext_module, _test_module - Refactored how the fixtures are set up - Module names now randomized and clash-proof via `uuid.uuid4()` _run_subproc() - Moved code outputting captured streams from `_run_test_module()` to here - Added timing code --- tests/test_child_procs.py | 224 ++++++++++++++++++++++++++++++++------ 1 file changed, 189 insertions(+), 35 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 94915fc0..970b7cf2 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -12,10 +12,11 @@ from pathlib import Path from tempfile import TemporaryDirectory from textwrap import dedent, indent +from time import monotonic +from uuid import uuid4 import pytest import ubelt as ub -from _pytest.fixtures import FixtureFunctionDefinition from line_profiler.line_profiler import LineStats @@ -34,14 +35,14 @@ def my_external_sum(x: list[int]) -> int: return result """).strip('\n') -TEST_MODULE_BODY = dedent(f""" +TEST_MODULE_TEMPLATE = dedent(""" from __future__ import annotations from argparse import ArgumentParser from collections.abc import Callable from multiprocessing import Pool -from my_ext_module import my_external_sum +from {EXT_MODULE} import my_external_sum def my_local_sum(x: list[int]) -> int: @@ -92,8 +93,15 @@ def main(args: list[str] | None = None) -> None: """).strip('\n') +# ============================== Fixtures ============================== + + @dataclass class _ModuleFixture: + """ + Convenience wrapper around a Python source file which represents an + importable module. + """ path: Path monkeypatch: pytest.MonkeyPatch dependencies: Collection[_ModuleFixture] = () @@ -102,6 +110,20 @@ def install( self, *, local: bool = False, children: bool = False, deps_only: bool = False, ) -> None: + """ + Set the module at :py:attr:`~.path` up to be importable. + + Args: + local (bool): + Make it importable for the CURRENT process (via + :py:data:`sys.path`). + children (bool): + Make it importable for CHILD processes (via + ``os.environ['PYTHONPATH']``). + deps_only (bool): + If true, only does the equivalent setup for + dependencies. + """ for dep in self.dependencies: dep.install(local=local, children=children) if deps_only: @@ -112,33 +134,50 @@ def install( if children: self.monkeypatch.setenv('PYTHONPATH', path, prepend=os.pathsep) + @staticmethod + def propose_name(prefix: str) -> Generator[str, None, None]: + """ + Propose a valid module name that isn't already occupied. + """ + while True: + name = '_'.join([prefix] + str(uuid4()).split('-')) + if name not in sys.modules: + assert name.isidentifier() + yield name + @property def name(self) -> str: return self.path.stem -def _module_path_fixture( - name: str, body: str, **kwargs, -) -> FixtureFunctionDefinition: - def my_fixture_func() -> Generator[Path, None, None]: - with TemporaryDirectory() as mydir_str: - my_dir = Path(mydir_str) - my_dir.mkdir(exist_ok=True) - my_module = my_dir / f'{name}.py' - my_module.write_text(body) - yield my_module - - my_fixture_func.__name__ = name - return pytest.fixture(my_fixture_func, **kwargs) +# Only write the files once per test session -# Only write the files once per test session -_ext_module = _module_path_fixture( - 'my_ext_module', EXTERNAL_MODULE_BODY, scope='session', -) -_test_module = _module_path_fixture( - 'my_test_module', TEST_MODULE_BODY, scope='session', -) +@pytest.fixture(scope='session') +def _ext_module() -> Generator[Path, None, None]: + name = next(_ModuleFixture.propose_name('my_ext_module')) + with TemporaryDirectory() as mydir_str: + my_dir = Path(mydir_str) + my_dir.mkdir(exist_ok=True) + my_module = my_dir / f'{name}.py' + my_module.write_text(EXTERNAL_MODULE_BODY) + yield my_module + + +@pytest.fixture(scope='session') +def _test_module(_ext_module: Path) -> Generator[Path, None, None]: + name = next(_ModuleFixture.propose_name('my_test_module')) + body = TEST_MODULE_TEMPLATE.format( + EXT_MODULE=_ext_module.stem, + NUM_NUMBERS=NUM_NUMBERS, + NUM_PROCS=NUM_PROCS, + ) + with TemporaryDirectory() as mydir_str: + my_dir = Path(mydir_str) + my_dir.mkdir(exist_ok=True) + my_module = my_dir / f'{name}.py' + my_module.write_text(body) + yield my_module @pytest.fixture @@ -162,7 +201,7 @@ def test_module( """ Yields: :py:class:`_ModuleFixture` helper object containing the code at - :py:data:`TEST_MODULE_BODY` + :py:data:`TEST_MODULE_TEMPLATE` """ yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) @@ -230,6 +269,8 @@ def _run_subproc( # Note: somehow `mypy` doesn't agree with simply unpacking the # `*args` into `subprocess.run()`... status: int | str = '???' + proc: subprocess.CompletedProcess | None = None + time = monotonic() try: proc = subprocess.run( # type: ignore[call-overload] cmd, *args, env=env, **kwargs, @@ -241,7 +282,19 @@ def _run_subproc( status = proc.returncode return proc finally: - print(f'-- Process end (return status: {status})--') + time = monotonic() - time + if proc is not None: + for name, captured, stream in [ + ('stdout', proc.stdout, sys.stdout), + ('stderr', proc.stderr, sys.stderr), + ]: + if captured is None: + continue + print(f'{name}:\n{indent(captured, " ")}', file=stream) + print( + f'-- Process end (time elapsed: {time:.2f} s / ' + f'return status: {status})--' + ) def _run_test_module( @@ -309,15 +362,8 @@ def check_output(output: str, tag: str, nhits: int) -> None: runner_args.extend(['--outfile', outfile]) proc = run_helper( runner_args, test_args, test_module, - text=True, capture_output=True, + text=True, capture_output=True, check=check, ) - try: - if check: - proc.check_returncode() - finally: - print(f'stdout:\n{indent(proc.stdout, " ")}') - print(f'stderr:\n{indent(proc.stderr, " ")}', file=sys.stderr) - # Checks: # - The result is correctly calculated assert proc.stdout.splitlines()[0] == str(nnums * (nnums + 1) // 2) @@ -390,7 +436,7 @@ def test_multiproc_script_sanity_check( ) @pytest.mark.parametrize( ('runner', 'outfile', 'profile', - 'label'), # Dummy argument to make `pytest` output more legible + 'label1'), # Dummy argument to make `pytest` output more legible # This is essentially a no-op since it doesn't actually do # line-profiling, but we check that code path for completeness [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] @@ -407,7 +453,7 @@ def test_running_multiproc_script( runner: str | list[str], outfile: str | None, profile: bool, - label: str, + label1: str, label2: str, ) -> None: """ @@ -423,3 +469,111 @@ def test_running_multiproc_script( thereafter. """ run_func(test_module, tmp_path_factory, runner, outfile, profile) + + +@pytest.mark.parametrize( + ('run_func', 'label1'), + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')] +) +@pytest.mark.parametrize( + ('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')], +) +@pytest.mark.parametrize( + ('preimports', 'label3'), + [(True, 'with-preimports'), (False, 'no-preimports')], +) +@pytest.mark.parametrize( + ('use_local_func', 'label4'), [(True, 'local'), (False, 'external')], +) +@pytest.mark.parametrize( + # XXX: should we explicitly test the single-proc case? We already + # have quite a lot of subtests tho... + ('nnums', 'nprocs'), [(2000, 3)], +) +def test_profiling_multiproc_script( + run_func: Callable[..., subprocess.CompletedProcess], + test_module: _ModuleFixture, + ext_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + prof_child_procs: bool, + preimports: bool, + use_local_func: bool, + nnums: int, + nprocs: int, + # Dummy arguments to make `pytest` output more legible + label1: str, + label2: str, + label3: str, + label4: str, +) -> None: + """ + Check that `kernprof` can PROFILE the test module in various + contexts, optionally extending profiling into child processes. + + Note: + This test function is heavily parametrized. Here is why that is + necessary: + + - ``run_func`` tests the different :cmd:`kernprof` modes (see + :py:func:`~.test_running_multiproc_script`). + + - ``use_local_func`` tests that we can consistently set up + profiling in both functions locally-defined in the profiled + code and imported by it. + + - ``preimports`` tests that both mechanisms for setting up + profiling targets work: + + - :py:const:`True`: child processes import the module + generated by + :py:mod:`line_profiler.autoprofile.eager_preimports`, like + the main :py:mod:`kernprof` process does. + + - :py:const:`False`: child processes rewrite the executed code + before passing it to :py:mod:`runpy`, similar to what + :py:mod:`line_profiler.autoprofile.autoprofile` does. + + These code paths go through different + :py:mod:`multiprocessing` components that we have patched and + thus needs separate testing. + + - ``prof_child_procs`` of course toggles whether to do the + patches to set up profiling in child processes. + """ + # How many calls do we expect? + nhits = dict.fromkeys( + ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, + ) + # Make sure we're profiling the right function + tag = 'LOCAL' if use_local_func else 'EXT' + if prof_child_procs: + # - `nprocs` child calls summing the `nnums` numbers + # - One call in the main proc summing the `nprocs` results from + # children + nhits[tag + '-INVOCATION'] = nprocs + 1 + nhits[tag + '-LOOP'] = nnums + nprocs + else: + nhits[tag + '-INVOCATION'] = 1 + nhits[tag + '-LOOP'] = nprocs + + runner = ['kernprof', '-l'] + runner.extend([ + '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), + '--{}preimports'.format('' if preimports else 'no-'), + ]) + if not use_local_func: + # Also make sure to include the external module in `--prof-mod` + runner.append(f'--prof-mod={ext_module.name}') + run_func( + test_module, tmp_path_factory, + runner=runner, + outfile='out.lprof', + profile=True, + use_local_func=use_local_func, + nhits=nhits, + nnums=nnums, + nprocs=nprocs, + ) From ddb211c0fd265a7c0521c4c6c1ec5b92d4ffb50c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 12 Apr 2026 05:42:26 +0200 Subject: [PATCH 024/165] WIP: flesh out the child-proc tests more tests/test_child_procs.py TEST_MODULE_BODY, [_]test_module() Added CLI flag to select `multiprocessing` start methods _Params New convenience class for test parametrization test_multiproc_script_sanity_check() - Streamlined parametrization (15 subtests -> 10) - Added subtests for various `multiprocessing` start methods test_multiproc_script_sanity_check() - Streamlined parametrization (24 subtests -> 21) - Added subtests for various `multiprocessing` start methods --- tests/test_child_procs.py | 310 ++++++++++++++++++++++++++++++++++---- 1 file changed, 277 insertions(+), 33 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 970b7cf2..cb87c634 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1,18 +1,22 @@ from __future__ import annotations +import enum import os import shlex import subprocess import sys from collections.abc import ( - Callable, Collection, Generator, Mapping, Sequence, + Callable, Collection, Generator, Iterable, Mapping, Sequence, ) from dataclasses import dataclass from functools import partial +from multiprocessing import get_all_start_methods from pathlib import Path from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic +from typing import Any, Literal, TypeVar, cast, final, overload +from typing_extensions import Self from uuid import uuid4 import pytest @@ -21,8 +25,12 @@ from line_profiler.line_profiler import LineStats +T = TypeVar('T') +C = TypeVar('C', bound=Callable[..., Any]) + NUM_NUMBERS = 100 NUM_PROCS = 4 +START_METHODS = set(get_all_start_methods()) EXTERNAL_MODULE_BODY = dedent(""" from __future__ import annotations @@ -40,7 +48,8 @@ def my_external_sum(x: list[int]) -> int: from argparse import ArgumentParser from collections.abc import Callable -from multiprocessing import Pool +from multiprocessing import get_context, Pool +from typing import Literal from {EXT_MODULE} import my_external_sum @@ -56,6 +65,7 @@ def my_local_sum(x: list[int]) -> int: def sum_in_child_procs( length: int, n: int, my_sum: Callable[[list[int]], int], + start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, ) -> int: my_list: list[int] = list(range(1, length + 1)) sublists: list[list[int]] = [] @@ -66,7 +76,11 @@ def sum_in_child_procs( while my_list: sublist, my_list = my_list[:sublength], my_list[sublength:] sublists.append(sublist) - with Pool(n) as pool: + if start_method: + pool = get_context(start_method).Pool(n) + else: + pool = Pool(n) + with pool: subsums = pool.map(my_sum, sublists) pool.close() pool.join() @@ -77,6 +91,10 @@ def main(args: list[str] | None = None) -> None: parser = ArgumentParser() parser.add_argument('-l', '--length', type=int, default={NUM_NUMBERS}) parser.add_argument('-n', type=int, default={NUM_PROCS}) + parser.add_argument( + '-s', '--start-method', + choices=['fork', 'forkserver', 'spawn'], default=None, + ) parser.add_argument( '--local', action='store_const', @@ -206,6 +224,197 @@ def test_module( yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) +# ========================== Helper functions ========================== + + +class _NotSupplied(enum.Enum): + NOT_SUPPLIED = enum.auto() + + +@final +@dataclass +class _Params: + """ + Convenience wrapper around :py:func:`pytest.mark.parametrize`. + """ + params: tuple[str, ...] + values: list[tuple[Any, ...]] + defaults: tuple[Any, ...] + + def __post_init__(self) -> None: + n = len(self.params) + assert all(p.isidentifier() for p in self.params) # Validity + assert len(set(self.params)) == n # Uniqueness + assert len(self.defaults) == n # Consistency + self.values = list(self._unique(self.values)) + assert all(len(v) == n for v in self.values) + + def __mul__(self, other: Self) -> Self: + """ + Form a Cartesian product between the two instances with disjoint + :py:attr:`~.params`, like stacking the + :py:func:`pytest.mark.parametrize `decorators. + + Example: + >>> p1 = _Params.new(('a', 'b'), [(0, 0), (1, 2), (3, 4)], + ... defaults=(1, 2)) + >>> p2 = _Params.new('c', [0, 5, 6]) + >>> p1 * p2 # doctest: +NORMALIZE_WHITESPACE + _Params(params=('a', 'b', 'c'), + values=[(0, 0, 0), (0, 0, 5), (0, 0, 6), + (1, 2, 0), (1, 2, 5), (1, 2, 6), + (3, 4, 0), (3, 4, 5), (3, 4, 6)], + defaults=(1, 2, 0)) + """ + assert not set(self.params) & set(other.params) + return type(self)( + self.params + other.params, + [sv + ov for sv in self.values for ov in other.values], + self.defaults + other.defaults, + ) + + def __add__(self, other: Self) -> Self: + """ + Concatenate two instances: + + - For parameters appearing in both, their lists of values are + concatenated. + + - For parameters appearing in either instance, the missing + values are taken from the other instance's + :py:attr:`~.defaults`. + + Note: + In the case of clashes, the :py:attr:`~.defaults` and the + order of the :py:attr:`~.params` of ``self`` (the left + operand) take precedence. + + Example: + >>> p1 = _Params.new(('a', 'b', 'c'), + ... [(0, 0, 0), # defaults + ... (1, 2, 3), (4, 5, 6)]) + >>> p2 = _Params.new(('c', 'd'), [(7, 8), (9, 10)], + ... defaults=(-1, -1)) + >>> p1 + p2 # doctest: +NORMALIZE_WHITESPACE + _Params(params=('a', 'b', 'c', 'd'), + values=[(0, 0, 0, -1), + (1, 2, 3, -1), + (4, 5, 6, -1), + (0, 0, 7, 8), + (0, 0, 9, 10)], + defaults=(0, 0, 0, -1)) + """ + self_defaults = dict(zip(self.params, self.defaults)) + other_defaults = dict(zip(other.params, other.defaults)) + new_params = tuple(self._unique(self.params + other.params)) + + defaults = {**other_defaults, **self_defaults} + new_defaults_tuple = tuple(defaults[p] for p in new_params) + + new_values: list[tuple[Any, ...]] = [] + for old_values, old_params in [ + (self.values, self.params), (other.values, other.params), + ]: + indices: list[ + tuple[Literal[True], int] | tuple[Literal[False], str] + ] = [ + (True, old_params.index(p)) if p in old_params else (False, p) + for p in new_params + ] + new_values.extend( + tuple( + ( + value[cast(int, index)] + if available else + defaults[cast(str, index)] + ) for available, index in indices + ) + for value in old_values + ) + return type(self)(new_params, new_values, new_defaults_tuple) + + def __call__(self, func: C) -> C: + """ + Mark a callable as with :py:func:`pytest.mark.parametrize`. + """ + # Note: `pytest` automatically assumes single-param values to + # be unpackes, so comply here + if len(self.params) == 1: + marker = pytest.mark.parametrize( + self.params[0], [v[0] for v in self.values], + ) + else: + marker = pytest.mark.parametrize(self.params, self.values) + return marker(func) + + @staticmethod + def _unique(items: Iterable[T]) -> Generator[T, None, None]: + seen: set[T] = set() + for item in items: + if item in seen: + continue + seen.add(item) + yield item + + @overload + @classmethod + def new( + cls, + params: Sequence[str] | str, + values: Sequence[Sequence[Any]], + defaults: Sequence[Any] | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + ) -> Self: + ... + + @overload + @classmethod + def new( + cls, + params: str, + values: Sequence[Any], + defaults: Any | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + ) -> Self: + ... + + @classmethod + def new( + cls, + params: Sequence[str] | str, + values: Sequence[Sequence[Any]] | Sequence[Any], + defaults: ( + Sequence[Any] | Any | _NotSupplied + ) = _NotSupplied.NOT_SUPPLIED, + ) -> Self: + """ + Instantiator more akin to :py:func:`pytest.mark.parametrize`: + + - ``params`` can be provided as a comma-separated string + + - Single parameters can be unpacked (singular param-name string + and param-value sequences) + + - If ``defaults`` are not given, it is implicitly set to the + FIRST item in ``values``. + """ + if isinstance(params, str): + param_list: tuple[str, ...] = tuple( + p.strip() for p in params.split(',') + ) + unpacked = len(param_list) == 1 + else: + param_list = tuple(params) + unpacked = False + if defaults == _NotSupplied.NOT_SUPPLIED: + defaults, *_ = values + if unpacked: + default_values: tuple[Any, ...] = defaults, + value_tuple_list: list[tuple[Any, ...]] = [(v,) for v in values] + else: + default_values = tuple(defaults) # type: ignore[arg-type] + value_tuple_list = [tuple(v) for v in values] + return cls(param_list, value_tuple_list, default_values) + + def _run_as_script( runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, **kwargs @@ -307,6 +516,7 @@ def _run_test_module( *, profiled_code_is_tempfile: bool = False, use_local_func: bool = False, + start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, nnums: int | None = None, nprocs: int | None = None, check: bool = True, @@ -350,6 +560,14 @@ def check_output(output: str, tag: str, nhits: int) -> None: test_args: list[str] = [] if use_local_func: test_args.append('--local') + if start_method: + if start_method in START_METHODS: + test_args.extend(['-s', start_method]) + else: + pytest.skip( + f'`multiprocessing` start method {start_method!r} ' + 'not available on the platform' + ) if nnums is None: nnums = NUM_NUMBERS else: @@ -393,27 +611,38 @@ def check_output(output: str, tag: str, nhits: int) -> None: _run_test_module, _run_as_literal_code, profiled_code_is_tempfile=True, ) +# =============================== Tests ================================ -@pytest.mark.parametrize( - ('run_func', 'use_local_func', - 'label'), # Dummy argument to make `pytest` output more legible - [(run_module, True, 'module-local'), (run_module, False, 'module-ext'), - (run_script, True, 'script-local'), (run_script, False, 'script-ext')] + +_fuzz_sanity = ( + _Params.new(('run_func', 'label1'), + [(run_module, 'module'), (run_script, 'script')]) + * _Params.new(('use_local_func', 'label2'), + [(True, 'local'), (False, 'ext')]) # Python can't pickle things unless they resided in a retrievable # location (so not the script supplied by `python -c`) - + [(run_literal_code, False, 'literal-code-ext')], -) -@pytest.mark.parametrize( - ('nnums', 'nprocs'), [(None, None), (None, 3), (200, None)], + + _Params.new(('run_func', 'label1', 'use_local_func', 'label2'), + [(run_literal_code, 'literal-code', False, 'ext')]) + # Also fuzz the parallelization-related stuff + + _Params.new('start_method', ['fork', 'forkserver', 'spawn'], + defaults=None) + + _Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], + defaults=(None, None)) ) + + +@_fuzz_sanity def test_multiproc_script_sanity_check( run_func: Callable[..., subprocess.CompletedProcess], test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, use_local_func: bool, - nnums: int, - nprocs: int, - label: str, + start_method: Literal['fork', 'forkserver', 'spawn'] | None, + nnums: int | None, + nprocs: int | None, + # Dummy arguments to make `pytest` output more legible + label1: str, + label2: str, ) -> None: """ Sanity check that the test module functions as expected when run @@ -423,20 +652,20 @@ def test_multiproc_script_sanity_check( test_module, tmp_path_factory, runner=sys.executable, profile=False, use_local_func=use_local_func, + start_method=start_method, nnums=nnums, nprocs=nprocs, ) @pytest.mark.parametrize( - ('run_func', - 'label2'), # Dummy argument to make `pytest` output more legible + ('run_func', 'label1'), [(run_module, 'module'), (run_script, 'script'), (run_literal_code, 'literal-code')] ) @pytest.mark.parametrize( ('runner', 'outfile', 'profile', - 'label1'), # Dummy argument to make `pytest` output more legible + 'label2'), # Dummy argument to make `pytest` output more legible # This is essentially a no-op since it doesn't actually do # line-profiling, but we check that code path for completeness [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] @@ -453,6 +682,7 @@ def test_running_multiproc_script( runner: str | list[str], outfile: str | None, profile: bool, + # Dummy arguments to make `pytest` output more legible label1: str, label2: str, ) -> None: @@ -471,23 +701,29 @@ def test_running_multiproc_script( run_func(test_module, tmp_path_factory, runner, outfile, profile) -@pytest.mark.parametrize( - ('run_func', 'label1'), - [(run_module, 'module'), - (run_script, 'script'), - (run_literal_code, 'literal-code')] +_fuzz_prof_mp_1 = ( + _Params.new(('run_func', 'label1'), + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')], + defaults=(run_script, 'script')) + + _Params.new(('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')]) + + _Params.new('start_method', ['fork', 'forkserver', 'spawn'], + defaults=None) ) -@pytest.mark.parametrize( - ('prof_child_procs', 'label2'), - [(True, 'with-child-prof'), (False, 'no-child-prof')], -) -@pytest.mark.parametrize( - ('preimports', 'label3'), - [(True, 'with-preimports'), (False, 'no-preimports')], -) -@pytest.mark.parametrize( - ('use_local_func', 'label4'), [(True, 'local'), (False, 'external')], +_fuzz_prof_mp_2 = ( + _Params.new(('preimports', 'label3'), + [(True, 'with-preimports'), (False, 'no-preimports')], + defaults=(False, 'no-preimports')) + + _Params.new(('use_local_func', 'label4'), + [(True, 'local'), (False, 'external')], + defaults=(False, 'external')) ) + + +@_fuzz_prof_mp_1 +@_fuzz_prof_mp_2 @pytest.mark.parametrize( # XXX: should we explicitly test the single-proc case? We already # have quite a lot of subtests tho... @@ -501,6 +737,7 @@ def test_profiling_multiproc_script( prof_child_procs: bool, preimports: bool, use_local_func: bool, + start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int, nprocs: int, # Dummy arguments to make `pytest` output more legible @@ -573,7 +810,14 @@ def test_profiling_multiproc_script( outfile='out.lprof', profile=True, use_local_func=use_local_func, + start_method=start_method, nhits=nhits, nnums=nnums, nprocs=nprocs, ) + + +# TODO: test for profiling under the following circumstances: +# - `os.system()` +# - `subprocess.run()` +# - Nested subprocesses From b4d28a2c6653dcd49fd9f37e2645bc9e13ad78a1 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 12 Apr 2026 23:17:29 +0200 Subject: [PATCH 025/165] WIP: test when the parallelly-run func fails tests/test_child_procs.py test_module(), ext_module() Updated so that we can toggle for the function sent to `multiprocessing` to raise an error with the `--force-failure` CLI flag _run_test_module() - Now raising a new `ResultMismatch` error class (instead of using base assertions) for: - If `test_module()` writes the wrong number to stdout - If `nhits` are provided and the profiling results differ therefrom - Added argument `fail` for using the aforementioned `--force-failure` flag test_multiproc_script_sanity_check() Now also chceking the cases where the test module is run with `--force-failure` test_profiling_multiproc_script() Now also chceking the cases where the test module is run with `--force-failure` (FIXME: profiling bugged when the function errors out, and doesn't fail with a consistent pattern) --- tests/test_child_procs.py | 199 ++++++++++++++++++++++++++++---------- 1 file changed, 150 insertions(+), 49 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index cb87c634..09473e36 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1,6 +1,7 @@ from __future__ import annotations import enum +import multiprocessing import os import shlex import subprocess @@ -8,9 +9,9 @@ from collections.abc import ( Callable, Collection, Generator, Iterable, Mapping, Sequence, ) +from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass from functools import partial -from multiprocessing import get_all_start_methods from pathlib import Path from tempfile import TemporaryDirectory from textwrap import dedent, indent @@ -30,16 +31,18 @@ NUM_NUMBERS = 100 NUM_PROCS = 4 -START_METHODS = set(get_all_start_methods()) +START_METHODS = set(multiprocessing.get_all_start_methods()) EXTERNAL_MODULE_BODY = dedent(""" from __future__ import annotations -def my_external_sum(x: list[int]) -> int: +def my_external_sum(x: list[int], fail: bool = False) -> int: result: int = 0 # GREP_MARKER[EXT-INVOCATION] for item in x: result += item # GREP_MARKER[EXT-LOOP] + if fail: + raise RuntimeError('forced failure') return result """).strip('\n') @@ -54,18 +57,21 @@ def my_external_sum(x: list[int]) -> int: from {EXT_MODULE} import my_external_sum -def my_local_sum(x: list[int]) -> int: +def my_local_sum(x: list[int], fail: bool = False) -> int: result: int = 0 # GREP_MARKER[LOCAL-INVOCATION] # The reversing is to prevent bytecode aliasing with # `my_external_sum()` (see issue #424, PR #425) for item in reversed(x): result += item # GREP_MARKER[LOCAL-LOOP] + if fail: + raise RuntimeError('forced failure') return result def sum_in_child_procs( length: int, n: int, my_sum: Callable[[list[int]], int], start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, + fail: bool = False, ) -> int: my_list: list[int] = list(range(1, length + 1)) sublists: list[list[int]] = [] @@ -81,10 +87,10 @@ def sum_in_child_procs( else: pool = Pool(n) with pool: - subsums = pool.map(my_sum, sublists) + subsums = pool.starmap(my_sum, [(sl, fail) for sl in sublists]) pool.close() pool.join() - return my_sum(subsums) + return my_sum(subsums, fail) def main(args: list[str] | None = None) -> None: @@ -95,6 +101,7 @@ def main(args: list[str] | None = None) -> None: '-s', '--start-method', choices=['fork', 'forkserver', 'spawn'], default=None, ) + parser.add_argument('-f', '--force-failure', action='store_true') parser.add_argument( '--local', action='store_const', @@ -103,7 +110,11 @@ def main(args: list[str] | None = None) -> None: const=my_local_sum, ) options = parser.parse_args(args) - print(sum_in_child_procs(options.length, options.n, options.my_sum)) + print(sum_in_child_procs( + options.length, options.n, options.my_sum, + start_method=options.start_method, + fail=options.force_failure, + )) if __name__ == '__main__': @@ -231,6 +242,30 @@ class _NotSupplied(enum.Enum): NOT_SUPPLIED = enum.auto() +class ResultMismatch(ValueError): + def __init__( + self, + expected: Any, + actual: Any | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + ) -> None: + msg = f'expected: {expected}' + if actual != _NotSupplied.NOT_SUPPLIED: + msg = f'{msg}, got {actual}' + super().__init__(msg) + self.expected = expected + self.actual = actual + + @property + def rich_message(self) -> str: + msg = '{}: {}'.format(type(self).__name__, self.args[0]) + if self.__traceback__ is not None: + tb = self.__traceback__ + msg = '{}:{}: {}'.format( + tb.tb_frame.f_code.co_filename, tb.tb_lineno, msg, + ) + return msg + + @final @dataclass class _Params: @@ -516,6 +551,7 @@ def _run_test_module( *, profiled_code_is_tempfile: bool = False, use_local_func: bool = False, + fail: bool = False, start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, nnums: int | None = None, nprocs: int | None = None, @@ -540,7 +576,11 @@ def check_output(output: str, tag: str, nhits: int) -> None: actual_nhits += int(n) except Exception: pass - assert actual_nhits == nhits + if actual_nhits == nhits: + return + raise ResultMismatch( + f'{nhits} hit(s) on line(s) tagged with {tag!r}', actual_nhits, + ) if isinstance(runner, str): runner_args: list[str] = [runner] @@ -560,6 +600,8 @@ def check_output(output: str, tag: str, nhits: int) -> None: test_args: list[str] = [] if use_local_func: test_args.append('--local') + if fail: + test_args.append('--force-failure') if start_method: if start_method in START_METHODS: test_args.extend(['-s', start_method]) @@ -580,11 +622,21 @@ def check_output(output: str, tag: str, nhits: int) -> None: runner_args.extend(['--outfile', outfile]) proc = run_helper( runner_args, test_args, test_module, - text=True, capture_output=True, check=check, + text=True, capture_output=True, check=(check and not fail), ) # Checks: - # - The result is correctly calculated - assert proc.stdout.splitlines()[0] == str(nnums * (nnums + 1) // 2) + if fail: + # - The process has failed as expected + if check: + assert proc.returncode + else: + # - The result is correctly calculated + expected = nnums * (nnums + 1) // 2 + output_lines = proc.stdout.splitlines() + if output_lines[0] != str(expected): + raise ResultMismatch( + f'result {expected}', f'output lines: {output_lines}', + ) # - Profiling results are written to the specified file prof_result: LineStats | None = None if outfile is None: @@ -614,6 +666,21 @@ def check_output(output: str, tag: str, nhits: int) -> None: # =============================== Tests ================================ +def _get_mp_start_method_fuzzer(label_name: str) -> _Params: + """ + Returns: + :py:class:`_Params` object which does a full Cartesian-product + fuzz between ``fail`` (true or false) and ``start_method`` + ('fork', 'forkserver', and 'spawn'; default :py:const:`None`) + """ + fuzz_fail = _Params.new(('fail', label_name), + [(True, 'failure'), (False, 'success')], + defaults=(False, 'success')) + fuzz_start = _Params.new('start_method', ['fork', 'forkserver', 'spawn'], + defaults=None) + return fuzz_fail * fuzz_start + + _fuzz_sanity = ( _Params.new(('run_func', 'label1'), [(run_module, 'module'), (run_script, 'script')]) @@ -623,9 +690,9 @@ def check_output(output: str, tag: str, nhits: int) -> None: # location (so not the script supplied by `python -c`) + _Params.new(('run_func', 'label1', 'use_local_func', 'label2'), [(run_literal_code, 'literal-code', False, 'ext')]) - # Also fuzz the parallelization-related stuff - + _Params.new('start_method', ['fork', 'forkserver', 'spawn'], - defaults=None) + # Also fuzz the parallelization-related stuff, esp. check what + # happens if an exception is raised inside the parallelly-run func + + _get_mp_start_method_fuzzer('label3') + _Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], defaults=(None, None)) ) @@ -637,12 +704,12 @@ def test_multiproc_script_sanity_check( test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, use_local_func: bool, + fail: bool, start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int | None, nprocs: int | None, # Dummy arguments to make `pytest` output more legible - label1: str, - label2: str, + label1: str, label2: str, label3: str, ) -> None: """ Sanity check that the test module functions as expected when run @@ -651,6 +718,7 @@ def test_multiproc_script_sanity_check( run_func( test_module, tmp_path_factory, runner=sys.executable, profile=False, + fail=fail, use_local_func=use_local_func, start_method=start_method, nnums=nnums, nprocs=nprocs, @@ -683,8 +751,7 @@ def test_running_multiproc_script( outfile: str | None, profile: bool, # Dummy arguments to make `pytest` output more legible - label1: str, - label2: str, + label1: str, label2: str, ) -> None: """ Check that `kernprof` can RUN the test module in various contexts @@ -709,14 +776,13 @@ def test_running_multiproc_script( defaults=(run_script, 'script')) + _Params.new(('prof_child_procs', 'label2'), [(True, 'with-child-prof'), (False, 'no-child-prof')]) - + _Params.new('start_method', ['fork', 'forkserver', 'spawn'], - defaults=None) + + _get_mp_start_method_fuzzer('label3') ) _fuzz_prof_mp_2 = ( - _Params.new(('preimports', 'label3'), + _Params.new(('preimports', 'label4'), [(True, 'with-preimports'), (False, 'no-preimports')], defaults=(False, 'no-preimports')) - + _Params.new(('use_local_func', 'label4'), + + _Params.new(('use_local_func', 'label5'), [(True, 'local'), (False, 'external')], defaults=(False, 'external')) ) @@ -737,14 +803,12 @@ def test_profiling_multiproc_script( prof_child_procs: bool, preimports: bool, use_local_func: bool, + fail: bool, start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int, nprocs: int, # Dummy arguments to make `pytest` output more legible - label1: str, - label2: str, - label3: str, - label4: str, + label1: str, label2: str, label3: str, label4: str, label5: str, ) -> None: """ Check that `kernprof` can PROFILE the test module in various @@ -757,10 +821,6 @@ def test_profiling_multiproc_script( - ``run_func`` tests the different :cmd:`kernprof` modes (see :py:func:`~.test_running_multiproc_script`). - - ``use_local_func`` tests that we can consistently set up - profiling in both functions locally-defined in the profiled - code and imported by it. - - ``preimports`` tests that both mechanisms for setting up profiling targets work: @@ -777,24 +837,61 @@ def test_profiling_multiproc_script( :py:mod:`multiprocessing` components that we have patched and thus needs separate testing. + - ``use_local_func`` tests that we can consistently set up + profiling in both functions locally-defined in the profiled + code and imported by it. + + - ``fail`` tests that our patches and hook doesn't choke when + exceptions occur in child processes, and profiling data can + still be collected. + + - ``start_method`` tests whether all available + :py:mod:`multiprocessing` start methods are covered. + - ``prof_child_procs`` of course toggles whether to do the patches to set up profiling in child processes. + + Known bugs: + - When the function sent to :py:mod:`multiprocessing` raises an + exception, profiling results are inconsistently gather from + child processes. + + For now we XFAIL these cases. """ + def xfail_on_result_mismatch(info: pytest.ExceptionInfo) -> None: + xc = info.value + assert isinstance(xc, ResultMismatch) + pytest.xfail(xc.rich_message) + + # XXX: we handle known bugs here... + ctx: AbstractContextManager[pytest.ExceptionInfo | None] + if fail: + ctx = pytest.raises( + ResultMismatch, match='hit.*tagged', + ) + else: + ctx = nullcontext() + # How many calls do we expect? nhits = dict.fromkeys( ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, ) # Make sure we're profiling the right function tag = 'LOCAL' if use_local_func else 'EXT' + tag_call = tag + '-INVOCATION' + tag_loop = tag + '-LOOP' + if not fail: + # The final sum in the parent process should always be profiled + # unless the child processes failed and we never returned from + # `Pool.starmap()` + nhits[tag_call] += 1 + nhits[tag_loop] += nprocs if prof_child_procs: - # - `nprocs` child calls summing the `nnums` numbers - # - One call in the main proc summing the `nprocs` results from - # children - nhits[tag + '-INVOCATION'] = nprocs + 1 - nhits[tag + '-LOOP'] = nnums + nprocs - else: - nhits[tag + '-INVOCATION'] = 1 - nhits[tag + '-LOOP'] = nprocs + # When profiling extends into child processes, each of them + # invokes the sum function once and when combined they loop thru + # all the items + nhits[tag_call] += nprocs + nhits[tag_loop] += nnums runner = ['kernprof', '-l'] runner.extend([ @@ -804,17 +901,21 @@ def test_profiling_multiproc_script( if not use_local_func: # Also make sure to include the external module in `--prof-mod` runner.append(f'--prof-mod={ext_module.name}') - run_func( - test_module, tmp_path_factory, - runner=runner, - outfile='out.lprof', - profile=True, - use_local_func=use_local_func, - start_method=start_method, - nhits=nhits, - nnums=nnums, - nprocs=nprocs, - ) + with ctx as maybe_xc: + run_func( + test_module, tmp_path_factory, + runner=runner, + outfile='out.lprof', + profile=True, + use_local_func=use_local_func, + fail=fail, + start_method=start_method, + nhits=nhits, + nnums=nnums, + nprocs=nprocs, + ) + if maybe_xc is not None: + xfail_on_result_mismatch(maybe_xc) # TODO: test for profiling under the following circumstances: From d1947fd4d64ed4f2e2672a8c89e4c89955de3438 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Apr 2026 04:09:51 +0200 Subject: [PATCH 026/165] More debug output, flexible `multiprocessing` patching --- .../multiprocessing_patches.py | 101 ++++++++++++------ .../_child_process_profiling/pth_hook.py | 1 + 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 9bfc5abf..1e6c1ba2 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,9 +16,9 @@ import multiprocessing.process from collections.abc import Callable -from functools import partial, partialmethod +from functools import partial, wraps from typing import Any, TypeVar -from typing_extensions import Concatenate, ParamSpec +from typing_extensions import ParamSpec from .cache import LineProfilingCache from .pth_hook import _setup_in_child_process @@ -58,39 +58,70 @@ def __setstate__(*_) -> None: _apply_mp_patches(lp_cache, _no_op) -def bootstrap( - self: multiprocessing.process.BaseProcess, - vanilla_impl: Callable[ - Concatenate[multiprocessing.process.BaseProcess, PS], T - ], - /, - *args: PS.args, - **kwargs: PS.kwargs -) -> T: +def cleanup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: """ - Wrap around - :py:meth:`multiprocessing.process.BaseProcess._bootstrap`, - writing the profiling results after it is run. + Wrap around :py:class:`multiprocessing.process.BaseProcess` methods + like ``._bootstrap()``, writing the profiling results after it is + run. Args: - self (multiprocessing.process.BaseProcess) - :py:class:`~.BaseProcess` vanilla_impl (Callable) - Vanilla :py:meth:`~.BaseProcess._bootstrap` - *args - **kwargs - Passed to :py:meth:`~.BaseProcess._bootstrap` + Vanilla implementation of the method Returns: - Return value of ``vanilla_impl(*args, **kwargs)`` + Wrapper around ``vanilla_impl`` Side effects: - Profiling results are written + Profiling results are written as the wrapper function exits, + before the result of ``vanilla_impl()`` is returned """ - try: - return vanilla_impl(self, *args, **kwargs) - finally: # Write profiling results - LineProfilingCache.load().cleanup() + @wraps(vanilla_impl) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + try: + return vanilla_impl(*args, **kwargs) + finally: # Write profiling results + # FIXME: somehow this finally clause is not consistently and + # fully executed when an error occurs in the function passed + # to `multiprocessing`... maybe the interpreter is being + # actively exited/torned down as we speak + cache = LineProfilingCache.load() + cache._debug_output(f'Calling cleanup hook via: {name}') + cache.cleanup() + + name = vanilla_impl.__name__ + return wrapper + + +def setup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: + """ + Wrap around :py:class:`multiprocessing.process.BaseProcess` methods + like ``.start()``, setting up profiling before it is run. + + Args: + vanilla_impl (Callable) + Vanilla implementation of the method + + Returns: + Wrapper around ``vanilla_impl`` + + Side effects: + Profiling set up when the wrapper function is called, before + ``vanilla_impl`` itself is invoked + + See also: + :py:func:`cleanup_wrapper` + """ + @wraps(vanilla_impl) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + cache = LineProfilingCache.load() + if cache.profiler is None: + cache._debug_output(f'Calling setup hook via: {name}') + _setup_in_child_process(cache, False, 'multiprocessing') + assert cache.profiler is not None + return vanilla_impl(*args, **kwargs) + + name = vanilla_impl.__name__ + return wrapper def get_preparation_data( @@ -165,13 +196,17 @@ def replace( # Patch `multiprocessing.process.BaseProcess._bootstrap()` Proc = multiprocessing.process.BaseProcess - bootstrap_wrapper = partialmethod( - bootstrap, Proc._bootstrap, # type: ignore[attr-defined] - ) - replace( - Proc, '_bootstrap', bootstrap_wrapper, - f'{Proc.__module__}.{Proc.__qualname__}', - ) + for wrapper_maker, methods in [ + # (setup_wrapper, ['start']), + (setup_wrapper, []), + (cleanup_wrapper, ['_bootstrap']), + ]: + for method in methods: + vanilla = getattr(Proc, method) + replace( + Proc, method, wrapper_maker(vanilla), + f'{Proc.__module__}.{Proc.__qualname__}', + ) # Patch `multiprocessing.spawn` try: diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index db6daea4..93a912ca 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -223,6 +223,7 @@ def _setup_in_child_process(cache, wrap_os_fork=False, context='', prof=None): ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) ctx.install() cache.add_cleanup(ctx.uninstall) + cache._debug_output(f'Set up `.profiler` at {id(prof):#x}') # Do the preimports at `cache.preimports_module` where appropriate if cache.preimports_module: From f1da2db9508f0e7ae4ee77b619d93594a95f739a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Apr 2026 05:37:47 +0200 Subject: [PATCH 027/165] Fix attribute-cleanup logic in `multiprocessing_patches.apply()` --- .../_child_process_profiling/multiprocessing_patches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 1e6c1ba2..d2421122 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -184,9 +184,9 @@ def replace( try: old = getattr(obj, attr) except AttributeError: - add_cleanup(setattr, obj, attr, old) - else: add_cleanup(delattr, obj, attr) + else: + add_cleanup(setattr, obj, attr, old) setattr(obj, attr, value) if obj_name is None: obj_name = repr(obj) From 843f4100b249077e5541d892a6f5e387b641347a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Apr 2026 09:41:39 +0200 Subject: [PATCH 028/165] WIP: let child processes fully clean up before terminating them --- .../multiprocessing_patches.py | 224 +++++++++++++++--- 1 file changed, 197 insertions(+), 27 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index d2421122..4d3d3865 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,9 +16,9 @@ import multiprocessing.process from collections.abc import Callable -from functools import partial, wraps +from functools import partial, partialmethod, wraps from typing import Any, TypeVar -from typing_extensions import ParamSpec +from typing_extensions import Concatenate, ParamSpec from .cache import LineProfilingCache from .pth_hook import _setup_in_child_process @@ -28,6 +28,7 @@ __all__ = ('apply',) +S = TypeVar('S') T = TypeVar('T') PS = ParamSpec('PS') @@ -49,16 +50,16 @@ def __getstate__(_) -> int: def __setstate__(*_) -> None: # We're in a child process created by `multiprocessing`, so set - # up shop here... + # up shop here. lp_cache = LineProfilingCache.load() _setup_in_child_process(lp_cache, False, 'multiprocessing') - # ... and we don't care about polluting the `multiprocessing` - # namespace either, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, _no_op) + _apply_mp_patches(lp_cache, True, True) -def cleanup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: +def cleanup_wrapper( + vanilla_impl: Callable[PS, T], name: str | None = None, +) -> Callable[PS, T]: """ Wrap around :py:class:`multiprocessing.process.BaseProcess` methods like ``._bootstrap()``, writing the profiling results after it is @@ -67,6 +68,9 @@ def cleanup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: Args: vanilla_impl (Callable) Vanilla implementation of the method + name (str | None) + Optional name to use in debug messages; if not provided, use + ``vanilla_impl.__name__`` where available. Returns: Wrapper around ``vanilla_impl`` @@ -74,25 +78,29 @@ def cleanup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: Side effects: Profiling results are written as the wrapper function exits, before the result of ``vanilla_impl()`` is returned + + See also: + :py:func:`setup_wrapper` """ @wraps(vanilla_impl) def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: try: return vanilla_impl(*args, **kwargs) finally: # Write profiling results - # FIXME: somehow this finally clause is not consistently and - # fully executed when an error occurs in the function passed - # to `multiprocessing`... maybe the interpreter is being - # actively exited/torned down as we speak cache = LineProfilingCache.load() cache._debug_output(f'Calling cleanup hook via: {name}') + for msg in 'FOO', 'BAR', 'BAZ': + cache._debug_output(msg) cache.cleanup() - name = vanilla_impl.__name__ + if name is None: + name = getattr(vanilla_impl, '__name__', '???') return wrapper -def setup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: +def setup_wrapper( + vanilla_impl: Callable[PS, T], name: str | None = None, +) -> Callable[PS, T]: """ Wrap around :py:class:`multiprocessing.process.BaseProcess` methods like ``.start()``, setting up profiling before it is run. @@ -100,6 +108,9 @@ def setup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: Args: vanilla_impl (Callable) Vanilla implementation of the method + name (str | None) + Optional name to use in debug messages; if not provided, use + ``vanilla_impl.__name__`` where available. Returns: Wrapper around ``vanilla_impl`` @@ -120,10 +131,112 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: assert cache.profiler is not None return vanilla_impl(*args, **kwargs) - name = vanilla_impl.__name__ + if name is None: + name = getattr(vanilla_impl, '__name__', '???') return wrapper +def get_target_property() -> property: + """ + Returns: + Property object which wraps around the ``._target`` attribute of + (i.e. ``target`` arguemnt to) + :py:class:`multiprocessing.process.BaseProcess` + + Note: + This is a hack to make sure that profiling output is written + ASAP after the call to the target is finished. + + More intuitive solutions that just didn't work are: + + * Wrap ``target`` at initialization time: + By replacing the callable with a wrapper, the whole process + object becomes un-pickle-able. + + * Wrap :py:meth:`multiprocessing.process.BaseProcess._bootstrap` + as :py:mod:`coverage` does: + For currently unclear reasons, if the function set to + :py:mod:`multiprocessing` raises an error, the cleanup clauses + in a try-finally block enclosing the call to the original + ``_bootstrap()`` implementation fails to cleanly, + consistently, and fully execute. Something seems to be + starting process/interpreter teardown prematurely in child + processes... + """ + def getter( + self: multiprocessing.process.BaseProcess, + ) -> Callable[..., Any] | None: + target = vars(self).get(loc) + if target is None: + return None + return cleanup_wrapper(target, name='') + + def setter( + self: multiprocessing.process.BaseProcess, + target: Callable[..., Any] | None, + ) -> None: + vars(self)[loc] = target + + def deleter( + self: multiprocessing.process.BaseProcess, + ) -> None: + vars(self).pop(loc, None) + + loc = '_target' + return property(getter, setter, deleter) + + +def _cache_hook( + vanilla_impl: Callable[PS, T], + get_logging_message: Callable[PS, str], + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> T: + msg = get_logging_message(*args, **kwargs) + LineProfilingCache.load()._debug_output(msg) + return vanilla_impl(*args, **kwargs) + + +def tee_log( + vanilla_impl: Callable[Concatenate[str, PS], None], + marker: str, + /, + msg: str, + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around logging functions like + :py:func:`multiprocessing.util.debug` so that we can tee log + messages from the package to our own logs. + """ + def get_msg(msg: str, *_, **__) -> str: + return f'`multiprocessing` logging ({marker}): {msg}' + + _cache_hook( + vanilla_impl, get_msg, # type: ignore[arg-type] + msg, *args, **kwargs, + ) + + +def log_method_call( + self: S, + vanilla_impl: Callable[Concatenate[S, PS], T], + name: str, + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> T: + def get_msg(self: S, *_, **__) -> str: + return f'Called: `.{name}()` method of {self!r}' + + return _cache_hook( + vanilla_impl, get_msg, # type: ignore[arg-type] + self, *args, **kwargs, + ) + + def get_preparation_data( vanilla_impl: Callable[PS, dict[str, Any]], /, @@ -172,11 +285,13 @@ def apply(lp_cache: LineProfilingCache) -> None: - Cleanup callbacks registered via `lp_cache.add_cleanup()` """ if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, lp_cache.add_cleanup) + _apply_mp_patches(lp_cache, False) def _apply_mp_patches( - lp_cache: LineProfilingCache, add_cleanup: Callable[..., None], + lp_cache: LineProfilingCache, + in_child_process: bool, + debug: bool = False, ) -> None: def replace( obj: Any, attr: str, value: Any, obj_name: str | None = None, @@ -194,19 +309,33 @@ def replace( obj_name, attr, value, )) + # In a child process, we don't care about polluting the + # `multiprocessing` namespace, so don't bother with cleanup + if in_child_process: + add_cleanup: Callable[..., None] = _no_op + else: + add_cleanup = lp_cache.add_cleanup + # Patch `multiprocessing.process.BaseProcess._bootstrap()` Proc = multiprocessing.process.BaseProcess - for wrapper_maker, methods in [ - # (setup_wrapper, ['start']), - (setup_wrapper, []), - (cleanup_wrapper, ['_bootstrap']), - ]: - for method in methods: - vanilla = getattr(Proc, method) - replace( - Proc, method, wrapper_maker(vanilla), - f'{Proc.__module__}.{Proc.__qualname__}', - ) + if False: + wrapper_maker: Callable[[Callable[PS, T]], Callable[PS, T]] + for wrapper_maker, methods in [ # type: ignore[assignment] + (setup_wrapper, []), + (cleanup_wrapper, ['_bootstrap']), + ]: + for method in methods: + vanilla = getattr(Proc, method) + replace( + Proc, method, wrapper_maker(vanilla), + f'{Proc.__module__}.{Proc.__qualname__}', + ) + else: + # Patch `multiprocessing.process.BaseProcess._target` + replace( + Proc, '_target', get_target_property(), + f'{Proc.__module__}.{Proc.__qualname__}', + ) # Patch `multiprocessing.spawn` try: @@ -226,6 +355,47 @@ def replace( spawn, 'runpy', create_runpy_wrapper(lp_cache), spawn.__name__, ) + # Log Popen calls + # XXX: these seem to mitigate (but not completely eliminate) the + # issue of incomplete profiling data; the point seems to be deleying + # the call to `Popen.terminate()` in the parent process so that + # "bad" child processes have the chance to complete their cleanup + # calls. Do we have a more robust way of doing this? + if False: + from importlib import import_module + + for submodule in [ + 'popen_fork', 'popen_spawn_posix', 'popen_spawn_win32', + ]: + try: + Popen = import_module('multiprocessing.' + submodule).Popen + except ImportError: + continue + for method in 'kill', 'terminate', 'interrupt', 'close', 'wait': + method_wrapper = partialmethod( + log_method_call, getattr(Popen, method), method, + ) + replace( + Popen, method, method_wrapper, + f'{Popen.__module__}.{Popen.__qualname__}', + ) + + # Intercept `multiprocessing` debug messages + if debug: + from multiprocessing import util + + for logging_func in [ + 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', + ]: + try: + vanilla = getattr(util, logging_func) + except AttributeError: + continue + replace( + util, logging_func, partial(tee_log, vanilla, logging_func), + 'multiprocessing.util', + ) + # Mark `multiprocessing` as having been patched setattr(multiprocessing, _PATCHED_MARKER, True) add_cleanup(vars(multiprocessing).pop, _PATCHED_MARKER, None) From 46b7d3781967e5104cf264d446f2885776ba6efa Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Apr 2026 23:44:07 +0200 Subject: [PATCH 029/165] Clean up `~~.multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py @cleanup_wrapper, @setup_wrapper get_target_property(), log_method_call() Removed _Poller New helper class for polling a callable wrap_{start,terminate}() New method wrappers for wrapping the eponymous methods of `multiprocessing.process.BaseProcess`; this fixes the bug where if the parallel function errors out in the child process, it may be terminated before profiling data can be gathered wrap_bootstrap() Refactored from `bootstrap()` _apply_mp_patches() Cleaned up testing code --- .../multiprocessing_patches.py | 401 +++++++++--------- 1 file changed, 208 insertions(+), 193 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 4d3d3865..fa79cb78 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -14,11 +14,18 @@ """ from __future__ import annotations -import multiprocessing.process +import multiprocessing +import os from collections.abc import Callable -from functools import partial, partialmethod, wraps +from contextlib import AbstractContextManager, nullcontext +from functools import partial, wraps +from importlib import import_module +from multiprocessing.process import BaseProcess +from pathlib import Path +from tempfile import mkstemp +from time import sleep from typing import Any, TypeVar -from typing_extensions import Concatenate, ParamSpec +from typing_extensions import Concatenate, ParamSpec, Self from .cache import LineProfilingCache from .pth_hook import _setup_in_child_process @@ -33,6 +40,12 @@ PS = ParamSpec('PS') _PATCHED_MARKER = '_line_profiler_patched_multiprocessing' +_PROCESS_TERM_LOCK_LOC = '_line_profiler_process_terminate_lock' +_POLLING_COOLDOWN = 1. / 32 # Seconds +# NOTE: Set this to `None` or `True` to tee the `multiprocessing` +# internal logging messages to the log files; if `None`, logs are only +# written if `LineProfilingCache.load().debug` is set to true. +_INTERCEPT_MP_LOG_MESSAGES = False class PickleHook: @@ -53,137 +66,182 @@ def __setstate__(*_) -> None: # up shop here. lp_cache = LineProfilingCache.load() _setup_in_child_process(lp_cache, False, 'multiprocessing') + # In a child process, we don't care about polluting the + # `multiprocessing` namespace, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, True, True) + _apply_mp_patches(lp_cache, _no_op) -def cleanup_wrapper( - vanilla_impl: Callable[PS, T], name: str | None = None, -) -> Callable[PS, T]: +class _Poller: """ - Wrap around :py:class:`multiprocessing.process.BaseProcess` methods - like ``._bootstrap()``, writing the profiling results after it is - run. + Poll a callable until it returns true-y. + """ + def __init__( + self, func: Callable[[], Any], cooldown: float | None = None, + ) -> None: + if not (cooldown and cooldown > 0): + cooldown = 0 + self._func: Callable[[], Any] = func + self._cooldown = cooldown + + def sleep(self): + cd = self._cooldown + if cd > 0: + sleep(cd) + + def with_cooldown(self, cooldown: float | None) -> Self: + return type(self)(self._func, cooldown) + + @classmethod + def poll_until( + cls, func: Callable[PS, Any], /, *args: PS.args, **kwargs: PS.kwargs + ) -> Self: + if args or kwargs: + func = partial(func, *args, **kwargs) + return cls(func) + + @classmethod + def poll_while( + cls, func: Callable[PS, Any], /, *args: PS.args, **kwargs: PS.kwargs + ) -> Self: + def negated( + func: Callable[PS, Any], *a: PS.args, **k: PS.kwargs + ) -> bool: + return not func(*a, **k) + + return cls(partial(negated, func, *args, **kwargs)) + + def __enter__(self) -> Self: + while not self._func(): + self.sleep() + return self + + def __exit__(self, *_, **__) -> None: + pass - Args: - vanilla_impl (Callable) - Vanilla implementation of the method - name (str | None) - Optional name to use in debug messages; if not provided, use - ``vanilla_impl.__name__`` where available. - Returns: - Wrapper around ``vanilla_impl`` +def _method_wrapper( + wrapper: Callable[Concatenate[S, Callable[Concatenate[S, PS], T], PS], T] +) -> Callable[ + [Callable[Concatenate[S, PS], T]], Callable[Concatenate[S, PS], T] +]: + def inner_wrapper( + vanilla_impl: Callable[Concatenate[S, PS], T], + ) -> Callable[Concatenate[S, PS], T]: + @wraps(vanilla_impl) + def wrapped_impl(self: S, *a: PS.args, **k: PS.kwargs) -> T: + return wrapper(self, vanilla_impl, *a, **k) - Side effects: - Profiling results are written as the wrapper function exits, - before the result of ``vanilla_impl()`` is returned + return wrapped_impl - See also: - :py:func:`setup_wrapper` + for field in 'name', 'qualname', 'doc': + dunder = f'__{field}__' + value = getattr(wrapper, dunder, None) + if value is not None: + setattr(inner_wrapper, dunder, value) + return inner_wrapper + + +@_method_wrapper +def wrap_start( + self: BaseProcess, vanilla_impl: Callable[[BaseProcess], None], +) -> None: """ - @wraps(vanilla_impl) - def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: - try: - return vanilla_impl(*args, **kwargs) - finally: # Write profiling results - cache = LineProfilingCache.load() - cache._debug_output(f'Calling cleanup hook via: {name}') - for msg in 'FOO', 'BAR', 'BAZ': - cache._debug_output(msg) - cache.cleanup() - - if name is None: - name = getattr(vanilla_impl, '__name__', '???') - return wrapper - - -def setup_wrapper( - vanilla_impl: Callable[PS, T], name: str | None = None, -) -> Callable[PS, T]: + Wrap around :py:meth:`BaseProcess.start` to specify the location for + a lock file, which is managed by the child process and checked by + the parent. This is to ensure that the child can exit gracefully and + complete any necessary cleanup. """ - Wrap around :py:class:`multiprocessing.process.BaseProcess` methods - like ``.start()``, setting up profiling before it is run. + cache = LineProfilingCache.load() + handle, tempfile = mkstemp( + dir=cache.cache_dir, prefix='process-term-lock-', suffix='.lock', + ) + try: + setattr(self, _PROCESS_TERM_LOCK_LOC, Path(tempfile)) + vanilla_impl(self) + finally: + os.close(handle) - Args: - vanilla_impl (Callable) - Vanilla implementation of the method - name (str | None) - Optional name to use in debug messages; if not provided, use - ``vanilla_impl.__name__`` where available. - Returns: - Wrapper around ``vanilla_impl`` +@_method_wrapper +def wrap_terminate( + self: BaseProcess, vanilla_impl: Callable[[BaseProcess], None], +) -> None: + """ + Wrap around :py:meth:`BaseProcess.terminate` to make sure that we + don't actually kill the child (OS-level) process before it has the + chance to properly clean up. This is done by blocking the call as + long as a lock file exists, which is specified by the parent process + and managed by the child. - Side effects: - Profiling set up when the wrapper function is called, before - ``vanilla_impl`` itself is invoked + Note: + We're technically polling in a hot loop, but: - See also: - :py:func:`cleanup_wrapper` - """ - @wraps(vanilla_impl) - def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: - cache = LineProfilingCache.load() - if cache.profiler is None: - cache._debug_output(f'Calling setup hook via: {name}') - _setup_in_child_process(cache, False, 'multiprocessing') - assert cache.profiler is not None - return vanilla_impl(*args, **kwargs) + - We're only calling this when explicitly terminating processes, + which isn't that bad; and - if name is None: - name = getattr(vanilla_impl, '__name__', '???') - return wrapper + - Such calls typically only happen: + - When e.g. the parallel function exectued in child + processes raised an error, so we're already on a "bad" + path; and -def get_target_property() -> property: + - AFTER the performance-critical part of the code (the + parallelly-run function). + + To circumvent this we may use dedicated FS-watching APIs like + :py:mod:`watchdog` (which use syscalls to do this), but we'll + think about introducing extra dependencies when we REALLY have + to. """ - Returns: - Property object which wraps around the ``._target`` attribute of - (i.e. ``target`` arguemnt to) - :py:class:`multiprocessing.process.BaseProcess` + # XXX: why can `coverage` get away with not doing all this lock-file + # hijinks and just patching `BaseProcess._bootstrap()`? + lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) + if lock_file: + lock: AbstractContextManager[Any] = ( + _Poller.poll_while(lock_file.exists) + .with_cooldown(_POLLING_COOLDOWN) + ) + else: + lock = nullcontext() + with lock: + try: + delattr(self, _PROCESS_TERM_LOCK_LOC) + except AttributeError: + pass + vanilla_impl(self) - Note: - This is a hack to make sure that profiling output is written - ASAP after the call to the target is finished. - - More intuitive solutions that just didn't work are: - - * Wrap ``target`` at initialization time: - By replacing the callable with a wrapper, the whole process - object becomes un-pickle-able. - - * Wrap :py:meth:`multiprocessing.process.BaseProcess._bootstrap` - as :py:mod:`coverage` does: - For currently unclear reasons, if the function set to - :py:mod:`multiprocessing` raises an error, the cleanup clauses - in a try-finally block enclosing the call to the original - ``_bootstrap()`` implementation fails to cleanly, - consistently, and fully execute. Something seems to be - starting process/interpreter teardown prematurely in child - processes... + +@_method_wrapper +def wrap_bootstrap( + self: BaseProcess, + vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], /, + *args: PS.args, **kwargs: PS.kwargs +) -> T: """ - def getter( - self: multiprocessing.process.BaseProcess, - ) -> Callable[..., Any] | None: - target = vars(self).get(loc) - if target is None: - return None - return cleanup_wrapper(target, name='') - - def setter( - self: multiprocessing.process.BaseProcess, - target: Callable[..., Any] | None, - ) -> None: - vars(self)[loc] = target + Wrap around :py:meth:`BaseProcess._bootstrap` to: - def deleter( - self: multiprocessing.process.BaseProcess, - ) -> None: - vars(self).pop(loc, None) + - Run ``LineProfilingCache.load().cleanup()`` so that profiling + results can be gathered; and - loc = '_target' - return property(getter, setter, deleter) + - Write a lock file before executing ``vanilla_impl()`` and deleted + it thereafter, to ensure that a parant process doesn't prematurely + ``.terminate()`` a failed child before the profiling results can + be gathered. + """ + cache = LineProfilingCache.load() + lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) + + if lock_file: + lock_file.touch() + cache.add_cleanup(lock_file.unlink, missing_ok=True) + try: + return vanilla_impl(self, *args, **kwargs) + finally: + cache._debug_output( + 'Calling cleanup hook via `BaseProcess._bootstrap`' + ) + cache.cleanup() def _cache_hook( @@ -220,23 +278,6 @@ def get_msg(msg: str, *_, **__) -> str: ) -def log_method_call( - self: S, - vanilla_impl: Callable[Concatenate[S, PS], T], - name: str, - /, - *args: PS.args, - **kwargs: PS.kwargs -) -> T: - def get_msg(self: S, *_, **__) -> str: - return f'Called: `.{name}()` method of {self!r}' - - return _cache_hook( - vanilla_impl, get_msg, # type: ignore[arg-type] - self, *args, **kwargs, - ) - - def get_preparation_data( vanilla_impl: Callable[PS, dict[str, Any]], /, @@ -279,19 +320,26 @@ def apply(lp_cache: LineProfilingCache) -> None: Side effects: - :py:mod:`multiprocessing` marked as having been set up - - :py:meth:`multiprocessing.process.BaseProcess._bootstrap` - patched - - :py:func:`multiprocessing.spawn.get_preparation_data` patched - - Cleanup callbacks registered via `lp_cache.add_cleanup()` + + - The following methods and functions patched: + - :py:meth:`multiprocessing.process.BaseProcess.start` + + - :py:meth:`multiprocessing.process.BaseProcess.terminate` + + - :py:meth:`multiprocessing.process.BaseProcess._bootstrap` + + - :py:func:`multiprocessing.spawn.get_preparation_data` + + - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` """ if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, False) + _apply_mp_patches(lp_cache, lp_cache.add_cleanup) def _apply_mp_patches( lp_cache: LineProfilingCache, - in_child_process: bool, - debug: bool = False, + add_cleanup: Callable[..., Any], + debug: bool | None = _INTERCEPT_MP_LOG_MESSAGES, ) -> None: def replace( obj: Any, attr: str, value: Any, obj_name: str | None = None, @@ -309,33 +357,25 @@ def replace( obj_name, attr, value, )) - # In a child process, we don't care about polluting the - # `multiprocessing` namespace, so don't bother with cleanup - if in_child_process: - add_cleanup: Callable[..., None] = _no_op - else: - add_cleanup = lp_cache.add_cleanup - - # Patch `multiprocessing.process.BaseProcess._bootstrap()` - Proc = multiprocessing.process.BaseProcess - if False: - wrapper_maker: Callable[[Callable[PS, T]], Callable[PS, T]] - for wrapper_maker, methods in [ # type: ignore[assignment] - (setup_wrapper, []), - (cleanup_wrapper, ['_bootstrap']), - ]: - for method in methods: - vanilla = getattr(Proc, method) - replace( - Proc, method, wrapper_maker(vanilla), - f'{Proc.__module__}.{Proc.__qualname__}', - ) - else: - # Patch `multiprocessing.process.BaseProcess._target` - replace( - Proc, '_target', get_target_property(), - f'{Proc.__module__}.{Proc.__qualname__}', - ) + # Patch `multiprocessing.process.BaseProcess` methods + Method = Callable[Concatenate[S, PS], T] + patches: dict[str, Callable[[Method], Method]] + for submodule, target, patches in [ # type: ignore[assignment] + ('process', 'BaseProcess', { + 'start': wrap_start, + 'terminate': wrap_terminate, + '_bootstrap': wrap_bootstrap, + }), + ]: + try: + mod = import_module('multiprocessing.' + submodule) + except ImportError: + continue + Class = getattr(mod, target) + name = f'{Class.__module__}.{Class.__qualname__}' + for method, method_wrapper in patches.items(): + vanilla = getattr(Class, method) + replace(Class, method, method_wrapper(vanilla), name) # Patch `multiprocessing.spawn` try: @@ -351,36 +391,12 @@ def replace( # Patch `runpy` (do it locally instead of tempering with the # global `runpy` mmodule) if hasattr(spawn, 'runpy'): - replace( - spawn, 'runpy', create_runpy_wrapper(lp_cache), spawn.__name__, - ) - - # Log Popen calls - # XXX: these seem to mitigate (but not completely eliminate) the - # issue of incomplete profiling data; the point seems to be deleying - # the call to `Popen.terminate()` in the parent process so that - # "bad" child processes have the chance to complete their cleanup - # calls. Do we have a more robust way of doing this? - if False: - from importlib import import_module - - for submodule in [ - 'popen_fork', 'popen_spawn_posix', 'popen_spawn_win32', - ]: - try: - Popen = import_module('multiprocessing.' + submodule).Popen - except ImportError: - continue - for method in 'kill', 'terminate', 'interrupt', 'close', 'wait': - method_wrapper = partialmethod( - log_method_call, getattr(Popen, method), method, - ) - replace( - Popen, method, method_wrapper, - f'{Popen.__module__}.{Popen.__qualname__}', - ) + runpy_wrapper = create_runpy_wrapper(lp_cache) + replace(spawn, 'runpy', runpy_wrapper, spawn.__name__) # Intercept `multiprocessing` debug messages + if debug is None: + debug = lp_cache.debug if debug: from multiprocessing import util @@ -397,8 +413,7 @@ def replace( ) # Mark `multiprocessing` as having been patched - setattr(multiprocessing, _PATCHED_MARKER, True) - add_cleanup(vars(multiprocessing).pop, _PATCHED_MARKER, None) + replace(multiprocessing, _PATCHED_MARKER, True) def _no_op(*_, **__) -> None: From 519dbd51457db52f660a68898fe697888847408b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Apr 2026 23:56:11 +0200 Subject: [PATCH 030/165] Fix `tests/test_child_procs.py` tests/test_child_procs.py::test_profiling_multiproc_script() Removed XFAIL-ing for cases where the profiled function fails (because the bug has been fixed) --- tests/test_child_procs.py | 49 ++++++++++----------------------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 09473e36..a6effffe 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -9,7 +9,6 @@ from collections.abc import ( Callable, Collection, Generator, Iterable, Mapping, Sequence, ) -from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass from functools import partial from pathlib import Path @@ -850,28 +849,7 @@ def test_profiling_multiproc_script( - ``prof_child_procs`` of course toggles whether to do the patches to set up profiling in child processes. - - Known bugs: - - When the function sent to :py:mod:`multiprocessing` raises an - exception, profiling results are inconsistently gather from - child processes. - - For now we XFAIL these cases. """ - def xfail_on_result_mismatch(info: pytest.ExceptionInfo) -> None: - xc = info.value - assert isinstance(xc, ResultMismatch) - pytest.xfail(xc.rich_message) - - # XXX: we handle known bugs here... - ctx: AbstractContextManager[pytest.ExceptionInfo | None] - if fail: - ctx = pytest.raises( - ResultMismatch, match='hit.*tagged', - ) - else: - ctx = nullcontext() - # How many calls do we expect? nhits = dict.fromkeys( ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, @@ -901,21 +879,18 @@ def xfail_on_result_mismatch(info: pytest.ExceptionInfo) -> None: if not use_local_func: # Also make sure to include the external module in `--prof-mod` runner.append(f'--prof-mod={ext_module.name}') - with ctx as maybe_xc: - run_func( - test_module, tmp_path_factory, - runner=runner, - outfile='out.lprof', - profile=True, - use_local_func=use_local_func, - fail=fail, - start_method=start_method, - nhits=nhits, - nnums=nnums, - nprocs=nprocs, - ) - if maybe_xc is not None: - xfail_on_result_mismatch(maybe_xc) + run_func( + test_module, tmp_path_factory, + runner=runner, + outfile='out.lprof', + profile=True, + use_local_func=use_local_func, + fail=fail, + start_method=start_method, + nhits=nhits, + nnums=nnums, + nprocs=nprocs, + ) # TODO: test for profiling under the following circumstances: From e7410e23cc161c6728149e5d0af9672b942b00f4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 01:18:14 +0200 Subject: [PATCH 031/165] Test for non-`multiprocessing` child processes tests/test_child_procs.py::test_profiling_bare_python() New test for checking the profiling of child processes created outside of `multiprocessing` (e.g. `subprocess.run()`, `os.system()`) --- tests/test_child_procs.py | 130 ++++++++++++++++++++++++++++++-------- 1 file changed, 104 insertions(+), 26 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index a6effffe..9947dbc7 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -32,7 +32,12 @@ NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) -EXTERNAL_MODULE_BODY = dedent(""" + +def strip(s: str) -> str: + return dedent(s).strip('\n') + + +EXTERNAL_MODULE_BODY = strip(""" from __future__ import annotations @@ -43,9 +48,9 @@ def my_external_sum(x: list[int], fail: bool = False) -> int: if fail: raise RuntimeError('forced failure') return result -""").strip('\n') +""") -TEST_MODULE_TEMPLATE = dedent(""" +TEST_MODULE_TEMPLATE = strip(""" from __future__ import annotations from argparse import ArgumentParser @@ -118,7 +123,7 @@ def main(args: list[str] | None = None) -> None: if __name__ == '__main__': main() -""").strip('\n') +""") # ============================== Fixtures ============================== @@ -564,23 +569,6 @@ def _run_test_module( profliing_stats (LineStats | None): Line-profiling stats (where available) """ - def check_output(output: str, tag: str, nhits: int) -> None: - # The line should be preixed with 5 numbers: - # lineno, nhits, time, time-per-hit, % time - actual_nhits = 0 - for line in output.splitlines(): - if line.endswith(f'# GREP_MARKER[{tag}]'): - try: - _, n, _, _, _, *_ = line.split() - actual_nhits += int(n) - except Exception: - pass - if actual_nhits == nhits: - return - raise ResultMismatch( - f'{nhits} hit(s) on line(s) tagged with {tag!r}', actual_nhits, - ) - if isinstance(runner, str): runner_args: list[str] = [runner] else: @@ -652,10 +640,28 @@ def check_output(output: str, tag: str, nhits: int) -> None: # code is in a tempfile the profiling data will be dropped in # the written outfile) for tag, num in (nhits or {}).items(): - check_output(proc.stdout, tag, num) + _check_output(proc.stdout, tag, num) return proc, prof_result +def _check_output(output: str, tag: str, nhits: int) -> None: + # The line should be preixed with 5 numbers: + # lineno, nhits, time, time-per-hit, % time + actual_nhits = 0 + for line in output.splitlines(): + if line.endswith(f'# GREP_MARKER[{tag}]'): + try: + _, n, _, _, _, *_ = line.split() + actual_nhits += int(n) + except Exception: + pass + if actual_nhits == nhits: + return + raise ResultMismatch( + f'{nhits} hit(s) on line(s) tagged with {tag!r}', actual_nhits, + ) + + run_module = partial(_run_test_module, _run_as_module) run_script = partial(_run_test_module, _run_as_script) run_literal_code = partial( @@ -893,7 +899,79 @@ def test_profiling_multiproc_script( ) -# TODO: test for profiling under the following circumstances: -# - `os.system()` -# - `subprocess.run()` -# - Nested subprocesses +@pytest.mark.parametrize(('use_subprocess', 'label1'), + [(True, 'subprocess.run'), (False, 'os.system')]) +@pytest.mark.parametrize(('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')]) +@pytest.mark.parametrize(('fail', 'label3'), + [(True, 'failure'), (False, 'success')]) +@pytest.mark.parametrize('n', [200]) +def test_profiling_bare_python( + tmp_path_factory: pytest.TempPathFactory, + ext_module: _ModuleFixture, + use_subprocess: bool, + prof_child_procs: bool, + fail: bool, + n: int, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, label3: str, +) -> None: + """ + Check that `kernprof` can profile the target functions if the code + invokes another bare Python process (via either :py:func:`os.system` + or :py:func:`subprocess.run`) that calls them. + """ + ext_module.install(children=True) + temp_dir = tmp_path_factory.mktemp('mytemp') + + script_path = temp_dir / 'my-script.py' + script_content = strip(""" + from {EXT_MODULE} import my_external_sum + + + if __name__ == '__main__': + numbers = list(range(1, 1 + {N})) + result = my_external_sum(numbers, {FAIL}) + """.format( + EXT_MODULE=ext_module.name, + N=n, + FAIL=fail, + )) + script_path.write_text(script_content) + + out_file = temp_dir / 'out.lprof' + cmd = [ + 'kernprof', '-lv', '--preimports', + f'--prof-mod={ext_module.name}', + f'--outfile={out_file}', + '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), + ] + sub_cmd = [sys.executable, str(script_path)] + if use_subprocess: + code = strip(f""" + import subprocess + + + subprocess.run({sub_cmd!r}, check=True) + """) + else: + code = strip(""" + import os + + + if os.system({!r}): + raise RuntimeError('called process failed') + """.format(shlex.join(sub_cmd))) + cmd.extend(['-c', code]) + proc = _run_subproc(cmd, text=True, capture_output=True) + + nhits = {'EXT-INVOCATION': 1, 'EXT-LOOP': n} + if not prof_child_procs: + for k in nhits: + nhits[k] = 0 + + # Check that the code errors out when expected + assert bool(fail) == bool(proc.returncode) + # Check that the profiling output is as expected + for tag, num in nhits.items(): + _check_output(proc.stdout, tag, num) From de705253688033d31e6f7558ebd4fbd9a1f0ea05 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 02:25:40 +0200 Subject: [PATCH 032/165] Unify tempfile creation line_profiler/_child_process_profiling/ cache.py::LineProfilingCache.make_tempfile() New convenience method for creating tempfiles with `mkstemp()` multiprocessing_patches.py::wrap_start() pth_hook.py::_setup_in_child_process() Simplfied implementations to just use `LineProfilingCache.make_tempfile()` --- .../_child_process_profiling/cache.py | 18 ++++++++++++++++++ .../multiprocessing_patches.py | 13 +++---------- .../_child_process_profiling/pth_hook.py | 9 ++------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 7ef9d0f0..a0d975c1 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -15,6 +15,7 @@ from operator import setitem from pathlib import Path from pickle import HIGHEST_PROTOCOL +from tempfile import mkstemp from typing import Any, ClassVar, cast from typing_extensions import Self, ParamSpec @@ -226,6 +227,23 @@ def _debug_output(self, msg: str) -> None: except OSError: # Cache dir may have been rm-ed during cleanup pass + def make_tempfile(self, **kwargs) -> Path: + """ + Create a fresh tempfile under :py:attr:`~.cache_dir`. The other + arguments are passed as-is to :py:func:`tempfile.mkstemp`. + + Returns: + path (Path): + Path to the created file. + """ + handle, path = mkstemp(dir=self.cache_dir, **kwargs) + try: + path_obj = Path(path) + self._debug_output(f'Created tempfile: {path_obj.name!r}') + return path_obj + finally: + os.close(handle) + def _replace_loaded_instance(self) -> bool: if self._consistent_with_loaded_instance: type(self)._loaded_instance = self diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index fa79cb78..f793affc 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -15,14 +15,12 @@ from __future__ import annotations import multiprocessing -import os from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext from functools import partial, wraps from importlib import import_module from multiprocessing.process import BaseProcess from pathlib import Path -from tempfile import mkstemp from time import sleep from typing import Any, TypeVar from typing_extensions import Concatenate, ParamSpec, Self @@ -153,14 +151,9 @@ def wrap_start( complete any necessary cleanup. """ cache = LineProfilingCache.load() - handle, tempfile = mkstemp( - dir=cache.cache_dir, prefix='process-term-lock-', suffix='.lock', - ) - try: - setattr(self, _PROCESS_TERM_LOCK_LOC, Path(tempfile)) - vanilla_impl(self) - finally: - os.close(handle) + tempfile = cache.make_tempfile(prefix='process-term-lock-', suffix='.lock') + setattr(self, _PROCESS_TERM_LOCK_LOC, tempfile) + vanilla_impl(self) @_method_wrapper diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 93a912ca..69004a05 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -203,7 +203,6 @@ def _setup_in_child_process(cache, wrap_os_fork=False, context='', prof=None): import os from atexit import register - from tempfile import mkstemp from ..autoprofile.autoprofile import ( # Note: we need this to equip the profiler with the # `.add_imported_function_or_module()` pseudo-method @@ -234,16 +233,12 @@ def _setup_in_child_process(cache, wrap_os_fork=False, context='', prof=None): # Occupy a tempfile slot in `cache.cache_dir` and set the profiler # up to write thereto when the process terminates - handle, prof_outfile = mkstemp( + prof_outfile = cache.make_tempfile( prefix='child-prof-output-{}-{}-{:#x}-' .format(cache.main_pid, os.getpid(), id(prof)), suffix='.lprof', - dir=cache.cache_dir, ) - try: # Whatever else we do, write the profiling stats first - cache._add_cleanup(prof.dump_stats, -1, prof_outfile) - finally: - os.close(handle) + cache._add_cleanup(prof.dump_stats, -1, prof_outfile) # Set up `os.fork()` wrapping if needed (i.e. in a spawned process) if wrap_os_fork: From a3cfa2f7484927f2ca3ba270637799be89a4bc0f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 03:37:23 +0200 Subject: [PATCH 033/165] Failsafe: timeout the lock-file waiting line_profiler/_child_process_profiling/multiprocessing_patches.py _Poller __init__() - Updated default and typing for `cooldown` - New arguments `timeout` and `on_timeout` for controlling timeout duration and behaviors with_timeout() New method creating a new instance a la `.with_cooldown()` __enter__() Added timeout handling Timeout New `RuntimeError` subclass raised if `timeout` is positive and reached wrap_terminate() Now only allowing the `BaseProcess.terminate()` call to be blocked by at most 1 s by the lock file, before issuing a warning and proceeding anyway tests/test_child_procs.py::test_profiling_multiproc_script() Now timing out the `kernprof` process after 5 s in case the lock files caused a deadlock --- .../multiprocessing_patches.py | 113 +++++++++++++++--- tests/test_child_procs.py | 9 ++ 2 files changed, 106 insertions(+), 16 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index f793affc..0563912c 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -15,16 +15,18 @@ from __future__ import annotations import multiprocessing +import warnings from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext from functools import partial, wraps from importlib import import_module from multiprocessing.process import BaseProcess from pathlib import Path -from time import sleep -from typing import Any, TypeVar +from time import sleep, monotonic +from typing import Any, Literal, TypeVar, NoReturn from typing_extensions import Concatenate, ParamSpec, Self +from .. import _diagnostics as diagnostics from .cache import LineProfilingCache from .pth_hook import _setup_in_child_process from .runpy_patches import create_runpy_wrapper @@ -39,11 +41,26 @@ _PATCHED_MARKER = '_line_profiler_patched_multiprocessing' _PROCESS_TERM_LOCK_LOC = '_line_profiler_process_terminate_lock' -_POLLING_COOLDOWN = 1. / 32 # Seconds + +# NOTE: polling behaviors for blocking `BaseProcess.terminate()` with +# the lock file: +# - _POLLING_COOLDOWN: +# Seconds between polling for the lock file; set to zero to poll in a +# hot loop (not recommended) +# - _POLLING_TIMEOUT: second to block before timing out anyway; set to +# zero to disable timing out +# - _POLLING_ON_TIMEOUT: what to do on timeout if enabled: +# - 'error': error out with `_Poller.Timeout` +# - 'warn': issue a warning and move on +# - 'ignore': move on without a warning +_POLLING_COOLDOWN = 1. / 32 +_POLLING_TIMEOUT = 1. +_POLLING_ON_TIMEOUT: Literal['error', 'warn', 'ignore'] = 'warn' + # NOTE: Set this to `None` or `True` to tee the `multiprocessing` # internal logging messages to the log files; if `None`, logs are only # written if `LineProfilingCache.load().debug` is set to true. -_INTERCEPT_MP_LOG_MESSAGES = False +_INTERCEPT_MP_LOG_MESSAGES: bool | None = False class PickleHook: @@ -75,20 +92,41 @@ class _Poller: Poll a callable until it returns true-y. """ def __init__( - self, func: Callable[[], Any], cooldown: float | None = None, + self, + func: Callable[[], Any], + cooldown: float = 0, + timeout: float = 0, + on_timeout: Literal['ignore', 'warn', 'error'] = 'error', ) -> None: - if not (cooldown and cooldown > 0): + if cooldown < 0: cooldown = 0 + if timeout < 0: + timeout = 0 self._func: Callable[[], Any] = func self._cooldown = cooldown + self._timeout = timeout + self._on_timeout = on_timeout def sleep(self): cd = self._cooldown if cd > 0: sleep(cd) - def with_cooldown(self, cooldown: float | None) -> Self: - return type(self)(self._func, cooldown) + def with_cooldown(self, cooldown: float) -> Self: + return type(self)( + self._func, cooldown, self._timeout, self._on_timeout, + ) + + def with_timeout( + self, + timeout: float | None = None, + on_timeout: Literal['ignore', 'warn', 'error'] | None = None, + ) -> Self: + if timeout is None: + timeout = self._timeout + if on_timeout is None: + on_timeout = self._on_timeout + return type(self)(self._func, self._cooldown, timeout, on_timeout) @classmethod def poll_until( @@ -110,13 +148,47 @@ def negated( return cls(partial(negated, func, *args, **kwargs)) def __enter__(self) -> Self: - while not self._func(): + def error(msg: str) -> NoReturn: + raise type(self).Timeout(msg) + + def warn(msg: str) -> None: + warnings.warn(msg) + diagnostics.log.warning(msg) + + def ignore(_): + pass + + timeout = self._timeout + callback = self._func + + handle_timeout: Callable[[str], Any] = { + 'error': error, 'warn': warn, 'ignore': ignore, + }[self._on_timeout] + fmt = '.3g' + timeout_msg_header = f'{type(self).__name__} at {id(self):#x}' + + start = monotonic() + while not callback(): + elapsed = monotonic() - start + if timeout and elapsed >= timeout: + handle_timeout( + f'{timeout_msg_header}: ' + f'timed out ({elapsed:{fmt}} s >= {timeout:{fmt}} s) ' + f'waiting for callback {callback!r} to return true' + ) + break self.sleep() return self def __exit__(self, *_, **__) -> None: pass + class Timeout(RuntimeError): + """ + Raised when a :py:class:`_Poller` is timed out when polling. + """ + pass + def _method_wrapper( wrapper: Callable[Concatenate[S, Callable[Concatenate[S, PS], T], PS], T] @@ -187,21 +259,30 @@ def wrap_terminate( think about introducing extra dependencies when we REALLY have to. """ - # XXX: why can `coverage` get away with not doing all this lock-file - # hijinks and just patching `BaseProcess._bootstrap()`? + # XXX: why can `coverage` get away with not doing all these + # lock-file hijinks and just patching `BaseProcess._bootstrap()`? + + def discard_lock() -> None: + assert lock_file is not None + # This should have already happened unless we timed out + lock_file.unlink(missing_ok=True) + try: + delattr(self, _PROCESS_TERM_LOCK_LOC) + except AttributeError: + pass + lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) if lock_file: lock: AbstractContextManager[Any] = ( _Poller.poll_while(lock_file.exists) .with_cooldown(_POLLING_COOLDOWN) + .with_timeout(_POLLING_TIMEOUT, _POLLING_ON_TIMEOUT) ) + callback = discard_lock else: - lock = nullcontext() + lock, callback = nullcontext(), _no_op with lock: - try: - delattr(self, _PROCESS_TERM_LOCK_LOC) - except AttributeError: - pass + callback() vanilla_impl(self) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 9947dbc7..ad6a91c5 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -561,6 +561,7 @@ def _run_test_module( nprocs: int | None = None, check: bool = True, nhits: Mapping[str, int] | None = None, + **kwargs ) -> tuple[subprocess.CompletedProcess, LineStats | None]: """ Returns: @@ -610,6 +611,7 @@ def _run_test_module( proc = run_helper( runner_args, test_args, test_module, text=True, capture_output=True, check=(check and not fail), + **kwargs ) # Checks: if fail: @@ -856,6 +858,12 @@ def test_profiling_multiproc_script( - ``prof_child_procs`` of course toggles whether to do the patches to set up profiling in child processes. """ + # XXX: owing to the shenanigans in + # `line_profiler._child_process_profiling.multiprocessing_patches`, + # there is a risk that failing child processes are not properly + # `.terminate()`-ed. So just put in a timeout... + timeout = 5 # Seconds + # How many calls do we expect? nhits = dict.fromkeys( ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, @@ -896,6 +904,7 @@ def test_profiling_multiproc_script( nhits=nhits, nnums=nnums, nprocs=nprocs, + timeout=timeout, ) From 5abf4e212283c166b367fd4aa465585e6ec5a806 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 06:14:13 +0200 Subject: [PATCH 034/165] Added check in tests for .pth-file cleanup --- tests/test_child_procs.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index ad6a91c5..90456606 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -12,6 +12,7 @@ from dataclasses import dataclass from functools import partial from pathlib import Path +from sysconfig import get_path from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic @@ -570,6 +571,11 @@ def _run_test_module( profliing_stats (LineStats | None): Line-profiling stats (where available) """ + def get_pth_files(pattern: str = '*.pth') -> set[str]: + return {path.name for path in lib_path.glob(pattern)} + + lib_path = Path(get_path('purelib')) + if isinstance(runner, str): runner_args: list[str] = [runner] else: @@ -608,6 +614,7 @@ def _run_test_module( with ub.ChDir(tmp_path_factory.mktemp('mytemp')): if outfile is not None: runner_args.extend(['--outfile', outfile]) + old_pth_files = get_pth_files() proc = run_helper( runner_args, test_args, test_module, text=True, capture_output=True, check=(check and not fail), @@ -626,6 +633,9 @@ def _run_test_module( raise ResultMismatch( f'result {expected}', f'output lines: {output_lines}', ) + # - Temporary `.pth` file(s) created by `~~.pth_hook` has been + # cleaned up + assert get_pth_files() == old_pth_files # - Profiling results are written to the specified file prof_result: LineStats | None = None if outfile is None: From 0837d7c4c2b30ae7272ba3bef15835adbf54a45f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 14:33:46 +0200 Subject: [PATCH 035/165] Fix kernprof docstring --- kernprof.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernprof.py b/kernprof.py index 50097fde..5909981d 100755 --- a/kernprof.py +++ b/kernprof.py @@ -138,6 +138,9 @@ def main(): If the script/module profiled is in `--prof-mod`, autoprofile all its imports. Only works with line profiling (`-l`/`--line- by-line`). (Default: False) + --prof-child-procs [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] + Extend profiling into child Python processes. Only works with + line profiling (`-l`/`--line-by-line`). (Default: False) output options: -o, --outfile OUTFILE @@ -152,7 +155,7 @@ def main(): -q, --quiet Decrease verbosity level (default: 0). At level -1, disable helpful messages (e.g. "Wrote profile results to <...>"); at level -2, silence the stdout; at level -3, silence the stderr. ---rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] + --rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] Use rich formatting if viewing output. (Default: False; short form: -r) -u, --unit UNIT Output unit (in seconds) in which the timing info is @@ -1059,7 +1062,7 @@ def _write_preimports(prof, options, exclude, keep=False): from line_profiler.autoprofile.autoprofile import ( _extend_line_profiler_for_profiling_imports as upgrade_profiler, ) - from line_profiler.curated_profiling import ClassifiedPreimportTargets + from line_profiler.curated_profiling import ClassifiedPreimportTargets # We could've done everything in-memory with `io.StringIO` and `exec()`, # but that results in indecipherable tracebacks should anything goes wrong; From 7b4eefa9928fd2c6be1d2e5a614943eadb88f2f8 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 17:04:56 +0200 Subject: [PATCH 036/165] Shuffled code inside `~._child_process_profiling` line_profiler/_child_process_profiling/ - The functions `pth_hook.py::_setup_in_child_process()` and `::_wrap_os_fork()` have been relocated to eponymous instance methods of `cache.py::LineProfilingCache` - The implementations of `pth_hook.py::{write,load}_pth_hook()` and `multiprocessing_patches.py::PickleHook.__setstate__()` are updated accordingly --- .../_child_process_profiling/cache.py | 121 +++++++++++++++- .../multiprocessing_patches.py | 3 +- .../_child_process_profiling/pth_hook.py | 135 +----------------- 3 files changed, 124 insertions(+), 135 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index a0d975c1..637cf804 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -4,6 +4,7 @@ """ from __future__ import annotations +import atexit import dataclasses import os try: @@ -11,7 +12,7 @@ except ImportError: import pickle # type: ignore[assignment,no-redef] from collections.abc import Collection, Callable -from functools import partial, cached_property +from functools import partial, cached_property, wraps from operator import setitem from pathlib import Path from pickle import HIGHEST_PROTOCOL @@ -20,6 +21,14 @@ from typing_extensions import Self, ParamSpec from .. import _diagnostics as diagnostics +from ..autoprofile.autoprofile import ( + # Note: we need this to equip the profiler with the + # `.add_imported_function_or_module()` pseudo-method + # (see `kernprof.py::_write_preimports()`), which is required for + # the preimports to work + _extend_line_profiler_for_profiling_imports as upgrade_profiler, +) +from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth @@ -227,6 +236,116 @@ def _debug_output(self, msg: str) -> None: except OSError: # Cache dir may have been rm-ed during cleanup pass + def _setup_in_child_process( + self, + wrap_os_fork: bool = False, + context: str = '', + prof: LineProfiler | None = None, + ) -> bool: + """ + Set up shop in a forked/spawned child process so that + (line-)profiling can extend therein. + + Args: + wrap_os_fork (bool): + Whether to wrap :py:func:`os.fork` which handles + profiling; already-forked child processes should set + this to false + context (str): + Optional context from which the function is called, to + be used in log messages + prof (LineProfiler | None): + Optional profiler instance to associate with the cache; + if not provided, an instance is created + + Returns: + has_set_up (bool): + False the instance has already been set up prior to + calling this function, true otherwise + """ + if not context: + context = '...' + self._debug_output(f'Setting up ({context})...') + if self.profiler is not None: # Already set up + self._debug_output(f'Setup aborted ({context})') + return False + + # Create a profiler instance and manage it with + # `CuratedProfilerContext` + if prof is None: + prof = LineProfiler() + self.profiler = prof + upgrade_profiler(prof) + ctx = CuratedProfilerContext(prof, insert_builtin=self.insert_builtin) + ctx.install() + self.add_cleanup(ctx.uninstall) + self._debug_output(f'Set up `.profiler` at {id(prof):#x}') + + # Do the preimports at `cache.preimports_module` where + # appropriate + if self.preimports_module: + self._debug_output('Loading preimports...') + with open(self.preimports_module, mode='rb') as fobj: + code = compile(fobj.read(), self.preimports_module, 'exec') + exec(code, {}) # Use a fresh, empty namespace + + # Occupy a tempfile slot in `.cache_dir` and set the profiler + # up to write thereto when the process terminates (with high + # priority) + prof_outfile = self.make_tempfile( + prefix='child-prof-output-{}-{}-{:#x}-' + .format(self.main_pid, os.getpid(), id(prof)), + suffix='.lprof', + ) + self._add_cleanup(prof.dump_stats, -1, prof_outfile) + + # Set up `os.fork()` wrapping if needed (i.e. in a spawned + # process) + if wrap_os_fork: + self._wrap_os_fork() + + # Set `.cleanup()` as an atexit hook to handle everything when + # the child process is about to terminate + atexit.register(self.cleanup) + + self._debug_output(f'Setup successful ({context})') + return True + + def _wrap_os_fork(self) -> None: + """ + Create a wrapper around :py:func:`os.fork` which handles + profiling. + + Side effects: + - :py:func:`os.fork` (if available) replaced with the + wrapper + - :py:meth:`~.cleanup` callback registered undoing that + """ + try: + fork = os.fork + except AttributeError: # Can't fork on this platform + return + + @wraps(fork) + def wrapper() -> int: + result = fork() + if result: + return result + # If we're here, we are in the fork + forked = self.copy() # Ditch inherited cleanups + if forked._replace_loaded_instance(): + forked._debug_output( + 'Superseded cached `.load()`-ed instance in forked process' + ) + # Note: we can reuse the profiler instance in the fork, but + # it needs to go through setup so that the separate + # profiling results are dumped into another output file + forked._setup_in_child_process(False, 'fork', self.profiler) + return result + + os.fork = wrapper + self.add_cleanup(setattr, os, 'fork', fork) + def make_tempfile(self, **kwargs) -> Path: """ Create a fresh tempfile under :py:attr:`~.cache_dir`. The other diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 0563912c..ad84e1b6 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -28,7 +28,6 @@ from .. import _diagnostics as diagnostics from .cache import LineProfilingCache -from .pth_hook import _setup_in_child_process from .runpy_patches import create_runpy_wrapper @@ -80,7 +79,7 @@ def __setstate__(*_) -> None: # We're in a child process created by `multiprocessing`, so set # up shop here. lp_cache = LineProfilingCache.load() - _setup_in_child_process(lp_cache, False, 'multiprocessing') + lp_cache._setup_in_child_process(False, 'multiprocessing') # In a child process, we don't care about polluting the # `multiprocessing` namespace, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 69004a05..ee845a52 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -20,16 +20,11 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import Callable # noqa: F401 from pathlib import Path # noqa: F401 - from typing import Any # noqa: F401 - from ..line_profiler import LineProfiler # noqa: F401 from .cache import LineProfilingCache # noqa: F401 -__all__ = ( - 'write_pth_hook', 'load_pth_hook', '_setup_in_child_process' -) +__all__ = ('write_pth_hook', 'load_pth_hook') INHERITED_PID_ENV_VARNAME = ( 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID' @@ -83,7 +78,7 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path finally: # Not closing the handle causes issues on Windows os.close(handle) - _wrap_os_fork(cache) + cache._wrap_os_fork() return fpath @@ -118,8 +113,7 @@ def load_pth_hook(ppid): # type: (int) -> None return try: cache = LineProfilingCache.load() - _setup_in_child_process(cache, True, 'pth') - # _setup_in_child_process(LineProfilingCache.load()) + cache._setup_in_child_process(True, 'pth') except Exception as e: if DEBUG: msg = f'{type(e)}: {e}' @@ -127,126 +121,3 @@ def load_pth_hook(ppid): # type: (int) -> None log.warning(msg) finally: load_pth_hook.called = True # type: ignore[attr-defined] - - -def _wrap_os_fork(cache): # type: (LineProfilingCache) -> None - """ - Create a wrapper around :py:func:`os.fork` which handles profiling. - - Args: - cache (:py:class:`~.LineProfilingCache`): - Cache object - - Side effects: - - :py:func:`os.fork` (if available) replaced with the wrapper - - Cleanup callback registered at ``cache`` undoing that - """ - import os - from functools import wraps - - try: - fork = os.fork - except AttributeError: # Can't fork on this platform - return - - @wraps(fork) - def wrapper(): # type: () -> int - result = fork() - if result: - return result - # If we're here, we are in the fork - forked = cache.copy() - if forked._replace_loaded_instance(): - forked._debug_output( - 'Superseded cached `.load()`-ed instance in forked process' - ) - # Note: we can reuse the profiler instance in the fork, but it - # needs to go through setup so that the separate profiling - # results are dumped into another output file - _setup_in_child_process(forked, False, 'fork', cache.profiler) - return result - - os.fork = wrapper - cache.add_cleanup(setattr, os, 'fork', fork) - - -def _setup_in_child_process(cache, wrap_os_fork=False, context='', prof=None): - # type: (LineProfilingCache, bool, str, LineProfiler | None) -> bool - """ - Set up shop in a forked/spawned child process so that - (line-)profiling can extend therein. - - Args: - cache (LineProfilingCache): - Cache object - wrap_os_fork (bool): - Whether to wrap :py:func:`os.fork` which handles profiling; - already-forked child processes should set this to false - context (str): - Optional context from which the function is called, to be - used in log messages - prof (LineProfiler | None): - Optional profiler instance to associate with the cache; - if not provided, an instance is created - - Returns: - has_set_up (bool): - False is ``cache`` has already been set up prior to calling - this function, true otherwise - """ - if not context: - context = '...' - cache._debug_output(f'Setting up ({context})...') - if cache.profiler is not None: # Already set up - cache._debug_output(f'Setup aborted ({context})') - return False - - import os - from atexit import register - from ..autoprofile.autoprofile import ( - # Note: we need this to equip the profiler with the - # `.add_imported_function_or_module()` pseudo-method - # (see `kernprof.py::_write_preimports()`), which is required - # for the preimports to work - _extend_line_profiler_for_profiling_imports as upgrade_profiler, - ) - from ..curated_profiling import CuratedProfilerContext - from ..line_profiler import LineProfiler # noqa: F811 - - # Create a profiler instance and manage it with - # `CuratedProfilerContext` - if prof is None: - prof = LineProfiler() - cache.profiler = prof - upgrade_profiler(prof) - ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) - ctx.install() - cache.add_cleanup(ctx.uninstall) - cache._debug_output(f'Set up `.profiler` at {id(prof):#x}') - - # Do the preimports at `cache.preimports_module` where appropriate - if cache.preimports_module: - cache._debug_output('Loading preimports...') - with open(cache.preimports_module, mode='rb') as fobj: - code = compile(fobj.read(), cache.preimports_module, 'exec') - exec(code, {}) # Use a fresh, empty namespace - - # Occupy a tempfile slot in `cache.cache_dir` and set the profiler - # up to write thereto when the process terminates - prof_outfile = cache.make_tempfile( - prefix='child-prof-output-{}-{}-{:#x}-' - .format(cache.main_pid, os.getpid(), id(prof)), - suffix='.lprof', - ) - cache._add_cleanup(prof.dump_stats, -1, prof_outfile) - - # Set up `os.fork()` wrapping if needed (i.e. in a spawned process) - if wrap_os_fork: - _wrap_os_fork(cache) - - # Set `cache.cleanup()` as an atexit hook to handle everything when - # the child process is about to terminate - register(cache.cleanup) - - cache._debug_output(f'Setup successful ({context})') - return True From 48e8239735b51e93835d12240116e8249696c915 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 19:08:41 +0200 Subject: [PATCH 037/165] Fix `subprocess.run()` wrapper in tests tests/test_child_procs.py::_run_subproc() - If any of the output streams is captured, we call `subprocess.run(..., check=False)` to get the chance to intercept and print the output, and only call `.check_returncode()` on the `CompletedProcess` afterwards - Fixed bug where if `text=False` we attempt to format the captured stream-content bytes as strings --- tests/test_child_procs.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 90456606..14179c59 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -486,6 +486,7 @@ def _run_subproc( cmd: Sequence[str] | str, /, *args, + check: bool = False, env: Mapping[str, str] | None = None, **kwargs ) -> subprocess.CompletedProcess: @@ -497,6 +498,16 @@ def _run_subproc( cmd_str = cmd else: cmd_str = shlex.join(cmd) + + # If we're capturing outputs, it may be for the best to wait until + # we've processed the output streams to check the return code... + check_rc_in_run = check + for arg in 'stdout', 'stdin': + if kwargs.get(arg) not in {None, subprocess.DEVNULL}: + check_rc_in_run = False + if kwargs.get('capture_output'): + check_rc_in_run = False + print('Command:', cmd_str) if env is not None: diff: list[str] = [] @@ -522,23 +533,29 @@ def _run_subproc( time = monotonic() try: proc = subprocess.run( # type: ignore[call-overload] - cmd, *args, env=env, **kwargs, + cmd, *args, env=env, check=check_rc_in_run, **kwargs, ) except Exception: status = 'error' raise else: + assert proc is not None + if check and not check_rc_in_run: # Perform missing check + proc.check_returncode() status = proc.returncode return proc finally: time = monotonic() - time if proc is not None: + captured: str | bytes | None for name, captured, stream in [ ('stdout', proc.stdout, sys.stdout), ('stderr', proc.stderr, sys.stderr), ]: if captured is None: continue + if isinstance(captured, bytes): # `text=False` + captured = captured.decode() print(f'{name}:\n{indent(captured, " ")}', file=stream) print( f'-- Process end (time elapsed: {time:.2f} s / ' From c256fa0a4ba74f6a88f97d13bbd75bb2d0a720d0 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 19:30:57 +0200 Subject: [PATCH 038/165] CHANGELOG and help-text update --- CHANGELOG.rst | 2 ++ kernprof.py | 5 +++-- line_profiler/_child_process_profiling/__init__.py | 8 +++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 30156f63..118e2fca 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ Changes is available (#427) * FIX: Bytecodes of profiled functions now always labeled to prevent confusion with non-profiled "twins" (#425) +* FEAT: Experimental support for profiling child processes with + ``kernprof --prof-child-procs`` (#431) 5.0.2 diff --git a/kernprof.py b/kernprof.py index 5909981d..e0b91cf2 100755 --- a/kernprof.py +++ b/kernprof.py @@ -140,7 +140,8 @@ def main(): by-line`). (Default: False) --prof-child-procs [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] Extend profiling into child Python processes. Only works with - line profiling (`-l`/`--line-by-line`). (Default: False) + line profiling (`-l`/`--line-by-line`). (EXPERIMENTAL; + default: False) output options: -o, --outfile OUTFILE @@ -690,7 +691,7 @@ def _add_core_parser_arguments(parser): action='store_true', help='Extend profiling into child Python processes. ' 'Only works with line profiling (`-l`/`--line-by-line`). ' - f'(Default: {default.conf_dict["prof_child_procs"]})', + f'(EXPERIMENTAL; default: {default.conf_dict["prof_child_procs"]})', ) out_opts = parser.add_argument_group('output options') if default.conf_dict['outfile']: diff --git a/line_profiler/_child_process_profiling/__init__.py b/line_profiler/_child_process_profiling/__init__.py index 4ff89c0a..58e0806f 100644 --- a/line_profiler/_child_process_profiling/__init__.py +++ b/line_profiler/_child_process_profiling/__init__.py @@ -2,7 +2,9 @@ Tooling for profiling child Python processes and gathering their profiling results. -Note: - All contents of this subpackage is to be considered implementation - details. +Notes: + - THIS IS AN EXPERIMENTAL FEATURE. + + - All contents of this subpackage is to be considered implementation + details. """ From 4088eb343285c447523eb06d06261f07b5a70080 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 15 Apr 2026 09:29:43 +0200 Subject: [PATCH 039/165] Typing fixes --- .../_child_process_profiling/cache.py | 26 +++++++++++++++---- .../multiprocessing_patches.py | 4 +-- .../_child_process_profiling/pth_hook.py | 2 +- .../_child_process_profiling/runpy_patches.py | 16 +++++------- line_profiler/curated_profiling.py | 21 +++++++++++---- line_profiler/line_profiler.py | 6 +++-- 6 files changed, 51 insertions(+), 24 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 637cf804..49837f34 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -166,7 +166,7 @@ def load(cls) -> Self: ) diagnostics.log.debug(msg) instance = cls._from_path(cls._get_filename(cache_dir)) - cls._loaded_instance = instance + instance._replace_loaded_instance(force=True) return instance def dump(self) -> None: @@ -343,7 +343,19 @@ def wrapper() -> int: forked._setup_in_child_process(False, 'fork', self.profiler) return result - os.fork = wrapper + # Note: type checkers have vastly different opinions on + # `os.fork = wrapper`: + # - `ty` wouldn't shut up about shadowing unless we explicitly + # type-annotate the assignment (error type: + # "invalid-assignment") + # - `mypy` is cool with the bare assignment, but complains about + # declaring types in "assignment to non-self-attribute" + # (error type: "misc") + # The only way to satisfy both seems to be either an + # unqualified "type: ignore" comment, or circumventing the + # attribute checks by `setattr()` or assigning to the module + # namespace dict. + os.fork = wrapper # type: ignore self.add_cleanup(setattr, os, 'fork', fork) def make_tempfile(self, **kwargs) -> Path: @@ -363,9 +375,13 @@ def make_tempfile(self, **kwargs) -> Path: finally: os.close(handle) - def _replace_loaded_instance(self) -> bool: - if self._consistent_with_loaded_instance: - type(self)._loaded_instance = self + def _replace_loaded_instance(self, force: bool = False) -> bool: + cls = type(self) + if force or self._consistent_with_loaded_instance: + # Note: `ty` REALLY hates assigning an instance to + # `ClassVar[Self]` (#3274); no choice but to ignore it for + # the time being... + cls._loaded_instance = self # type: ignore return True return False diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index ad84e1b6..dffd92e5 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,7 +16,7 @@ import multiprocessing import warnings -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import AbstractContextManager, nullcontext from functools import partial, wraps from importlib import import_module @@ -432,7 +432,7 @@ def replace( # Patch `multiprocessing.process.BaseProcess` methods Method = Callable[Concatenate[S, PS], T] - patches: dict[str, Callable[[Method], Method]] + patches: Mapping[str, Callable[[Method], Method]] for submodule, target, patches in [ # type: ignore[assignment] ('process', 'BaseProcess', { 'start': wrap_start, diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index ee845a52..827bd277 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -120,4 +120,4 @@ def load_pth_hook(ppid): # type: (int) -> None warnings.warn(msg) log.warning(msg) finally: - load_pth_hook.called = True # type: ignore[attr-defined] + load_pth_hook.called = True # type: ignore diff --git a/line_profiler/_child_process_profiling/runpy_patches.py b/line_profiler/_child_process_profiling/runpy_patches.py index 6ebd3496..659f4de8 100644 --- a/line_profiler/_child_process_profiling/runpy_patches.py +++ b/line_profiler/_child_process_profiling/runpy_patches.py @@ -88,6 +88,7 @@ def _run( cache: LineProfilingCache, runpy: ModuleType, func: Callable[Concatenate[str, PS], T], + name: str, resolve_target_to_path: Callable[[str], str], CodeWriter: type[AstTreeProfiler], target: str, @@ -96,7 +97,7 @@ def _run( ) -> T: cache._debug_output('Calling via {}: `runpy.{}({})`'.format( THIS_MODULE, - func.__name__, + name, ', '.join( [repr(a) for a in (target, *args)] + [f'{k}={v!r}' for k, v in kwargs.items()] @@ -109,7 +110,7 @@ def _run( except Exception as e: cache._debug_output( f'{THIS_MODULE}: Failed to check whether code loaded by ' - f'`runpy.{func.__name__}(...)` is to be rewritten ' + f'`runpy.{name}(...)` is to be rewritten ' f'({type(e).__name__}: {e})' ) profile = False @@ -119,16 +120,13 @@ def _run( # `exec()` into the `runpy` namespace which just rewrites # `cache.rewrite_module` and executes it if profile: - runpy.exec = ( # type: ignore[attr-defined] - partial(_exec, cache, CodeWriter) - ) + # Dodge attr-defined errors and their ilk + vars(runpy)['exec'] = partial(_exec, cache, CodeWriter) try: return func(target, *args, **kwargs) finally: - try: + if hasattr(runpy, 'exec'): del runpy.exec - except AttributeError: - pass def create_runpy_wrapper(cache: LineProfilingCache) -> ModuleType: @@ -144,6 +142,6 @@ def create_runpy_wrapper(cache: LineProfilingCache) -> ModuleType: ]: impl = getattr(runpy, func) res = cast(Callable[[str], str], resolver) # Help `mypy` out - wrapper = partial(_run, cache, runpy, impl, res, CodeWriter) + wrapper = partial(_run, cache, runpy, impl, func, res, CodeWriter) setattr(runpy, func, wrapper) return runpy diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index ecad97a9..8ed720c0 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -12,7 +12,7 @@ from collections.abc import Callable, Collection from io import StringIO from textwrap import indent -from typing import Any, TextIO +from typing import Any, TextIO, cast from typing_extensions import Self from . import _diagnostics as diagnostics, profile as _global_profiler @@ -21,6 +21,7 @@ is_dotted_path, write_eager_import_module, ) from .cli_utils import short_string_path +from .line_profiler import LineProfiler from .profiler_mixin import ByCountProfilerMixin @@ -71,7 +72,10 @@ def write_preimport_module( if not self: return None - write_module_kwargs = { + # Note: `ty` (but not `mypy`) keeps complaining about the our + # splatting this dict; explicitly use `Any` to tell it to shut + # up. + write_module_kwargs: dict[str, Any] = { 'dotted_paths': self.regular, 'recurse': self.recurse, **kwargs, @@ -168,7 +172,14 @@ def __init__( self.insert_builtin = insert_builtin self.builtin_loc = builtin_loc self._installed = False - self._global_install = _global_profiler._kernprof_overwrite + self._kpo = _global_profiler._kernprof_overwrite + + def _global_install(self, prof: ByCountProfilerMixin | None) -> None: + # Wrapper to convince type-checkers it is okay to pass these + # stuff to `._kernprof_overwrite()`. We don't want to patch + # that method's signature because passing non `LineProfiler` + # objects to it should be the exception, not the norm. + self._kpo(cast(LineProfiler, prof)) def install(self) -> None: def del_builtin_profile() -> None: @@ -180,7 +191,7 @@ def set_builtin_profile(old: Any) -> None: if self._installed: return # Overwrite the explicit profiler (`@line_profiler.profile`) - self._global_install(self.prof) # type: ignore[arg-type] + self._global_install(self.prof) # type: ignore # Set up hooks to deal with inserting `.prof` as a builtin name if self.insert_builtin: try: @@ -205,7 +216,7 @@ def uninstall(self) -> None: for _i in range(getattr(self.prof, 'enable_count', 0)): self.prof.disable_by_count() # Restore the state of the global `@line_profiler.profile` - self._global_install(None) # type: ignore[arg-type] + self._global_install(None) self._installed = False def __enter__(self) -> None: diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 03899261..50ef0d88 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -375,8 +375,10 @@ def get_empty_instance(cls) -> Self: instance (LineStats): New instance without any profiling data. """ - unit = LineProfiler().timer_unit # type: ignore[attr-defined] - return cls({}, unit) + prof = LineProfiler() + if TYPE_CHECKING: + assert hasattr(prof, 'timer_unit') + return cls({}, cast(float, prof.timer_unit)) @classmethod def from_files( From fa5b4a8200d93c95243868d35c321b0de1f93758 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 16 Apr 2026 02:10:23 +0200 Subject: [PATCH 040/165] Cache refactoring + logging improvement kernprof.py _manage_profiler.__exit__() Now gathering the debug logs of the child processes and writing them to the main logger _prepare_child_profiling_cache() Simplified implementation line_profiler/_child_process_profiling/cache.py _CallbackRepr New `reprlib.Repr` subclass for handling reprs of the cleanup callbacks (mostly for truncating the repr of `os.environ`, and that of objects it appears in) LineProfilingCache [_add_]cleanup() Updated to use `_CallbackRepr` to represent the callbacks _dump_debug_logs() New method for gathering debug log files from child processes _debug_output() Added timestamps to the messages _setup_in_main_process() New method consisting mostly of code relocated from `kernprof.py::_prepare_child_profiling_cache()` line_profiler/_child_process_profiling/pth_hook.py::write_pth_hook() No longer wrapping `os.fork()` (relocated to `LineProfilingCache._setup_in_main_process()`) --- kernprof.py | 39 +-- .../_child_process_profiling/cache.py | 262 +++++++++++++++++- .../_child_process_profiling/pth_hook.py | 4 - 3 files changed, 264 insertions(+), 41 deletions(-) diff --git a/kernprof.py b/kernprof.py index e0b91cf2..70ae3a59 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1225,6 +1225,9 @@ def __exit__(self, *_, **__): try: extra_stats = None if self.set_up_child_profiling: + if self.cache.debug: + # Recover debug output from child processes + self.cache._dump_debug_logs() try: extra_stats = self.cache.gather_stats() finally: @@ -1340,10 +1343,6 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): Handle the (line-)profiling of spawned/forked child Python processes. """ - from line_profiler._child_process_profiling import ( - pth_hook, multiprocessing_patches, - ) - # Create the cache dir and cache file here; the cache instance will # be responsible for managing their lifetimes, while derivative # instances in child processes will merely inherit and use them @@ -1356,11 +1355,10 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): insert_builtin=options.builtin, debug=options.debug, ) - clean_up = functools.partial(cache.add_cleanup, _remove) + clean_up = functools.partial(cache.add_cleanup, _remove, missing_ok=True) if not diagnostics.KEEP_TEMPDIRS: - clean_up(cache.cache_dir, recursive=True, missing_ok=True) - cache.dump() - clean_up(cache.filename, missing_ok=True) + clean_up(cache.cache_dir, recursive=True) + clean_up(cache.filename) # This file is handed to us at the end of # `_manage_profiler.__enter__()`; @@ -1368,29 +1366,10 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): # child-process profiling is used, it is to persist for the lifetime # of the cache (so that child processes can do the same preimports) if not (preimports_file is None or diagnostics.KEEP_TEMPDIRS): - clean_up(preimports_file, missing_ok=True) - - # Note: the following functions/methods all clean up after - # themselves, so there is no need to explicitly call - # `cache.add_cleanup()` - - # Inject environment variables so that child Python processes can - # inherit them and resume profiling - cache.inject_env_vars() - - # Create the .pth file which allows for setting up profiling in all - # the child Python processes (e.g. those created by `os.system()` or - # `subprocess.run()` - pth_hook.write_pth_hook(cache) - - # Patch `multiprocessing` so that child processes are properly - # handled across all "start methods" - multiprocessing_patches.apply(cache) + clean_up(preimports_file) - # Misc. setup: - # - Set up the cache instance to be the default one `.load()`-ed - cache._replace_loaded_instance() - # - Set `cache.profiler` + # Handle various setup tasks (see docs thereof) + cache._setup_in_main_process() cache.profiler = prof return cache diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 49837f34..4b06993e 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -12,13 +12,18 @@ except ImportError: import pickle # type: ignore[assignment,no-redef] from collections.abc import Collection, Callable +from datetime import datetime from functools import partial, cached_property, wraps +from importlib import import_module from operator import setitem from pathlib import Path from pickle import HIGHEST_PROTOCOL +from reprlib import Repr from tempfile import mkstemp -from typing import Any, ClassVar, cast -from typing_extensions import Self, ParamSpec +from textwrap import indent +from types import MethodType +from typing import Any, ClassVar, TypeVar, TypedDict, cast +from typing_extensions import Self, ParamSpec, Unpack from .. import _diagnostics as diagnostics from ..autoprofile.autoprofile import ( @@ -41,15 +46,183 @@ PS = ParamSpec('PS') +# Note: `typing.AnyStr` deprecated since 3.13 +AnyStr = TypeVar('AnyStr', str, bytes) INHERITED_CACHE_ENV_VARNAME_PREFIX = ( 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_DIR' ) CACHE_FILENAME = 'line_profiler_cache.pkl' +_DEBUG_LOG_FILENAME_PATTERN = 'debug_log_{main_pid}_{current_pid}.log' + + +class _ReprAttributes(TypedDict, total=False): + """ + Note: + We use this typed dict instead of directly supplying them in the + :py:meth:`_CallbackRepr.__init__()` signature, because we don't + want to bother with the default values there. + """ + maxlevel: int + maxtuple: int + maxlist: int + maxarray: int + maxdict: int + maxset: int + maxfrozenset: int + maxdeque: int + maxstring: int + maxlog: int + maxother: int + fillvalue: str + indent: str | int | None + + +class _CallbackRepr(Repr): + """ + :py:class:`reprlib.Repr` subclass to help with representing cleanup + callbacks, special-casing certain relevant object types (see + examples below). + + Example: + >>> from functools import partial + >>> + >>> + >>> class MyEnviron(dict): + ... def some_method(self) -> None: + ... ... + ... + >>> + >>> class MyRepr(_CallbackRepr): + ... # Since we can't instantiate a new `os._Environ`, test + ... # the relevant method with a mock + ... repr_MyEnviron = _CallbackRepr.repr__Environ + ... + >>> + >>> r = MyRepr(maxenv=3, maxargs=4, maxstring=15) + + Environ-dict formatting: + + >>> my_env = MyEnviron( + ... foo='1', + ... bar='2', + ... this_varname_is_long_but_isnt_truncated=( + ... "THIS VALUE IS TRUNCATED BECAUSE IT'S TOO LONG" + ... ), + ... baz='4', + ... ) + >>> print(r.repr(my_env)) + environ({'foo': '1', 'bar': '2', \ +'this_varname_is_long_but_isnt_truncated': 'THIS ... LONG', ...}) + + Partial-object formatting: + + >>> r.indent = 2 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial( + , + base=8, + ) + + >>> callback_2 = partial(min, 5, 4, 3, 2, 1) + >>> r.indent = '----' + >>> print(r.repr(callback_2)) + functools.partial( + ----, + ----5, + ----4, + ----3, + ----2, + ----..., + ) + + Bound-method formatting: + + >>> r.indent = ' ' + >>> r.maxenv = 2 + >>> print(r.repr(my_env.some_method)) + + >>> r.indent = None + >>> r.maxenv = 0 + >>> print(r.repr(my_env.some_method)) + + """ + def __init__( + self, + *, + maxargs: int = 5, + maxenv: int = 3, + **kwargs: Unpack[_ReprAttributes] + ) -> None: + super().__init__() # kwargs are 3.12+ + for k, v in kwargs.items(): + setattr(self, k, v) + self.maxargs = maxargs + self.maxenv = maxenv + + def repr__Environ(self, env: os._Environ[AnyStr], level: int) -> str: + get: Callable[[AnyStr], str] = partial(self.repr1, level=level-1) + # Truncate envvar values, but not their names + envvars = ['{!r}: {}'.format(k, get(v)) for k, v in env.items()] + return self._format_items(envvars, ('environ({', '})'), self.maxenv) + + def repr_method(self, method: MethodType, level: int) -> str: + instance = self.repr1(method.__self__, level-1) + func = getattr(method.__func__, '__qualname__', '?') + prefix, suffix = f'' + # Take care of possible multi-line reprs + return _indent_with_prefix(instance, prefix) + suffix + + def repr_partial(self, ptl: partial, level: int) -> str: + get: Callable[[Any], str] = partial(self.repr1, level=level-1) + args = [get(arg) for arg in ptl.args] + args.extend('{}={}'.format(k, get(v)) for k, v in ptl.keywords.items()) + args.insert(0, get(ptl.func)) + name = '{0.__module__}.{0.__qualname__}'.format(type(ptl)) + # The +1 is to account for `ptl.func` + return self._format_items(args, (name + '(', ')'), self.maxargs + 1) + + def _format_items( + self, + items: Collection[str], + delims: tuple[str, str], + maxlen: int | None = None, + ) -> str: + start, end = delims + if maxlen is not None and len(items) > maxlen: + items = list(items)[:maxlen] + ['...'] + if self.indent is None or not items: + return '{}{}{}'.format(start, ', '.join(items), end) + return '\n'.join([ + start, *(indent(item + ',', self.indent) for item in items), end, + ]) + + @property + def indent(self) -> str | None: + return self._indent + + @indent.setter + def indent(self, indent: str | int | None) -> None: + if indent is None or isinstance(indent, str): + self._indent = indent + return + self._indent = ' ' * indent + + +_CALLBACK_REPR = _CallbackRepr(maxother=cast(int, float('inf'))).repr @dataclasses.dataclass class LineProfilingCache: + """ + Helper object for coordinating a line-profiling session, caching the + info required to make profiling persist into child processes. + """ cache_dir: os.PathLike[str] | str profiling_targets: Collection[str] = dataclasses.field( default_factory=list, @@ -78,12 +251,13 @@ def cleanup(self) -> None: callbacks = self._cleanup_stacks.pop(priority) while callbacks: callback = callbacks.pop() + callback_repr = _CALLBACK_REPR(callback) try: callback() except Exception as e: - msg = f'failed: {callback}: {type(e).__name__}: {e}' + msg = f'failed: {callback_repr}: {type(e).__name__}: {e}' else: - msg = f'succeeded: {callback}' + msg = f'succeeded: {callback_repr}' self._debug_output('Cleanup ' + msg) def add_cleanup( @@ -105,7 +279,7 @@ def _add_cleanup( header = 'Cleanup callback added' if priority: header = f'{header} (priority: {priority})' - self._debug_output(f'{header}: {callback}') + self._debug_output(f'{header}: {_CALLBACK_REPR(callback)}') def copy( self, *, @@ -199,6 +373,23 @@ def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: return LineStats.get_empty_instance() return LineStats.from_files(*fnames, on_defective='ignore') + def _dump_debug_logs(self) -> None: + """ + Gather the debug logfiles in child processes and write them to + the logger; to be called in the main process. + """ + pattern = _DEBUG_LOG_FILENAME_PATTERN.format( + main_pid=self.main_pid, current_pid='*', + ) + for log in sorted(Path(self.cache_dir).glob(pattern)): + if log == self._debug_log: # Don't double dip + continue + *_, child_pid = log.stem.rpartition('_') + msg = 'Cache log messages from child process {}:\n{}'.format( + child_pid, indent(log.read_text(), ' '), + ) + diagnostics.log.debug(msg) + def inject_env_vars( self, env: dict[str, str] | None = None, ) -> None: @@ -232,10 +423,56 @@ def _debug_output(self, msg: str) -> None: return try: with self._debug_log.open(mode='a') as fobj: - print(msg, file=fobj) + prefix = self._debug_message_timestamp + ' ' + print(_indent_with_prefix(msg, prefix), file=fobj) except OSError: # Cache dir may have been rm-ed during cleanup pass + def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: + """ + Set up shop in the main process so that (line-)profiling can + extend into child processes. + + Args: + wrap_os_fork (bool): + Whether to wrap :py:func:`os.fork` which handles + profiling + + Side effects: + + - Instance data written to :py:attr:`~.cache_dir` + + - Environment variables injected + (see :py:meth:`~.inject_env_vars()`) + + - A ``.pth`` file written so that child processes + automaticaly runs setup code (see + :py:func:`line_profiler._child_process_hook.pth_hook.\ +write_pth_hook`) + + - :py:func:`os.fork` wrapped so that profiling set up in + forked processes is properly handled (if + ``wrap_os_fork=True``) + + - :py:mod:`multiprocessing` wrapped so that child processes + managed by the package are properly handled + + - Instance to be returned if :py:func:`~.load()` is called + from now on + """ + this_subpkg, *_, _ = (lambda: None).__module__.rpartition('.') + + self.dump() + self.inject_env_vars() + pth_hook = import_module(this_subpkg + '.pth_hook') + pth_hook.write_pth_hook(self) + + self._wrap_os_fork() + mp_patches = import_module(this_subpkg + '.multiprocessing_patches') + mp_patches.apply(self) + + self._replace_loaded_instance() + def _setup_in_child_process( self, wrap_os_fork: bool = False, @@ -421,9 +658,15 @@ def filename(self) -> str: def _debug_log(self) -> Path | None: if not self.debug: return None - fname = f'debug_log_{self.main_pid}_{os.getpid()}.log' + fname = _DEBUG_LOG_FILENAME_PATTERN.format( + main_pid=self.main_pid, current_pid=os.getpid(), + ) return Path(self.cache_dir) / fname + @property + def _debug_message_timestamp(self) -> str: + return f'[cache-debug-log {datetime.now()} DEBUG]' + @cached_property def _debug_message_header(self) -> str: pid = os.getpid() @@ -436,3 +679,8 @@ def _debug_message_header(self) -> str: @cached_property def _consistent_with_loaded_instance(self) -> bool: return type(self).load()._get_init_args() == self._get_init_args() + + +def _indent_with_prefix(string: str, prefix: str, fill_char: str = ' ') -> str: + width = len(prefix) + return prefix + indent(string, fill_char * width)[width:] diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 827bd277..3065f52d 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -48,8 +48,6 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path - To be called in the main process. - The ``cache`` is responsible for deleting the written .pth file via the registered cleanup callback. - - For convenience, we also wrap :py:func:`os.fork` when this - function is called. """ import os from pathlib import Path # noqa: F811 @@ -78,8 +76,6 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path finally: # Not closing the handle causes issues on Windows os.close(handle) - cache._wrap_os_fork() - return fpath From d6e99b3c4dc926f9a3152461208419c38cb37ace Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 16 Apr 2026 03:53:01 +0200 Subject: [PATCH 041/165] Factor out common utils line_profiler/_child_process_profiling/ cache.py, pth_hook.py Relocated shared code snippets to `misc_utils` misc_utils.py New submodules for util functions --- .../_child_process_profiling/cache.py | 21 +++------- .../_child_process_profiling/misc_utils.py | 40 +++++++++++++++++++ .../_child_process_profiling/pth_hook.py | 10 ++--- 3 files changed, 49 insertions(+), 22 deletions(-) create mode 100644 line_profiler/_child_process_profiling/misc_utils.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 4b06993e..1c997ec0 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -19,7 +19,6 @@ from pathlib import Path from pickle import HIGHEST_PROTOCOL from reprlib import Repr -from tempfile import mkstemp from textwrap import indent from types import MethodType from typing import Any, ClassVar, TypeVar, TypedDict, cast @@ -35,6 +34,7 @@ ) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats +from .misc_utils import block_indent, make_tempfile # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth # hook, which must run with minimal overhead when a Python process isn't @@ -176,7 +176,7 @@ def repr_method(self, method: MethodType, level: int) -> str: func = getattr(method.__func__, '__qualname__', '?') prefix, suffix = f'' # Take care of possible multi-line reprs - return _indent_with_prefix(instance, prefix) + suffix + return block_indent(instance, prefix) + suffix def repr_partial(self, ptl: partial, level: int) -> str: get: Callable[[Any], str] = partial(self.repr1, level=level-1) @@ -424,7 +424,7 @@ def _debug_output(self, msg: str) -> None: try: with self._debug_log.open(mode='a') as fobj: prefix = self._debug_message_timestamp + ' ' - print(_indent_with_prefix(msg, prefix), file=fobj) + print(block_indent(msg, prefix), file=fobj) except OSError: # Cache dir may have been rm-ed during cleanup pass @@ -604,13 +604,9 @@ def make_tempfile(self, **kwargs) -> Path: path (Path): Path to the created file. """ - handle, path = mkstemp(dir=self.cache_dir, **kwargs) - try: - path_obj = Path(path) - self._debug_output(f'Created tempfile: {path_obj.name!r}') - return path_obj - finally: - os.close(handle) + path = make_tempfile(dir=self.cache_dir, **kwargs) + self._debug_output(f'Created tempfile: {path.name!r}') + return path def _replace_loaded_instance(self, force: bool = False) -> bool: cls = type(self) @@ -679,8 +675,3 @@ def _debug_message_header(self) -> str: @cached_property def _consistent_with_loaded_instance(self) -> bool: return type(self).load()._get_init_args() == self._get_init_args() - - -def _indent_with_prefix(string: str, prefix: str, fill_char: str = ' ') -> str: - width = len(prefix) - return prefix + indent(string, fill_char * width)[width:] diff --git a/line_profiler/_child_process_profiling/misc_utils.py b/line_profiler/_child_process_profiling/misc_utils.py new file mode 100644 index 00000000..050380bf --- /dev/null +++ b/line_profiler/_child_process_profiling/misc_utils.py @@ -0,0 +1,40 @@ +""" +Misc. utility functions used by the subpackage. +""" +import os +from pathlib import Path +from tempfile import mkstemp +from textwrap import indent + + +__all__ = ('block_indent', 'make_tempfile') + + +def block_indent(string: str, prefix: str, fill_char: str = ' ') -> str: + r""" + Example: + >>> string = 'foo\nbar\nbaz' + >>> print(string) + foo + bar + baz + >>> print(block_indent(string, '++++', '-')) + ++++foo + ----bar + ----baz + """ + width = len(prefix) + return prefix + indent(string, fill_char * width)[width:] + + +def make_tempfile(**kwargs) -> Path: + """ + Convenience wrapper around :py:func:`tempfile.mkstemp`, discarding + and closing the integer handle (which if left unattended causes + problems on some platforms). + """ + handle, fname = mkstemp(**kwargs) + try: + return Path(fname) + finally: + os.close(handle) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 3065f52d..5a19cbce 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -50,20 +50,18 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path file via the registered cleanup callback. """ import os - from pathlib import Path # noqa: F811 from sysconfig import get_path - from tempfile import mkstemp + from .misc_utils import make_tempfile if not os.path.exists(cache.filename): cache.dump() assert os.path.exists(cache.filename) - handle, fname = mkstemp( + fpath = make_tempfile( prefix='_line_profiler_profiling_hook_', suffix='.pth', dir=get_path('purelib'), ) - fpath = Path(fname) try: pth_content = 'import {0}; {0}.load_pth_hook({1})'.format( (lambda: None).__module__, cache.main_pid, @@ -71,10 +69,8 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path fpath.write_text(pth_content) cache.add_cleanup(fpath.unlink, missing_ok=True) except Exception: - os.remove(fpath) + fpath.unlink(missing_ok=True) raise - finally: # Not closing the handle causes issues on Windows - os.close(handle) return fpath From 9363e5e2d11fa95818b1adb52f93f9585cbbca43 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 16 Apr 2026 16:58:20 +0200 Subject: [PATCH 042/165] Refactored `LineProfilingCache` logging line_profiler/_child_process_profiling/_cache_logging.py ::CacheLoggingEntry New object for IO to/from `LineProfilingCache._debug_log` line_profiler/_child_process_profiling/cache.py::LineProfilingCache cleanup() More verbose logging messages to help with: - Distinguishing cases where no callback has been registered - Indicating whether the cleanup callbacks have been successfully exhausted (e.g. may not be the case for child processes managed by `multiprocessing`) _gather_debug_log_entries() New method for reading in `CacheLoggingEntry`-s from all log files of the session _debug_message_{header,timestamp} Refactored away (functionalities absorbed into `CacheLoggingEntry`) --- .../_cache_logging.py | 338 ++++++++++++++++++ .../_child_process_profiling/cache.py | 89 +++-- 2 files changed, 398 insertions(+), 29 deletions(-) create mode 100644 line_profiler/_child_process_profiling/_cache_logging.py diff --git a/line_profiler/_child_process_profiling/_cache_logging.py b/line_profiler/_child_process_profiling/_cache_logging.py new file mode 100644 index 00000000..91b0635f --- /dev/null +++ b/line_profiler/_child_process_profiling/_cache_logging.py @@ -0,0 +1,338 @@ +""" +Logging utilities. +""" +from __future__ import annotations + +import os +import re +from collections.abc import Generator +from datetime import datetime +from itertools import pairwise +from pathlib import Path +from string import Formatter as StringParser +from textwrap import dedent +from typing import TYPE_CHECKING, NamedTuple, TextIO, overload +from typing_extensions import Self + +from .. import _diagnostics as diagnostics +from .misc_utils import block_indent + + +__all__ = ('CacheLoggingEntry',) + + +FILENAME_PATTERN = 'debug_log_{main_pid}_{current_pid}.log' +TIMESTAMP_PATTERN = '[cache-debug-log {timestamp} DEBUG]' +HEADER_PATTERN = 'PID {current_pid} ({main_pid}): Cache {obj_id:#x}' + +TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S' +TIMESTAMP_MICROSECOND_SEP = ',' +TIMESTAMP_MICROSECOND_PLACES = 3 +TIMESTAMP_SPACING = ' ' + +HEADER_SEP = ': ' +HEADER_MAIN_INDICATOR = 'main process' + + +def get_logger_header(current_pid: int, main_pid: int, obj_id: int) -> str: + """ + Returns: + msg_header (str): + Message header, to be prefixed to messages sent to + :py:data:`line_profiler._diagnostics.log`. + """ + return HEADER_PATTERN.format( + current_pid=current_pid, + main_pid=( + HEADER_MAIN_INDICATOR if main_pid == current_pid else main_pid + ), + obj_id=obj_id, + ) + + +def format_timestamp(ts: datetime) -> str: + """ + Replicate the :py:mod:`logging`'s default formatting for timestamps. + + Example: + >>> ts = datetime(2000, 1, 23, 4, 5, 6, 789000) + >>> as_str = format_timestamp(ts) + >>> print(as_str) + 2000-01-23 04:05:06,789 + >>> assert parse_timestamp(as_str) == ts + """ + return '{}{}{:0{}d}'.format( + ts.strftime(TIMESTAMP_FORMAT), + TIMESTAMP_MICROSECOND_SEP, + int(ts.microsecond / 1000), + TIMESTAMP_MICROSECOND_PLACES, + ) + + +def parse_timestamp(ts: str) -> datetime: + """ + Turn a formatted string timestamp back to a + :py:class:`datetime.datetime` object. + """ + assert TIMESTAMP_MICROSECOND_SEP in ts + base, _, fractional = ts.rpartition(TIMESTAMP_MICROSECOND_SEP) + # The microsecond field %f must be 6 digits long + if len(fractional) < 6: + fractional = f'{fractional:<06}' + else: + fractional = fractional[:6] + parse_format = f'{TIMESTAMP_FORMAT}{TIMESTAMP_MICROSECOND_SEP}%f' + ts = f'{base}{TIMESTAMP_MICROSECOND_SEP}{fractional}' + return datetime.strptime(ts, parse_format) + + +def add_timestamp(msg: str, timestamp: datetime | None = None) -> str: + """ + Returns: + msg_with_timestamp (str): + (Block-indented) message with timestamp, to be written to + the :py:attr:`LineProfilingCache._debug_log`. + """ + if timestamp is None: + timestamp = datetime.now() + ts_formatted = TIMESTAMP_PATTERN.format( + timestamp=format_timestamp(timestamp), + ) + return block_indent(msg, ts_formatted + TIMESTAMP_SPACING) + + +def parse_id(uint: str) -> int: + """ + Example: + >>> n = 123456 + >>> for formatter in str, bin, oct, hex: + ... assert parse_id(formatter(n)) == n + """ + for prefix, base in ('0b', 2), ('0o', 8), ('0x', 16): + if uint.startswith(prefix): + return int(uint[len(prefix):], base=base) + return int(uint) + + +@overload +def fmt_to_regex(fmt: str, /, *auto_numbered_fields: str) -> str: + ... + + +@overload +def fmt_to_regex(fmt: str, /, **named_fields: str) -> str: + ... + + +def fmt_to_regex( + fmt: str, /, *auto_numbered_fields: str, **named_fields: str +) -> str: + """ + Example: + >>> import re + + Simple case: + + >>> pattern = fmt_to_regex( + ... '{func}({args})', func=r'[_\\w][_\\w\\d]+', args='.*', + ... ) + >>> print(pattern) + (?P[_\\w][_\\w\\d]+)\\((?P.*)\\) + >>> regex = re.compile('^' + pattern, re.MULTILINE) + >>> assert not regex.search('0(1)') + >>> match = regex.search(' \\nint(-1.5)') + >>> assert match.group('func', 'args') == ('int', '-1.5') + + Repeated fields: + + >>> palindrome_5l = re.compile(fmt_to_regex( + ... '{first}{second}{third}{second}{first}', + ... first='.', second='.', third='.', + ... )) + >>> print(palindrome_5l.pattern) + (?P.)(?P.)(?P.)(?P=second)(?P=first) + >>> assert not palindrome_5l.match('abbbe') + >>> match = palindrome_5l.match('aBcBa') + >>> assert match.group('first', 'second', 'third') == ( + ... 'a', 'B', 'c', + ... ) + + Auto-numbered fields: + + >>> print(fmt_to_regex( + ... '[{} {}-{}-{} {}:{}:{},{} {}]', + ... # Logger name + ... '.+', + ... # Date + ... r'\\d\\d', r'\\d\\d', r'\\d\\d', + ... # Time + milliseconds + ... r'\\d\\d', r'\\d\\d', r'\\d\\d', r'\\d\\d\\d', + ... # Category + ... 'DEBUG|INFO|WARNING|ERROR|CRITICAL', + ... )) + \\[(.+)\\ (\\d\\d)\\-(\\d\\d)\\-(\\d\\d)\\ \ +(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\ \ +(DEBUG|INFO|WARNING|ERROR|CRITICAL)\\] + """ + chunks: list[str] = [] + seen_fields: set[str] = set() + for i, (prefix, field, *_) in enumerate(StringParser().parse(fmt)): + chunks.append(re.escape(prefix)) + if field is None: + break # Suffix -> we're done + if field: # Named fields + assert field.isidentifier() + if field in seen_fields: + chunks.append(f'(?P={field})') + else: + chunks.append(f'(?P<{field}>{named_fields[field]})') + seen_fields.add(field) + else: # Auto-numbered fields + chunks.append(f'({auto_numbered_fields[i]})') + return ''.join(chunks) + + +class CacheLoggingEntry(NamedTuple): + """ + Logging entry written to a log file by + :py:meth:`LineProfilingCache._debug_output`. + + Example: + >>> from datetime import datetime + >>> + >>> + >>> entry = CacheLoggingEntry( + ... datetime(1900, 1, 1, 0, 0, 0, 0), + ... 12345, + ... 12345, + ... 12345678, + ... 'This is a log message;\\nit has multiple lines', + ... ) + >>> print(entry.to_text()) + [cache-debug-log 1900-01-01 00:00:00,000 DEBUG] PID 12345 \ +(main process): Cache 0xbc614e: This is a log message; + it has \ +multiple lines + >>> another_entry = CacheLoggingEntry( + ... datetime(2000, 12, 31, 12, 34, 56, 789000), + ... 12345, + ... 54321, + ... 87654321, + ... 'FOO BAR BAZ', + ... ) + >>> print(another_entry.to_text()) + [cache-debug-log 2000-12-31 12:34:56,789 DEBUG] PID 54321 \ +(12345): Cache 0x5397fb1: FOO BAR BAZ + >>> log_text = '\\n'.join([ + ... e.to_text() for e in [entry, another_entry] + ... ]) + >>> assert CacheLoggingEntry.from_text(log_text) == [ + ... entry, another_entry, + ... ] + """ + timestamp: datetime + main_pid: int + current_pid: int + cache_id: int + msg: str + + def to_text(self) -> str: + return add_timestamp(self._get_header() + self.msg, self.timestamp) + + def _get_header(self) -> str: + return get_logger_header( + self.current_pid, self.main_pid, self.cache_id, + ) + HEADER_SEP + + def write(self, tee: os.PathLike[str] | str | None = None) -> None: + log_msg = self._get_header() + self.msg + diagnostics.log.debug(log_msg) + if tee is None: + return + with Path(tee).open(mode='a') as fobj: + print(add_timestamp(log_msg, self.timestamp), file=fobj) + + @classmethod + def new(cls, main_pid: int, cache_id: int, msg: str) -> Self: + return cls(datetime.now(), main_pid, os.getpid(), cache_id, msg) + + @classmethod + def from_file(cls, file: os.PathLike[str] | str | TextIO) -> list[Self]: + try: + path = Path(file) # type: ignore + except TypeError: # File object + # If we're here, `file` is a file object + if TYPE_CHECKING: + assert isinstance(file, TextIO) + content = file.read() + else: + content = path.read_text() + return cls.from_text(content) + + @classmethod + def from_text(cls, text: str) -> list[Self]: + def gen_timestamps(text: str) -> Generator[re.Match, None, None]: + last_ts_match: re.Match | None = None + while True: + ts_match = timestamp_regex.search( + text, last_ts_match.end() if last_ts_match else 0, + ) + if ts_match: + yield ts_match + last_ts_match = ts_match + else: + return + + def gen_message_blocks(text: str) -> Generator[ + tuple[datetime, re.Match, str], None, None + ]: + timestamps = list(gen_timestamps(text)) + if not timestamps: + return + + # Handle all the entries up till the 2nd-to-last one + for this_match, next_match in pairwise(timestamps): + ts = parse_timestamp(this_match.group('timestamp')) + # The -1 is for stripping the trailing newline + text_block = text[this_match.start():next_match.start() - 1] + yield (ts, this_match, text_block) + # Handle the last entry + last_match = timestamps[-1] + yield ( + parse_timestamp(last_match.group('timestamp')), + last_match, + text[last_match.start():], + ) + + def get_entries(text: str) -> Generator[Self, None, None]: + for timestamp, ts_match, text_block in gen_message_blocks(text): + # Strip the block indent + ts_text = ts_match.group(0) + assert text_block.startswith(ts_text) + ts_width = len(ts_text) + text_block = dedent(' ' * ts_width + text_block[ts_width:]) + # Strip the header and parse the relevant info from it + header_match = header_regex.match(text_block) + assert header_match + current_pid = int(header_match.group('current_pid')) + main_pid_ = header_match.group('main_pid') + if main_pid_ == HEADER_MAIN_INDICATOR: + main_pid = current_pid + else: + main_pid = int(main_pid_) + cache_id = parse_id(header_match.group('obj_id')) + # The rest of the block is the message proper + msg = text_block[header_match.end():] + yield cls(timestamp, main_pid, current_pid, cache_id, msg) + + timestamp_pattern = fmt_to_regex( + f'{TIMESTAMP_PATTERN}{TIMESTAMP_SPACING}', timestamp='.+?', + ) + timestamp_regex = re.compile('^' + timestamp_pattern, re.MULTILINE) + header_regex = re.compile(fmt_to_regex( + HEADER_PATTERN + HEADER_SEP, + current_pid=r'\d+', + main_pid=r'\d+|' + re.escape(HEADER_MAIN_INDICATOR), + obj_id='.+?', + )) + return list(get_entries(text)) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 1c997ec0..95ef42bd 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -11,8 +11,7 @@ import _pickle as pickle except ImportError: import pickle # type: ignore[assignment,no-redef] -from collections.abc import Collection, Callable -from datetime import datetime +from collections.abc import Collection, Callable, Iterable from functools import partial, cached_property, wraps from importlib import import_module from operator import setitem @@ -34,6 +33,7 @@ ) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats +from ._cache_logging import CacheLoggingEntry from .misc_utils import block_indent, make_tempfile # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth @@ -247,8 +247,18 @@ def cleanup(self) -> None: Pop all the cleanup callbacks from the internal stack added via :py:meth:`~.add_cleanup` and call them in order. """ - for priority in sorted(self._cleanup_stacks): - callbacks = self._cleanup_stacks.pop(priority) + stacks = self._cleanup_stacks + num_callbacks = sum(len(stack) for stack in stacks.values()) + if not num_callbacks: + self._debug_output('Cleanup aborted (no registered callbacks)') + return + # Bookend the cleanup loop with log messages to help detect if + # child processes are prematurely terminated + self._debug_output( + f'Starting cleanup ({num_callbacks} callback(s))...', + ) + for priority in sorted(stacks): + callbacks = stacks.pop(priority) while callbacks: callback = callbacks.pop() callback_repr = _CALLBACK_REPR(callback) @@ -258,7 +268,10 @@ def cleanup(self) -> None: msg = f'failed: {callback_repr}: {type(e).__name__}: {e}' else: msg = f'succeeded: {callback_repr}' - self._debug_output('Cleanup ' + msg) + self._debug_output('- Cleanup ' + msg) + self._debug_output( + f'... cleanup completed ({num_callbacks} callback(s))', + ) def add_cleanup( self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, @@ -375,13 +388,18 @@ def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: def _dump_debug_logs(self) -> None: """ - Gather the debug logfiles in child processes and write them to - the logger; to be called in the main process. + Gather the debug logfiles in child processes and write their + contents to the logger + (:py:data:`line_profiler._diagnostics.log`). + + Notes: + - The content of each child-process log file is not + re-parsed and is written to the logger as a single + multi-line message. + + - To be called in the main process. """ - pattern = _DEBUG_LOG_FILENAME_PATTERN.format( - main_pid=self.main_pid, current_pid='*', - ) - for log in sorted(Path(self.cache_dir).glob(pattern)): + for log in sorted(self._get_debug_logfiles()): if log == self._debug_log: # Don't double dip continue *_, child_pid = log.stem.rpartition('_') @@ -390,6 +408,34 @@ def _dump_debug_logs(self) -> None: ) diagnostics.log.debug(msg) + def _gather_debug_log_entries( + self, chronological: bool = False, + ) -> list[CacheLoggingEntry]: + """ + Gather and return all entries from debug logfiles sorted by + timestamps. + """ + log_files: Iterable[Path] = self._get_debug_logfiles() + if chronological: # Sorting on the entries -> chronological + to_list: Callable[ + [Iterable[CacheLoggingEntry]], list[CacheLoggingEntry] + ] = sorted + else: + # Otherwise, just sort by filename (entries in each file are + # still chronological) + log_files = sorted(log_files) + to_list = list + return to_list( + entry for log in log_files + for entry in CacheLoggingEntry.from_file(log) + ) + + def _get_debug_logfiles(self) -> Iterable[Path]: + pattern = _DEBUG_LOG_FILENAME_PATTERN.format( + main_pid=self.main_pid, current_pid='*', + ) + return Path(self.cache_dir).glob(pattern) + def inject_env_vars( self, env: dict[str, str] | None = None, ) -> None: @@ -417,14 +463,8 @@ def inject_env_vars( env[name] = value def _debug_output(self, msg: str) -> None: - msg = f'{self._debug_message_header}: {msg}' - diagnostics.log.debug(msg) - if not self._debug_log: - return try: - with self._debug_log.open(mode='a') as fobj: - prefix = self._debug_message_timestamp + ' ' - print(block_indent(msg, prefix), file=fobj) + self._make_debug_entry(msg).write(self._debug_log) except OSError: # Cache dir may have been rm-ed during cleanup pass @@ -659,18 +699,9 @@ def _debug_log(self) -> Path | None: ) return Path(self.cache_dir) / fname - @property - def _debug_message_timestamp(self) -> str: - return f'[cache-debug-log {datetime.now()} DEBUG]' - @cached_property - def _debug_message_header(self) -> str: - pid = os.getpid() - return 'PID {} ({}): Cache {:#x}'.format( - pid, - 'main process' if self.main_pid == pid else self.main_pid, - id(self), - ) + def _make_debug_entry(self) -> Callable[[str], CacheLoggingEntry]: + return partial(CacheLoggingEntry.new, self.main_pid, id(self)) @cached_property def _consistent_with_loaded_instance(self) -> bool: From 248dc0f3de66ddae5e254dc15335bf1bd45513bf Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 16 Apr 2026 20:19:31 +0200 Subject: [PATCH 043/165] WIP: write combined debug logs for debugging kernprof.py _add_core_parser_arguments() Added temporary/undocumented flag `--debug-log=...` for gathering the cache logs and writing them to one place _manage_profiler Now registering a delayed cleanup callback for writing the debug log on `.__enter__()`, so that the gathering of log entries happens as late as possible (right before the cache dir is wiped) and includes most of the cleanup in the main process _prepare_child_profiling_cache() Now explicitly deferring the deletion of the cache dir to the very end line_profiler/_child_process_profiling/cache.py ::LineProfilingCache.cleanup() Made debug log messages more verbose, indicating the number of callbacks made as the result of each is reported tests/test_child_procs.py ::test_profiling_{bare_python,multiproc_script}() Now gathering cache logs via `--debug-log` and printing them out XXX: do we drop these functionalities when/if the bugs are fixed? --- kernprof.py | 35 +++++- .../_child_process_profiling/cache.py | 20 ++-- tests/test_child_procs.py | 103 +++++++++++------- 3 files changed, 107 insertions(+), 51 deletions(-) diff --git a/kernprof.py b/kernprof.py index 70ae3a59..6fc4e725 100755 --- a/kernprof.py +++ b/kernprof.py @@ -202,7 +202,7 @@ def main(): import shutil import tempfile import time -from argparse import ArgumentParser +from argparse import ArgumentParser, SUPPRESS from operator import methodcaller from runpy import run_module from pathlib import Path @@ -237,6 +237,10 @@ def main(): DIAGNOSITICS_VERBOSITY = 2 +CLEANUP_PRIORITIES = { # Bigger number -> more delayed + 'rm_cache_dir': 1024, + 'gather_logs': 1, +} def execfile(filename, globals=None, locals=None): @@ -783,6 +787,8 @@ def _add_core_parser_arguments(parser): 'Minimum value (and the value implied if the bare option ' f'is given) is 1 s. (Default: {def_out_int})', ) + # Hidden option for dumping the debug logs to a desinated location + add_argument(out_opts, '--debug-log', help=SUPPRESS) def _build_parsers(args=None): @@ -1219,16 +1225,24 @@ def __enter__(self): self.cache = _prepare_child_profiling_cache( self.options, self.prof, preimports_file, script_file, ) + # Add a deferred callback for gathering debug logfiles + # (should run right before `.cache.cache_dir` is wiped) + if self.options.debug_log: + self.cache._add_cleanup( + self._gather_debug_log, + CLEANUP_PRIORITIES['gather_logs'], + self.options.debug_log, + ) return self.prof, script_file def __exit__(self, *_, **__): try: extra_stats = None if self.set_up_child_profiling: - if self.cache.debug: - # Recover debug output from child processes - self.cache._dump_debug_logs() try: + if self.cache.debug: + # Recover debug output from child processes + self.cache._dump_debug_logs() extra_stats = self.cache.gather_stats() finally: self.cache.cleanup() @@ -1236,6 +1250,11 @@ def __exit__(self, *_, **__): finally: self._ctx.uninstall() + def _gather_debug_log(self, logfile): + with open(logfile, mode='w') as fobj: + for entry in self.cache._gather_debug_log_entries(): + print(entry.to_text(), file=fobj) + @property def set_up_child_profiling(self): return bool( @@ -1353,11 +1372,15 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): profile_imports=options.prof_imports, preimports_module=preimports_file, insert_builtin=options.builtin, - debug=options.debug, + debug=bool(options.debug or options.debug_log), ) clean_up = functools.partial(cache.add_cleanup, _remove, missing_ok=True) if not diagnostics.KEEP_TEMPDIRS: - clean_up(cache.cache_dir, recursive=True) + # Defer the scrubbing of the cache dir + cache._add_cleanup( + _remove, CLEANUP_PRIORITIES['rm_cache_dir'], cache.cache_dir, + recursive=True, + ) clean_up(cache.filename) # This file is handed to us at the end of diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 95ef42bd..330b400a 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -248,29 +248,35 @@ def cleanup(self) -> None: :py:meth:`~.add_cleanup` and call them in order. """ stacks = self._cleanup_stacks - num_callbacks = sum(len(stack) for stack in stacks.values()) - if not num_callbacks: + ncallbacks_total = sum(len(stack) for stack in stacks.values()) + if not ncallbacks_total: self._debug_output('Cleanup aborted (no registered callbacks)') return # Bookend the cleanup loop with log messages to help detect if # child processes are prematurely terminated self._debug_output( - f'Starting cleanup ({num_callbacks} callback(s))...', + f'Starting cleanup ({ncallbacks_total} callback(s))...', ) + ncallbacks_run = 0 for priority in sorted(stacks): callbacks = stacks.pop(priority) while callbacks: callback = callbacks.pop() callback_repr = _CALLBACK_REPR(callback) + ncallbacks_run += 1 try: callback() except Exception as e: - msg = f'failed: {callback_repr}: {type(e).__name__}: {e}' + state = 'failed' + msg = f'{callback_repr}: {type(e).__name__}: {e}' else: - msg = f'succeeded: {callback_repr}' - self._debug_output('- Cleanup ' + msg) + state, msg = 'succeeded', f'{callback_repr}' + self._debug_output( + f'- Cleanup {state} ' + f'({ncallbacks_run}/{ncallbacks_total}): {msg}', + ) self._debug_output( - f'... cleanup completed ({num_callbacks} callback(s))', + f'... cleanup completed ({ncallbacks_total} callback(s))', ) def add_cleanup( diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 14179c59..16352b3d 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -32,6 +32,7 @@ NUM_NUMBERS = 100 NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) +_WRITE_DEBUG_LOGS = True def strip(s: str) -> str: @@ -578,6 +579,7 @@ def _run_test_module( nnums: int | None = None, nprocs: int | None = None, check: bool = True, + debug_log: str | None = None, nhits: Mapping[str, int] | None = None, **kwargs ) -> tuple[subprocess.CompletedProcess, LineStats | None]: @@ -631,45 +633,54 @@ def get_pth_files(pattern: str = '*.pth') -> set[str]: with ub.ChDir(tmp_path_factory.mktemp('mytemp')): if outfile is not None: runner_args.extend(['--outfile', outfile]) + if debug_log: + runner_args.extend(['--debug-log', debug_log]) old_pth_files = get_pth_files() proc = run_helper( runner_args, test_args, test_module, text=True, capture_output=True, check=(check and not fail), **kwargs ) - # Checks: - if fail: - # - The process has failed as expected - if check: - assert proc.returncode - else: - # - The result is correctly calculated - expected = nnums * (nnums + 1) // 2 - output_lines = proc.stdout.splitlines() - if output_lines[0] != str(expected): - raise ResultMismatch( - f'result {expected}', f'output lines: {output_lines}', - ) - # - Temporary `.pth` file(s) created by `~~.pth_hook` has been - # cleaned up - assert get_pth_files() == old_pth_files - # - Profiling results are written to the specified file - prof_result: LineStats | None = None - if outfile is None: - assert not list(Path.cwd().iterdir()) - else: - assert os.path.exists(outfile) - assert os.stat(outfile).st_size - if profile: - prof_result = LineStats.from_files(outfile) - # - If we're keeping track, the function is called the expected - # number of times and has run the expected # of loops - # (Note: we do it by parsing the output of `kernprof -v` - # instead of reading the `--outfile`, because if the profiled - # code is in a tempfile the profiling data will be dropped in - # the written outfile) - for tag, num in (nhits or {}).items(): - _check_output(proc.stdout, tag, num) + try: + # Checks: + if fail: + # - The process has failed as expected + if check: + assert proc.returncode + else: + # - The result is correctly calculated + expected = nnums * (nnums + 1) // 2 + output_lines = proc.stdout.splitlines() + if output_lines[0] != str(expected): + raise ResultMismatch( + f'result {expected}', f'output lines: {output_lines}', + ) + # - Temporary `.pth` file(s) created by `~~.pth_hook` has + # been cleaned up + assert get_pth_files() == old_pth_files + # - Profiling results are written to the specified file + prof_result: LineStats | None = None + if outfile is None: + assert not list(Path.cwd().iterdir()) + else: + assert os.path.exists(outfile) + assert os.stat(outfile).st_size + if profile: + prof_result = LineStats.from_files(outfile) + # - If we're keeping track, the function is called the + # expected number of times and has run the expected # of + # loops (Note: we do it by parsing the output of + # `kernprof -v` instead of reading the `--outfile`, + # because if the profiled code is in a tempfile the + # profiling data will be dropped in the written outfile) + for tag, num in (nhits or {}).items(): + _check_output(proc.stdout, tag, num) + finally: + if debug_log is not None: + with open(debug_log) as fobj: + print('-- Combined debug logs --', file=sys.stderr) + print(indent(fobj.read(), ' '), end='', file=sys.stderr) + print('-- End of debug logs --', file=sys.stderr) return proc, prof_result @@ -932,6 +943,9 @@ def test_profiling_multiproc_script( nnums=nnums, nprocs=nprocs, timeout=timeout, + debug_log=( + 'debug.log' if prof_child_procs and _WRITE_DEBUG_LOGS else None + ), ) @@ -976,12 +990,16 @@ def test_profiling_bare_python( script_path.write_text(script_content) out_file = temp_dir / 'out.lprof' + debug_log_file = temp_dir / 'debug.log' + write_debug = _WRITE_DEBUG_LOGS and prof_child_procs cmd = [ 'kernprof', '-lv', '--preimports', f'--prof-mod={ext_module.name}', f'--outfile={out_file}', '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), ] + if write_debug: + cmd.append(f'--debug-log={debug_log_file}') sub_cmd = [sys.executable, str(script_path)] if use_subprocess: code = strip(f""" @@ -1006,8 +1024,17 @@ def test_profiling_bare_python( for k in nhits: nhits[k] = 0 - # Check that the code errors out when expected - assert bool(fail) == bool(proc.returncode) - # Check that the profiling output is as expected - for tag, num in nhits.items(): - _check_output(proc.stdout, tag, num) + try: + # Check that the code errors out when expected + assert bool(fail) == bool(proc.returncode) + # Check that the profiling output is as expected + for tag, num in nhits.items(): + _check_output(proc.stdout, tag, num) + finally: + if write_debug: + print('-- Combined debug logs --', file=sys.stderr) + print( + indent(debug_log_file.read_text(), ' '), + end='', file=sys.stderr, + ) + print('-- End of debug logs --', file=sys.stderr) From e6d8b8c8ba09102b529f80d9eda4323b2c825e89 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 17 Apr 2026 00:00:04 +0200 Subject: [PATCH 044/165] Compatibility fix line_profiler/_child_process_profiling/cache.py::_CallbackRepr __doc__ Updated so that the parts that test `.indent` is skipped on Python < 3.12 indent No longer a property; private methods now use `._get_indent()` to indirectly access it _get_indent() Wrapper around `.indent`; falls back to `None` on legacy versions (< 3.12) without the attribute --- .../_child_process_profiling/cache.py | 59 ++++++++++++------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 330b400a..01bf03e5 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -7,6 +7,7 @@ import atexit import dataclasses import os +import sys try: import _pickle as pickle except ImportError: @@ -86,8 +87,8 @@ class _CallbackRepr(Repr): Example: >>> from functools import partial - >>> - >>> + >>> from sys import version_info + >>> class MyEnviron(dict): ... def some_method(self) -> None: ... ... @@ -117,6 +118,28 @@ class _CallbackRepr(Repr): Partial-object formatting: + >>> r.maxenv = 0 + >>> print(r.repr(my_env.some_method)) + + + Bound-method formatting: + + >>> r.maxargs = 0 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial(, ...) + + Indentation (Python 3.12+): + + >>> if version_info < (3, 12): + ... from pytest import skip + ... + ... skip( + ... '`Repr.indent` not available on {}.{},{}' + ... .format(*sys.version_info) + ... ) + + >>> r = MyRepr(maxenv=2, maxargs=4) >>> r.indent = 2 >>> callback_1 = partial(int, base=8) >>> print(r.repr(callback_1)) @@ -137,8 +160,6 @@ class _CallbackRepr(Repr): ----..., ) - Bound-method formatting: - >>> r.indent = ' ' >>> r.maxenv = 2 >>> print(r.repr(my_env.some_method)) @@ -147,10 +168,6 @@ class _CallbackRepr(Repr): 'bar': '2', ..., })> - >>> r.indent = None - >>> r.maxenv = 0 - >>> print(r.repr(my_env.some_method)) - """ def __init__( self, @@ -196,22 +213,24 @@ def _format_items( start, end = delims if maxlen is not None and len(items) > maxlen: items = list(items)[:maxlen] + ['...'] - if self.indent is None or not items: + indent_prefix: str | None = self._get_indent() + if indent_prefix is None or not items: return '{}{}{}'.format(start, ', '.join(items), end) return '\n'.join([ - start, *(indent(item + ',', self.indent) for item in items), end, + start, *(indent(item + ',', indent_prefix) for item in items), end, ]) - @property - def indent(self) -> str | None: - return self._indent - - @indent.setter - def indent(self, indent: str | int | None) -> None: - if indent is None or isinstance(indent, str): - self._indent = indent - return - self._indent = ' ' * indent + if sys.version_info >= (3, 12): + # Note: `.indent` only available since 3.12 + def _get_indent(self) -> str | None: + indent = self.indent + if indent is None or isinstance(indent, str): + return indent + return ' ' * indent + else: + @staticmethod + def _get_indent() -> None: + return None _CALLBACK_REPR = _CallbackRepr(maxother=cast(int, float('inf'))).repr From 108fc33a7bdd3aab7e5a31d417556d81128086b6 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 17 Apr 2026 01:16:09 +0200 Subject: [PATCH 045/165] Compat fix (test) tests/test_child_procs.py::{_run_subproc,test_profiling_bare_python}() Fixed bug where command lines are malformed on Windows owing to the use of `shlex.join()`; the (undocumented but stable and correct) `subprocess.list2cmdline()` is used instead if we're on the platform --- tests/test_child_procs.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 16352b3d..802f2c4f 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -456,6 +456,19 @@ def new( return cls(param_list, value_tuple_list, default_values) +# `shlex.join()` doesn't work properly on Windows, so use +# `subprocess.list2cmdline()` instead; +# though an "intentionally" undocumented API (cpython issue #10308), +# it's been around since 2.4, seems stable enough, and does exactly what +# is needed +if sys.platform == 'win32': + concat_command_line: Callable[ + [Sequence[str]], str + ] = subprocess.list2cmdline +else: + concat_command_line = shlex.join + + def _run_as_script( runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, **kwargs @@ -498,7 +511,7 @@ def _run_subproc( if isinstance(cmd, str): cmd_str = cmd else: - cmd_str = shlex.join(cmd) + cmd_str = concat_command_line(cmd) # If we're capturing outputs, it may be for the best to wait until # we've processed the output streams to check the return code... @@ -1015,7 +1028,7 @@ def test_profiling_bare_python( if os.system({!r}): raise RuntimeError('called process failed') - """.format(shlex.join(sub_cmd))) + """.format(concat_command_line(sub_cmd))) cmd.extend(['-c', code]) proc = _run_subproc(cmd, text=True, capture_output=True) From 093b6e9e80295ae3c36980fa271d78a8f0b07116 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 01:52:27 +0200 Subject: [PATCH 046/165] Better handling of `multiprocessing` line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_{start,terminate,bootstrap}() Wrapped methods now write debug-log entries when: - Before the wrapped call is made - After the wrapped call returns/errors out wrap_terminate() Now always calling `vanilla_impl()` (`= Process.terminate()`) regardless of whether the preceding polling and callback errors out or not, to avoid orphaning child processes --- .../multiprocessing_patches.py | 85 +++++++++++++------ 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index dffd92e5..11accc69 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,14 +16,14 @@ import multiprocessing import warnings -from collections.abc import Callable, Mapping +from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext from functools import partial, wraps from importlib import import_module from multiprocessing.process import BaseProcess from pathlib import Path from time import sleep, monotonic -from typing import Any, Literal, TypeVar, NoReturn +from typing import Any, Generic, Literal, Protocol, TypeVar, NoReturn, cast from typing_extensions import Concatenate, ParamSpec, Self from .. import _diagnostics as diagnostics @@ -34,7 +34,6 @@ __all__ = ('apply',) -S = TypeVar('S') T = TypeVar('T') PS = ParamSpec('PS') @@ -62,6 +61,11 @@ _INTERCEPT_MP_LOG_MESSAGES: bool | None = False +class _Wrapper(Protocol, Generic[PS, T]): + def __call__(self, func: Callable[PS, T], /) -> Callable[PS, T]: + ... + + class PickleHook: """ Object which, when unpickled, sets up profiling in the @@ -190,16 +194,41 @@ class Timeout(RuntimeError): def _method_wrapper( - wrapper: Callable[Concatenate[S, Callable[Concatenate[S, PS], T], PS], T] -) -> Callable[ - [Callable[Concatenate[S, PS], T]], Callable[Concatenate[S, PS], T] -]: - def inner_wrapper( - vanilla_impl: Callable[Concatenate[S, PS], T], - ) -> Callable[Concatenate[S, PS], T]: + wrapper: Callable[Concatenate[LineProfilingCache, Callable[PS, T], PS], T], +) -> Callable[[Callable[PS, T]], Callable[PS, T]]: + def inner_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: @wraps(vanilla_impl) - def wrapped_impl(self: S, *a: PS.args, **k: PS.kwargs) -> T: - return wrapper(self, vanilla_impl, *a, **k) + def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: + cache = LineProfilingCache.load() + debug = cache._debug_output + + arg_reprs: list[str] = [repr(arg) for arg in args] + arg_reprs.extend(f'{k}={v!r}' for k, v in kwargs.items()) + formatted_call = '{}({})'.format(name, ', '.join(arg_reprs)) + + debug(f'Wrapped method call made: {formatted_call}...') + try: + result = wrapper(cache, vanilla_impl, *args, **kwargs) + except Exception as e: + debug( + 'Wrapped method call failed: ' + f'{formatted_call} -> {type(e).__name__}: {e}', + ) + raise + else: + debug( + 'Wrapped method call succeeded: ' + f'{formatted_call} -> {result!r}', + ) + return result + + if ( + hasattr(vanilla_impl, '__module__') + and hasattr(vanilla_impl, '__qualname__') + ): + name = '{0.__module__}.{0.__qualname__}'.format(vanilla_impl) + else: + name = f'' return wrapped_impl @@ -213,7 +242,9 @@ def wrapped_impl(self: S, *a: PS.args, **k: PS.kwargs) -> T: @_method_wrapper def wrap_start( - self: BaseProcess, vanilla_impl: Callable[[BaseProcess], None], + cache: LineProfilingCache, + vanilla_impl: Callable[[BaseProcess], None], + self: BaseProcess, ) -> None: """ Wrap around :py:meth:`BaseProcess.start` to specify the location for @@ -221,7 +252,6 @@ def wrap_start( the parent. This is to ensure that the child can exit gracefully and complete any necessary cleanup. """ - cache = LineProfilingCache.load() tempfile = cache.make_tempfile(prefix='process-term-lock-', suffix='.lock') setattr(self, _PROCESS_TERM_LOCK_LOC, tempfile) vanilla_impl(self) @@ -229,7 +259,9 @@ def wrap_start( @_method_wrapper def wrap_terminate( - self: BaseProcess, vanilla_impl: Callable[[BaseProcess], None], + cache: LineProfilingCache, # TODO: config timeouts + vanilla_impl: Callable[[BaseProcess], None], + self: BaseProcess, ) -> None: """ Wrap around :py:meth:`BaseProcess.terminate` to make sure that we @@ -281,14 +313,18 @@ def discard_lock() -> None: else: lock, callback = nullcontext(), _no_op with lock: - callback() - vanilla_impl(self) + try: + callback() + finally: # Always call `Process.terminate()` to avoid orphans + vanilla_impl(self) @_method_wrapper def wrap_bootstrap( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], self: BaseProcess, - vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], /, + /, *args: PS.args, **kwargs: PS.kwargs ) -> T: """ @@ -302,7 +338,6 @@ def wrap_bootstrap( ``.terminate()`` a failed child before the profiling results can be gathered. """ - cache = LineProfilingCache.load() lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) if lock_file: @@ -431,13 +466,13 @@ def replace( )) # Patch `multiprocessing.process.BaseProcess` methods - Method = Callable[Concatenate[S, PS], T] - patches: Mapping[str, Callable[[Method], Method]] - for submodule, target, patches in [ # type: ignore[assignment] + # Note: the type checkers seem to need some help figuring the + # `patches` out... so do explicit `cast()`s + for submodule, target, patches in [ ('process', 'BaseProcess', { - 'start': wrap_start, - 'terminate': wrap_terminate, - '_bootstrap': wrap_bootstrap, + 'start': cast(_Wrapper[..., None], wrap_start), + 'terminate': cast(_Wrapper[..., None], wrap_terminate), + '_bootstrap': cast(_Wrapper[..., Any], wrap_bootstrap), }), ]: try: From c864fa04a111954dfc8ec7f2d365d5217f1c60e6 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 02:35:19 +0200 Subject: [PATCH 047/165] Cache the active config line_profiler/_child_process_profiling/cache.py::LineProfilingCache Added init field/attribute `config` for storing the config source line_profiler/toml_config.py::ConfigSource.get_subconfig() Fixed bug where the `copy` argument is silently ignored --- kernprof.py | 1 + line_profiler/_child_process_profiling/cache.py | 1 + line_profiler/toml_config.py | 8 ++++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/kernprof.py b/kernprof.py index 6fc4e725..110c88a3 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1367,6 +1367,7 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): # instances in child processes will merely inherit and use them cache = LineProfilingCache( cache_dir=tempfile.mkdtemp(), + config=options.config, profiling_targets=options.prof_mod, rewrite_module=script_file, profile_imports=options.prof_imports, diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 01bf03e5..04819f37 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -243,6 +243,7 @@ class LineProfilingCache: info required to make profiling persist into child processes. """ cache_dir: os.PathLike[str] | str + config: os.PathLike[str] | str | None = None profiling_targets: Collection[str] = dataclasses.field( default_factory=list, ) diff --git a/line_profiler/toml_config.py b/line_profiler/toml_config.py index 781dc60d..236e86da 100644 --- a/line_profiler/toml_config.py +++ b/line_profiler/toml_config.py @@ -106,7 +106,10 @@ def get_subconfig( get_subtable(self.conf_dict, headers, allow_absence=allow_absence), ) new_subtable = [*self.subtable, *headers] - return type(self)(new_dict, self.path, new_subtable) + new_instance = type(self)(new_dict, self.path, new_subtable) + if copy: + new_instance = new_instance.copy() + return new_instance @classmethod def from_default(cls, *, copy: bool = True) -> ConfigSource: @@ -355,7 +358,8 @@ def iter_configs(dir_path): def get_subtable( - table: Mapping[K, Mapping], keys: Sequence[K], *, allow_absence: bool = True + table: Mapping[K, Mapping], keys: Sequence[K], *, + allow_absence: bool = True, ) -> Mapping: """ Arguments: From 8cf8ffb2319d12b495087f7933661dfc74df2cb6 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 05:14:29 +0200 Subject: [PATCH 048/165] Move config for `multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_terminate() Now configuring the `_Poller` object based on `cache.config` wrap_get_preparation_data() Refactored from `get_preparation_data()`; now also emitting extra debug output like the other `wrap_*()` functions _apply_mp_patches(debug=None) Instead of using `lp_cache.debug`, now using `cache.config` to decide on whether to intercept the `multiprocessing` logs line_profiler/rc/line_profiler.toml ::[tool.line_profiler.multiprocessing] New private subtable for configuring the `multiprocessing_patches` behavior (poller timeouts, logging, etc.) --- .../multiprocessing_patches.py | 130 ++++++++++++------ line_profiler/rc/line_profiler.toml | 33 +++++ 2 files changed, 118 insertions(+), 45 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 11accc69..03eba201 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,17 +16,22 @@ import multiprocessing import warnings -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import AbstractContextManager, nullcontext -from functools import partial, wraps +from functools import lru_cache, partial, wraps from importlib import import_module from multiprocessing.process import BaseProcess +from os import PathLike from pathlib import Path from time import sleep, monotonic -from typing import Any, Generic, Literal, Protocol, TypeVar, NoReturn, cast +from types import MappingProxyType +from typing import ( + Any, Generic, Literal, Protocol, TypeVar, Union, NoReturn, cast, +) from typing_extensions import Concatenate, ParamSpec, Self from .. import _diagnostics as diagnostics +from ..toml_config import ConfigSource from .cache import LineProfilingCache from .runpy_patches import create_runpy_wrapper @@ -40,26 +45,6 @@ _PATCHED_MARKER = '_line_profiler_patched_multiprocessing' _PROCESS_TERM_LOCK_LOC = '_line_profiler_process_terminate_lock' -# NOTE: polling behaviors for blocking `BaseProcess.terminate()` with -# the lock file: -# - _POLLING_COOLDOWN: -# Seconds between polling for the lock file; set to zero to poll in a -# hot loop (not recommended) -# - _POLLING_TIMEOUT: second to block before timing out anyway; set to -# zero to disable timing out -# - _POLLING_ON_TIMEOUT: what to do on timeout if enabled: -# - 'error': error out with `_Poller.Timeout` -# - 'warn': issue a warning and move on -# - 'ignore': move on without a warning -_POLLING_COOLDOWN = 1. / 32 -_POLLING_TIMEOUT = 1. -_POLLING_ON_TIMEOUT: Literal['error', 'warn', 'ignore'] = 'warn' - -# NOTE: Set this to `None` or `True` to tee the `multiprocessing` -# internal logging messages to the log files; if `None`, logs are only -# written if `LineProfilingCache.load().debug` is set to true. -_INTERCEPT_MP_LOG_MESSAGES: bool | None = False - class _Wrapper(Protocol, Generic[PS, T]): def __call__(self, func: Callable[PS, T], /) -> Callable[PS, T]: @@ -193,6 +178,27 @@ class Timeout(RuntimeError): pass +def _get_config( + config: PathLike[str] | str | bool | None = None, +) -> Mapping[str, Any]: + if config not in (True, False, None): + config = str(config) + return _get_config_cached(cast(Union[str, bool, None], config)) + + +@lru_cache() +def _get_config_cached( + config: PathLike[str] | str | bool | None = None, +) -> Mapping[str, Any]: + cd = dict( + ConfigSource.from_config(config) + .get_subconfig('multiprocessing', copy=True) + .conf_dict + ) + assert isinstance(cd.get('polling'), Mapping) + return MappingProxyType({**cd, 'polling': MappingProxyType(cd['polling'])}) + + def _method_wrapper( wrapper: Callable[Concatenate[LineProfilingCache, Callable[PS, T], PS], T], ) -> Callable[[Callable[PS, T]], Callable[PS, T]]: @@ -206,18 +212,18 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: arg_reprs.extend(f'{k}={v!r}' for k, v in kwargs.items()) formatted_call = '{}({})'.format(name, ', '.join(arg_reprs)) - debug(f'Wrapped method call made: {formatted_call}...') + debug(f'Wrapped call made: {formatted_call}...') try: result = wrapper(cache, vanilla_impl, *args, **kwargs) except Exception as e: debug( - 'Wrapped method call failed: ' + 'Wrapped call failed: ' f'{formatted_call} -> {type(e).__name__}: {e}', ) raise else: debug( - 'Wrapped method call succeeded: ' + 'Wrapped call succeeded: ' f'{formatted_call} -> {result!r}', ) return result @@ -302,21 +308,53 @@ def discard_lock() -> None: except AttributeError: pass - lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) - if lock_file: - lock: AbstractContextManager[Any] = ( - _Poller.poll_while(lock_file.exists) - .with_cooldown(_POLLING_COOLDOWN) - .with_timeout(_POLLING_TIMEOUT, _POLLING_ON_TIMEOUT) - ) - callback = discard_lock - else: - lock, callback = nullcontext(), _no_op - with lock: + def get_poller_args( + config: PathLike[str] | str | bool | None = None, + ) -> tuple[float, float, str | None]: + values = _get_config(config)['polling'] + try: + cooldown = max(float(values['cooldown']), 0) + except (TypeError, ValueError): + cooldown = 0 + try: + timeout = max(float(values['timeout']), 0) + except (TypeError, ValueError): + timeout = 0 try: + on_timeout: str | None = values['on_timeout'].lower() + except Exception: # Fallback (use `_Poller`'s default) + on_timeout = None + return cooldown, timeout, on_timeout + + def wait_for_deletion( + path: Path, config: PathLike[str] | str | None = None, + ) -> _Poller: + cooldown, timeout, on_timeout = get_poller_args(config) + # `False` -> no resolution, force loading the vanilla file + *_, default_on_timeout = get_poller_args(False) + if on_timeout not in ('ignore', 'warn', 'error'): + on_timeout = default_on_timeout + return ( + _Poller.poll_while(path.exists) + .with_cooldown(cooldown) + .with_timeout( + timeout, cast(Literal['ignore', 'warn', 'error'], on_timeout), + ) + ) + + try: + lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) + if lock_file: + lock: AbstractContextManager[Any] = wait_for_deletion( + lock_file, cache.config, + ) + callback = discard_lock + else: + lock, callback = nullcontext(), _no_op + with lock: callback() - finally: # Always call `Process.terminate()` to avoid orphans - vanilla_impl(self) + finally: # Always call `Process.terminate()` to avoid orphans + vanilla_impl(self) @_method_wrapper @@ -386,7 +424,11 @@ def get_msg(msg: str, *_, **__) -> str: ) -def get_preparation_data( +@_method_wrapper +def wrap_get_preparation_data( + # We don't use the cache here, but `@_method_wrapper` expects it in + # the signature (and we want the debug output) + _, vanilla_impl: Callable[PS, dict[str, Any]], /, *args: PS.args, @@ -447,7 +489,7 @@ def apply(lp_cache: LineProfilingCache) -> None: def _apply_mp_patches( lp_cache: LineProfilingCache, add_cleanup: Callable[..., Any], - debug: bool | None = _INTERCEPT_MP_LOG_MESSAGES, + debug: bool | None = None, ) -> None: def replace( obj: Any, attr: str, value: Any, obj_name: str | None = None, @@ -492,9 +534,7 @@ def replace( pass else: # Patch `get_preparation_data()` - gpd_wrapper = partial( # type: ignore[call-arg] - get_preparation_data, spawn.get_preparation_data, - ) + gpd_wrapper = wrap_get_preparation_data(spawn.get_preparation_data) replace(spawn, 'get_preparation_data', gpd_wrapper, spawn.__name__) # Patch `runpy` (do it locally instead of tempering with the # global `runpy` mmodule) @@ -504,7 +544,7 @@ def replace( # Intercept `multiprocessing` debug messages if debug is None: - debug = lp_cache.debug + debug = _get_config(lp_cache.config)['intercept_logs'] if debug: from multiprocessing import util diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index a5b7740c..5b39f5cb 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -209,3 +209,36 @@ hits = 9 time = 12 perhit = 8 percent = 8 + +# XXX: --- Start of implementation details --- +# `line_profiler._child_process_profiling.multiprocessing_patches` +# settings + +[tool.line_profiler.multiprocessing] + +# - `intercept_logs` (bool): +# Whether to patch the `multiprocessing.util` logging funcions so that +# the internal log messages are captured and writtent to the debug +# logs +intercept_logs = false + +[tool.line_profiler.multiprocessing.polling] + +# - `polling.cooldown` (float): +# Cooldown time (seconds) before successive polls on lock files (set +# to <= 0 to disable cooldowns) +cooldown = 0.03125 # 1/32-nd of a second +# - `polling.timeout` (float): +# Time (seconds) before the main process disregards the existence of +# lock file and unblocks calls to `.terminate()` a child process +# anyway (set to <= 0 to disable timeouts) +timeout = 1 +# - `polling.on_timeout` (Literal['error', 'warn', 'ignore']) +# What to do when the above timeout is exhausted, before actually +# `.terminate()`-ing the child process: +# - 'error': raise an error +# - `warn`: issue a warning +# - `ignore`: nothing +on_timeout = 'error' + +# XXX: --- End of implementation details --- From d7ee4c9df16cfc6c63cd7caba00ef556063dc051 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 07:17:53 +0200 Subject: [PATCH 049/165] Streamlined profiler installation line_profiler/curated_profiling.py::CuratedProfilerContext.install() Now responsible for equiping `.prof` with the `.add_imported_function_or_module()` pseudo-method kernprof.py::_write_preimports() line_profiler/_child_process_profiling/cache.py::LineProfilingCache Removed code doing the same --- kernprof.py | 4 ---- line_profiler/_child_process_profiling/cache.py | 8 -------- line_profiler/curated_profiling.py | 6 ++++++ 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/kernprof.py b/kernprof.py index 110c88a3..96888468 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1066,15 +1066,11 @@ def _write_preimports(prof, options, exclude, keep=False): Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. """ - from line_profiler.autoprofile.autoprofile import ( - _extend_line_profiler_for_profiling_imports as upgrade_profiler, - ) from line_profiler.curated_profiling import ClassifiedPreimportTargets # We could've done everything in-memory with `io.StringIO` and `exec()`, # but that results in indecipherable tracebacks should anything goes wrong; # so we write to a tempfile and `execfile()` it - upgrade_profiler(prof) temp_mod_path = _touch_tempfile( dir=options.tmpdir, prefix='kernprof-eager-preimports-', suffix='.py' ) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 04819f37..6cebe16f 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -25,13 +25,6 @@ from typing_extensions import Self, ParamSpec, Unpack from .. import _diagnostics as diagnostics -from ..autoprofile.autoprofile import ( - # Note: we need this to equip the profiler with the - # `.add_imported_function_or_module()` pseudo-method - # (see `kernprof.py::_write_preimports()`), which is required for - # the preimports to work - _extend_line_profiler_for_profiling_imports as upgrade_profiler, -) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats from ._cache_logging import CacheLoggingEntry @@ -578,7 +571,6 @@ def _setup_in_child_process( if prof is None: prof = LineProfiler() self.profiler = prof - upgrade_profiler(prof) ctx = CuratedProfilerContext(prof, insert_builtin=self.insert_builtin) ctx.install() self.add_cleanup(ctx.uninstall) diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index 8ed720c0..6e21ffe8 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -16,6 +16,9 @@ from typing_extensions import Self from . import _diagnostics as diagnostics, profile as _global_profiler +from .autoprofile.autoprofile import ( + _extend_line_profiler_for_profiling_imports as upgrade_profiler, +) from .autoprofile.util_static import modpath_to_modname from .autoprofile.eager_preimports import ( is_dotted_path, write_eager_import_module, @@ -190,6 +193,9 @@ def set_builtin_profile(old: Any) -> None: if self._installed: return + # Equip the profiler instance with the + # `.add_imported_function_or_module()` pseudo-method + upgrade_profiler(self.prof) # Overwrite the explicit profiler (`@line_profiler.profile`) self._global_install(self.prof) # type: ignore # Set up hooks to deal with inserting `.prof` as a builtin name From 8fe9bcf4cfd6c5df9ae7b5d9193296a4cd671c7a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 08:18:12 +0200 Subject: [PATCH 050/165] WIP: boost test coverage line_profiler/_child_process_profiling/multiprocessing_patches.py ::_Poller.__doc__ Added doctest tests/test_child_procs.py test_module_clone New fixture which is identical in content to `test_module` but backed by a different physical file create_cache() New fixture function for creating a `LineProfilingCache` instance curated_profiler New fixture for a `LineProfiler` managed by a `CuratedProfilerContext` test_runpy_patches() New test for `~._child_process_profiling.runpy_patches.create_runpy_wrapper()` --- .../multiprocessing_patches.py | 28 +++ tests/test_child_procs.py | 186 +++++++++++++++++- 2 files changed, 212 insertions(+), 2 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 03eba201..24029022 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -78,6 +78,34 @@ def __setstate__(*_) -> None: class _Poller: """ Poll a callable until it returns true-y. + + Example: + >>> from itertools import count + >>> from typing import Iterator + >>> + >>> + >>> def count_until(limit: int) -> bool: + ... def counter_is_big_enough( + ... counter: Iterator[int], limit: int, + ... ) -> bool: + ... return next(counter) >= limit + ... + ... return _Poller.poll_until( + ... counter_is_big_enough, count(), limit, + ... ) + ... + >>> + >>> with count_until(10).with_cooldown(.01).with_timeout(.25): + ... print('We counted up to 10') + We counted up to 10 + >>> with count_until(30).with_cooldown(.01).with_timeout(.25): + ... print('We counted up to 30') \ +# doctest: +NORMALIZE_WHITESPACE + Traceback (most recent call last): + ... + line_profiler..._Poller.Timeout: ... + timed out (... s >= 0.25 s) waiting for + callback ...counter_is_big_enough... to return true """ def __init__( self, diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 802f2c4f..91548298 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -11,6 +11,7 @@ ) from dataclasses import dataclass from functools import partial +from io import StringIO from pathlib import Path from sysconfig import get_path from tempfile import TemporaryDirectory @@ -23,7 +24,12 @@ import pytest import ubelt as ub -from line_profiler.line_profiler import LineStats +from line_profiler._child_process_profiling.cache import LineProfilingCache +from line_profiler._child_process_profiling.runpy_patches import ( + create_runpy_wrapper, +) +from line_profiler.curated_profiling import CuratedProfilerContext +from line_profiler.line_profiler import LineProfiler, LineStats T = TypeVar('T') @@ -241,6 +247,80 @@ def test_module( yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) +@pytest.fixture +def test_module_clone( + tmp_path_factory: pytest.TempPathFactory, + monkeypatch: pytest.MonkeyPatch, + _test_module: Path, + ext_module: _ModuleFixture, +) -> Generator[_ModuleFixture, None, None]: + """ + Yields: + :py:class:`_ModuleFixture` helper object containing the same + code as :py:data:`test_module` + """ + tmpdir = tmp_path_factory.mktemp('my_path') + name = next(_ModuleFixture.propose_name('my_cloned_module')) + path = tmpdir / f'{name}.py' + path.write_text(_test_module.read_text()) + yield _ModuleFixture(path, monkeypatch, [ext_module]) + + +@pytest.fixture +def create_cache( + tmp_path_factory: pytest.TempPathFactory, + curated_profiler: LineProfiler, +) -> Generator[Callable[..., LineProfilingCache], None, None]: + """ + Wrapper around the :py:class:`LineProfilingCache` instantiator + which: + + - Automatically creates a tempdir and provides it as the + :py:attr:`LineProfilingCache.cache_dir`, + + - Automatically creates an instance of :py:class:`LineProfiler` that + is curated by a :py:class:`CuratedProfilerContext` and provides it + as the :py:attr:`LineProfilingCache.profiler`, and + + - At teardown: + + - Restores the value of the class' internal reference to the + :py:meth:`LineProfilingCache.load`-ed instance. + + - Calls the `.cleanup()` method of each instance created. + """ + def instantiate(**kwargs) -> LineProfilingCache: + tmpdir = tmp_path_factory.mktemp('my_cache_dir') + cache = LineProfilingCache(tmpdir, **kwargs) + cache.profiler = curated_profiler + instances.append(cache) + return cache + + old_value = LineProfilingCache._loaded_instance + instances: list[LineProfilingCache] = [] + try: + yield instantiate + finally: + LineProfilingCache._loaded_instance = old_value # type: ignore + for cache in instances: + try: + cache.cleanup() + except Exception: + pass + + +@pytest.fixture +def curated_profiler() -> Generator[LineProfiler, None, None]: + """ + Yields: + Fresh instance of :py:class:`LineProfiler` that is managed by a + :py:class:`CuratedProfilerContext` + """ + prof = LineProfiler() + with CuratedProfilerContext(prof, insert_builtin=True): + yield prof + + # ========================== Helper functions ========================== @@ -721,7 +801,109 @@ def _check_output(output: str, tag: str, nhits: int) -> None: _run_test_module, _run_as_literal_code, profiled_code_is_tempfile=True, ) -# =============================== Tests ================================ +# ============================= Unit tests ============================= + + +@pytest.mark.parametrize( + ('run_profiled_code', 'label1'), + [(True, 'run-profiled'), (False, 'run-unrelated')]) +@pytest.mark.parametrize( + ('as_module', 'label2'), + [(True, 'run_module'), (False, 'run_path')]) +@pytest.mark.parametrize( + ('debug', 'label3'), + [(True, 'with-debug'), (False, 'no-debug')]) +def test_runpy_patches( + capsys: pytest.CaptureFixture[str], + ext_module: _ModuleFixture, + test_module: _ModuleFixture, + test_module_clone: _ModuleFixture, + create_cache: Callable[..., LineProfilingCache], + run_profiled_code: bool, + as_module: bool, + debug: bool, + label1: str, label2: str, label3: str, +) -> None: + """ + Test that the :py:mod:`runpy` clone created by + :py:func:`line_profiler._child_process_profiling\ +.create_runpy_wrapper` + correctly sets up profiling when its ``run_*()`` functions are + called. + """ + class restore_argv: + def __enter__(self) -> None: + self.argv = list(sys.argv) + + def __exit__(self, *_, **__) -> None: + sys.argv[:] = self.argv + + cache = create_cache( + rewrite_module=test_module.path, + profiling_targets=[str(ext_module.path)], + profile_imports=True, + debug=debug, + ) + assert cache.profiler is not None + runpy = create_runpy_wrapper(cache) + + nnums = 42 + nprocs = 2 + # If we're running some unrelated code, the profiler should not be + # involved + if run_profiled_code: + module = test_module + num_invocations, num_loops = 1, nprocs + expected_funcs: list[str] = ['my_external_sum'] + else: + module = test_module_clone + num_invocations, num_loops = 0, 0 + expected_funcs = [] + if as_module: + first_arg = module.name + runner = partial(runpy.run_module, alter_sys=True) + called_func = 'run_module' + else: + first_arg = str(module.path) + runner = runpy.run_path + called_func = 'run_path' + + # Check that the code is run + module.install(local=True, deps_only=not as_module) + with restore_argv(): + sys.argv[:] = [first_arg, f'--length={nnums}', '-n', str(nprocs)] + runner(first_arg, run_name='__main__') + stdout = capsys.readouterr().out + assert stdout.rstrip('\n') == str(nnums * (nnums + 1) // 2) + + # Check that profiler has received the appropriate targets + funcs = [func.__name__ for func in getattr(cache.profiler, 'functions')] + assert funcs == expected_funcs + + # Check that calls in the current process are profiled iif the + # correct file is executed + with StringIO() as sio: + cache.profiler.print_stats(sio) + stats = sio.getvalue() + _check_output(stats, 'EXT-INVOCATION', num_invocations) + _check_output(stats, 'EXT-LOOP', num_loops) + + # Check the debug-log entries are correctly gathered + log = [entry.msg.lower() for entry in cache._gather_debug_log_entries()] + if debug: + assert ( + sum(1 for msg in log if called_func in msg if 'call' in msg) + == 1 + ), log + assert ( + sum(1 for msg in log if 'exec' in msg if 'call' in msg) + == int(run_profiled_code) + ), log + else: + assert not log # No logs written + + +# ========================= Integration tests ========================== def _get_mp_start_method_fuzzer(label_name: str) -> _Params: From 82ed140b5cf6a392bc260a140df61508e46dece4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 16:37:07 +0200 Subject: [PATCH 051/165] Minor refactoring line_profiler/_child_process_profiling/cache.py::LineProfilingCache __repr__() Now hiding cluttering fields from the repr _setup_in_main_process() Fixed bug where `wrap_os_fork` is not respected _write_pth_hook() Now using `.patch()` instead of manually overriding `os.fork()` patch() New convenience method for replacing/setting an attribute on an object line_profiler/_child_process_profiling/multiprocessing_patches.py _apply_mp_patches() - Updated call signature - Simplified implementation to use `lp_cache.patch()` --- .../_child_process_profiling/cache.py | 46 ++++++++++++------- .../multiprocessing_patches.py | 42 ++++++----------- 2 files changed, 43 insertions(+), 45 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 6cebe16f..3d2484b7 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -249,9 +249,11 @@ class LineProfilingCache: insert_builtin: bool = True debug: bool = diagnostics.DEBUG - profiler: LineProfiler | None = dataclasses.field(default=None, init=False) + profiler: LineProfiler | None = dataclasses.field( + default=None, init=False, repr=False, + ) _cleanup_stacks: dict[float, list[Callable[[], Any]]] = dataclasses.field( - default_factory=dict, init=False, + default_factory=dict, init=False, repr=False, ) _loaded_instance: ClassVar[Self | None] = None @@ -526,7 +528,8 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: pth_hook = import_module(this_subpkg + '.pth_hook') pth_hook.write_pth_hook(self) - self._wrap_os_fork() + if wrap_os_fork: + self._wrap_os_fork() mp_patches = import_module(this_subpkg + '.multiprocessing_patches') mp_patches.apply(self) @@ -638,20 +641,29 @@ def wrapper() -> int: forked._setup_in_child_process(False, 'fork', self.profiler) return result - # Note: type checkers have vastly different opinions on - # `os.fork = wrapper`: - # - `ty` wouldn't shut up about shadowing unless we explicitly - # type-annotate the assignment (error type: - # "invalid-assignment") - # - `mypy` is cool with the bare assignment, but complains about - # declaring types in "assignment to non-self-attribute" - # (error type: "misc") - # The only way to satisfy both seems to be either an - # unqualified "type: ignore" comment, or circumventing the - # attribute checks by `setattr()` or assigning to the module - # namespace dict. - os.fork = wrapper # type: ignore - self.add_cleanup(setattr, os, 'fork', fork) + self.patch(os, 'fork', wrapper, name='os') + + def patch( + self, obj: Any, attr: str, value: Any, *, + name: str | None = None, cleanup: bool = True, + ) -> None: + """ + Patch ``attr`` on ``obj`` with ``value``. If ``cleanup`` is + true, register a cleanup callback to either reset or delete the + attribute. + """ + add_cleanup = self.add_cleanup if cleanup else (lambda *_, **__: None) + try: + old = getattr(obj, attr) + except AttributeError: + add_cleanup(delattr, obj, attr) + else: + add_cleanup(setattr, obj, attr, old) + setattr(obj, attr, value) + if name is None: + name = repr(obj) + msg = 'Patched `{}.{}` -> `{}`'.format(name, attr, value) + self._debug_output(msg) def make_tempfile(self, **kwargs) -> Path: """ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 24029022..ca7be7cf 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -42,7 +42,7 @@ T = TypeVar('T') PS = ParamSpec('PS') -_PATCHED_MARKER = '_line_profiler_patched_multiprocessing' +_PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' _PROCESS_TERM_LOCK_LOC = '_line_profiler_process_terminate_lock' @@ -72,7 +72,7 @@ def __setstate__(*_) -> None: # In a child process, we don't care about polluting the # `multiprocessing` namespace, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, _no_op) + _apply_mp_patches(lp_cache, add_cleanup=False) class _Poller: @@ -293,7 +293,7 @@ def wrap_start( @_method_wrapper def wrap_terminate( - cache: LineProfilingCache, # TODO: config timeouts + cache: LineProfilingCache, vanilla_impl: Callable[[BaseProcess], None], self: BaseProcess, ) -> None: @@ -511,29 +511,15 @@ def apply(lp_cache: LineProfilingCache) -> None: - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` """ if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, lp_cache.add_cleanup) + _apply_mp_patches(lp_cache) def _apply_mp_patches( lp_cache: LineProfilingCache, - add_cleanup: Callable[..., Any], + add_cleanup: bool = True, debug: bool | None = None, ) -> None: - def replace( - obj: Any, attr: str, value: Any, obj_name: str | None = None, - ) -> None: - try: - old = getattr(obj, attr) - except AttributeError: - add_cleanup(delattr, obj, attr) - else: - add_cleanup(setattr, obj, attr, old) - setattr(obj, attr, value) - if obj_name is None: - obj_name = repr(obj) - lp_cache._debug_output('Patched `{}.{}` -> `{}`'.format( - obj_name, attr, value, - )) + replace = partial(lp_cache.patch, cleanup=add_cleanup) # Patch `multiprocessing.process.BaseProcess` methods # Note: the type checkers seem to need some help figuring the @@ -551,9 +537,10 @@ def replace( continue Class = getattr(mod, target) name = f'{Class.__module__}.{Class.__qualname__}' + patch_class = partial(replace, Class, name=name) for method, method_wrapper in patches.items(): vanilla = getattr(Class, method) - replace(Class, method, method_wrapper(vanilla), name) + patch_class(method, method_wrapper(vanilla)) # Patch `multiprocessing.spawn` try: @@ -561,14 +548,15 @@ def replace( except ImportError: # Incompatible platforms pass else: + patch_spawn = partial(replace, spawn, name=spawn.__name__) # Patch `get_preparation_data()` gpd_wrapper = wrap_get_preparation_data(spawn.get_preparation_data) - replace(spawn, 'get_preparation_data', gpd_wrapper, spawn.__name__) + patch_spawn('get_preparation_data', gpd_wrapper) # Patch `runpy` (do it locally instead of tempering with the # global `runpy` mmodule) if hasattr(spawn, 'runpy'): runpy_wrapper = create_runpy_wrapper(lp_cache) - replace(spawn, 'runpy', runpy_wrapper, spawn.__name__) + patch_spawn('runpy', runpy_wrapper) # Intercept `multiprocessing` debug messages if debug is None: @@ -576,6 +564,7 @@ def replace( if debug: from multiprocessing import util + patch_util = partial(replace, util, name=util.__name__) for logging_func in [ 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', ]: @@ -583,13 +572,10 @@ def replace( vanilla = getattr(util, logging_func) except AttributeError: continue - replace( - util, logging_func, partial(tee_log, vanilla, logging_func), - 'multiprocessing.util', - ) + patch_util(logging_func, partial(tee_log, vanilla, logging_func)) # Mark `multiprocessing` as having been patched - replace(multiprocessing, _PATCHED_MARKER, True) + replace(multiprocessing, _PATCHED_MARKER, True, name='multiprocessing') def _no_op(*_, **__) -> None: From acd48b32ae8b5cf278f572fa62a66805547950bb Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 22:54:57 +0200 Subject: [PATCH 052/165] More in-process unit tests for `test_child_procs` tests/test_child_procs.py create_cache() - Now permits toggling whether the `curated_profiler` is supplied by a private argument - Now permits passing a boolean to `preimports_module`, processing the `profiling_targets` to write a preimports-module tempfile on-the-fly - Added callbacks for outputting the `.profiler` state, `.gather_stats()` results, and debug logs test_*() - Added more robust protection of global states (e.g. for modules prone to being patched) - Refactored debug-log output matching test_cache_dump_load() New unit test for `line_profiler._child_process_profiling.cache .LineProfilingCache.dump()` and `.load()` test_cache_cleanup_order() New unit test for `~~.LineProfilingCache.cleanup()` test_cache_setup_{main,child}_process() New unit tests for `~~.LineProfilingCache._setup_in_{main,child}_process()` test_apply_mp_patches() New unit test for `line_profiler._child_process_profiling .multiprocessing_patches.apply()` TODO: One subtest in `test_apply_mp_patches()` involving `forkserver` runs fine in isolation and errors out otherwise; investigate --- tests/test_child_procs.py | 739 +++++++++++++++++++++++++++++++++++--- 1 file changed, 689 insertions(+), 50 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 91548298..42c6e12b 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1,24 +1,31 @@ from __future__ import annotations +import ast +import dataclasses import enum -import multiprocessing +import inspect +import multiprocessing.pool import os +import re import shlex import subprocess import sys +import sysconfig +from abc import ABC, abstractmethod from collections.abc import ( Callable, Collection, Generator, Iterable, Mapping, Sequence, ) -from dataclasses import dataclass -from functools import partial +from contextlib import ExitStack +from functools import lru_cache, partial, wraps from io import StringIO +from importlib import import_module from pathlib import Path -from sysconfig import get_path +from runpy import run_path from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic -from typing import Any, Literal, TypeVar, cast, final, overload -from typing_extensions import Self +from typing import Any, Generic, Literal, TypeVar, cast, final, overload +from typing_extensions import Self, ParamSpec from uuid import uuid4 import pytest @@ -28,11 +35,15 @@ from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) -from line_profiler.curated_profiling import CuratedProfilerContext +from line_profiler.curated_profiling import ( + CuratedProfilerContext, ClassifiedPreimportTargets, +) from line_profiler.line_profiler import LineProfiler, LineStats T = TypeVar('T') +TCtx_ = TypeVar('TCtx_') +PS = ParamSpec('PS') C = TypeVar('C', bound=Callable[..., Any]) NUM_NUMBERS = 100 @@ -137,7 +148,7 @@ def main(args: list[str] | None = None) -> None: # ============================== Fixtures ============================== -@dataclass +@dataclasses.dataclass class _ModuleFixture: """ Convenience wrapper around a Python source file which represents an @@ -269,7 +280,7 @@ def test_module_clone( @pytest.fixture def create_cache( tmp_path_factory: pytest.TempPathFactory, - curated_profiler: LineProfiler, + request: pytest.FixtureRequest, ) -> Generator[Callable[..., LineProfilingCache], None, None]: """ Wrapper around the :py:class:`LineProfilingCache` instantiator @@ -278,35 +289,121 @@ def create_cache( - Automatically creates a tempdir and provides it as the :py:attr:`LineProfilingCache.cache_dir`, - - Automatically creates an instance of :py:class:`LineProfiler` that - is curated by a :py:class:`CuratedProfilerContext` and provides it - as the :py:attr:`LineProfilingCache.profiler`, and + - Extends the argument ``preimports_module`` to allow for taking + boolean values: + + - ``True``: a temporary preimports module is automatically written + based on ``profiling_targets`` and supplied to the base + constructor. + + - ``False``: equivalent to ``None``. + + - Unless the argument ``_use_curated_profiler: bool = True`` is set + to :py:const`True`, automatically creates an instance of + :py:class:`LineProfiler` that is curated by a + :py:class:`CuratedProfilerContext` and provides it as the + :py:attr:`LineProfilingCache.profiler`, and - At teardown: + - Removes tempdirs and tempfiles generated. + - Restores the value of the class' internal reference to the :py:meth:`LineProfilingCache.load`-ed instance. - Calls the `.cleanup()` method of each instance created. + + - Prints these diagnostics for each instance: + + - The stats on the ``.profiler`` associated with each instance + (if any) + + - The stats gathered by + :py:meth:`LineProfilingCache.gather_stats()` + + - The debug logs (if ``.debug`` is true) """ - def instantiate(**kwargs) -> LineProfilingCache: + def instantiate( + *, + profiling_targets: Collection[str] = (), + preimports_module: os.PathLike[str] | str | bool | None = None, + _use_curated_profiler: bool = True, + **kwargs + ) -> LineProfilingCache: tmpdir = tmp_path_factory.mktemp('my_cache_dir') - cache = LineProfilingCache(tmpdir, **kwargs) - cache.profiler = curated_profiler + pim: os.PathLike[str] | str | None + if preimports_module in (True, False): + if preimports_module: + targets = ( + ClassifiedPreimportTargets.from_targets(profiling_targets) + ) + if targets: + pim = tmpdir / 'preimports.py' + with pim.open(mode='w') as fobj: + targets.write_preimport_module(fobj) + else: + pim = None + else: + pim = None + else: + # The type checker needs some convincing... + assert not isinstance(preimports_module, bool) + pim = preimports_module + cache = LineProfilingCache( + tmpdir, + profiling_targets=profiling_targets, + preimports_module=pim, + **kwargs, + ) + if _use_curated_profiler: + cache.profiler = request.getfixturevalue('curated_profiler') instances.append(cache) return cache - old_value = LineProfilingCache._loaded_instance + def print_result( + cache: LineProfilingCache, topic: str, result: str, + ) -> None: + header = f'{topic} (cache instance {id(cache):#x}' + print(header, indent(result, ' '), sep='\n') + + def print_profiler_stats(cache: LineProfilingCache) -> None: + if cache.profiler is None: + result = '' + else: + with StringIO() as sio: + cache.profiler.print_stats(sio) + result = sio.getvalue() + print_result(cache, 'Native profiler stats', result) + + def print_gathered_stats(cache: LineProfilingCache) -> None: + with StringIO() as sio: + cache.gather_stats().print(sio) + result = sio.getvalue() + print_result(cache, 'Gathered profiler stats', result) + + def print_debug_logs(cache: LineProfilingCache) -> None: + result = '\n'.join( + entry.to_text() for entry in cache._gather_debug_log_entries() + ) + print_result(cache, 'Gathered debug logs', result) + instances: list[LineProfilingCache] = [] + handlers: list[Callable[[LineProfilingCache], None]] + handlers = [print_profiler_stats, print_gathered_stats, print_debug_logs] try: - yield instantiate + with _preserve_obj_attributes( + LineProfilingCache, ['_loaded_instance'], + ): + yield instantiate finally: - LineProfilingCache._loaded_instance = old_value # type: ignore for cache in instances: - try: - cache.cleanup() - except Exception: - pass + callbacks: list[Callable[[], Any]] = [cache.cleanup] + callbacks.extend(partial(func, cache) for func in handlers) + for callback in callbacks: + try: + callback() + except Exception: + pass @pytest.fixture @@ -353,7 +450,7 @@ def rich_message(self) -> str: @final -@dataclass +@dataclasses.dataclass class _Params: """ Convenience wrapper around :py:func:`pytest.mark.parametrize`. @@ -536,6 +633,207 @@ def new( return cls(param_list, value_tuple_list, default_values) +class _CallableContextManager(ABC, Generic[TCtx_]): + @abstractmethod + def __enter__(self) -> TCtx_: + ... + + @abstractmethod + def __exit__(self, *a, **k) -> Any: + ... + + def __call__(self, func: Callable[PS, T]) -> Callable[PS, T]: + """ + Wrap ``func()`` so that its calls always happen in the context + of the instance. + """ + @wraps(func) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + with self: + return func(*args, **kwargs) + + return wrapper + + +class _preserve_obj_attributes(_CallableContextManager[dict[str, Any]]): + def __init__(self, obj: Any, attrs: Collection[str]) -> None: + self.obj = obj + self.attrs = set(attrs) + self._callbacks: list[Callable[[], None]] = [] + + def __enter__(self) -> dict[str, Any]: + def delete(obj: Any, attr: str) -> None: + try: + delattr(obj, attr) + except AttributeError: + pass + + result: dict[str, Any] = {} + for attr in self.attrs: + old = getattr(self.obj, attr, _NotSupplied.NOT_SUPPLIED) + if old is _NotSupplied.NOT_SUPPLIED: + callback = partial(delete, self.obj, attr) + else: + callback = partial(setattr, self.obj, attr, old) + result[attr] = old + self._callbacks.append(callback) + return result + + def __exit__(self, *_, **__) -> None: + for callback in self._callbacks[::-1]: + try: + callback() + except Exception: + pass + + +class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): + """ + Example: + >>> from functools import wraps + >>> from line_profiler.curated_profiling import ( + ... CuratedProfilerContext, + ... ) + >>> from line_profiler import line_profiler + + >>> assert not hasattr(CuratedProfilerContext, 'foo') + >>> old_main = line_profiler.main + >>> + >>> + >>> def foo(_) -> None: + ... pass + ... + >>> + >>> @wraps(old_main) + ... def main(*a, **k): + ... return old_main(*a, **k) + ... + >>> + >>> with _preserve_attributes({ + ... 'line_profiler.curated_profiling' + ... '.CuratedProfilerContext': {'foo'}, + ... 'line_profiler.line_profiler': {'main'}, + ... }) as old_values: + ... assert old_values == { + ... 'line_profiler.curated_profiling' + ... '.CuratedProfilerContext': { + ... 'foo': _NotSupplied.NOT_SUPPLIED, + ... }, + ... 'line_profiler.line_profiler': {'main': old_main}, + ... } + ... CuratedProfilerContext.foo = foo + ... line_profiler.main = main + ... print('ok') + ... + ok + >>> assert not hasattr(CuratedProfilerContext, 'foo') + >>> assert main is not line_profiler.main is old_main + """ + def __init__(self, targets: Mapping[str, Collection[str]]) -> None: + self.targets = { + target: set(attrs) for target, attrs in targets.items() + } + self._stacks: list[ExitStack] = [] + + def __enter__(self) -> dict[str, dict[str, Any]]: + stack = ExitStack() + self._stacks.append(stack) + result: dict[str, Any] = {} + for target, attrs in self.targets.items(): + result[target] = stack.enter_context(_preserve_obj_attributes( + _import_target(target), attrs, + )) + return result + + def __exit__(self, *_, **__) -> None: + self._stacks.pop().close() + + +class _preserve_pth_files(_CallableContextManager[frozenset[str]]): + def __enter__(self) -> frozenset[str]: + self.old = self.get_pth_files() + return self.old + + def __exit__(self, *_, **__) -> None: + for new_pth_file in self.get_pth_files() - self.old: + (self._get_path() / new_pth_file).unlink(missing_ok=True) + del self.old + + @classmethod + def get_pth_files(cls, name_only: bool = True) -> frozenset[str]: + return frozenset( + pth.name if name_only else str(pth) + for pth in cls._get_path().glob('*.pth') + ) + + @staticmethod + def _get_path() -> Path: + return Path(sysconfig.get_path('purelib')) + + +def _import_target(target: str) -> Any: + try: + return import_module(target) + except ImportError: # Not a module + assert '.' in target + module, _, attr = target.rpartition('.') + return getattr(import_module(module), attr) + + +def _search_cache_logs( + cache: LineProfilingCache, + expecting_logs: bool, + patterns: Mapping[str, bool] | Collection[str], + match_individual_messages: bool = False, + flags: int = 0, +) -> None: + entries = cache._gather_debug_log_entries() + if bool(entries) != expecting_logs: + raise ResultMismatch( + 'logs' if expecting_logs else 'no logs', + repr(entries) if entries else 'nothing' + ) + if not expecting_logs: + return + text_chunks: list[str] = [entry.to_text() for entry in entries] + if not match_individual_messages: + text_chunks = ['\n'.join(text_chunks)] + if isinstance(patterns, Mapping): + to_match: dict[str, bool] = { + str(pat): bool(should_match) + for pat, should_match in patterns.items() + } + else: + to_match = dict.fromkeys(patterns, True) + for pat, should_match in to_match.items(): + pattern = re.compile(pat, flags) + if any(pattern.search(chunk) for chunk in text_chunks) == should_match: + continue + raise ResultMismatch( + f'pattern {pattern!r} to {"" if should_match else "not "}match ' + f'{cache!r}\'s logs: {text_chunks!r}' + ) + + +@lru_cache() +def _find_return_lines(func: str) -> list[int]: + class FindReturns(ast.NodeVisitor): + def __init__(self) -> None: + self.found: set[int] = set() + + def visit_Return(self, node: ast.Return) -> None: + self.found.add(node.lineno) + self.generic_visit(node) + + func_obj = _import_target(func) + assert inspect.isfunction(func_obj) + lines, start = inspect.getsourcelines(func_obj) + tree = ast.parse(''.join(lines)) + finder = FindReturns() + finder.visit(tree) + return sorted(lineno + start - 1 for lineno in finder.found) + + # `shlex.join()` doesn't work properly on Windows, so use # `subprocess.list2cmdline()` instead; # though an "intentionally" undocumented API (cpython issue #10308), @@ -683,11 +981,6 @@ def _run_test_module( profliing_stats (LineStats | None): Line-profiling stats (where available) """ - def get_pth_files(pattern: str = '*.pth') -> set[str]: - return {path.name for path in lib_path.glob(pattern)} - - lib_path = Path(get_path('purelib')) - if isinstance(runner, str): runner_args: list[str] = [runner] else: @@ -728,7 +1021,7 @@ def get_pth_files(pattern: str = '*.pth') -> set[str]: runner_args.extend(['--outfile', outfile]) if debug_log: runner_args.extend(['--debug-log', debug_log]) - old_pth_files = get_pth_files() + old_pth_files = _preserve_pth_files.get_pth_files() proc = run_helper( runner_args, test_args, test_module, text=True, capture_output=True, check=(check and not fail), @@ -750,7 +1043,7 @@ def get_pth_files(pattern: str = '*.pth') -> set[str]: ) # - Temporary `.pth` file(s) created by `~~.pth_hook` has # been cleaned up - assert get_pth_files() == old_pth_files + assert _preserve_pth_files.get_pth_files() == old_pth_files # - Profiling results are written to the specified file prof_result: LineStats | None = None if outfile is None: @@ -803,16 +1096,33 @@ def _check_output(output: str, tag: str, nhits: int) -> None: # ============================= Unit tests ============================= - -@pytest.mark.parametrize( - ('run_profiled_code', 'label1'), - [(True, 'run-profiled'), (False, 'run-unrelated')]) -@pytest.mark.parametrize( - ('as_module', 'label2'), - [(True, 'run_module'), (False, 'run_path')]) -@pytest.mark.parametrize( - ('debug', 'label3'), - [(True, 'with-debug'), (False, 'no-debug')]) +# XXX: Tests in this section concerns implementation details, and the +# tested APIs and behaviors MUST NOT be relied upon by end-users. + + +_MP_PATCHED_NAMES = { + 'multiprocessing.process.BaseProcess': frozenset({ + '_bootstrap', 'start', 'terminate' + }), + 'multiprocessing.spawn': frozenset({'get_preparation_data', 'runpy'}), +} +# NOTE: we need a function which isn't used by the codebase itself +# (esp. during cache cleanup); otherwise the profiling results may +# be skewed +_SAFE_TARGET = 'calendar.weekday' +_SAFE_TARGET_ARGS = [ + (1970, 1, 1), + (2000, 12, 31), + (2008, 9, 16), # Where the repo started +] + + +@pytest.mark.parametrize(('run_profiled_code', 'label1'), + [(True, 'run-profiled'), (False, 'run-unrelated')]) +@pytest.mark.parametrize(('as_module', 'label2'), + [(True, 'run_module'), (False, 'run_path')]) +@pytest.mark.parametrize(('debug', 'label3'), + [(True, 'with-debug'), (False, 'no-debug')]) def test_runpy_patches( capsys: pytest.CaptureFixture[str], ext_module: _ModuleFixture, @@ -889,19 +1199,348 @@ def __exit__(self, *_, **__) -> None: _check_output(stats, 'EXT-LOOP', num_loops) # Check the debug-log entries are correctly gathered - log = [entry.msg.lower() for entry in cache._gather_debug_log_entries()] + _search_cache_logs( + cache, + debug, + { + rf'calling .*{called_func}\(': True, + r'calling .*exec\(': run_profiled_code, + }, + match_individual_messages=True, + flags=re.IGNORECASE, + ) + + +def test_cache_dump_load( + create_cache: Callable[..., LineProfilingCache], +) -> None: + """ + Test that: + + - We can round-trip the cache via :py:meth:`LineProfilingCache.dump` + and :py:meth:`LineProfilingCache.load` + + - The same instance is :py:meth:`LineProfilingCache.load`-ed in + subsequent calls + """ + original = create_cache( + profiling_targets=['foo', 'bar', 'baz'], main_pid=123456, + ) + envvars: set[str] = set(os.environ) + try: + original.inject_env_vars() # Needed for `.load()` + try: + # Env vars should be inserted + assert set(os.environ) == envvars.union(original.environ) > envvars + original.dump() + loaded = original.load() + reloaded = original.load() + assert original is not loaded is reloaded + # Compare init fields + for field in dataclasses.fields(LineProfilingCache): + if not field.init: + continue + assert ( + getattr(original, field.name) + == getattr(loaded, field.name) + ) + finally: # Explicitly cleanup + original.cleanup() + finally: # Env vars restored after cleanup + assert set(os.environ) == envvars + + +@pytest.mark.parametrize(('debug', 'label'), + [(True, 'with-debug'), (False, 'no-debug')]) +def test_cache_cleanup_order( + create_cache: Callable[..., LineProfilingCache], debug: bool, label: str, +) -> None: + """ + Test that :py:meth`LineProfilingCache.cleanup` executes the cleanup + callback stacks in order. + """ + strings: list[str] = [] + cache = create_cache(debug=debug) + + cache.add_cleanup(strings.append, 'first') + # Decreased priority + cache._add_cleanup(strings.append, 1, 'second') + # Increased priority + cache._add_cleanup(strings.append, -1, 'third') + cache.add_cleanup(strings.append, 'fourth') + + cache.cleanup() + assert strings == ['third', 'fourth', 'first', 'second'] + + pattern = '\n'.join( + rf'.*succeeded \({i + 1}/{len(strings)}\): .*append.*{string!r}.*' + for i, string in enumerate(strings) + ) + _search_cache_logs(cache, debug, [pattern]) + + +@pytest.mark.parametrize(('wrap_os_fork', 'label1'), + [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) +@pytest.mark.parametrize(('debug', 'label2'), + [(True, 'with-debug'), (False, 'no-debug')]) +def test_cache_setup_main_process( + create_cache: Callable[..., LineProfilingCache], + wrap_os_fork: bool, + debug: bool, + label1: str, label2: str, +) -> None: + """ + Test that :py:meth:`LineProfilingCache._setup_in_main_process` works + as expected. + """ + cache = create_cache(debug=debug) + patches: dict[str, dict[str, bool]] = {'os': {'fork': wrap_os_fork}} + patches.update( + (target, dict.fromkeys(attrs, True)) + for target, attrs in _MP_PATCHED_NAMES.items() + ) + targets: dict[str, Any] = { + target: _import_target(target) for target in patches + } + with ExitStack() as stack: + patched = stack.enter_context(_preserve_attributes(patches)) + original_pths = stack.enter_context(_preserve_pth_files()) + cache._setup_in_main_process(wrap_os_fork=wrap_os_fork) + # There should be exactly one extra `.pth` file + new_pth_hook, = _preserve_pth_files.get_pth_files() - original_pths + # Check whether the patches are applied + for target, maybe_patches in patches.items(): + obj = targets[target] + for attr, is_patched in maybe_patches.items(): + assert ( + getattr(obj, attr) is patched[target][attr] + ) != is_patched + # Check whether the patches are reversed + cache.cleanup() + for target, orig_attrs in patched.items(): + obj = targets[target] + for attr, orig_value in orig_attrs.items(): + if orig_value is _NotSupplied.NOT_SUPPLIED: + assert not hasattr(obj, attr) + else: + assert getattr(obj, attr) is orig_value + # Check that the instance is set as the `.load()`-ed one + assert cache is cache.load() + + # Check the debug-log output + patterns: dict[str, bool] = dict.fromkeys( + [ + r'\(main process\)', + r'Injecting env var.*\$\{LINE_PROFILER_\w+\}', + re.escape(new_pth_hook), + ], + True, + ) + for target, maybe_patches in patches.items(): + patterns.update( + ('Patched.*' + re.escape(f'{target}.{attr}'), is_patched) + for attr, is_patched in maybe_patches.items() + ) + _search_cache_logs(cache, debug, patterns) + + +@pytest.mark.parametrize(('wrap_os_fork', 'label1'), + [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) +@pytest.mark.parametrize(('preimports', 'label2'), + [(True, 'with-preimports'), (False, 'no-preimports')]) +@pytest.mark.parametrize(('new_profiler', 'label3'), + [(True, 'no-profiler'), (False, 'with-profiler')]) +@pytest.mark.parametrize(('debug', 'label4'), + [(True, 'with-debug'), (False, 'no-debug')]) +def test_cache_setup_child( + create_cache: Callable[..., LineProfilingCache], + curated_profiler: LineProfiler, + wrap_os_fork: bool, + preimports: bool, + new_profiler: bool, + debug: bool, + label1: str, label2: str, label3: str, label4: str, +) -> None: + """ + Test that :py:meth:`LineProfilingCache._setup_in_child_process` + works as expected. + """ + def list_profiled_funcs() -> list[str]: + return [ + f'{func.__module__}.{func.__qualname__}' + for func in getattr(cache.profiler, 'functions', []) + ] + + # Make sure we get a different PID from the current process + curr_pid = os.getpid() + main_pid = (curr_pid - 42) % (2 * 16) + assert main_pid != curr_pid + + cache = create_cache( + profiling_targets=[_SAFE_TARGET], + preimports_module=preimports, + _use_curated_profiler=not new_profiler, + main_pid=main_pid, + debug=debug, + ) + assert (cache.profiler is None) == new_profiler + + seen_funcs = list_profiled_funcs() + if preimports: + preimport_targets = list(cache.profiling_targets) + else: + preimport_targets = [] + + with _preserve_obj_attributes(os, ['fork']) as preserved: + old_fork = preserved['fork'] + # Check that we're only setting up if there isn't already a + # profiler + assert cache._setup_in_child_process( + wrap_os_fork=wrap_os_fork, context='test_cache_setup_child', + ) == new_profiler + assert cache.profiler + if not new_profiler: + return + + # Check that the profiler has been presented with the profiling + # target + assert list_profiled_funcs() == (seen_funcs + preimport_targets) + + # Check that on cache cleanup: + # - Profiling data is collected + # - `os.fork()` is restored + _import_target(_SAFE_TARGET)(*_SAFE_TARGET_ARGS[0]) + stats = cache.profiler.get_stats() + for callback, has_prof_data, fork_patched in [ + (lambda: None, False, wrap_os_fork), + (cache.cleanup, preimports, False), + ]: + callback() + gathered = cache.gather_stats() + assert any(gathered.timings.values()) == has_prof_data, gathered + assert (os.fork is not old_fork) == fork_patched + + # Check that profiling results have been written to the cache + # directory + stats_file, = Path(cache.cache_dir).glob('*.lprof') + assert LineStats.from_files(stats_file) == stats == gathered + + # Check the debug-log output + patterns = { + f'Set up .*profiler.* {id(cache.profiler):#x}': True, + 'Loading preimports': preimports, + 'Created .*' + re.escape(stats_file.name): True, + 'Cleanup succeeded.*: .*dump_stats': True, + 'Loading results .*' + re.escape(stats_file.name): True, + } + _search_cache_logs(cache, debug, patterns) + + +@pytest.mark.parametrize('start_method', + ['fork', 'forkserver', 'spawn', 'dummy']) +@pytest.mark.parametrize(('debug', 'label'), + [(True, 'with-debug'), (False, 'no-debug')]) +@_preserve_pth_files() +@_preserve_attributes(_MP_PATCHED_NAMES) +def test_apply_mp_patches( + tmp_path_factory: pytest.TempPathFactory, + create_cache: Callable[..., LineProfilingCache], + start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], + debug: bool, + label: str, +) -> None: + """ + Test that :py:func:`line_profiler._child_process_profiling\ +.multiprocessing_patches.apply` + works as expected. + """ + def gpd_wrapper(name: str) -> dict[str, Any]: + impl = _import_target('multiprocessing.spawn.get_preparation_data') + return impl(name) + + def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: + try: + LineStats.from_files(path, on_defective='error') + except Exception: + return False + return True + + config: Path | None = None if debug: - assert ( - sum(1 for msg in log if called_func in msg if 'call' in msg) - == 1 - ), log - assert ( - sum(1 for msg in log if 'exec' in msg if 'call' in msg) - == int(run_profiled_code) - ), log + config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' + config.write_text( + '[tool.line_profiler.multiprocessing]\n' + 'intercept_logs = true' + ) + + cache = create_cache( + profiling_targets=[_SAFE_TARGET], + preimports_module=True, + config=config, + debug=True, + ) + old_data = gpd_wrapper('my_name') + # Note: + # - The reversibility of the patches have already been tested in + # `test_cache_setup_main_process()`, so we just actually test the + # patched-in components themselves here. + # - `._setup_in_main_process()` doesn't include actually doing the + # preimports. To may the results more consistent between + # `start_method='dummy'` and the others, manually do them below. + cache._setup_in_main_process() # This calls `apply()` + assert cache.preimports_module is not None + run_path(str(cache.preimports_module), {'profile': cache.profiler}) + + # Check for the insertion of the `PickleHook` by the patched + # `get_preparation_data()` + new_data = gpd_wrapper('my_name') + inserted_key, = set(new_data) - set(old_data) + assert type(new_data[inserted_key]).__name__ == 'PickleHook' + + func = _import_target(_SAFE_TARGET) + return_lines = _find_return_lines(_SAFE_TARGET) + Pool: Callable[..., multiprocessing.pool.Pool] + if start_method == 'dummy': + Pool = _import_target('multiprocessing.dummy.Pool') + elif start_method not in START_METHODS: + pytest.skip( + f'`multiprocessing` start method {start_method!r} ' + 'not available on the platform' + ) else: - assert not log # No logs written + Pool = multiprocessing.get_context(start_method).Pool + + with Pool(2) as pool: + pool.starmap(func, _SAFE_TARGET_ARGS) + pool.close() + pool.join() + + # Check that calls in children are traced + if start_method == 'dummy': + assert cache.profiler is not None + stats = cache.profiler.get_stats() + else: + stats = cache.gather_stats() + line_entries = stats.timings[ + inspect.getfile(func), inspect.getsourcelines(func)[1], func.__name__, + ] + num_returns = sum( + nhits for lineno, nhits, _ in line_entries if lineno in return_lines + ) + assert num_returns == len(_SAFE_TARGET_ARGS) + + # Check the debug logs to see if we have done everything right, esp. + # the logging interception part not covered by other tests + patterns: dict[str, bool] = { + 'Cleanup succeeded.*: .*dump_stats.*' + re.escape(path.name): True + for path in Path(cache.cache_dir).glob('*.lprof') + if is_valid_stats_file(path) + } + patterns[re.escape('`multiprocessing` logging (debug)')] = debug + _search_cache_logs(cache, True, patterns) + +# XXX: End of tests for implementation details # ========================= Integration tests ========================== From e91719df48351c04c7967b4048b396970c95d554 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 20 Apr 2026 00:03:26 +0200 Subject: [PATCH 053/165] WIP debug the new unit tests tests/test_child_procs.py Added debugging output to the private context managers for preserving attributes and purging stray .pth files --- tests/test_child_procs.py | 81 ++++++++++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 19 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 42c6e12b..c309973c 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -49,7 +49,7 @@ NUM_NUMBERS = 100 NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) -_WRITE_DEBUG_LOGS = True +_DEBUG = True def strip(s: str) -> str: @@ -363,7 +363,7 @@ def instantiate( def print_result( cache: LineProfilingCache, topic: str, result: str, ) -> None: - header = f'{topic} (cache instance {id(cache):#x}' + header = f'{topic} (cache instance {id(cache):#x}):' print(header, indent(result, ' '), sep='\n') def print_profiler_stats(cache: LineProfilingCache) -> None: @@ -382,9 +382,12 @@ def print_gathered_stats(cache: LineProfilingCache) -> None: print_result(cache, 'Gathered profiler stats', result) def print_debug_logs(cache: LineProfilingCache) -> None: - result = '\n'.join( - entry.to_text() for entry in cache._gather_debug_log_entries() - ) + if cache.debug: + result = '\n'.join( + entry.to_text() for entry in cache._gather_debug_log_entries() + ) + else: + result = '' print_result(cache, 'Gathered debug logs', result) instances: list[LineProfilingCache] = [] @@ -634,6 +637,8 @@ def new( class _CallableContextManager(ABC, Generic[TCtx_]): + debug: bool + @abstractmethod def __enter__(self) -> TCtx_: ... @@ -654,27 +659,53 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: return wrapper + def _debug(self, msg: str, **kwargs) -> None: + if not self.debug: + return + header = f'{os.environ["PYTEST_CURRENT_TEST"]}: {type(self).__name__}' + print(f'{header}: {msg}', **kwargs) + class _preserve_obj_attributes(_CallableContextManager[dict[str, Any]]): - def __init__(self, obj: Any, attrs: Collection[str]) -> None: + def __init__( + self, obj: Any, attrs: Collection[str], debug: bool = _DEBUG, + ) -> None: self.obj = obj self.attrs = set(attrs) self._callbacks: list[Callable[[], None]] = [] + self.debug = debug def __enter__(self) -> dict[str, Any]: - def delete(obj: Any, attr: str) -> None: + def get_repr(attr: str) -> str: try: - delattr(obj, attr) + value = getattr(self.obj, attr) + except ValueError: + return '' + else: + return repr(value) + + def delete(attr: str) -> None: + try: + self._debug('Deleted attr `.{} = {}` on `{!r}`'.format( + attr, get_repr(attr), self.obj, + )) + delattr(self.obj, attr) except AttributeError: pass + def reset(attr: str, value: Any) -> None: + self._debug('Reset attr `.{} = {} -> {!r}` on `{!r}`'.format( + attr, get_repr(attr), value, self.obj, + )) + setattr(self.obj, attr, value) + result: dict[str, Any] = {} for attr in self.attrs: old = getattr(self.obj, attr, _NotSupplied.NOT_SUPPLIED) if old is _NotSupplied.NOT_SUPPLIED: - callback = partial(delete, self.obj, attr) + callback = partial(delete, attr) else: - callback = partial(setattr, self.obj, attr, old) + callback = partial(reset, attr, old) result[attr] = old self._callbacks.append(callback) return result @@ -709,12 +740,13 @@ class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): ... return old_main(*a, **k) ... >>> - >>> with _preserve_attributes({ + >>> preserved = { ... 'line_profiler.curated_profiling' ... '.CuratedProfilerContext': {'foo'}, ... 'line_profiler.line_profiler': {'main'}, - ... }) as old_values: - ... assert old_values == { + ... } + >>> with _preserve_attributes(preserved, debug=False) as old: + ... assert old == { ... 'line_profiler.curated_profiling' ... '.CuratedProfilerContext': { ... 'foo': _NotSupplied.NOT_SUPPLIED, @@ -727,13 +759,19 @@ class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): ... ok >>> assert not hasattr(CuratedProfilerContext, 'foo') - >>> assert main is not line_profiler.main is old_main + >>> assert old_main is \ +old['line_profiler.line_profiler']['main'] + >>> assert old_main is line_profiler.main + >>> assert main is not line_profiler.main """ - def __init__(self, targets: Mapping[str, Collection[str]]) -> None: + def __init__( + self, targets: Mapping[str, Collection[str]], debug: bool = _DEBUG, + ) -> None: self.targets = { target: set(attrs) for target, attrs in targets.items() } self._stacks: list[ExitStack] = [] + self.debug = debug def __enter__(self) -> dict[str, dict[str, Any]]: stack = ExitStack() @@ -741,7 +779,7 @@ def __enter__(self) -> dict[str, dict[str, Any]]: result: dict[str, Any] = {} for target, attrs in self.targets.items(): result[target] = stack.enter_context(_preserve_obj_attributes( - _import_target(target), attrs, + _import_target(target), attrs, debug=self.debug, )) return result @@ -750,12 +788,16 @@ def __exit__(self, *_, **__) -> None: class _preserve_pth_files(_CallableContextManager[frozenset[str]]): + def __init__(self, debug: bool = _DEBUG) -> None: + self.debug = debug + def __enter__(self) -> frozenset[str]: self.old = self.get_pth_files() return self.old def __exit__(self, *_, **__) -> None: for new_pth_file in self.get_pth_files() - self.old: + self._debug(f'Deleting stray .pth file: {new_pth_file!r}') (self._get_path() / new_pth_file).unlink(missing_ok=True) del self.old @@ -1511,9 +1553,10 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: Pool = multiprocessing.get_context(start_method).Pool with Pool(2) as pool: - pool.starmap(func, _SAFE_TARGET_ARGS) + par_result = pool.starmap(func, _SAFE_TARGET_ARGS) pool.close() pool.join() + assert par_result == [func(*args) for args in _SAFE_TARGET_ARGS] # Check that calls in children are traced if start_method == 'dummy': @@ -1778,7 +1821,7 @@ def test_profiling_multiproc_script( nprocs=nprocs, timeout=timeout, debug_log=( - 'debug.log' if prof_child_procs and _WRITE_DEBUG_LOGS else None + 'debug.log' if prof_child_procs and _DEBUG else None ), ) @@ -1825,7 +1868,7 @@ def test_profiling_bare_python( out_file = temp_dir / 'out.lprof' debug_log_file = temp_dir / 'debug.log' - write_debug = _WRITE_DEBUG_LOGS and prof_child_procs + write_debug = _DEBUG and prof_child_procs cmd = [ 'kernprof', '-lv', '--preimports', f'--prof-mod={ext_module.name}', From d0dfd8aff6e86021bee14b783235285ffc43afb5 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 20 Apr 2026 03:56:12 +0200 Subject: [PATCH 054/165] Restart the fork-server process after each session line_profiler/_child_process_profiling/multiprocessing_patches.py ::apply() Now registering a cleanup callback to restart the stored `multiprocessing.forkserver.ForkServer` used globally, fixing the bug where child processes started by the `forkserver` start method cannot don't have profiling set up from the second profiling session/`LineProfilingCache` instance onwards --- .../multiprocessing_patches.py | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index ca7be7cf..4ae261f4 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -69,10 +69,8 @@ def __setstate__(*_) -> None: # up shop here. lp_cache = LineProfilingCache.load() lp_cache._setup_in_child_process(False, 'multiprocessing') - # In a child process, we don't care about polluting the - # `multiprocessing` namespace, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, add_cleanup=False) + _apply_mp_patches(lp_cache, main_process=False) class _Poller: @@ -509,6 +507,19 @@ def apply(lp_cache: LineProfilingCache) -> None: - :py:func:`multiprocessing.spawn.get_preparation_data` - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` + + Note: + When ``lp_cache.cleanup()`` is run, the global + :py:class:`multiprocessing.forkserver.ForkServer` object will be + rebooted. This is necessary because the server process staticly + inherits the environment when it is first spun up + (see :py:func:`multiprocessing.forkserver.ensure_running`). + Thus, if in the same Python process we ever start up two + separate profliing sessions managed by different caches, the + child processes forked from the server will fail to inherit the + updated environment variables injected by the newer cache + instance, leading to the setup code in this subpackage not being + loaded. """ if not getattr(multiprocessing, _PATCHED_MARKER, False): _apply_mp_patches(lp_cache) @@ -516,10 +527,12 @@ def apply(lp_cache: LineProfilingCache) -> None: def _apply_mp_patches( lp_cache: LineProfilingCache, - add_cleanup: bool = True, + main_process: bool = True, debug: bool | None = None, ) -> None: - replace = partial(lp_cache.patch, cleanup=add_cleanup) + # In a child process, we don't care about polluting the + # `multiprocessing` namespace, so don't bother with cleanup + replace = partial(lp_cache.patch, cleanup=main_process) # Patch `multiprocessing.process.BaseProcess` methods # Note: the type checkers seem to need some help figuring the @@ -574,6 +587,21 @@ def _apply_mp_patches( continue patch_util(logging_func, partial(tee_log, vanilla, logging_func)) + # Stop the current `ForkServer` server process as a part of cache + # cleanup (this uses `ForkServer._stop()` which is private API, but + # it's the same hack used in Python's own test suite -- see the + # comment to said method) + if main_process: + try: + from multiprocessing import forkserver + except ImportError: # Incompatible platform + pass + else: + server_instance: forkserver.ForkServer = forkserver._forkserver + stop = getattr(server_instance, '_stop', None) + assert callable(stop) # Appease the type checker + lp_cache.add_cleanup(stop) + # Mark `multiprocessing` as having been patched replace(multiprocessing, _PATCHED_MARKER, True, name='multiprocessing') From 565c30698934d5e8300b9fd751ea51c465a915bc Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 20 Apr 2026 05:45:37 +0200 Subject: [PATCH 055/165] Simplify `multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_start() Removed wrap_terminate() Now polling on the `Process.Popen` object instead of a lock file, simplifying the implementation wrap_bootstrap() Simplified implementation apply() No longer wrapping `BaseProcess.start` line_profiler/rc/line_profiler.toml ::[tool.line_profiler.multiprocessing] - Shortened `timeout` to .25 s - Set `on_timeout` to `'warn'` --- .../multiprocessing_patches.py | 120 +++++------------- line_profiler/rc/line_profiler.toml | 8 +- 2 files changed, 36 insertions(+), 92 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 4ae261f4..1b37720e 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -17,12 +17,10 @@ import multiprocessing import warnings from collections.abc import Callable, Mapping -from contextlib import AbstractContextManager, nullcontext from functools import lru_cache, partial, wraps from importlib import import_module from multiprocessing.process import BaseProcess from os import PathLike -from pathlib import Path from time import sleep, monotonic from types import MappingProxyType from typing import ( @@ -41,9 +39,9 @@ T = TypeVar('T') PS = ParamSpec('PS') +_OnTimeout = Literal['ignore', 'warn', 'error'] _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' -_PROCESS_TERM_LOCK_LOC = '_line_profiler_process_terminate_lock' class _Wrapper(Protocol, Generic[PS, T]): @@ -110,7 +108,7 @@ def __init__( func: Callable[[], Any], cooldown: float = 0, timeout: float = 0, - on_timeout: Literal['ignore', 'warn', 'error'] = 'error', + on_timeout: _OnTimeout = 'error', ) -> None: if cooldown < 0: cooldown = 0 @@ -134,7 +132,7 @@ def with_cooldown(self, cooldown: float) -> Self: def with_timeout( self, timeout: float | None = None, - on_timeout: Literal['ignore', 'warn', 'error'] | None = None, + on_timeout: _OnTimeout | None = None, ) -> Self: if timeout is None: timeout = self._timeout @@ -272,23 +270,6 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: return inner_wrapper -@_method_wrapper -def wrap_start( - cache: LineProfilingCache, - vanilla_impl: Callable[[BaseProcess], None], - self: BaseProcess, -) -> None: - """ - Wrap around :py:meth:`BaseProcess.start` to specify the location for - a lock file, which is managed by the child process and checked by - the parent. This is to ensure that the child can exit gracefully and - complete any necessary cleanup. - """ - tempfile = cache.make_tempfile(prefix='process-term-lock-', suffix='.lock') - setattr(self, _PROCESS_TERM_LOCK_LOC, tempfile) - vanilla_impl(self) - - @_method_wrapper def wrap_terminate( cache: LineProfilingCache, @@ -298,42 +279,16 @@ def wrap_terminate( """ Wrap around :py:meth:`BaseProcess.terminate` to make sure that we don't actually kill the child (OS-level) process before it has the - chance to properly clean up. This is done by blocking the call as - long as a lock file exists, which is specified by the parent process - and managed by the child. + chance to properly clean up. Note: - We're technically polling in a hot loop, but: - - - We're only calling this when explicitly terminating processes, - which isn't that bad; and - - - Such calls typically only happen: - - - When e.g. the parallel function exectued in child - processes raised an error, so we're already on a "bad" - path; and - - - AFTER the performance-critical part of the code (the - parallelly-run function). - - To circumvent this we may use dedicated FS-watching APIs like - :py:mod:`watchdog` (which use syscalls to do this), but we'll - think about introducing extra dependencies when we REALLY have - to. + We're technically polling in a loop, but it isn't actually + *that* bad: typically ``.terminate()`` is only called when we're + on the bad path (e.g. the parallel workload errored out), and + after the performance-critical part of the code (said workload). """ # XXX: why can `coverage` get away with not doing all these # lock-file hijinks and just patching `BaseProcess._bootstrap()`? - - def discard_lock() -> None: - assert lock_file is not None - # This should have already happened unless we timed out - lock_file.unlink(missing_ok=True) - try: - delattr(self, _PROCESS_TERM_LOCK_LOC) - except AttributeError: - pass - def get_poller_args( config: PathLike[str] | str | bool | None = None, ) -> tuple[float, float, str | None]: @@ -352,8 +307,21 @@ def get_poller_args( on_timeout = None return cooldown, timeout, on_timeout - def wait_for_deletion( - path: Path, config: PathLike[str] | str | None = None, + def process_has_returned(proc: BaseProcess, timeout: float) -> bool: + popen = getattr(proc, '_popen', None) + if popen is None: + msg, result = 'No associated process', True + else: + result = popen.wait(timeout) is not None + if result: + msg = f'Process {popen.pid} has returned' + else: + msg = f'Waiting for process {popen.pid} to return...' + cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') + return result + + def wait_for_return( + config: PathLike[str] | str | None = None, ) -> _Poller: cooldown, timeout, on_timeout = get_poller_args(config) # `False` -> no resolution, force loading the vanilla file @@ -361,24 +329,13 @@ def wait_for_deletion( if on_timeout not in ('ignore', 'warn', 'error'): on_timeout = default_on_timeout return ( - _Poller.poll_while(path.exists) - .with_cooldown(cooldown) - .with_timeout( - timeout, cast(Literal['ignore', 'warn', 'error'], on_timeout), - ) + _Poller.poll_until(process_has_returned, self, cooldown) + .with_timeout(timeout, cast(_OnTimeout, on_timeout)) ) try: - lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) - if lock_file: - lock: AbstractContextManager[Any] = wait_for_deletion( - lock_file, cache.config, - ) - callback = discard_lock - else: - lock, callback = nullcontext(), _no_op - with lock: - callback() + with wait_for_return(cache.config): + pass finally: # Always call `Process.terminate()` to avoid orphans vanilla_impl(self) @@ -392,27 +349,15 @@ def wrap_bootstrap( *args: PS.args, **kwargs: PS.kwargs ) -> T: """ - Wrap around :py:meth:`BaseProcess._bootstrap` to: - - - Run ``LineProfilingCache.load().cleanup()`` so that profiling - results can be gathered; and - - - Write a lock file before executing ``vanilla_impl()`` and deleted - it thereafter, to ensure that a parant process doesn't prematurely - ``.terminate()`` a failed child before the profiling results can - be gathered. + Wrap around :py:meth:`BaseProcess._bootstrap` to run + ``LineProfilingCache.load().cleanup()`` so that profiling results + can be gathered. """ - lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) - - if lock_file: - lock_file.touch() - cache.add_cleanup(lock_file.unlink, missing_ok=True) try: return vanilla_impl(self, *args, **kwargs) finally: - cache._debug_output( - 'Calling cleanup hook via `BaseProcess._bootstrap`' - ) + msg = 'Calling cleanup hook via `BaseProcess._bootstrap`' + cache._debug_output(msg) cache.cleanup() @@ -539,7 +484,6 @@ def _apply_mp_patches( # `patches` out... so do explicit `cast()`s for submodule, target, patches in [ ('process', 'BaseProcess', { - 'start': cast(_Wrapper[..., None], wrap_start), 'terminate': cast(_Wrapper[..., None], wrap_terminate), '_bootstrap': cast(_Wrapper[..., Any], wrap_bootstrap), }), diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 5b39f5cb..a420aad8 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -230,15 +230,15 @@ intercept_logs = false cooldown = 0.03125 # 1/32-nd of a second # - `polling.timeout` (float): # Time (seconds) before the main process disregards the existence of -# lock file and unblocks calls to `.terminate()` a child process -# anyway (set to <= 0 to disable timeouts) -timeout = 1 +# lock file and unblocks calls to `.terminate()` a (most probably +# errored-out) child process anyway (set to <= 0 to disable timeouts) +timeout = 0.25 # - `polling.on_timeout` (Literal['error', 'warn', 'ignore']) # What to do when the above timeout is exhausted, before actually # `.terminate()`-ing the child process: # - 'error': raise an error # - `warn`: issue a warning # - `ignore`: nothing -on_timeout = 'error' +on_timeout = 'warn' # XXX: --- End of implementation details --- From 492a5a1bb10ee5c6fa16d2c5682216a204cefd29 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 20 Apr 2026 06:31:39 +0200 Subject: [PATCH 056/165] iMisc. test fixes tests/test_child_procs.py test_cache_setup_main_process() Fixed bug where we checked `BaseProcess.start()`, which is no longer patched test_cache_setup_{main,child}() Fixed bug where we directly check for `os.fork()` without accounting for its nonexistence on Windows test_apply_mp_patches() Fixed bug where the expected number of function calls is wrong when `multiprocessing.dummy` is used --- tests/test_child_procs.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index c309973c..3094c244 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1144,7 +1144,7 @@ def _check_output(output: str, tag: str, nhits: int) -> None: _MP_PATCHED_NAMES = { 'multiprocessing.process.BaseProcess': frozenset({ - '_bootstrap', 'start', 'terminate' + '_bootstrap', 'terminate', }), 'multiprocessing.spawn': frozenset({'get_preparation_data', 'runpy'}), } @@ -1354,9 +1354,11 @@ def test_cache_setup_main_process( for target, maybe_patches in patches.items(): obj = targets[target] for attr, is_patched in maybe_patches.items(): - assert ( - getattr(obj, attr) is patched[target][attr] - ) != is_patched + orig_value = patched[target][attr] + if orig_value is _NotSupplied.NOT_SUPPLIED: + assert not hasattr(obj, attr) + else: + assert (getattr(obj, attr) is orig_value) != is_patched # Check whether the patches are reversed cache.cleanup() for target, orig_attrs in patched.items(): @@ -1460,7 +1462,10 @@ def list_profiled_funcs() -> list[str]: callback() gathered = cache.gather_stats() assert any(gathered.timings.values()) == has_prof_data, gathered - assert (os.fork is not old_fork) == fork_patched + if hasattr(os, 'fork'): + assert (os.fork is not old_fork) == fork_patched + else: # E.g. Windows + assert old_fork == _NotSupplied.NOT_SUPPLIED # Check that profiling results have been written to the cache # directory @@ -1530,6 +1535,7 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: # preimports. To may the results more consistent between # `start_method='dummy'` and the others, manually do them below. cache._setup_in_main_process() # This calls `apply()` + assert cache.profiler is not None assert cache.preimports_module is not None run_path(str(cache.preimports_module), {'profile': cache.profiler}) @@ -1544,6 +1550,10 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: Pool: Callable[..., multiprocessing.pool.Pool] if start_method == 'dummy': Pool = _import_target('multiprocessing.dummy.Pool') + # Twice the counted calls because we're also collecting the + # checking calls in this process + expected_ncalls = len(_SAFE_TARGET_ARGS) * 2 + get_stats: Callable[[], LineStats] = cache.profiler.get_stats elif start_method not in START_METHODS: pytest.skip( f'`multiprocessing` start method {start_method!r} ' @@ -1551,6 +1561,8 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: ) else: Pool = multiprocessing.get_context(start_method).Pool + expected_ncalls = len(_SAFE_TARGET_ARGS) + get_stats = cache.gather_stats with Pool(2) as pool: par_result = pool.starmap(func, _SAFE_TARGET_ARGS) @@ -1559,18 +1571,13 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: assert par_result == [func(*args) for args in _SAFE_TARGET_ARGS] # Check that calls in children are traced - if start_method == 'dummy': - assert cache.profiler is not None - stats = cache.profiler.get_stats() - else: - stats = cache.gather_stats() - line_entries = stats.timings[ + line_entries = get_stats().timings[ inspect.getfile(func), inspect.getsourcelines(func)[1], func.__name__, ] num_returns = sum( nhits for lineno, nhits, _ in line_entries if lineno in return_lines ) - assert num_returns == len(_SAFE_TARGET_ARGS) + assert num_returns == expected_ncalls # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests From b820e50c6bfd50bd1d81dfb273083a77e8edf64c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 22 Apr 2026 00:24:17 +0200 Subject: [PATCH 057/165] Fix thread profiling + refactoring line_profiler/_child_process_profiling/cache.py ::LineProfilingCache._method_wrapper New decorator (relocated from `~.multiprocessing_patches`) line_profiler/_child_process_profiling/multiprocessing_patches.py Replaced uses of `@_method_wrapper` with `@LineProfilingCache._method_wrapper` line_profiler/_child_process_profiling/threading_patches.py::apply() New function for patching `threading.Thread` to set up thread profiling: when the thread object `.run()`s the target function, the profiler's `.enable_count` is synced to that of the starting thread. This fixes the bug where calls to a profiled function on new threads may not be profiled despite profiling being currently on tests/test_child_procs.py create_cache() Now also writing the profiler `id()` in the output test_cache_setup_main_process() Updated tested list of patched attributes test_apply_mp_patches() Updated preserved attributes --- .../_child_process_profiling/cache.py | 121 +++++++++++++++++- .../multiprocessing_patches.py | 61 +-------- .../threading_patches.py | 108 ++++++++++++++++ tests/test_child_procs.py | 26 ++-- 4 files changed, 248 insertions(+), 68 deletions(-) create mode 100644 line_profiler/_child_process_profiling/threading_patches.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 3d2484b7..5ccca6fe 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -21,8 +21,8 @@ from reprlib import Repr from textwrap import indent from types import MethodType -from typing import Any, ClassVar, TypeVar, TypedDict, cast -from typing_extensions import Self, ParamSpec, Unpack +from typing import Any, ClassVar, TypeVar, TypedDict, cast, overload +from typing_extensions import Concatenate, ParamSpec, Self, Unpack from .. import _diagnostics as diagnostics from ..curated_profiling import CuratedProfilerContext @@ -39,6 +39,7 @@ __all__ = ('LineProfilingCache',) +T = TypeVar('T') PS = ParamSpec('PS') # Note: `typing.AnyStr` deprecated since 3.13 AnyStr = TypeVar('AnyStr', str, bytes) @@ -515,8 +516,9 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: forked processes is properly handled (if ``wrap_os_fork=True``) - - :py:mod:`multiprocessing` wrapped so that child processes - managed by the package are properly handled + - :py:mod:`multiprocessing` and :py:mod:`threading` patched + so that child processes and threads managed thereby are + properly handled - Instance to be returned if :py:func:`~.load()` is called from now on @@ -532,6 +534,8 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: self._wrap_os_fork() mp_patches = import_module(this_subpkg + '.multiprocessing_patches') mp_patches.apply(self) + th_patches = import_module(this_subpkg + '.threading_patches') + th_patches.apply(self) self._replace_loaded_instance() @@ -704,6 +708,115 @@ def _get_init_args(self) -> dict[str, Any]: def _get_filename(cache_dir: os.PathLike[str] | str) -> str: return os.path.join(cache_dir, CACHE_FILENAME) + @overload + @classmethod + def _method_wrapper( + cls, + wrapper: Callable[Concatenate[Self, Callable[PS, T], PS], T], + debug: bool | None = None, + ) -> Callable[[Callable[PS, T]], Callable[PS, T]]: + ... + + @overload + @classmethod + def _method_wrapper( + cls, wrapper: None = None, debug: bool | None = None, + ) -> Callable[ + [Callable[Concatenate[Self, Callable[PS, T], PS], T]], + Callable[[Callable[PS, T]], Callable[PS, T]] + ]: + ... + + @classmethod + def _method_wrapper( + cls, + wrapper: ( + Callable[Concatenate[Self, Callable[PS, T], PS], T] | None + ) = None, + debug: bool | None = None, + ) -> ( + Callable[ + [Callable[Concatenate[Self, Callable[PS, T], PS], T]], + Callable[[Callable[PS, T]], Callable[PS, T]] + ] + | Callable[[Callable[PS, T]], Callable[PS, T]] + ): + """ + Convenience wrapper decorator for functions which use the + :py:meth:`load`-ed session instance and wrap another callable. + + Args: + wrapper (Callable[..., T]) + Callable with the call signature + ``(cache, vanilla_impl, *args, **kwargs) -> retval``; + ``*args``, ``**kwargs``, and ``retval`` should be + consistent with that of ``vanilla_impl()``'s. + debug (bool | None) + Whether to format and write debug messages before and + after the call to the ``wrapper`` callable; + if ``debug`` is not set, it will be taken from the + session instance. + + Returns: + inner_wrapper (Callable[[Callable[PS, T]], Callable[PS, T]]) + Wrapper(-maker) which takes the ``vanilla_impl`` and + return a wrapper around it. + """ + if wrapper is None: + # `ty` doesn't quite support `partial` yet, see issue #1536 + return cast( + Callable[[Callable[PS, T]], Callable[PS, T]], + partial(cls._method_wrapper, debug=debug), + ) + + def inner_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: + @wraps(vanilla_impl) + def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: + cache = cls.load() + write = cache._debug_output + debug_: bool | None = debug + if debug_ is None: + debug_ = cache.debug + + if debug_: + arg_reprs: list[str] = [repr(arg) for arg in args] + arg_reprs.extend(f'{k}={v!r}' for k, v in kwargs.items()) + formatted_call = f'{name}({", ".join(arg_reprs)})' + write(f'Wrapped call made: {formatted_call}...') + try: + result = wrapper(cache, vanilla_impl, *args, **kwargs) + except Exception as e: + if debug_: + write( + 'Wrapped call failed: ' + f'{formatted_call} -> {type(e).__name__}: {e}', + ) + raise + else: + if debug_: + write( + 'Wrapped call succeeded: ' + f'{formatted_call} -> {result!r}', + ) + return result + + if ( + hasattr(vanilla_impl, '__module__') + and hasattr(vanilla_impl, '__qualname__') + ): + name = '{0.__module__}.{0.__qualname__}'.format(vanilla_impl) + else: + name = f'' + + return wrapped_impl + + for field in 'name', 'qualname', 'doc': + dunder = f'__{field}__' + value = getattr(wrapper, dunder, None) + if value is not None: + setattr(inner_wrapper, dunder, value) + return inner_wrapper + @property def environ(self) -> dict[str, str]: """ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 1b37720e..ef76ca45 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -17,7 +17,7 @@ import multiprocessing import warnings from collections.abc import Callable, Mapping -from functools import lru_cache, partial, wraps +from functools import lru_cache, partial from importlib import import_module from multiprocessing.process import BaseProcess from os import PathLike @@ -223,54 +223,7 @@ def _get_config_cached( return MappingProxyType({**cd, 'polling': MappingProxyType(cd['polling'])}) -def _method_wrapper( - wrapper: Callable[Concatenate[LineProfilingCache, Callable[PS, T], PS], T], -) -> Callable[[Callable[PS, T]], Callable[PS, T]]: - def inner_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: - @wraps(vanilla_impl) - def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: - cache = LineProfilingCache.load() - debug = cache._debug_output - - arg_reprs: list[str] = [repr(arg) for arg in args] - arg_reprs.extend(f'{k}={v!r}' for k, v in kwargs.items()) - formatted_call = '{}({})'.format(name, ', '.join(arg_reprs)) - - debug(f'Wrapped call made: {formatted_call}...') - try: - result = wrapper(cache, vanilla_impl, *args, **kwargs) - except Exception as e: - debug( - 'Wrapped call failed: ' - f'{formatted_call} -> {type(e).__name__}: {e}', - ) - raise - else: - debug( - 'Wrapped call succeeded: ' - f'{formatted_call} -> {result!r}', - ) - return result - - if ( - hasattr(vanilla_impl, '__module__') - and hasattr(vanilla_impl, '__qualname__') - ): - name = '{0.__module__}.{0.__qualname__}'.format(vanilla_impl) - else: - name = f'' - - return wrapped_impl - - for field in 'name', 'qualname', 'doc': - dunder = f'__{field}__' - value = getattr(wrapper, dunder, None) - if value is not None: - setattr(inner_wrapper, dunder, value) - return inner_wrapper - - -@_method_wrapper +@LineProfilingCache._method_wrapper def wrap_terminate( cache: LineProfilingCache, vanilla_impl: Callable[[BaseProcess], None], @@ -340,7 +293,7 @@ def wait_for_return( vanilla_impl(self) -@_method_wrapper +@LineProfilingCache._method_wrapper def wrap_bootstrap( cache: LineProfilingCache, vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], @@ -395,10 +348,11 @@ def get_msg(msg: str, *_, **__) -> str: ) -@_method_wrapper +@LineProfilingCache._method_wrapper def wrap_get_preparation_data( - # We don't use the cache here, but `@_method_wrapper` expects it in - # the signature (and we want the debug output) + # We don't use the cache here, but + # `@LineProfilingCache._method_wrapper` expects it in the signature + # (and we want the debug output) _, vanilla_impl: Callable[PS, dict[str, Any]], /, @@ -443,7 +397,6 @@ def apply(lp_cache: LineProfilingCache) -> None: - :py:mod:`multiprocessing` marked as having been set up - The following methods and functions patched: - - :py:meth:`multiprocessing.process.BaseProcess.start` - :py:meth:`multiprocessing.process.BaseProcess.terminate` diff --git a/line_profiler/_child_process_profiling/threading_patches.py b/line_profiler/_child_process_profiling/threading_patches.py new file mode 100644 index 00000000..d5a2e94d --- /dev/null +++ b/line_profiler/_child_process_profiling/threading_patches.py @@ -0,0 +1,108 @@ +""" +Patch :py:mod:`threading` so that profiling extends into processes +it creates. +""" +from __future__ import annotations + +import threading +from collections.abc import Callable +from functools import wraps +from typing import TYPE_CHECKING, Any, TypeVar +from typing_extensions import ParamSpec + +from ..line_profiler import LineProfiler +from .cache import LineProfilingCache + + +__all__ = ('apply',) + + +T = TypeVar('T') +PS = ParamSpec('PS') + +_PATCHED_MARKER = '__line_profiler_patched_threading__' + + +def make_syncing_wrapper( + func: Callable[PS, T], prof: LineProfiler, enable_count: int, +) -> Callable[PS, T]: + """ + Wrap the callable ``func`` so that when we spin up a new thread, we + sync the + :py:attr:`line_profiler.line_profiler.LineProfiler.enable_count` of + the active profiler (stored at the cache instance loaded from + :py:meth:`LineProfilingCache.load`) with ``enable_count``. + """ + @wraps(func) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + if TYPE_CHECKING: + assert hasattr(prof, 'enable_count') + assert isinstance(prof.enable_count, int) + # Note: `prof.enable_count` is most likely to be zero on the new + # thread + thread_enable_count: int = prof.enable_count + for _ in range(enable_count - thread_enable_count): + prof.enable_by_count() + try: + return func(*args, **kwargs) + finally: + # Reset enable counts to avoid problems if the thread id is + # ever reused + for _ in range(prof.enable_count - thread_enable_count): + prof.disable_by_count() + + return wrapper + + +# Threads are supposed to be lightweight, so don't waste time formatting +# debug messages during startup + + +@LineProfilingCache._method_wrapper(debug=False) +def wrap_init( + cache: LineProfilingCache, + vanilla_impl: Callable[..., None], + self: threading.Thread, + group: None = None, + target: Callable[..., Any] | None = None, + *a, **k +) -> None: + """ + Wrap the initializer of :py:class:`threading.Thread` so that the + profiler's :py:attr:`LineProfiler.enable_count` is synced up on + newly spun-up threads. + """ + prof = cache.profiler + enable_count: int | None = getattr(prof, 'enable_count', None) + if target is not None and enable_count: + if TYPE_CHECKING: + assert prof is not None + target = make_syncing_wrapper(target, prof, enable_count) + vanilla_impl(self, group, target, *a, **k) + + +def apply(lp_cache: LineProfilingCache) -> None: + """ + Set up profiling in threads started by :py:mod:`threading` by + applying patches to the module. + + Args: + lp_cache (LineProfilingCache) + Cache instance governing the profiling run + + Side effects: + - :py:mod:`threading` marked as having been set up + + - The following methods and functions patched: + + - :py:meth:`threading.Thread.__init__` + + - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` + """ + if getattr(threading, _PATCHED_MARKER, False): + return + init_wrapper = wrap_init(threading.Thread.__init__) + lp_cache.patch( + threading.Thread, '__init__', init_wrapper, name='threading.Thread', + ) + lp_cache.patch(threading, _PATCHED_MARKER, True, name='threading') diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 3094c244..68ad3045 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -361,19 +361,23 @@ def instantiate( return cache def print_result( - cache: LineProfilingCache, topic: str, result: str, + cache: LineProfilingCache, topic: str, result: str, *notes: str, ) -> None: - header = f'{topic} (cache instance {id(cache):#x}):' + header = '{} ({}):'.format( + topic, '; '.join([f'cache instance {id(cache):#x}', *notes]), + ) print(header, indent(result, ' '), sep='\n') def print_profiler_stats(cache: LineProfilingCache) -> None: if cache.profiler is None: result = '' + notes = [] else: with StringIO() as sio: cache.profiler.print_stats(sio) result = sio.getvalue() - print_result(cache, 'Native profiler stats', result) + notes = [f'profiler instance {id(cache.profiler):#x}'] + print_result(cache, 'Native profiler stats', result, *notes) def print_gathered_stats(cache: LineProfilingCache) -> None: with StringIO() as sio: @@ -1142,11 +1146,13 @@ def _check_output(output: str, tag: str, nhits: int) -> None: # tested APIs and behaviors MUST NOT be relied upon by end-users. -_MP_PATCHED_NAMES = { +_GLOBAL_PATCHES = { 'multiprocessing.process.BaseProcess': frozenset({ '_bootstrap', 'terminate', }), 'multiprocessing.spawn': frozenset({'get_preparation_data', 'runpy'}), + 'threading.Thread': frozenset({'__init__'}), + 'os': frozenset({'fork'}), } # NOTE: we need a function which isn't used by the codebase itself # (esp. during cache cleanup); otherwise the profiling results may @@ -1336,11 +1342,11 @@ def test_cache_setup_main_process( as expected. """ cache = create_cache(debug=debug) - patches: dict[str, dict[str, bool]] = {'os': {'fork': wrap_os_fork}} - patches.update( - (target, dict.fromkeys(attrs, True)) - for target, attrs in _MP_PATCHED_NAMES.items() - ) + patches: dict[str, dict[str, bool]] = { + target: dict.fromkeys(attrs, True) + for target, attrs in _GLOBAL_PATCHES.items() + } + patches['os']['fork'] = wrap_os_fork targets: dict[str, Any] = { target: _import_target(target) for target in patches } @@ -1488,7 +1494,7 @@ def list_profiled_funcs() -> list[str]: @pytest.mark.parametrize(('debug', 'label'), [(True, 'with-debug'), (False, 'no-debug')]) @_preserve_pth_files() -@_preserve_attributes(_MP_PATCHED_NAMES) +@_preserve_attributes(_GLOBAL_PATCHES) def test_apply_mp_patches( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], From a5447e7038ca882f3ab93b721e8b7d080dfadc93 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 22 Apr 2026 02:25:12 +0200 Subject: [PATCH 058/165] Partial rollback of `threading` fix line_profiler/_child_process_profiling/threading_patches.py SHOULD_PATCH_THREADING Alias to `line_profiler._line_profiler.USE_LEGACY_TRACE` apply() Now a no-op if `SHOULD_PATCH_THREADING` is false (because the fix only seems to work when using `LINE_PROFILER_CORE=ctrace`, and causes flakiness otherwise) tests/test_child_procs.py test_cache_setup_main_process() Now only checking for `threading` patches if `SHOULD_PATCH_THREADING` is true test_apply_mp_patches() Now only protecting against `threading` patches if `SHOULD_PATCH_THREADING` is true --- .../_child_process_profiling/threading_patches.py | 15 ++++++++++++++- tests/test_child_procs.py | 8 ++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/line_profiler/_child_process_profiling/threading_patches.py b/line_profiler/_child_process_profiling/threading_patches.py index d5a2e94d..c8a5b62c 100644 --- a/line_profiler/_child_process_profiling/threading_patches.py +++ b/line_profiler/_child_process_profiling/threading_patches.py @@ -10,11 +10,14 @@ from typing import TYPE_CHECKING, Any, TypeVar from typing_extensions import ParamSpec +from .._line_profiler import ( # type: ignore + USE_LEGACY_TRACE as SHOULD_PATCH_THREADING, +) from ..line_profiler import LineProfiler from .cache import LineProfilingCache -__all__ = ('apply',) +__all__ = ('apply', 'SHOULD_PATCH_THREADING') T = TypeVar('T') @@ -32,6 +35,10 @@ def make_syncing_wrapper( :py:attr:`line_profiler.line_profiler.LineProfiler.enable_count` of the active profiler (stored at the cache instance loaded from :py:meth:`LineProfilingCache.load`) with ``enable_count``. + + Note: + This only seems to work as intended when using the legacy trace + system... """ @wraps(func) def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: @@ -98,7 +105,13 @@ def apply(lp_cache: LineProfilingCache) -> None: - :py:meth:`threading.Thread.__init__` - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` + + Note: + This is a no-op when using :py:mod:`sys.monitoring`-based + profiling. """ + if not SHOULD_PATCH_THREADING: + return if getattr(threading, _PATCHED_MARKER, False): return init_wrapper = wrap_init(threading.Thread.__init__) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 68ad3045..90d51562 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -35,6 +35,9 @@ from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) +from line_profiler._child_process_profiling.threading_patches import ( + SHOULD_PATCH_THREADING, +) from line_profiler.curated_profiling import ( CuratedProfilerContext, ClassifiedPreimportTargets, ) @@ -1145,15 +1148,16 @@ def _check_output(output: str, tag: str, nhits: int) -> None: # XXX: Tests in this section concerns implementation details, and the # tested APIs and behaviors MUST NOT be relied upon by end-users. - _GLOBAL_PATCHES = { 'multiprocessing.process.BaseProcess': frozenset({ '_bootstrap', 'terminate', }), 'multiprocessing.spawn': frozenset({'get_preparation_data', 'runpy'}), - 'threading.Thread': frozenset({'__init__'}), 'os': frozenset({'fork'}), } +if SHOULD_PATCH_THREADING: + _GLOBAL_PATCHES['threading.Thread'] = frozenset({'__init__'}) + # NOTE: we need a function which isn't used by the codebase itself # (esp. during cache cleanup); otherwise the profiling results may # be skewed From 575f32c8aa8920c750dfcd66e42eff7b145c3d71 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 22 Apr 2026 08:02:33 +0200 Subject: [PATCH 059/165] Fix forkserver profiling in newer Pythons line_profiler/_child_process_profiling/cache.py::LineProfilingCache _setup_in_main_process() Refactored part into `._setup_common()` _setup_in_child_process() Now calling `._setup_common()`, making sure that the fork-server process is also set up for profiling _wrap_os_hook() `os.fork()` wrapper now writing debug-log messages indicating the parent and child PIDs line_profiler/_child_process_profiling/multiprocessing_patches.py PickleHook, wrap_get_preparation_data() Removed no-unused code tee_log() Updated call signature apply() Added argument `reboot_forkserver` _apply_mp_patches() - Updated call signature - Refactored internals - Now rebooting the fork-server process to make sure that it is set up for profiling tests/test_child_procs.py test_cache_setup_main_process() No longer checking patching/restoration of `multiprocessing.spawn.get_preparation_data()` because it is no longer patched test_apply_mp_patches() No longer checking the output `multiprocessing.spawn.get_preparation_data()` because it is no longer patched FIXME: performance regression in newer Python versions... --- .../_child_process_profiling/cache.py | 40 ++-- .../multiprocessing_patches.py | 214 ++++++++---------- tests/test_child_procs.py | 13 +- 3 files changed, 116 insertions(+), 151 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 5ccca6fe..51a464b3 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -20,7 +20,7 @@ from pickle import HIGHEST_PROTOCOL from reprlib import Repr from textwrap import indent -from types import MethodType +from types import MethodType, ModuleType from typing import Any, ClassVar, TypeVar, TypedDict, cast, overload from typing_extensions import Concatenate, ParamSpec, Self, Unpack @@ -44,6 +44,7 @@ # Note: `typing.AnyStr` deprecated since 3.13 AnyStr = TypeVar('AnyStr', str, bytes) +_THIS_SUBPACKAGE, *_ = (lambda: None).__module__.rpartition('.') INHERITED_CACHE_ENV_VARNAME_PREFIX = ( 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_DIR' ) @@ -51,6 +52,10 @@ _DEBUG_LOG_FILENAME_PATTERN = 'debug_log_{main_pid}_{current_pid}.log' +def _import_sibling(submodule: str) -> ModuleType: + return import_module(f'{_THIS_SUBPACKAGE}.{submodule}') + + class _ReprAttributes(TypedDict, total=False): """ Note: @@ -523,20 +528,10 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: - Instance to be returned if :py:func:`~.load()` is called from now on """ - this_subpkg, *_, _ = (lambda: None).__module__.rpartition('.') - self.dump() self.inject_env_vars() - pth_hook = import_module(this_subpkg + '.pth_hook') - pth_hook.write_pth_hook(self) - - if wrap_os_fork: - self._wrap_os_fork() - mp_patches = import_module(this_subpkg + '.multiprocessing_patches') - mp_patches.apply(self) - th_patches = import_module(this_subpkg + '.threading_patches') - th_patches.apply(self) - + _import_sibling('pth_hook').write_pth_hook(self) + self._setup_common(wrap_os_fork, reboot_forkserver=True) self._replace_loaded_instance() def _setup_in_child_process( @@ -601,10 +596,8 @@ def _setup_in_child_process( ) self._add_cleanup(prof.dump_stats, -1, prof_outfile) - # Set up `os.fork()` wrapping if needed (i.e. in a spawned - # process) - if wrap_os_fork: - self._wrap_os_fork() + # Various setups + self._setup_common(wrap_os_fork, reboot_forkserver=False) # Set `.cleanup()` as an atexit hook to handle everything when # the child process is about to terminate @@ -613,6 +606,16 @@ def _setup_in_child_process( self._debug_output(f'Setup successful ({context})') return True + def _setup_common( + self, wrap_os_fork: bool, reboot_forkserver: bool, + ) -> None: + if wrap_os_fork: + self._wrap_os_fork() + _import_sibling('multiprocessing_patches').apply( + self, reboot_forkserver, + ) + _import_sibling('threading_patches').apply(self) + def _wrap_os_fork(self) -> None: """ Create a wrapper around :py:func:`os.fork` which handles @@ -630,11 +633,14 @@ def _wrap_os_fork(self) -> None: @wraps(fork) def wrapper() -> int: + ppid = os.getpid() result = fork() if result: return result # If we're here, we are in the fork + pid = os.getpid() forked = self.copy() # Ditch inherited cleanups + forked._debug_output(f'Forked: {ppid} -> {pid}') if forked._replace_loaded_instance(): forked._debug_output( 'Superseded cached `.load()`-ed instance in forked process' diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index ef76ca45..c5561fa4 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -49,28 +49,6 @@ def __call__(self, func: Callable[PS, T], /) -> Callable[PS, T]: ... -class PickleHook: - """ - Object which, when unpickled, sets up profiling in the - :py:mod:`multiprocessing`-created process. - - See also: - :py:class:`coverage.multiproc.Stowaway` - """ - def __getstate__(_) -> int: - # Cannot return `None`, or nothing will be pickled and - # `.__getstate__()` will not be invoked in the child - return 1 - - def __setstate__(*_) -> None: - # We're in a child process created by `multiprocessing`, so set - # up shop here. - lp_cache = LineProfilingCache.load() - lp_cache._setup_in_child_process(False, 'multiprocessing') - if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, main_process=False) - - class _Poller: """ Poll a callable until it returns true-y. @@ -327,8 +305,8 @@ def _cache_hook( def tee_log( - vanilla_impl: Callable[Concatenate[str, PS], None], marker: str, + vanilla_impl: Callable[Concatenate[str, PS], None], /, msg: str, *args: PS.args, @@ -348,50 +326,21 @@ def get_msg(msg: str, *_, **__) -> str: ) -@LineProfilingCache._method_wrapper -def wrap_get_preparation_data( - # We don't use the cache here, but - # `@LineProfilingCache._method_wrapper` expects it in the signature - # (and we want the debug output) - _, - vanilla_impl: Callable[PS, dict[str, Any]], - /, - *args: PS.args, - **kwargs: PS.kwargs -) -> dict[str, Any]: - """ - Wrap around :py:func:`multiprocessing.spawn.get_preparation_data`, - slipping a :py:class:`PickleHook` into the returned dictionary so - that profiling is triggered upon unpickling. - - Args: - vanilla_impl - Vanilla - :py:func:`multiprocessing.spawn.get_preparation_data` - *args - **kwargs - Passed to - :py:func:`multiprocessing.spawn.get_preparation_data` - - Returns - Dictionary returned by - ``get_preparation_data(*args, **kwargs)`` with an extra key - """ - key = 'line_profiler_pickle_hook' # Doesn't matter - data = vanilla_impl(*args, **kwargs) - assert key not in data - data[key] = PickleHook() - return data - - -def apply(lp_cache: LineProfilingCache) -> None: +def apply( + lp_cache: LineProfilingCache, reboot_forkserver: bool = True, +) -> None: """ Set up profiling in :py:mod:`multiprocessing` child processes by applying patches to the module. Args: - lp_cache (LineProfilingCache) + lp_cache (LineProfilingCache): Cache instance governing the profiling run + reboot_forkserver (bool): + Whether to reboot the global + :py:class`multiprocessing.forkserver.ForkServer` instance + so as to ensure that profiling happens on processes forked + therefrom (see Note) Side effects: - :py:mod:`multiprocessing` marked as having been set up @@ -402,93 +351,112 @@ def apply(lp_cache: LineProfilingCache) -> None: - :py:meth:`multiprocessing.process.BaseProcess._bootstrap` - - :py:func:`multiprocessing.spawn.get_preparation_data` + - If ``reboot_forkserver=True``, fork-server process rebooted: + + - Immediately + + - When ``lp_cache.cleanup()`` is run - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` Note: - When ``lp_cache.cleanup()`` is run, the global - :py:class:`multiprocessing.forkserver.ForkServer` object will be - rebooted. This is necessary because the server process staticly - inherits the environment when it is first spun up + Rebooting the fork server is necessary because its process + staticly inherits the environment when it is first spun up (see :py:func:`multiprocessing.forkserver.ensure_running`). - Thus, if in the same Python process we ever start up two - separate profliing sessions managed by different caches, the - child processes forked from the server will fail to inherit the - updated environment variables injected by the newer cache - instance, leading to the setup code in this subpackage not being - loaded. + Thus, without the reboots: + + - If in the same Python process we ever start up two separate + profliing sessions managed by different caches, the child + processes forked from the server will fail to inherit the + updated environment variables injected by the newer cache + instance, leading to the setup code in this subpackage not + being loaded. + + - Since 3.13.8 and 3.14.1, the bug where the ``main_path`` + argument to :py:func:`multiprocessing.forkserver.main` is + unused has been fixed (see ``cpython`` issue `GH-126631`_). + This causes ``sys.modules['__main__']`` to be set up in the + fork-server process, meaning that children forked therefrom + will NOT redo the setup. Thus, the fork-server process itself + will also need to be properly set up for profiling. + + .. _GH-126631: https://github.com/python/cpython/issues/126631 """ if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache) + _apply_mp_patches(lp_cache, reboot_forkserver) + + +def _apply_patches_generic( + lp_cache: LineProfilingCache, + submodule: str, + targets: Mapping[str, Mapping[str, Callable[[Any], Any]]], + cleanup: bool = True, +) -> None: + submod_name = 'multiprocessing.' + submodule + try: + mod = import_module(submod_name) + except ImportError: + return + for target, patches in targets.items(): + if target: + try: + obj: Any = getattr(mod, target) + except AttributeError: + continue + name = f'{submod_name}.{target}' + else: + obj, name = mod, submod_name + replace = partial(lp_cache.patch, obj, cleanup=cleanup, name=name) + for method, method_wrapper in patches.items(): + try: + vanilla = getattr(obj, method) + except AttributeError: + continue + replace(method, method_wrapper(vanilla)) def _apply_mp_patches( lp_cache: LineProfilingCache, - main_process: bool = True, + reboot_forkserver: bool = True, debug: bool | None = None, ) -> None: # In a child process, we don't care about polluting the # `multiprocessing` namespace, so don't bother with cleanup - replace = partial(lp_cache.patch, cleanup=main_process) - + apply_patches = partial(_apply_patches_generic, lp_cache) # Patch `multiprocessing.process.BaseProcess` methods # Note: the type checkers seem to need some help figuring the # `patches` out... so do explicit `cast()`s - for submodule, target, patches in [ - ('process', 'BaseProcess', { - 'terminate': cast(_Wrapper[..., None], wrap_terminate), - '_bootstrap': cast(_Wrapper[..., Any], wrap_bootstrap), - }), - ]: - try: - mod = import_module('multiprocessing.' + submodule) - except ImportError: - continue - Class = getattr(mod, target) - name = f'{Class.__module__}.{Class.__qualname__}' - patch_class = partial(replace, Class, name=name) - for method, method_wrapper in patches.items(): - vanilla = getattr(Class, method) - patch_class(method, method_wrapper(vanilla)) - + apply_patches( + 'process', + {'BaseProcess': {'terminate': wrap_terminate, + '_bootstrap': wrap_bootstrap}}, + ) # Patch `multiprocessing.spawn` try: from multiprocessing import spawn - except ImportError: # Incompatible platforms + except ImportError: pass else: - patch_spawn = partial(replace, spawn, name=spawn.__name__) - # Patch `get_preparation_data()` - gpd_wrapper = wrap_get_preparation_data(spawn.get_preparation_data) - patch_spawn('get_preparation_data', gpd_wrapper) - # Patch `runpy` (do it locally instead of tempering with the - # global `runpy` mmodule) if hasattr(spawn, 'runpy'): - runpy_wrapper = create_runpy_wrapper(lp_cache) - patch_spawn('runpy', runpy_wrapper) - + lp_cache.patch( + spawn, 'runpy', create_runpy_wrapper(lp_cache), + name='multiprocessing.spawn', + ) # Intercept `multiprocessing` debug messages if debug is None: debug = _get_config(lp_cache.config)['intercept_logs'] if debug: - from multiprocessing import util - - patch_util = partial(replace, util, name=util.__name__) - for logging_func in [ - 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', - ]: - try: - vanilla = getattr(util, logging_func) - except AttributeError: - continue - patch_util(logging_func, partial(tee_log, vanilla, logging_func)) - - # Stop the current `ForkServer` server process as a part of cache - # cleanup (this uses `ForkServer._stop()` which is private API, but - # it's the same hack used in Python's own test suite -- see the - # comment to said method) - if main_process: + lfuncs = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] + lpatches = {func: partial(partial, tee_log, func) for func in lfuncs} + apply_patches('util', {'': lpatches}) + # Stop the current `ForkServer` server process: + # - Now, so that the (rebooted) fork-server process has profiling + # set up; and + # - Also as a part of cache cleanup + # (this uses `ForkServer._stop()` which is private API, but it's the + # same hack used in Python's own test suite -- see the comment to + # said method) + if reboot_forkserver: try: from multiprocessing import forkserver except ImportError: # Incompatible platform @@ -497,10 +465,12 @@ def _apply_mp_patches( server_instance: forkserver.ForkServer = forkserver._forkserver stop = getattr(server_instance, '_stop', None) assert callable(stop) # Appease the type checker + stop() lp_cache.add_cleanup(stop) - # Mark `multiprocessing` as having been patched - replace(multiprocessing, _PATCHED_MARKER, True, name='multiprocessing') + lp_cache.patch( + multiprocessing, _PATCHED_MARKER, True, name='multiprocessing', + ) def _no_op(*_, **__) -> None: diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 90d51562..ba1061a8 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1152,7 +1152,7 @@ def _check_output(output: str, tag: str, nhits: int) -> None: 'multiprocessing.process.BaseProcess': frozenset({ '_bootstrap', 'terminate', }), - 'multiprocessing.spawn': frozenset({'get_preparation_data', 'runpy'}), + 'multiprocessing.spawn': frozenset({'runpy'}), 'os': frozenset({'fork'}), } if SHOULD_PATCH_THREADING: @@ -1511,10 +1511,6 @@ def test_apply_mp_patches( .multiprocessing_patches.apply` works as expected. """ - def gpd_wrapper(name: str) -> dict[str, Any]: - impl = _import_target('multiprocessing.spawn.get_preparation_data') - return impl(name) - def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: try: LineStats.from_files(path, on_defective='error') @@ -1536,7 +1532,6 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: config=config, debug=True, ) - old_data = gpd_wrapper('my_name') # Note: # - The reversibility of the patches have already been tested in # `test_cache_setup_main_process()`, so we just actually test the @@ -1549,12 +1544,6 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: assert cache.preimports_module is not None run_path(str(cache.preimports_module), {'profile': cache.profiler}) - # Check for the insertion of the `PickleHook` by the patched - # `get_preparation_data()` - new_data = gpd_wrapper('my_name') - inserted_key, = set(new_data) - set(old_data) - assert type(new_data[inserted_key]).__name__ == 'PickleHook' - func = _import_target(_SAFE_TARGET) return_lines = _find_return_lines(_SAFE_TARGET) Pool: Callable[..., multiprocessing.pool.Pool] From 67785640705031fab15a29b36360998ae143ab00 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 22 Apr 2026 18:08:09 +0200 Subject: [PATCH 060/165] Fix test on Windows --- tests/test_child_procs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index ba1061a8..cadf145e 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1350,7 +1350,7 @@ def test_cache_setup_main_process( target: dict.fromkeys(attrs, True) for target, attrs in _GLOBAL_PATCHES.items() } - patches['os']['fork'] = wrap_os_fork + patches['os']['fork'] = wrap_os_fork and (sys.platform != 'win32') targets: dict[str, Any] = { target: _import_target(target) for target in patches } From 577e85b34c0fb5714402e062a31caf33b097c33b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 25 Apr 2026 04:08:57 +0200 Subject: [PATCH 061/165] Added `conftest.py` conftest.py Installs the marker `@pytest.mark.retry(...)` so that certain tests can be retried TODO: tests --- tests/conftest.py | 380 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..41deeb64 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,380 @@ +""" +A simple :py:deco:`pytest.mark.retry` decorator. +Function-scoped fixtures are re-fetched bewteen retries. +""" +import dataclasses +from collections.abc import ( + Callable, Collection, Generator, Hashable, Iterable, Mapping, +) +from functools import cached_property, partial +from operator import contains +from pathlib import Path +from typing import ( + TYPE_CHECKING, Any, ClassVar, Literal, Protocol, TypeVar, cast, final, +) + +import pytest +from _pytest.nodes import Node +from _pytest.unittest import TestCaseFunction +try: + from pytest import TerminalReporter +except ImportError: # pytest < ~8.4 + from _pytest.terminal import TerminalReporter + + +_Status = Literal['passed', 'failed', 'skipped'] +F = TypeVar('F', bound=TestCaseFunction) +FCls = TypeVar('FCls', bound=type[TestCaseFunction]) +T = TypeVar('T') + + +class _PyfuncCallImpl(Protocol): + def __call__(self, *, pyfuncitem: pytest.Function) -> Any: + ... + + +@final +@dataclasses.dataclass +class _RetryEntry: + func: pytest.Function + retries: int + status: _Status + + def _get_name(self, with_params: bool) -> str: + path, prefix = self._name_prefixes + if with_params: + name = self.func.name + else: + name = self.func.originalname + if prefix: + name = f'{prefix}.{name}' + if path: + name = f'{path}::{name}' + return name + + @classmethod + def add_entry(cls, func: pytest.Function, *args, **kwargs) -> _RetryEntry: + assert func.config is not None + entry = cls(func, *args, **kwargs) + entry.get_entries(func.config).append(entry) + return entry + + @staticmethod + def get_entries(config: pytest.Config) -> list[_RetryEntry]: + return config.stash.setdefault(_RETRY_ENTRIES_KEY, []) + + @property + def full_name(self) -> str: + return self._get_name(True) + + @property + def full_original_name(self) -> str: + return self._get_name(False) + + @cached_property + def _name_prefixes(self) -> tuple[str, str]: + chunks: list[str] = [] + node: Node | None = self.func.parent + path = '' + seen: set[int] = set() + while True: + if id(node) in seen: + break + else: + seen.add(id(node)) + if isinstance(node, (pytest.Module, pytest.Package)): + if node.path: + path = str(node.path.relative_to(Path.cwd())) + else: + path = repr(node) + break + name: str | None = getattr(node, 'name', None) + if not name: + break + chunks.append(name) + return path, '.'.join(reversed(chunks)) + + +_RETRY_ENTRIES_KEY = pytest.StashKey[list[_RetryEntry]]() + + +@final +@dataclasses.dataclass +class _RetryHelper: + retries: int = 1 + exceptions: type[Exception] | tuple[type[Exception], ...] = () + reset_fixtures: bool | Collection[str] = True + name: ClassVar[str] = 'retry' + + def __post_init__(self) -> None: + if not (int(self.retries) == self.retries > 0): + raise TypeError( + f'.entries = {self.retries!r}: expected a positive integer' + ) + if isinstance(self.exceptions, tuple): + xc: tuple[type[Exception], ...] = self.exceptions + else: + xc = self.exceptions, + if not all(issubclass(X, Exception) for X in xc): + raise TypeError( + f'.exceptions = {self.exceptions!r}: ' + 'expected an exception type or a tuple thereof' + ) + + def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: + """ + Manage the call(s) to a function. + + Args: + impl (Callable): + Implementation of + ``pytest_pyfunc_call(pyfuncitem: pytest.Function) \ +-> Any`` + func (pytest.Function): + Test function item + + Returns: + Value of the first successful call to + ``pytest_pyfunc_call(pyfuncitem=func)`` + """ + check_fixture_name: Callable[[str], bool] + reset_fixtures: Callable[[pytest.Function], None] + if self.reset_fixtures: + if isinstance(self.reset_fixtures, Collection): + check_fixture_name = partial(contains, self.reset_fixtures) + else: + check_fixture_name = lambda _: True # noqa: E731 + reset_fixtures = partial( + self._reset_between_retries, + reset_fixtures=True, + should_reset=check_fixture_name, + ) + else: + reset_fixtures = self._reset_between_retries + + result: Any = None + xc: Exception | None = None + for i in range(1 + self.retries): + if i: + reset_fixtures(func) + try: + result = impl(pyfuncitem=func) + except self.exceptions as e: + # `ty` doesn't agree that `e` is an exception... + xc = cast(Exception, e) + except Exception as e: # Uncaught exc. -> break to raise + xc = e + break + else: # Correct execution -> break to return + xc = None + break + if i: + if xc is None: + status: _Status = 'passed' + elif isinstance(xc, pytest.skip.Exception): + status = 'skipped' + else: + status = 'failed' + _RetryEntry.add_entry(func, i, status) + if xc is None: + return result + else: + raise xc + + @staticmethod + def _reset_between_retries( + func: pytest.Function, + reset_fixtures: bool = False, + should_reset: Callable[[str], bool] = lambda _: False, + ) -> None: + def clear_fixture_cache(fixture_def: pytest.FixtureDef[Any]) -> None: + if ( + fixture_def.scope == 'function' + and should_reset(fixture_def.argname) + and hasattr(fixture_def, 'cached_result') + ): + fixture_def.cached_result = None + + def unique( + items: Iterable[T], key: Callable[[T], Hashable] = id, + ) -> Generator[T, None, None]: + seen: set[Hashable] = set() + for item in items: + hashed = key(item) + if hashed in seen: + continue + seen.add(hashed) + yield item + + def iter_all_fixture_defs( + func: pytest.Function, + ) -> Generator[pytest.FixtureDef[Any], None, None]: + fdef_mapping: Mapping[Any, Iterable[pytest.FixtureDef[Any]]] + # Somehow `mypy` doesn't trust the below but `ty` does... + for fdef_mapping in [ # type:ignore[assignment] + func._fixtureinfo.name2fixturedefs, + func._request._arg2fixturedefs, + func.session._fixturemanager._arg2fixturedefs, + ]: + for fixture_defs in fdef_mapping.values(): + yield from fixture_defs + + if reset_fixtures: + # Beside clearing `.funcargs`, `._initrequest()` also resets + # the `TopRequest` instance that `func` has (`._request`) + func._initrequest() + for fixture_def in unique(iter_all_fixture_defs(func)): + clear_fixture_cache(fixture_def) + else: + # Fixture values will naturally refill, possibly from caches + func.funcargs.clear() + func.setup() + + @classmethod + def pytest_pyfunc_call(cls, pyfuncitem: pytest.Function) -> Any: + """ + Run the :py:class:`pytest.Function` object with the requisite + number of retries if necessary. + """ + pm = pyfuncitem.config.pluginmanager + impl: _PyfuncCallImpl = pm.subset_hook_caller( + 'pytest_pyfunc_call', [cls], + ) + helper = cls.get_helper(pyfuncitem) + if helper: + return helper.manage_call(impl, pyfuncitem) + return impl(pyfuncitem=pyfuncitem) + + @classmethod + def get_helper(cls, pyfuncitem: pytest.Function) -> _RetryHelper | None: + retries: int = 0 + xc: set[type[Exception]] = set() + reset_fixtures: bool | set[str] = True + for mark in pyfuncitem.iter_markers(): + if mark.name != cls.name: + continue + instance = cls(*mark.args, **mark.kwargs) + retries += instance.retries + if isinstance(instance.exceptions, tuple): + xc.update(instance.exceptions) + else: + xc.add(instance.exceptions) + if ( + reset_fixtures not in (True, False) + and instance.reset_fixtures not in (True, False) + ): # Both collections of fixture names + if TYPE_CHECKING: + assert not isinstance(reset_fixtures, bool) + assert not isinstance(instance.reset_fixtures, bool) + reset_fixtures.update(instance.reset_fixtures) + elif instance.reset_fixtures not in (True, False): + if TYPE_CHECKING: + assert not isinstance(instance.reset_fixtures, bool) + reset_fixtures = set(instance.reset_fixtures) + else: + reset_fixtures = bool(instance.reset_fixtures) + if not retries: + return None + if not xc: + xc = {Exception} + return cls(retries, tuple(xc), reset_fixtures) + + +def _pluralize(noun: str, count: int, plural: str | None = None) -> str: + if plural is None: + plural = noun + 's' + return f'{count} {noun if count == 1 else plural}' + + +def pytest_addhooks(pluginmanager: pytest.PytestPluginManager) -> None: + """ + Register :py:class:`_RetryHelper` as a plugin so that its + :py:meth:`pytest_pyfunc_call` method can safely call other + implementations without recursing to itself. + """ + pluginmanager.register(_RetryHelper) + + +def pytest_configure(config: pytest.Config) -> None: + """ + Register the :py:deco:`pytest.mark.retry` marker. + """ + help_text = ' '.join(""" + retry[(retries=1, exceptions=Exception, reset_fixtures=True)]: + + mark the test for retrying upon failure. + + Args: + retries (int): + Max number of retries for the (sub-)test; + exceptions (type[Exception] | tuple[type[Exception], ...]): + Error types which trigger a retry when caught; + reset_fixtures (bool | Collection[str]): + Names of function-scoped fixtures to reset between retries, + `True` reset all such fixtures, + `False` none thereof + """.split()) + config.addinivalue_line('markers', help_text) + + +# TODO: do we need to call `pytest_runtest_teardown()` between retries +# to make sure that the reset fixtures are properly cleaned up?` + + +def pytest_terminal_summary( + terminalreporter: TerminalReporter, config: pytest.Config, +) -> None: + """ + Write a summary section about rerun tests. + """ + def get_summary(status: _Status, entries: list[_RetryEntry]) -> str: + return f'{_pluralize("test", len(entries))} {status} with retries' + + def group_subtests( + entries: list[_RetryEntry] + ) -> dict[str, list[_RetryEntry]]: + result: dict[str, list[_RetryEntry]] = {} + for entry in entries: + result.setdefault(entry.full_original_name, []).append(entry) + return result + + formatting = {'yellow': True} + write_line: Callable[[str], None] = partial( + terminalreporter.write_line, **formatting # type: ignore + ) + write_header = partial( + terminalreporter.write_sep, '=', 'retries summary', **formatting + ) + write_newline = partial(write_line, '') + try: + verbosity: int = config.get_verbosity() # type: ignore + except AttributeError: # pytest < 8.0 + verbosity = int(config.option.verbose) + retry_entries: dict[_Status, list[_RetryEntry]] = {} + for entry in _RetryEntry.get_entries(config): + retry_entries.setdefault(entry.status, []).append(entry) + if not retry_entries: + return + if verbosity > 0: + write_newline() + write_header() + write_newline() + for status, entries in retry_entries.items(): + write_line(get_summary(status, entries) + ':') + for entry in entries: + write_line( + f' {entry.full_name}: ' + f'retried {_pluralize("time", entry.retries)}' + ) + write_newline() + else: + write_header() + for status, entries in retry_entries.items(): + tests: list[str] = [] + for name, children in group_subtests(entries).items(): + if len(children) == 1: + tests.append(children[0].full_name) + else: + msg = f'{name} ({_pluralize("subtest", len(children))})' + tests.append(msg) + write_line(f'{get_summary(status, entries)}: {", ".join(tests)}') From fd34eda47dbc4d7d3e9e3d64548fa8e99de999c4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 00:12:42 +0200 Subject: [PATCH 062/165] Test for `@pytest.mark.retry` tests/conftest.py::pytest_configure() Fixed formatting of the line added by `config.addinivalue_line()` so that usage of the marker does not result in a `PytestUnknownMarkWarning` tests/test_retry_tests.py test_fixture_scoping() Test that `@pytest.mark.retry(reset_fixtures=...)` is handled correctly test_fixture_teardown() Test that stale fixtures created by reruns are torn down correctly test_exception_restrictions() Test that `@pytest.mark.retry(exceptions=...)` is handled correctly --- tests/conftest.py | 6 +- tests/test_retry_tests.py | 599 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 603 insertions(+), 2 deletions(-) create mode 100644 tests/test_retry_tests.py diff --git a/tests/conftest.py b/tests/conftest.py index 41deeb64..76ebd9f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,8 @@ A simple :py:deco:`pytest.mark.retry` decorator. Function-scoped fixtures are re-fetched bewteen retries. """ +from __future__ import annotations + import dataclasses from collections.abc import ( Callable, Collection, Generator, Hashable, Iterable, Mapping, @@ -300,11 +302,11 @@ def pytest_configure(config: pytest.Config) -> None: Register the :py:deco:`pytest.mark.retry` marker. """ help_text = ' '.join(""" - retry[(retries=1, exceptions=Exception, reset_fixtures=True)]: + retry(retries=1, exceptions=Exception, reset_fixtures=True): mark the test for retrying upon failure. - Args: + Args (all optional): retries (int): Max number of retries for the (sub-)test; exceptions (type[Exception] | tuple[type[Exception], ...]): diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py new file mode 100644 index 00000000..0b6ee307 --- /dev/null +++ b/tests/test_retry_tests.py @@ -0,0 +1,599 @@ +""" +Tests to make sure that our :py:deco:`pytest.mark.retry` decorator +works. +""" +from __future__ import annotations + +import re +import textwrap +from collections.abc import Callable, Generator +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from shutil import rmtree +from typing import Literal +from typing_extensions import Self + +import pytest + + +pytest_plugins = ('pytester',) + +_Status = Literal['passed', 'failed', 'skipped'] +_RunPytest_Method = Literal[ + 'runpytest', 'runpytest_inprocess', 'runpytest_subprocess', +] +_RunPytest = Callable[..., pytest.RunResult] +_RunnerGetter = Callable[[str, str], _RunPytest] + + +TEST_COUNTERS = """ +from __future__ import annotations +from itertools import count +from typing import Literal + +import pytest + + +@pytest.fixture +def func_scoped_counter() -> count: + return count() + + +@pytest.fixture(scope='module') +def module_scoped_counter() -> count: + return count() + + +@pytest.mark.parametrize( + ('scope', 'n'), + [('func', 0), # This passes + ('func', 2), # This passes with 2 retries + ('func', 6), # This fails with 3 retries + ('module', 4), # This fails with 3 retries (counter now at 3) + ('module', 5)] # This passes with 1 retry (counter now at 5) +) +@pytest.mark.retry(3, reset_fixtures=False) +def test_dynamic_fixtures_persisted( + request: pytest.FixtureRequest, scope: Literal['func', 'module'], n: int, +) -> None: + ''' + Test counter fixtures that are requested dynamically via the + ``request`` fixture; function-scoped fixtures persist between + test retries. + ''' + counter = request.getfixturevalue(scope + '_scoped_counter') + assert next(counter) >= n + + +@pytest.mark.parametrize( + ('scope', 'n'), + [('func', 3), # This passes with 3 retries + ('func', 4), # This fails with 3 retries + ('module', 4), # This passes (counter now at 6) + ('module', 9)] # This passes with 2 retries (counter now at 9) +) +@pytest.mark.retry(3, reset_fixtures=False) +def test_static_fixtures_persisted( + func_scoped_counter: Iterable[int], + module_scoped_counter: Iterable[int], + scope: Literal['func', 'module'], + n: int, +) -> None: + ''' + Test counter fixtures that are requested by name; function-scoped + fixtures persist between test retries. + ''' + if scope == 'func': + counter = func_scoped_counter + else: + counter = module_scoped_counter + assert next(counter) >= n + + +@pytest.mark.parametrize( + ('scope', 'n'), + [('func', 0), # This passes + ('func', 1), # This fails with 1 retry + ('module', 11)] # This passes with 1 retry (counter now at 11) +) +@pytest.mark.retry # Counters reset between retries +def test_dynamic_fixtures_reset( + request: pytest.FixtureRequest, scope: Literal['func', 'module'], n: int, +) -> None: + ''' + Test counter fixtures that are requested dynamically via the + ``request`` fixture; function-scoped fixtures are reset between + test retries. + ''' + counter = request.getfixturevalue(scope + '_scoped_counter') + assert next(counter) >= n + + +@pytest.mark.parametrize( + ('scope', 'n'), + [('func', 0), # This passes + ('func', 1), # This fails with 2 retries + ('module', 14)] # This passes with 2 retries (counter now at 14) +) +@pytest.mark.retry(2) # Ditto above +def test_static_fixtures_reset( + func_scoped_counter: Iterable[int], + module_scoped_counter: Iterable[int], + scope: Literal['func', 'module'], + n: int, +) -> None: + ''' + Test counter fixtures that are requested by name; function-scoped + fixtures are reset between test retries. + ''' + if scope == 'func': + counter = func_scoped_counter + else: + counter = module_scoped_counter + assert next(counter) >= n +""" +TEST_TEARDOWN = """ +from __future__ import annotations + +import os +import tempfile +from collections.abc import Callable, Generator +from pathlib import Path + +import pytest + + +@pytest.fixture(scope='module') +def my_temp_dir(pytestconfig: pytest.Config) -> Generator[Path, None, None]: + path: Path | None = getattr(pytestconfig.option, 'my_temp_dir', None) + if path is None: + with tempfile.TemporaryDirectory() as tmpdir: + yield Path(tmpdir) + else: + yield path + + +def _tempfile(*args, **kwargs) -> Path: + handle, path = tempfile.mkstemp(*args, **kwargs) + try: + return Path(path) + finally: + os.close(handle) + + +@pytest.fixture +def maketemp( + my_temp_dir: Path, +) -> Generator[Callable[..., Path], None, None]: + paths: list[Path] = [] + try: + def _maketemp(*args, **kwargs) -> Path: + path = _tempfile(*args, **kwargs) + paths.append(path) + print(f'maketemp() @ {id(_maketemp):#x}: created tempfile {path}') + return path + + yield _maketemp + finally: + for path in paths: + path.unlink(missing_ok=True) + print(f'maketemp() @ {id(_maketemp):#x}: removed tempfile {path}') + + +@pytest.mark.retry(reset_fixtures=True) +def test_no_fixture_reset( + my_temp_dir: Path, maketemp: Callable[..., Path], +) -> None: + path = maketemp(dir=my_temp_dir) + assert False + + +@pytest.mark.retry(2, reset_fixtures=False) +def test_with_fixture_reset( + my_temp_dir: Path, maketemp: Callable[..., Path], +) -> None: + path = maketemp(dir=my_temp_dir) + assert False +""" +TEST_EXCEPTIONS = """ +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import pytest + + +@pytest.fixture +def items() -> Iterable[Any]: + return iter(['1', None, '', '-1']) + + +@pytest.mark.retry(3, reset_fixtures=('foo',)) # Not resetting `items` +def test_all_xc_types(items: Iterable[Any]) -> None: + ''' + This should pass after 3 retries because the last item fulfills the + criterion. + ''' + assert int(next(items)) < 0 + + +@pytest.mark.retry(3, exceptions=AssertionError, reset_fixtures=()) +def test_one_xc_type(items: Iterable[Any]) -> None: + ''' + This should fail after 1 retry because the second item triggers a + :py:class:`TypeError`. + ''' + assert int(next(items)) < 0 + + +@pytest.mark.retry(reset_fixtures=False) +@pytest.mark.retry(exceptions=TypeError) +@pytest.mark.retry(exceptions=AssertionError) +def test_two_xc_types(items: Iterable[Any]) -> None: + ''' + This should fail after 2 retries because the third item triggers a + :py:class:`ValueError`. + + Note: + The three decorators stack to give 3 retries and to accept both + :py:class:`AssertionError` and :py:class:`TypeError`. + ''' + assert int(next(items)) < 0 + + +@pytest.mark.retry( + 3, + exceptions=(AssertionError, TypeError, ValueError), + reset_fixtures=False, +) +def test_three_xc_types(items: Iterable[Any]) -> None: + ''' + This should pass after 3 retries because the last item fulfills the + criterion, and the preceding errors are all included in the + ``exceptions`` argument to the wrapper. + ''' + assert int(next(items)) < 0 +""" + + +@dataclass +class _TestOutcome: + name: str = '' + status: _Status = 'passed' + retries: int = 0 + + def subtest( + self, + *params: str, + status: _Status | None = None, + retries: int | None = None, + ) -> Self: + if status is None: + status = self.status + if retries is None: + retries = self.retries + name = f'{self.name}[{"-".join(params)}]' + return type(self)(name, status, retries) + + +@dataclass +class _TestModule: + """ + Helper object for running a test module. + """ + name: str + content: str + expected_outcomes: dict[str, list[_TestOutcome]] + pytester: pytest.Pytester + conftest: str | None = None + + def __post_init__(self) -> None: + self.content = self._strip(self.content) + if self.conftest: + self.conftest = self._strip(self.conftest) + + def run( + self, + *args: str, + check_results: bool = False, + check_summary: Literal['verbose', 'concise'] | None = None, + check_warnings: int | None = None, + runner: _RunPytest_Method = 'runpytest', + ) -> pytest.RunResult: + """ + Args: + *args (str): + Passed to :py:meth:`pytester.Pytester.runpytest` + check_results (bool): + If true, check that the test outcomes are as expected + using :py:meth:`pytester.Pytester.assert_outcomes` + check_summary (bool): + If true, check that the 'retries summary' report section + is written with the expected content indicating test + results and number of retries + check_warnings (int | None): + If an integer and if ``check_results`` is true, also + check that the number of captured warnings match + runner (Literal['runpytest', 'runpytest_inprocess', \ +'runpytest_subprocess']): + The :py:class:`pytest.Pytester` method used to run the + test module + + Returns: + :py:class:`pytest.RunResult` object returned by the + :py:class:`pytest.Pytester` method + """ + tempfiles: list[Path] = [] + tempdirs: list[Path] = [] + try: + conftests: list[str] = [self._get_proj_conftest().read_text()] + if self.conftest: + conftests.append(self.conftest) + # Create separate conftest.py in nested subdirs to avoid + # hook-func implementations stepping oer one another + path = self.pytester.path + for i, conftest in enumerate(conftests): + if i: + path /= 'nested' + path.mkdir() + tempdirs.append(path) + conftest_file = path / 'conftest.py' + conftest_file.write_text(conftest) + tempfiles.append(conftest_file) + module = path / f'{self.name}.py' + module.write_text(self.content) + tempfiles.append(module) + result = getattr(self.pytester, runner)(*args, str(module)) + if check_results: + self.check_results(result, check_warnings) + if check_summary is not None: + if check_summary == 'verbose': + self.check_verbose_summary(result) + else: + self.check_concise_summary(result) + return result + finally: + for path in tempfiles: + try: + path.unlink(missing_ok=True) + except OSError: + pass + else: + print('Removed temppath', path) + for path in reversed(tempdirs): + try: + rmtree(path) + except OSError: + pass + else: + print('Removed tempdir', path) + + def check_results( + self, result: pytest.RunResult, warnings: int | None = None, + ) -> None: + counts: dict[_Status, int] = {} + for outcomes in self.expected_outcomes.values(): + for outcome in outcomes: + counts[outcome.status] = counts.get(outcome.status, 0) + 1 + result.assert_outcomes(warnings=warnings, **counts) + + def check_verbose_summary(self, result: pytest.RunResult) -> None: + lines: list[str] = [] + counts: dict[_Status, int] = {} + for outcomes in self.expected_outcomes.values(): + for outcome in outcomes: + lines.append( + f'.*::{re.escape(outcome.name)} +{outcome.status.upper()}', + ) + if not outcome.retries: + continue + counts[outcome.status] = counts.get(outcome.status, 0) + 1 + lines.append(r'.*{}.*retried {} time{}'.format( + re.escape(outcome.name), + outcome.retries, + '' if outcome.retries == 1 else 's', + )) + lines.extend( + self._format_header(status, n) for status, n in counts.items() + ) + + print(f'Expecting these lines in the output: {lines!r}...') + result.stdout.re_match_lines_random(lines) + + def check_concise_summary(self, result: pytest.RunResult) -> None: + lines: list[str] = [] + counts: dict[_Status, int] = {} + test_names: dict[_Status, dict[str, set[str]]] = {} + consolidated_names: dict[_Status, set[str]] = {} + for parent_test, outcomes in self.expected_outcomes.items(): + for outcome in outcomes: + if outcome.status == 'failed': + lines.append( + f'{outcome.status.upper()} +' + f'.*::{re.escape(outcome.name)}', + ) + if not outcome.retries: + continue + counts[outcome.status] = counts.get(outcome.status, 0) + 1 + ( + test_names + .setdefault(outcome.status, {}) + .setdefault(parent_test, set()) + .add(outcome.name) + ) + + for status, tests in test_names.items(): + for parent_test, subtests in tests.items(): + names = consolidated_names.setdefault(status, set()) + n = len(subtests) + if n == 1: + names.add(*subtests) + else: + names.add('{} ({} subtest{})'.format( + parent_test, n, '' if n == 1 else 's', + )) + + print(f'Expecting these lines in the output: {lines!r}...') + result.stdout.re_match_lines_random(lines) + + for status, n in counts.items(): + header = self._format_header(status, n) + names = consolidated_names[status] + print(f'Expecting line in the output: "{header}: <...>"...') + print(f'Expecting these names in said line: {names!r}...') + line = self._find_line(header + ':', str(result.stdout)) + for test_name in names: + assert test_name in line + + @staticmethod + def _find_line(pattern: str, text: str) -> str: + pattern = f'^.*{pattern}.*' + maybe_match = re.search(pattern, text, re.MULTILINE) + if not maybe_match: + raise ValueError(f'Cannot find {pattern!r} in {text!r}') + return maybe_match.group() + + @staticmethod + def _format_header(status: _Status, n: int) -> str: + return '{} test{} {} with retries'.format( + n, '' if n == 1 else 's', status, + ) + + @staticmethod + def _get_proj_conftest() -> Path: + return Path(__file__).parent / 'conftest.py' + + @staticmethod + def _strip(text: str) -> str: + return textwrap.dedent(text).strip('\n') + + +@pytest.fixture +def counters_module( + pytester: pytest.Pytester, +) -> Generator[_TestModule, None, None]: + dynamic_p = _TestOutcome('test_dynamic_fixtures_persisted').subtest + static_p = _TestOutcome('test_static_fixtures_persisted').subtest + dynamic_r = _TestOutcome('test_dynamic_fixtures_reset').subtest + static_r = _TestOutcome('test_static_fixtures_reset').subtest + outcomes = { + 'test_dynamic_fixtures_persisted': [ + dynamic_p('func-0'), + dynamic_p('func-2', retries=2), + dynamic_p('func-6', status='failed', retries=3), + dynamic_p('module-4', status='failed', retries=3), + dynamic_p('module-5', retries=1), + ], + 'test_static_fixtures_persisted': [ + static_p('func-3', retries=3), + static_p('func-4', status='failed', retries=3), + static_p('module-4'), + static_p('module-9', retries=2), + ], + 'test_dynamic_fixtures_reset': [ + dynamic_r('func-0'), + dynamic_r('func-1', status='failed', retries=1), + dynamic_r('module-11', retries=1), + ], + 'test_static_fixtures_reset': [ + static_r('func-0'), + static_r('func-1', status='failed', retries=2), + static_r('module-14', retries=2), + ], + } + yield _TestModule('test_counters', TEST_COUNTERS, outcomes, pytester) + + +@pytest.fixture +def teardown_module( + pytester: pytest.Pytester, +) -> Generator[_TestModule, None, None]: + yield _TestModule( + 'test_teardown', + TEST_TEARDOWN, + { + 'test_no_fixture_reset': + [_TestOutcome('test_no_fixture_reset', 'failed', 1)], + 'test_with_fixture_reset': + [_TestOutcome('test_with_fixture_reset', 'failed', 2)], + }, + pytester, + conftest=""" + from __future__ import annotations + + from pathlib import Path + + import pytest + + + def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + '--my-temp-dir', + type=Path, + help=f'persisted tempdir location for {__file__!r}', + ) + """, + ) + + +@pytest.fixture +def exceptions_module( + pytester: pytest.Pytester, +) -> Generator[_TestModule, None, None]: + yield _TestModule( + 'test_exceptions', + TEST_EXCEPTIONS, + { + 'test_all_xc_types': + [_TestOutcome('test_all_xc_types', retries=3)], + 'test_one_xc_type': + [_TestOutcome('test_one_xc_type', 'failed', 1)], + 'test_two_xc_types': + [_TestOutcome('test_two_xc_types', 'failed', 2)], + 'test_three_xc_types': + [_TestOutcome('test_three_xc_types', retries=3)], + }, + pytester, + ) + + +@pytest.mark.parametrize('verbose', [True, False]) +def test_fixture_scoping(counters_module: _TestModule, verbose: bool) -> None: + """ + Test that the decorator correctly handles scoped fixtures. + """ + run = partial(counters_module.run, check_results=True, check_warnings=0) + if verbose: + run('--verbose', check_summary='verbose') + else: + run(check_summary='concise') + + +def test_fixture_teardown( + tmp_path_factory: pytest.TempPathFactory, teardown_module: _TestModule, +) -> None: + """ + Test that the decorator correctly handles teardown for additional + fixture copies incurred by retries. + """ + tempdir = tmp_path_factory.mktemp('my_temp') + print(tempdir) + teardown_module.run( + '--verbose', f'--my-temp-dir={tempdir}', + check_results=True, check_summary='verbose', check_warnings=0, + ) + files = {path.name for path in tempdir.iterdir()} + assert not files + + +def test_exception_restrictions(exceptions_module: _TestModule) -> None: + """ + Test that the decorator correctly handles failures owing to + different exception classes. + """ + exceptions_module.run( + '--verbose', + check_results=True, check_summary='verbose', check_warnings=0, + ) From 0ca2af206317d0b42c6600835b42ed7284d9a3fa Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 07:31:57 +0200 Subject: [PATCH 063/165] Check retried fixture teardown order tests/conftest.py::_RetryHelper._reset_between_retries() Now calling `FixtureDef.finish()` on the cleaned-up function-scoped fixtures tests/test_retry_tests.py::test_fixture_teardown() Added check that the teardown events happen in the expected order (old fixture value torn down before the new value is used) --- tests/conftest.py | 58 +++++++++++++--- tests/test_retry_tests.py | 141 +++++++++++++++++++++++++++++++++----- 2 files changed, 170 insertions(+), 29 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 76ebd9f5..f1f0ce19 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ from __future__ import annotations import dataclasses +import warnings from collections.abc import ( Callable, Collection, Generator, Hashable, Iterable, Mapping, ) @@ -16,7 +17,10 @@ ) import pytest +from _pytest.compat import NOTSET +from _pytest.fixtures import SubRequest from _pytest.nodes import Node +from _pytest.scope import Scope from _pytest.unittest import TestCaseFunction try: from pytest import TerminalReporter @@ -29,6 +33,8 @@ FCls = TypeVar('FCls', bound=type[TestCaseFunction]) T = TypeVar('T') +_FUNCTION_SCOPE = Scope.Function + class _PyfuncCallImpl(Protocol): def __call__(self, *, pyfuncitem: pytest.Function) -> Any: @@ -98,6 +104,7 @@ def _name_prefixes(self) -> tuple[str, str]: _RETRY_ENTRIES_KEY = pytest.StashKey[list[_RetryEntry]]() +_NEXTITEMS_KEY = pytest.StashKey[dict[pytest.Item, pytest.Item | None]]() @final @@ -189,13 +196,46 @@ def _reset_between_retries( reset_fixtures: bool = False, should_reset: Callable[[str], bool] = lambda _: False, ) -> None: - def clear_fixture_cache(fixture_def: pytest.FixtureDef[Any]) -> None: - if ( - fixture_def.scope == 'function' - and should_reset(fixture_def.argname) - and hasattr(fixture_def, 'cached_result') + """ + Note: + This makes HEAVY use of :py:mod`_pytest` internals. + """ + def cleanup_fixture(fdef: pytest.FixtureDef[Any]) -> None: + if not ( + fdef.scope == 'function' + and should_reset(fdef.argname) + and getattr(fdef, 'cached_result', None) is not None ): - fixture_def.cached_result = None + return + fdef.cached_result = None + finalize(fdef) + + def finalize(fdef: pytest.FixtureDef[Any]) -> None: + assert fdef.scope == 'function' + + # Plagiarized code from + # `FixtureRequest._get_active_fixture_def()` + try: + callspec = func.callspec + except AttributeError: + callspec = None + if callspec is not None and fdef.argname in callspec.params: + value = callspec.params[fdef.argname] + index = callspec.indices[fdef.argname] + else: + value, index = NOTSET, 0 + + with warnings.catch_warnings(): + warnings.simplefilter( + 'ignore', pytest.PytestDeprecationWarning, + ) + fdef.finish(SubRequest( + request=func._request, + scope=_FUNCTION_SCOPE, + param=value, + param_index=index, + fixturedef=fdef, + )) def unique( items: Iterable[T], key: Callable[[T], Hashable] = id, @@ -226,7 +266,7 @@ def iter_all_fixture_defs( # the `TopRequest` instance that `func` has (`._request`) func._initrequest() for fixture_def in unique(iter_all_fixture_defs(func)): - clear_fixture_cache(fixture_def) + cleanup_fixture(fixture_def) else: # Fixture values will naturally refill, possibly from caches func.funcargs.clear() @@ -319,10 +359,6 @@ def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line('markers', help_text) -# TODO: do we need to call `pytest_runtest_teardown()` between retries -# to make sure that the reset fixtures are properly cleaned up?` - - def pytest_terminal_summary( terminalreporter: TerminalReporter, config: pytest.Config, ) -> None: diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py index 0b6ee307..b18df4e0 100644 --- a/tests/test_retry_tests.py +++ b/tests/test_retry_tests.py @@ -5,13 +5,15 @@ from __future__ import annotations import re +import pprint import textwrap -from collections.abc import Callable, Generator +from collections.abc import Callable, Generator, Iterable from dataclasses import dataclass from functools import partial +from operator import attrgetter from pathlib import Path from shutil import rmtree -from typing import Literal +from typing import Any, Literal, cast from typing_extensions import Self import pytest @@ -139,6 +141,7 @@ def test_static_fixtures_reset( import os import tempfile from collections.abc import Callable, Generator +from functools import partial from pathlib import Path import pytest @@ -153,6 +156,11 @@ def my_temp_dir(pytestconfig: pytest.Config) -> Generator[Path, None, None]: else: yield path +@pytest.fixture(scope='module') +def my_log(pytestconfig: pytest.Config) -> Path | None: + path: Path | None = getattr(pytestconfig.option, 'my_log', None) + return path + def _tempfile(*args, **kwargs) -> Path: handle, path = tempfile.mkstemp(*args, **kwargs) @@ -164,25 +172,41 @@ def _tempfile(*args, **kwargs) -> Path: @pytest.fixture def maketemp( - my_temp_dir: Path, + my_temp_dir: Path, my_log: Path | None, ) -> Generator[Callable[..., Path], None, None]: paths: list[Path] = [] - try: - def _maketemp(*args, **kwargs) -> Path: - path = _tempfile(*args, **kwargs) - paths.append(path) - print(f'maketemp() @ {id(_maketemp):#x}: created tempfile {path}') - return path + def _maketemp(*args, **kwargs) -> Path: + path = _tempfile(*args, **kwargs) + paths.append(path) + log(f'created tempfile {path}') + return path + + log = partial(_log, _maketemp, my_log) + try: yield _maketemp finally: for path in paths: path.unlink(missing_ok=True) - print(f'maketemp() @ {id(_maketemp):#x}: removed tempfile {path}') + log(f'removed tempfile {path}') + + +def _log(maketemp: Any, my_log: Path | None, msg: str) -> None: + chunks: list[str] = [ + os.environ['PYTEST_CURRENT_TEST'], + f'maketemp() @ {id(maketemp):#x}', + msg, + ] + msg = ': '.join(chunks) + print(msg) + if my_log is None: + return + with my_log.open(mode='a') as fobj: + print(msg, file=fobj) @pytest.mark.retry(reset_fixtures=True) -def test_no_fixture_reset( +def test_with_fixture_reset( my_temp_dir: Path, maketemp: Callable[..., Path], ) -> None: path = maketemp(dir=my_temp_dir) @@ -190,7 +214,7 @@ def test_no_fixture_reset( @pytest.mark.retry(2, reset_fixtures=False) -def test_with_fixture_reset( +def test_no_fixture_reset( my_temp_dir: Path, maketemp: Callable[..., Path], ) -> None: path = maketemp(dir=my_temp_dir) @@ -470,6 +494,34 @@ def _strip(text: str) -> str: return textwrap.dedent(text).strip('\n') +def _identical_items_are_adjacent(items: Iterable[Any]) -> bool: + """ + Example: + >>> _identical_items_are_adjacent([]) + True + >>> _identical_items_are_adjacent([1]) + True + >>> _identical_items_are_adjacent([1, 10]) + True + >>> _identical_items_are_adjacent([1, 10, 1]) + False + >>> _identical_items_are_adjacent('AAcCb') + True + >>> _identical_items_are_adjacent('AcCAb') + False + """ + past: set[Any] = set() + sentinel = object() + last: Any = sentinel + for item in items: + if last is not sentinel and last != item: + past.add(last) + if item in past: + return False + last = item + return True + + @pytest.fixture def counters_module( pytester: pytest.Pytester, @@ -515,9 +567,9 @@ def teardown_module( TEST_TEARDOWN, { 'test_no_fixture_reset': - [_TestOutcome('test_no_fixture_reset', 'failed', 1)], + [_TestOutcome('test_no_fixture_reset', 'failed', 2)], 'test_with_fixture_reset': - [_TestOutcome('test_with_fixture_reset', 'failed', 2)], + [_TestOutcome('test_with_fixture_reset', 'failed', 1)], }, pytester, conftest=""" @@ -534,6 +586,11 @@ def pytest_addoption(parser: pytest.Parser) -> None: type=Path, help=f'persisted tempdir location for {__file__!r}', ) + parser.addoption( + '--my-log', + type=Path, + help=f'log file location for tempfile creation/deletion', + ) """, ) @@ -576,16 +633,64 @@ def test_fixture_teardown( ) -> None: """ Test that the decorator correctly handles teardown for additional - fixture copies incurred by retries. + fixture copies incurred by retries; in particular, superseded + function-scoped fixtures should be torn down before their + replacements are set up. """ + Stage = Literal['setup', 'call', 'teardown'] + + @dataclass + class LogEntry: + test: str + stage: Stage + fixture_id: int + msg: str + + @classmethod + def parse_line(cls, line: str) -> Self: + test, ident, *remainder = line.split(': ') + msg = ': '.join(remainder) + test_match = re.fullmatch( + r'(.+) +\((setup|call|teardown)\)', test, + ) + assert test_match + test, stage = test_match.group(1, 2) + assert stage in ('setup', 'call', 'teardown') + ident_match = re.fullmatch( + r'maketemp\(\) @ 0x([0-9a-f]+)', ident, + ) + assert ident_match + fixture_id = int(ident_match.group(1), base=16) + return cls(test, cast(Stage, stage), fixture_id, msg) + tempdir = tmp_path_factory.mktemp('my_temp') - print(tempdir) + log = tempdir / 'tempfiles.log' teardown_module.run( - '--verbose', f'--my-temp-dir={tempdir}', + '--verbose', f'--my-temp-dir={tempdir}', f'--my-log={log}', check_results=True, check_summary='verbose', check_warnings=0, ) + + # Check that all the tempfiles ahve been wiped files = {path.name for path in tempdir.iterdir()} - assert not files + assert not (files - {log.name}) + + # Check that tempfiles are deleted as soon as the fixture value + # that created them went obsolete, before the next rerun; + # we can verify that by checking that the ids of the `makefile()` + # fixtures appear in contiguous blocks + with log.open() as fobj: + entries = [LogEntry.parse_line(line.rstrip('\n')) for line in fobj] + pprint.pprint(entries) + field: tuple[str, ...] | str + for field in ('test', 'stage'), 'fixture_id': + if isinstance(field, str): + getter: Callable[[LogEntry], Any] = attrgetter(field) + else: + getter = attrgetter(*field) + values = [getter(entry) for entry in entries] + assert _identical_items_are_adjacent(values), ( + f'Inconsistency in {field} order: {values!r}' + ) def test_exception_restrictions(exceptions_module: _TestModule) -> None: From 532b0dff26b264ab3c3f9be1d42f04965bc12899 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 08:21:36 +0200 Subject: [PATCH 064/165] Update `test_child_procs.py` tests/test_child_procs.py ResultMismatch compare() New helper method for raising an error pytest_runtest_makereport() New helper method for truncating the traceback so that it stops at the frame calling `.compare()` test_apply_mp_patches() test_profiling_multiproc_script() Added `@pytest.mark.retry` decorators to deal with flakiness test_profiling_bare_python() - Added `@pytest.mark.retry` decorators to deal with flakiness - Added timeout (5 s) to the `subprocess.run()` call --- tests/test_child_procs.py | 118 +++++++++++++++++++++++++++++++------- 1 file changed, 96 insertions(+), 22 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index cadf145e..e03e41d2 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -5,6 +5,7 @@ import enum import inspect import multiprocessing.pool +import operator import os import re import shlex @@ -24,6 +25,7 @@ from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic +from types import TracebackType from typing import Any, Generic, Literal, TypeVar, cast, final, overload from typing_extensions import Self, ParamSpec from uuid import uuid4 @@ -38,6 +40,9 @@ from line_profiler._child_process_profiling.threading_patches import ( SHOULD_PATCH_THREADING, ) +from line_profiler._child_process_profiling.multiprocessing_patches import ( + _Poller, +) from line_profiler.curated_profiling import ( CuratedProfilerContext, ClassifiedPreimportTargets, ) @@ -45,6 +50,8 @@ T = TypeVar('T') +T1 = TypeVar('T1') +T2 = TypeVar('T2') TCtx_ = TypeVar('TCtx_') PS = ParamSpec('PS') C = TypeVar('C', bound=Callable[..., Any]) @@ -52,6 +59,13 @@ NUM_NUMBERS = 100 NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) + +# XXX: owing to the shenanigans in +# `line_profiler._child_process_profiling.multiprocessing_patches`, +# there is a risk that failing child processes are not properly +# `.terminate()`-ed. So just put in a timeout... +_NUM_RETRIES = 2 +_SUBPROC_TIMEOUT = 5 # Seconds _DEBUG = True @@ -428,6 +442,18 @@ def curated_profiler() -> Generator[LineProfiler, None, None]: yield prof +@pytest.fixture(autouse=True) +def _trim_mismatch_traceback(pytestconfig: pytest.Config) -> None: + """ + Truncate the traceback of raised :py:class`ResultMismatch` for more + useful error attribution. + """ + try: + pytestconfig.pluginmanager.register(ResultMismatch) + except ValueError: # Already registered + pass + + # ========================== Helper functions ========================== @@ -435,11 +461,13 @@ class _NotSupplied(enum.Enum): NOT_SUPPLIED = enum.auto() +@final class ResultMismatch(ValueError): def __init__( self, expected: Any, actual: Any | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + _trunc_tb: int = 0, ) -> None: msg = f'expected: {expected}' if actual != _NotSupplied.NOT_SUPPLIED: @@ -447,6 +475,54 @@ def __init__( super().__init__(msg) self.expected = expected self.actual = actual + self._trunc_tb = max(0, _trunc_tb) + + @classmethod + def compare( + cls, expected_: T1, actual_: T2, /, *, + comparator: Callable[[T1, T2], bool] = operator.eq, + expected: str | None = None, + actual: str | None = None, + ) -> None: + if comparator(expected_, actual_): + return + raise cls( + expected_ if expected is None else expected, + actual_ if actual is None else actual, + _trunc_tb=1, + ) + + @classmethod + def pytest_runtest_makereport( + cls, item: pytest.Item, call: pytest.CallInfo, + ) -> Any: + """ + Truncate the tracebacks of instances so that pytest outputs are + more useful and actually stops at the frame where the comparends + are shown. + """ + impl: Callable[..., Any] + impl = item.config.pluginmanager.subset_hook_caller( + 'pytest_runtest_makereport', [cls], + ) + make_report = partial(impl, item=item, call=call) + + xc = call.excinfo + if xc is None: + return make_report() + if not (isinstance(xc.value, cls) and xc.value._trunc_tb): + return make_report() + + tb_stack: list[TracebackType] = [xc.tb] + while tb_stack[-1].tb_next: + tb_stack.append(tb_stack[-1].tb_next) + if len(tb_stack) <= xc.value._trunc_tb: + return make_report() + tb_stack[-(xc.value._trunc_tb + 1)].tb_next = None + + del tb_stack # Help the GC + call.excinfo = xc.from_exception(xc.value.with_traceback(xc.tb)) + return make_report(call=call) @property def rich_message(self) -> str: @@ -837,11 +913,11 @@ def _search_cache_logs( flags: int = 0, ) -> None: entries = cache._gather_debug_log_entries() - if bool(entries) != expecting_logs: - raise ResultMismatch( - 'logs' if expecting_logs else 'no logs', - repr(entries) if entries else 'nothing' - ) + ResultMismatch.compare( + expecting_logs, bool(entries), + expected='logs' if expecting_logs else 'no logs', + actual=repr(entries) if entries else 'nothing', + ) if not expecting_logs: return text_chunks: list[str] = [entry.to_text() for entry in entries] @@ -1086,10 +1162,7 @@ def _run_test_module( # - The result is correctly calculated expected = nnums * (nnums + 1) // 2 output_lines = proc.stdout.splitlines() - if output_lines[0] != str(expected): - raise ResultMismatch( - f'result {expected}', f'output lines: {output_lines}', - ) + ResultMismatch.compare(str(expected), output_lines[0]) # - Temporary `.pth` file(s) created by `~~.pth_hook` has # been cleaned up assert _preserve_pth_files.get_pth_files() == old_pth_files @@ -1130,10 +1203,9 @@ def _check_output(output: str, tag: str, nhits: int) -> None: actual_nhits += int(n) except Exception: pass - if actual_nhits == nhits: - return - raise ResultMismatch( - f'{nhits} hit(s) on line(s) tagged with {tag!r}', actual_nhits, + ResultMismatch.compare( + nhits, actual_nhits, + expected=f'{nhits} hit(s) on line(s) tagged with {tag!r}', ) @@ -1406,6 +1478,7 @@ def test_cache_setup_main_process( [(True, 'no-profiler'), (False, 'with-profiler')]) @pytest.mark.parametrize(('debug', 'label4'), [(True, 'with-debug'), (False, 'no-debug')]) +@_preserve_attributes(_GLOBAL_PATCHES) def test_cache_setup_child( create_cache: Callable[..., LineProfilingCache], curated_profiler: LineProfiler, @@ -1493,6 +1566,7 @@ def list_profiled_funcs() -> list[str]: _search_cache_logs(cache, debug, patterns) +@pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) @pytest.mark.parametrize(('debug', 'label'), @@ -1576,7 +1650,7 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: num_returns = sum( nhits for lineno, nhits, _ in line_entries if lineno in return_lines ) - assert num_returns == expected_ncalls + ResultMismatch.compare(expected_ncalls, num_returns) # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests @@ -1718,6 +1792,8 @@ def test_running_multiproc_script( @_fuzz_prof_mp_1 @_fuzz_prof_mp_2 +@pytest.mark.retry(_NUM_RETRIES, + exceptions=(ResultMismatch, subprocess.TimeoutExpired)) @pytest.mark.parametrize( # XXX: should we explicitly test the single-proc case? We already # have quite a lot of subtests tho... @@ -1779,12 +1855,6 @@ def test_profiling_multiproc_script( - ``prof_child_procs`` of course toggles whether to do the patches to set up profiling in child processes. """ - # XXX: owing to the shenanigans in - # `line_profiler._child_process_profiling.multiprocessing_patches`, - # there is a risk that failing child processes are not properly - # `.terminate()`-ed. So just put in a timeout... - timeout = 5 # Seconds - # How many calls do we expect? nhits = dict.fromkeys( ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, @@ -1825,13 +1895,15 @@ def test_profiling_multiproc_script( nhits=nhits, nnums=nnums, nprocs=nprocs, - timeout=timeout, + timeout=_SUBPROC_TIMEOUT, debug_log=( 'debug.log' if prof_child_procs and _DEBUG else None ), ) +@pytest.mark.retry(_NUM_RETRIES, + exceptions=(ResultMismatch, subprocess.TimeoutExpired)) @pytest.mark.parametrize(('use_subprocess', 'label1'), [(True, 'subprocess.run'), (False, 'os.system')]) @pytest.mark.parametrize(('prof_child_procs', 'label2'), @@ -1900,7 +1972,9 @@ def test_profiling_bare_python( raise RuntimeError('called process failed') """.format(concat_command_line(sub_cmd))) cmd.extend(['-c', code]) - proc = _run_subproc(cmd, text=True, capture_output=True) + proc = _run_subproc( + cmd, text=True, capture_output=True, timeout=_SUBPROC_TIMEOUT, + ) nhits = {'EXT-INVOCATION': 1, 'EXT-LOOP': n} if not prof_child_procs: From 251f60e97489d8c33fea3857aa9c6e867ed17d66 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 16:58:36 +0200 Subject: [PATCH 065/165] Test fixes line_profiler/_child_process_profiling/multiprocessing_patches.py ::_Poller.__doc__ Increased timeout threshold on the passing test statement because something in CI seems to be slowing code execution of each polling loop down tests/conftest.py::_RetryEntry._name_prefixes Now gracefully handling when the curdir isn't a parent of the file the test function is defined in --- .../_child_process_profiling/multiprocessing_patches.py | 5 ++++- tests/conftest.py | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index c5561fa4..eacd0746 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -69,7 +69,10 @@ class _Poller: ... ) ... >>> - >>> with count_until(10).with_cooldown(.01).with_timeout(.25): + >>> with count_until(10).with_cooldown(.01).with_timeout(.5): + ... # Note: we shouldn't really need that much time, but + ... # something in CI seems to be slowing down the polling + ... # loop... ... print('We counted up to 10') We counted up to 10 >>> with count_until(30).with_cooldown(.01).with_timeout(.25): diff --git a/tests/conftest.py b/tests/conftest.py index f1f0ce19..a11bb3aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -92,7 +92,12 @@ def _name_prefixes(self) -> tuple[str, str]: seen.add(id(node)) if isinstance(node, (pytest.Module, pytest.Package)): if node.path: - path = str(node.path.relative_to(Path.cwd())) + npath = node.path + try: + npath = npath.relative_to(Path.cwd()) + except ValueError: # Not a subpath + pass + path = str(npath) else: path = repr(node) break From 466a05ed472c3b7af6f3487d9e531785dc426b5f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 18:42:09 +0200 Subject: [PATCH 066/165] WIP: improve coverage line_profiler/_child_process_profiling/multiprocessing_patches.py _Poller __doc__ - Lengthened timeout in passing case (again) since it is slow in CI - Extended example to test these components: - `.poll_while()` - `.with_timeout(None, ...)` - `on_timeout='ignore'` - `on_timeout='warn'` __init__() Minor refactoring to hack coverage __enter__() Now issuing a `_Poller.TimeoutWarning` TimeoutWarning New warning subclass wrap_bootstrap() `# nocover` since `coverage` is having trouble tracing calls in child processes _apply_patches_generic(), _apply_mp_patches() Added `# nocover` to unlikely exception branches --- .../multiprocessing_patches.py | 99 ++++++++++++++----- 1 file changed, 76 insertions(+), 23 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index eacd0746..97dd803c 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -54,35 +54,79 @@ class _Poller: Poll a callable until it returns true-y. Example: + >>> import warnings + >>> from contextlib import ExitStack + >>> from functools import partial >>> from itertools import count - >>> from typing import Iterator - >>> - >>> - >>> def count_until(limit: int) -> bool: + >>> from typing import Iterator, Literal + + >>> def count_until( + ... limit: int, mode: Literal['until', 'while'] = 'until', + ... ) -> bool: ... def counter_is_big_enough( ... counter: Iterator[int], limit: int, ... ) -> bool: ... return next(counter) >= limit ... - ... return _Poller.poll_until( - ... counter_is_big_enough, count(), limit, - ... ) + ... def counter_is_small_enough( + ... counter: Iterator[int], limit: int, + ... ) -> bool: + ... return next(counter) < limit ... - >>> - >>> with count_until(10).with_cooldown(.01).with_timeout(.5): + ... # The branches are ultimately equal in results, but we + ... # want to explicitly test both `.poll_until()` and + ... # `.poll_while()` + ... if mode == 'until': + ... get_poller = partial( + ... _Poller.poll_until, counter_is_big_enough, + ... ) + ... else: + ... get_poller = partial( + ... _Poller.poll_while, counter_is_small_enough, + ... ) + ... return get_poller(count(), limit) + + >>> with count_until(10).with_cooldown(.01).with_timeout(1): ... # Note: we shouldn't really need that much time, but ... # something in CI seems to be slowing down the polling ... # loop... ... print('We counted up to 10') We counted up to 10 - >>> with count_until(30).with_cooldown(.01).with_timeout(.25): - ... print('We counted up to 30') \ -# doctest: +NORMALIZE_WHITESPACE + + >>> with ( + ... count_until(100) + ... .with_cooldown(.01) + ... .with_timeout(.5) # `[on_]timeout` separately supplied + ... .with_timeout(on_timeout='ignore') + ... ): + ... print("We probably didn't count up to 100 but whatever") + We probably didn't count up to 100 but whatever + + >>> with ( # doctest: +NORMALIZE_WHITESPACE + ... count_until(30).with_cooldown(.01).with_timeout(.25) + ... ): + ... print('We counted up to 30') Traceback (most recent call last): ... line_profiler..._Poller.Timeout: ... timed out (... s >= 0.25 s) waiting for callback ...counter_is_big_enough... to return true + + >>> with ExitStack() as stack: # doctest: +NORMALIZE_WHITESPACE + ... enter = stack.enter_context + ... enter(warnings.catch_warnings()) + ... warnings.simplefilter('error', _Poller.TimeoutWarning) + ... enter( + ... count_until(30, 'while') + ... .with_cooldown(.01) + ... .with_timeout(.25, 'warn') + ... ) + ... print('We counted up to 30 again') + Traceback (most recent call last): + ... + line_profiler..._Poller.TimeoutWarning: ... + timed out (... s >= 0.25 s) waiting for + callback ...counter_is_small_enough... to return true """ def __init__( self, @@ -91,13 +135,9 @@ def __init__( timeout: float = 0, on_timeout: _OnTimeout = 'error', ) -> None: - if cooldown < 0: - cooldown = 0 - if timeout < 0: - timeout = 0 self._func: Callable[[], Any] = func - self._cooldown = cooldown - self._timeout = timeout + self._cooldown = max(0, cooldown) + self._timeout = max(0, timeout) self._on_timeout = on_timeout def sleep(self): @@ -145,7 +185,7 @@ def error(msg: str) -> NoReturn: raise type(self).Timeout(msg) def warn(msg: str) -> None: - warnings.warn(msg) + warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) diagnostics.log.warning(msg) def ignore(_): @@ -182,6 +222,12 @@ class Timeout(RuntimeError): """ pass + class TimeoutWarning(Timeout, UserWarning): + """ + Issued when a :py:class:`_Poller` is timed out when polling. + """ + pass + def _get_config( config: PathLike[str] | str | bool | None = None, @@ -274,7 +320,7 @@ def wait_for_return( vanilla_impl(self) -@LineProfilingCache._method_wrapper +@LineProfilingCache._method_wrapper # nocover def wrap_bootstrap( cache: LineProfilingCache, vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], @@ -286,6 +332,13 @@ def wrap_bootstrap( Wrap around :py:meth:`BaseProcess._bootstrap` to run ``LineProfilingCache.load().cleanup()`` so that profiling results can be gathered. + + Note: + This is only invoked in child processes, and :py:mod:`coverage` + seem to be having trouble with them in the current setup, + probably due to issues with .pth file precendence causing + :py:mod:`line_profiler` to be loaded before it. Hence the + ``# nocover``. """ try: return vanilla_impl(self, *args, **kwargs) @@ -398,13 +451,13 @@ def _apply_patches_generic( submod_name = 'multiprocessing.' + submodule try: mod = import_module(submod_name) - except ImportError: + except ImportError: # nocover return for target, patches in targets.items(): if target: try: obj: Any = getattr(mod, target) - except AttributeError: + except AttributeError: # nocover continue name = f'{submod_name}.{target}' else: @@ -462,7 +515,7 @@ def _apply_mp_patches( if reboot_forkserver: try: from multiprocessing import forkserver - except ImportError: # Incompatible platform + except ImportError: # Incompatible platform # nocover pass else: server_instance: forkserver.ForkServer = forkserver._forkserver From f74674d5de20fd0bd6747f93d65ae79ccb1ea6b2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 21:30:53 +0200 Subject: [PATCH 067/165] WIP: refactor cleanup functionalities line_profiler/cleanup.py::Cleanup New base class for objects that do cleanup TODO: migrate `CuratedProfileContext` and `LineProfilingCache` to this class line_profiler/line_profiler_utils.py Migrated `block_indent()` and `make_tempfile()` from `line_profiler/_child_process_profiling/misc_utils.py` to here --- line_profiler/cleanup.py | 582 +++++++++++++++++++++++++++ line_profiler/line_profiler_utils.py | 43 +- 2 files changed, 621 insertions(+), 4 deletions(-) create mode 100644 line_profiler/cleanup.py diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py new file mode 100644 index 00000000..6794e203 --- /dev/null +++ b/line_profiler/cleanup.py @@ -0,0 +1,582 @@ +""" +Utilities for cleaning up after ourselves. +""" +from __future__ import annotations + +import sys +import os +from collections.abc import ( + Callable, Generator, Iterable, Mapping, MutableMapping, Collection, +) +from functools import partial +from operator import setitem +from reprlib import Repr +from textwrap import indent +from pathlib import Path +from types import MethodType +from typing import Any, TypeVar, TypedDict, cast +from typing_extensions import ParamSpec, Self, Unpack + +from .line_profiler_utils import block_indent, make_tempfile +from . import _diagnostics as diagnostics + + +__all__ = ('Cleanup',) + +PS = ParamSpec('PS') +K = TypeVar('K') +V = TypeVar('V') +# Note: `typing.AnyStr` deprecated since 3.13 +AnyStr = TypeVar('AnyStr', str, bytes) +_Stacks = dict[float, list[Callable[[], Any]]] +_StackContexts = list[_Stacks] + + +class _ReprAttributes(TypedDict, total=False): + """ + Note: + We use this typed dict instead of directly supplying them in the + :py:meth:`_CallbackRepr.__init__()` signature, because we don't + want to bother with the default values there. + """ + maxlevel: int + maxtuple: int + maxlist: int + maxarray: int + maxdict: int + maxset: int + maxfrozenset: int + maxdeque: int + maxstring: int + maxlog: int + maxother: int + fillvalue: str + indent: str | int | None + + +class _CallbackRepr(Repr): + """ + :py:class:`reprlib.Repr` subclass to help with representing cleanup + callbacks, special-casing certain relevant object types (see + examples below). + + Example: + >>> from functools import partial + >>> from sys import version_info + + >>> class MyEnviron(dict): + ... def some_method(self) -> None: + ... ... + ... + >>> + >>> class MyRepr(_CallbackRepr): + ... # Since we can't instantiate a new `os._Environ`, test + ... # the relevant method with a mock + ... repr_MyEnviron = _CallbackRepr.repr__Environ + ... + >>> + >>> r = MyRepr(maxenv=3, maxargs=4, maxstring=15) + + Environ-dict formatting: + + >>> my_env = MyEnviron( + ... foo='1', + ... bar='2', + ... this_varname_is_long_but_isnt_truncated=( + ... "THIS VALUE IS TRUNCATED BECAUSE IT'S TOO LONG" + ... ), + ... baz='4', + ... ) + >>> print(r.repr(my_env)) + environ({'foo': '1', 'bar': '2', \ +'this_varname_is_long_but_isnt_truncated': 'THIS ... LONG', ...}) + + Partial-object formatting: + + >>> r.maxenv = 0 + >>> print(r.repr(my_env.some_method)) + + + Bound-method formatting: + + >>> r.maxargs = 0 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial(, ...) + + Indentation (Python 3.12+): + + >>> if version_info < (3, 12): + ... from pytest import skip + ... + ... skip( + ... '`Repr.indent` not available on {}.{},{}' + ... .format(*sys.version_info) + ... ) + + >>> r = MyRepr(maxenv=2, maxargs=4) + >>> r.indent = 2 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial( + , + base=8, + ) + + >>> callback_2 = partial(min, 5, 4, 3, 2, 1) + >>> r.indent = '----' + >>> print(r.repr(callback_2)) + functools.partial( + ----, + ----5, + ----4, + ----3, + ----2, + ----..., + ) + + >>> r.indent = ' ' + >>> r.maxenv = 2 + >>> print(r.repr(my_env.some_method)) + + """ + def __init__( + self, + *, + maxargs: int = 5, + maxenv: int = 3, + **kwargs: Unpack[_ReprAttributes] + ) -> None: + super().__init__() # kwargs are 3.12+ + valid_kwargs = ( + _ReprAttributes.__optional_keys__ + | _ReprAttributes.__required_keys__ + ) + for k, v in kwargs.items(): + if k in valid_kwargs: + setattr(self, k, v) + self.maxargs = maxargs + self.maxenv = maxenv + + def repr__Environ(self, env: os._Environ[AnyStr], level: int) -> str: + get: Callable[[AnyStr], str] = partial(self.repr1, level=level-1) + # Truncate envvar values, but not their names + envvars = ['{!r}: {}'.format(k, get(v)) for k, v in env.items()] + return self._format_items(envvars, ('environ({', '})'), self.maxenv) + + def repr_method(self, method: MethodType, level: int) -> str: + instance = self.repr1(method.__self__, level-1) + func = getattr(method.__func__, '__qualname__', '?') + prefix, suffix = f'' + # Take care of possible multi-line reprs + return block_indent(instance, prefix) + suffix + + def repr_partial(self, ptl: partial, level: int) -> str: + get: Callable[[Any], str] = partial(self.repr1, level=level-1) + args = [get(arg) for arg in ptl.args] + args.extend('{}={}'.format(k, get(v)) for k, v in ptl.keywords.items()) + args.insert(0, get(ptl.func)) + name = '{0.__module__}.{0.__qualname__}'.format(type(ptl)) + # The +1 is to account for `ptl.func` + return self._format_items(args, (name + '(', ')'), self.maxargs + 1) + + def _format_items( + self, + items: Collection[str], + delims: tuple[str, str], + maxlen: int | None = None, + ) -> str: + start, end = delims + if maxlen is not None and len(items) > maxlen: + items = list(items)[:maxlen] + ['...'] + indent_prefix: str | None = self._get_indent() + if indent_prefix is None or not items: + return '{}{}{}'.format(start, ', '.join(items), end) + return '\n'.join([ + start, *(indent(item + ',', indent_prefix) for item in items), end, + ]) + + if sys.version_info >= (3, 12): + # Note: `.indent` only available since 3.12 + def _get_indent(self) -> str | None: + indent = self.indent + if indent is None or isinstance(indent, str): + return indent + return ' ' * indent + else: + @staticmethod + def _get_indent() -> None: + return None + + +_CALLBACK_REPR = _CallbackRepr(maxother=cast(int, float('inf'))).repr + + +class Cleanup: + """ + Object which holds cleanup callbacks. Also provides convenience + methods for creating tempfiles, updating mappings, and setting + attributes on objects. + """ + def __init__(self, *_, **__) -> None: + self._contexts: _StackContexts = [] + + def __enter__(self) -> Self: + """ + Returns: + The instance + + Note: + This context manager is reentrant; entering the context + create a new set of cleanup stacks, which is then cleaned up + on :py:meth:`~.__exit__`. + + Example: + >>> strings = [] + >>> add = strings.append + >>> with Cleanup() as cleanup: + ... cleanup.add_cleanup(add, 'one') + ... # Increased priority + ... cleanup.add_cleanup_with_priority(add, 1, 'two') + ... add('three') + ... with cleanup: + ... # Decreased priority + ... cleanup.add_cleanup_with_priority( + ... add, -1, 'four', + ... ) + ... cleanup.add_cleanup(add, 'five') + ... add('six') + ... add('seven') + ... # Increased priority + ... cleanup.add_cleanup_with_priority(add, 1, 'eight') + ... + >>> strings # doctest: +NORMALIZE_WHITESPACE + ['three', 'six', 'five', 'four', 'seven', 'eight', 'two', + 'one'] + """ + self._contexts.append({}) + return self + + def __exit__(self, *_, **__) -> Any: + """ + Call ``~.cleanup(1)``, clearing the level of cleanup stacks we + previously :py:meth:`~.__enter__`-ed into. + """ + self.cleanup(1) + + # Cleanup methods + + def cleanup(self, levels: int | None = None) -> None: + """ + Pop cleanup callbacks from the internal stacks added via + :py:meth:`~.add_cleanup` etc. and call them in order. + + Args: + levels (int | None): + Number of stack levels to clear; passing :py:const`None` + clears the entire stack of callback stacks + """ + def pop_all_contexts( + contexts: _StackContexts, + ) -> Generator[_Stacks, None, None]: + while contexts: + yield contexts.pop() + + def pop_n_levels_of_contexts( + contexts: _StackContexts, n: int, + ) -> Generator[_Stacks, None, None]: + for _ in range(n): + try: + yield contexts.pop() + except IndexError: # Ran out of levels + return + + pop_contexts: Iterable[_Stacks] + if levels is None: + pop_contexts = pop_all_contexts(self._contexts) + else: + pop_contexts = pop_n_levels_of_contexts(self._contexts, levels) + cleanup = partial(self._cleanup, self._debug_output) + for stacks in pop_contexts: + cleanup(stacks) + + @staticmethod + def _cleanup(log: Callable[[str], Any], stacks: _Stacks) -> None: + ncallbacks_total = sum(len(stack) for stack in stacks.values()) + if not ncallbacks_total: + log('Cleanup aborted (no registered callbacks)') + return + # Bookend the cleanup loop with log messages to help detect if + # child processes are prematurely terminated + log(f'Starting cleanup ({ncallbacks_total} callback(s))...') + ncallbacks_run = 0 + for priority in sorted(stacks, reverse=True): + callbacks = stacks.pop(priority) + while callbacks: + callback = callbacks.pop() + callback_repr = _CALLBACK_REPR(callback) + ncallbacks_run += 1 + try: + callback() + except Exception as e: + state = 'failed' + msg = f'{callback_repr}: {type(e).__name__}: {e}' + else: + state, msg = 'succeeded', f'{callback_repr}' + log( + f'- Cleanup {state} ' + f'({ncallbacks_run}/{ncallbacks_total}): {msg}', + ) + log(f'... cleanup completed ({ncallbacks_total} callback(s))') + + def add_cleanup( + self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, + ) -> None: + """ + Shorthand for calling :py:meth:`~.add_cleanup_with_priority` + with ``priority=0``, which should be considered the default. + """ + self.add_cleanup_with_priority(callback, 0, *args, **kwargs) + + def add_cleanup_with_priority( + self, callback: Callable[PS, Any], priority: float, /, + *args: PS.args, **kwargs: PS.kwargs, + ) -> None: + """ + Add a cleanup callback to the internal stacks. + + Args: + callback (Callable[..., Any]): + Callback to be called at cleanup + priority (float): + Numeric priority value; callbacks with a HIGHER value + are invoked BEFORE those with bigger values + *args, **kwargs: + Arguments ``callback`` should be called with + + Example: + >>> strings = [] + >>> cleanup = Cleanup() + >>> # Default priority + >>> cleanup.add_cleanup(strings.append, 'first') + >>> # Decreased priority + >>> cleanup.add_cleanup_with_priority( + ... strings.append, -1, 'second', + ... ) + >>> # Increased priority + >>> cleanup.add_cleanup_with_priority( + ... strings.append, 1, 'third', + ... ) + >>> cleanup.add_cleanup(strings.append, 'fourth') + >>> assert not strings + >>> cleanup.cleanup() + >>> strings + ['third', 'fourth', 'first', 'second'] + """ + if args or kwargs: + callback = partial(callback, *args, **kwargs) + self._current_context.setdefault(priority, []).append(callback) + header = 'Cleanup callback added' + if priority: + header = f'{header} (priority: {priority})' + self._debug_output(f'{header}: {_CALLBACK_REPR(callback)}') + + # Convenience methods + + def update_mapping( + self, + mapping: MutableMapping[K, V], + updates: Mapping[K, V], + *, + _format_debug_msg: Callable[[Mapping[K, V], K, str], str] = ( + lambda mapping, key, change: 'Update {}[{!r}]: {}'.format( + object.__repr__(mapping), key, change, + ) + ), + ) -> None: + """ + Update a mapping with another and add cleanup callbacks to + reverse them. + + Args: + mapping (MutableMapping[K, V]): + Mapping to be updated + updates (Mapping[K, V]): + Mapping containing the updates + + Example: + >>> d1 = {1: 2, 3: 4} + >>> d2 = d1.copy() + >>> updates = {0: -1, 3: 5} + >>> with Cleanup() as cleanup: + ... cleanup.update_mapping(d1, updates) + ... for key, value in updates.items(): + ... assert d1[key] == value + ... + >>> assert d1 == d2 + """ + for key, value in updates.items(): + try: + old = mapping[key] + except KeyError: + self.add_cleanup(mapping.pop, key, None) + change = f'{value!r} (new)' + else: + self.add_cleanup(setitem, mapping, key, old) + change = f'{old!r} -> {value!r}' + self._debug_output(_format_debug_msg(mapping, key, change)) + mapping[key] = value + + def make_tempfile( + self, *, + delete: bool = True, + priority: float = 0, + _format_debug_msg: Callable[[Path], str] = ( + lambda path: f'Created tempfile: {path.name!r}' + ), + **kwargs + ) -> Path: + """ + Create a fresh tempfile with :py:func:`tempfile.mkstemp`. + + Args: + delete (bool): + Whether to remove the file on cleanup + priority (float): + Cleanup priority (see + :py:meth:`~.add_cleanup_with_priority`) + **kwargs: + Passed to :py:func:`tempfile.mkstemp` + + Returns: + path (Path): + Path to the created file. + + Example: + >>> prefix, suffix = 'my_file_', '.txt' + >>> with Cleanup() as cleanup: + ... path = cleanup.make_tempfile( + ... prefix=prefix, suffix=suffix, + ... ) + ... assert path.exists() + ... assert path.name.startswith(prefix) + ... assert path.name.endswith(suffix) + ... + >>> assert not path.exists() + """ + path = make_tempfile(**kwargs) + self._debug_output(_format_debug_msg(path)) + if delete: + self.add_cleanup_with_priority( + path.unlink, priority, missing_ok=True, + ) + return path + + def patch( + self, obj: Any, attr: str, value: Any, *, + name: str | None = None, + cleanup: bool = True, + priority: float = 0, + ) -> None: + """ + Patch an attribute on an object. + + Args: + obj (Any): + Object to be patched + attr (str): + Name of the attribute + value (Any): + Value to be assigned to said attribute of ``obj`` + name (str | None): + Optional name for ``obj`` to be used in debug messages + cleanup (bool): + Whether to reverse the patch (by resetting or deleting + the attribute) on cleanup + priority (float): + Cleanup priority (see + :py:meth:`~.add_cleanup_with_priority`) + + Example: + >>> class Object(object): + ... pass # Allow setting arbitrary attributes + ... + >>> + >>> obj = Object() + >>> obj.foo = 1 + >>> with Cleanup() as cleanup: + ... cleanup.patch(obj, 'foo', 2) + ... cleanup.patch(obj, 'bar', 3) + ... assert obj.foo == 2 + ... assert obj.bar == 3 + ... + >>> assert obj.foo == 1 + >>> assert not hasattr(obj, 'bar') + """ + add_cleanup = self.add_cleanup if cleanup else (lambda *_, **__: None) + try: + old = getattr(obj, attr) + except AttributeError: + add_cleanup(delattr, obj, attr) + else: + add_cleanup(setattr, obj, attr, old) + setattr(obj, attr, value) + if name is None: + name = self._get_name(obj) + msg = 'Patched `{}.{}` -> `{}`'.format(name, attr, value) + self._debug_output(msg) + + # Helper methods + + @staticmethod + def _get_name(obj: Any, /) -> str: + """ + Get an appropriate name for an arbitrary object. + + Example: + >>> import textwrap + >>> + >>> + >>> Cleanup._get_name(textwrap) + 'textwrap' + >>> Cleanup._get_name(textwrap.dedent) + 'textwrap.dedent' + >>> Cleanup._get_name(str) + 'str' + >>> Cleanup._get_name(print) + 'print' + >>> Cleanup._get_name(object()) # doctest: +ELLIPSIS + '' + """ + if hasattr(obj, '__qualname__'): + name = obj.__qualname__ + elif hasattr(obj, '__name__'): + name = obj.__name__ + else: + return repr(obj) + if hasattr(obj, '__module__'): + if obj.__module__ not in ('builtins', '__builtins__'): + name = f'{obj.__module__}.{name}' + return str(name) + + def _debug_output(self, msg: str, /) -> None: + """ + Write debugging output. + + Note: + This default implementation just writes to the logger. + """ + diagnostics.log.debug(msg) + + @property + def _current_context(self) -> _Stacks: + try: + return self._contexts[-1] + except IndexError: + ctx: _Stacks = {} + self._contexts.append(ctx) + return ctx diff --git a/line_profiler/line_profiler_utils.py b/line_profiler/line_profiler_utils.py index 887cdd55..ea86695e 100644 --- a/line_profiler/line_profiler_utils.py +++ b/line_profiler/line_profiler_utils.py @@ -5,10 +5,15 @@ from __future__ import annotations import enum -import typing +import os +from pathlib import Path +from tempfile import mkstemp +from textwrap import indent +from typing import TYPE_CHECKING +from typing_extensions import Self -if typing.TYPE_CHECKING: - from typing_extensions import Self + +__all__ = ('StringEnum', 'block_indent', 'make_tempfile') class _StrEnumBase(str, enum.Enum): @@ -49,7 +54,7 @@ def __str__(self) -> str: try: from enum import StrEnum as _StrEnum except ImportError: - if not typing.TYPE_CHECKING: # Don't confuse the typechecker + if not TYPE_CHECKING: # Don't confuse the typechecker _StrEnum = _StrEnumBase @@ -89,3 +94,33 @@ def _missing_(cls, value: object) -> Self | None: for name, instance in cls.__members__.items() } return members.get(value.casefold()) + + +def block_indent(string: str, prefix: str, fill_char: str = ' ') -> str: + r""" + Example: + >>> string = 'foo\nbar\nbaz' + >>> print(string) + foo + bar + baz + >>> print(block_indent(string, '++++', '-')) + ++++foo + ----bar + ----baz + """ + width = len(prefix) + return prefix + indent(string, fill_char * width)[width:] + + +def make_tempfile(**kwargs) -> Path: + """ + Convenience wrapper around :py:func:`tempfile.mkstemp`, discarding + and closing the integer handle (which if left unattended causes + problems on some platforms). + """ + handle, fname = mkstemp(**kwargs) + try: + return Path(fname) + finally: + os.close(handle) From 8450358bf698f6b8a422a5a91f5883ea6184a13f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 22:54:52 +0200 Subject: [PATCH 068/165] Move files around line_profiler/_threading_patches.py Moved from `line_profiler/_child_process_profiling/threading_patches.py` because the patch is needed for correctness even if not profiling child-process-based parallelism TODO: fix and adapt and use it with `CuratedProfilerContext` --- .../threading_patches.py => _threading_patches.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename line_profiler/{_child_process_profiling/threading_patches.py => _threading_patches.py} (100%) diff --git a/line_profiler/_child_process_profiling/threading_patches.py b/line_profiler/_threading_patches.py similarity index 100% rename from line_profiler/_child_process_profiling/threading_patches.py rename to line_profiler/_threading_patches.py From 9b052ce42478590612b16ad850910671854bf6c4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 23:38:41 +0200 Subject: [PATCH 069/165] Patch `threading` with `CuratedProfilerContext` line_profiler/_threading_patches.py make_thread_init_wrapper() Refactored from `wrap_init()` apply() Changed call signature (non-backwards compatible) line_profiler/curated_profiling.py::CuratedProfilerContext Now inheriting from `~.cleanup.Cleanup` install(), uninstall() - Simplified implementations - Added handling for `threading` patching __enter__() Now returning the instance to be consistent with the base class line_profiler/_child_process_profiling/cache.py Removed code applying (the now-nonexistent) `~.threading_patches.apply()` --- .../_child_process_profiling/cache.py | 6 +- line_profiler/_threading_patches.py | 73 ++++++++++--------- line_profiler/curated_profiling.py | 58 +++++++-------- 3 files changed, 66 insertions(+), 71 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 51a464b3..b70807f1 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -521,9 +521,8 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: forked processes is properly handled (if ``wrap_os_fork=True``) - - :py:mod:`multiprocessing` and :py:mod:`threading` patched - so that child processes and threads managed thereby are - properly handled + - :py:mod:`multiprocessing` patched so that child processes + managed thereby are properly handled - Instance to be returned if :py:func:`~.load()` is called from now on @@ -614,7 +613,6 @@ def _setup_common( _import_sibling('multiprocessing_patches').apply( self, reboot_forkserver, ) - _import_sibling('threading_patches').apply(self) def _wrap_os_fork(self) -> None: """ diff --git a/line_profiler/_threading_patches.py b/line_profiler/_threading_patches.py index c8a5b62c..32f13471 100644 --- a/line_profiler/_threading_patches.py +++ b/line_profiler/_threading_patches.py @@ -1,6 +1,6 @@ """ -Patch :py:mod:`threading` so that profiling extends into processes -it creates. +Patch :py:mod:`threading` so that profiling extends consistenly into +processes it creates. """ from __future__ import annotations @@ -8,13 +8,13 @@ from collections.abc import Callable from functools import wraps from typing import TYPE_CHECKING, Any, TypeVar -from typing_extensions import ParamSpec +from typing_extensions import ParamSpec, Concatenate -from .._line_profiler import ( # type: ignore +from ._line_profiler import ( # type: ignore USE_LEGACY_TRACE as SHOULD_PATCH_THREADING, ) -from ..line_profiler import LineProfiler -from .cache import LineProfilingCache +from .line_profiler import LineProfiler +from .cleanup import Cleanup __all__ = ('apply', 'SHOULD_PATCH_THREADING') @@ -61,41 +61,46 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: return wrapper -# Threads are supposed to be lightweight, so don't waste time formatting -# debug messages during startup - - -@LineProfilingCache._method_wrapper(debug=False) -def wrap_init( - cache: LineProfilingCache, - vanilla_impl: Callable[..., None], - self: threading.Thread, - group: None = None, - target: Callable[..., Any] | None = None, - *a, **k -) -> None: +def make_thread_init_wrapper( + prof: LineProfiler, + vanilla_impl: Callable[ + Concatenate[threading.Thread, None, Callable[..., Any] | None, PS], + None + ], +) -> Callable[ + Concatenate[threading.Thread, None, Callable[..., Any] | None, PS], None +]: """ Wrap the initializer of :py:class:`threading.Thread` so that the profiler's :py:attr:`LineProfiler.enable_count` is synced up on newly spun-up threads. """ - prof = cache.profiler - enable_count: int | None = getattr(prof, 'enable_count', None) - if target is not None and enable_count: - if TYPE_CHECKING: - assert prof is not None - target = make_syncing_wrapper(target, prof, enable_count) - vanilla_impl(self, group, target, *a, **k) + @wraps(vanilla_impl) + def wrapper( + self: threading.Thread, + group: None = None, + target: Callable[..., Any] | None = None, + *args: PS.args, + **kwargs: PS.kwargs + ) -> None: + enable_count: int | None = getattr(prof, 'enable_count', None) + if target is not None and enable_count: + if TYPE_CHECKING: + assert prof is not None + target = make_syncing_wrapper(target, prof, enable_count) + vanilla_impl(self, group, target, *args, **kwargs) + + return wrapper -def apply(lp_cache: LineProfilingCache) -> None: +def apply(cleanup: Cleanup, prof: LineProfiler) -> None: """ Set up profiling in threads started by :py:mod:`threading` by applying patches to the module. Args: - lp_cache (LineProfilingCache) - Cache instance governing the profiling run + cleanup (Cleanup) + Cleanup instance managing the profiling session Side effects: - :py:mod:`threading` marked as having been set up @@ -104,7 +109,7 @@ def apply(lp_cache: LineProfilingCache) -> None: - :py:meth:`threading.Thread.__init__` - - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` + - Cleanup callbacks registered via ``cleanup.add_cleanup()`` Note: This is a no-op when using :py:mod:`sys.monitoring`-based @@ -114,8 +119,6 @@ def apply(lp_cache: LineProfilingCache) -> None: return if getattr(threading, _PATCHED_MARKER, False): return - init_wrapper = wrap_init(threading.Thread.__init__) - lp_cache.patch( - threading.Thread, '__init__', init_wrapper, name='threading.Thread', - ) - lp_cache.patch(threading, _PATCHED_MARKER, True, name='threading') + init_wrapper = make_thread_init_wrapper(prof, threading.Thread.__init__) + cleanup.patch(threading.Thread, '__init__', init_wrapper) + cleanup.patch(threading, _PATCHED_MARKER, True) diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index 6e21ffe8..3b4c8ac4 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -6,16 +6,16 @@ import builtins import dataclasses -import functools import os import warnings -from collections.abc import Callable, Collection +from collections.abc import Collection from io import StringIO from textwrap import indent from typing import Any, TextIO, cast from typing_extensions import Self from . import _diagnostics as diagnostics, profile as _global_profiler +from ._threading_patches import apply as apply_threading_patches from .autoprofile.autoprofile import ( _extend_line_profiler_for_profiling_imports as upgrade_profiler, ) @@ -23,6 +23,7 @@ from .autoprofile.eager_preimports import ( is_dotted_path, write_eager_import_module, ) +from .cleanup import Cleanup from .cli_utils import short_string_path from .line_profiler import LineProfiler from .profiler_mixin import ByCountProfilerMixin @@ -150,20 +151,27 @@ def from_targets( return cls(filtered_targets, recurse_targets, invalid_targets) -class CuratedProfilerContext: +class CuratedProfilerContext(Cleanup): """ Context manager for handling various bookkeeping tasks when setting up and tearing down profiling: - Slipping ``prof`` into the builtin namespace (if ``insert_builtin`` is true) and :py::deco:`~.profile` + - Patch :py:class:`threading.Thread` so that line-profiling is + enabled on new threads if it is on the spawning threads - At exit, clearing the ``enable_count`` of ``prof``, properly disabling it - Note: - The attributes on this object are to be considered - implementation details, but not its methods and their - signatures. + Notes: + + - The attributes on this object are to be considered + implementation details, but not its methods and their + signatures. + + - In contrast to the base class (:py:class:`Cleanup`), while + this context manager is still reentrant, reentering in nested + `with: ...` statements is a no-op. """ def __init__( self, @@ -171,6 +179,7 @@ def __init__( insert_builtin: bool = False, builtin_loc: str = 'profile', ) -> None: + super().__init__() self.prof = prof self.insert_builtin = insert_builtin self.builtin_loc = builtin_loc @@ -185,48 +194,33 @@ def _global_install(self, prof: ByCountProfilerMixin | None) -> None: self._kpo(cast(LineProfiler, prof)) def install(self) -> None: - def del_builtin_profile() -> None: - delattr(builtins, self.builtin_loc) - - def set_builtin_profile(old: Any) -> None: - setattr(builtins, self.builtin_loc, old) - if self._installed: return # Equip the profiler instance with the # `.add_imported_function_or_module()` pseudo-method upgrade_profiler(self.prof) # Overwrite the explicit profiler (`@line_profiler.profile`) - self._global_install(self.prof) # type: ignore + self._global_install(self.prof) + self.add_cleanup(self._global_install, None) + # Patch `threading` + if isinstance(self.prof, LineProfiler): + apply_threading_patches(self, self.prof) # Set up hooks to deal with inserting `.prof` as a builtin name if self.insert_builtin: - try: - old = getattr(builtins, self.builtin_loc) - except AttributeError: - self._restore: Callable[[], None] = del_builtin_profile - else: - self._restore = functools.partial(set_builtin_profile, old) - set_builtin_profile(self.prof) - self._installed = True + self.patch(builtins, self.builtin_loc, self.prof) + self.patch(self, '_installed', True) def uninstall(self) -> None: if not self._installed: return - # Restore the `builtins` namespace - if ( - self.insert_builtin - and getattr(builtins, self.builtin_loc, None) is self.prof - ): - self._restore() # Fully disable the profiler for _i in range(getattr(self.prof, 'enable_count', 0)): self.prof.disable_by_count() - # Restore the state of the global `@line_profiler.profile` - self._global_install(None) - self._installed = False + self.cleanup() - def __enter__(self) -> None: + def __enter__(self) -> Self: self.install() + return self def __exit__(self, *_, **__) -> None: self.uninstall() From 09d4e182f0afde3fe03c0a3bc0e28dc6b5de44b2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 27 Apr 2026 00:44:44 +0200 Subject: [PATCH 070/165] Continue migration line_profiler/cleanup.py::Cleanup.make_tempfile() Updated default formatting for the debug-logging message line_profiler/_child_process_profiling/ _cache_logging.py Updated import location of `block_indent()` cache.py - Removed duplicate code (previously migrated to `line_profiler/cleanup.py`) - Made `LineProfilingCache` a `@final` class and a `line_profiler.cleanup.Cleanup` subclass - Updated signature of `LineProfilingCache.inject_env_vars()` - Removed `LineProfilingCache._add_cleanup()` pth_hook.py Minor internal refactoring --- .../_cache_logging.py | 2 +- .../_child_process_profiling/cache.py | 324 ++---------------- .../_child_process_profiling/pth_hook.py | 4 +- line_profiler/cleanup.py | 2 +- 4 files changed, 35 insertions(+), 297 deletions(-) diff --git a/line_profiler/_child_process_profiling/_cache_logging.py b/line_profiler/_child_process_profiling/_cache_logging.py index 91b0635f..992f68d0 100644 --- a/line_profiler/_child_process_profiling/_cache_logging.py +++ b/line_profiler/_child_process_profiling/_cache_logging.py @@ -15,7 +15,7 @@ from typing_extensions import Self from .. import _diagnostics as diagnostics -from .misc_utils import block_indent +from ..line_profiler_utils import block_indent __all__ = ('CacheLoggingEntry',) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index b70807f1..d7978f84 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -7,28 +7,25 @@ import atexit import dataclasses import os -import sys try: import _pickle as pickle except ImportError: import pickle # type: ignore[assignment,no-redef] -from collections.abc import Collection, Callable, Iterable +from collections.abc import Collection, Callable, MutableMapping, Iterable from functools import partial, cached_property, wraps from importlib import import_module -from operator import setitem from pathlib import Path from pickle import HIGHEST_PROTOCOL -from reprlib import Repr from textwrap import indent -from types import MethodType, ModuleType -from typing import Any, ClassVar, TypeVar, TypedDict, cast, overload -from typing_extensions import Concatenate, ParamSpec, Self, Unpack +from types import ModuleType +from typing import Any, ClassVar, TypeVar, cast, final, overload +from typing_extensions import Concatenate, ParamSpec, Self from .. import _diagnostics as diagnostics +from ..cleanup import Cleanup from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats from ._cache_logging import CacheLoggingEntry -from .misc_utils import block_indent, make_tempfile # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth # hook, which must run with minimal overhead when a Python process isn't @@ -56,187 +53,9 @@ def _import_sibling(submodule: str) -> ModuleType: return import_module(f'{_THIS_SUBPACKAGE}.{submodule}') -class _ReprAttributes(TypedDict, total=False): - """ - Note: - We use this typed dict instead of directly supplying them in the - :py:meth:`_CallbackRepr.__init__()` signature, because we don't - want to bother with the default values there. - """ - maxlevel: int - maxtuple: int - maxlist: int - maxarray: int - maxdict: int - maxset: int - maxfrozenset: int - maxdeque: int - maxstring: int - maxlog: int - maxother: int - fillvalue: str - indent: str | int | None - - -class _CallbackRepr(Repr): - """ - :py:class:`reprlib.Repr` subclass to help with representing cleanup - callbacks, special-casing certain relevant object types (see - examples below). - - Example: - >>> from functools import partial - >>> from sys import version_info - - >>> class MyEnviron(dict): - ... def some_method(self) -> None: - ... ... - ... - >>> - >>> class MyRepr(_CallbackRepr): - ... # Since we can't instantiate a new `os._Environ`, test - ... # the relevant method with a mock - ... repr_MyEnviron = _CallbackRepr.repr__Environ - ... - >>> - >>> r = MyRepr(maxenv=3, maxargs=4, maxstring=15) - - Environ-dict formatting: - - >>> my_env = MyEnviron( - ... foo='1', - ... bar='2', - ... this_varname_is_long_but_isnt_truncated=( - ... "THIS VALUE IS TRUNCATED BECAUSE IT'S TOO LONG" - ... ), - ... baz='4', - ... ) - >>> print(r.repr(my_env)) - environ({'foo': '1', 'bar': '2', \ -'this_varname_is_long_but_isnt_truncated': 'THIS ... LONG', ...}) - - Partial-object formatting: - - >>> r.maxenv = 0 - >>> print(r.repr(my_env.some_method)) - - - Bound-method formatting: - - >>> r.maxargs = 0 - >>> callback_1 = partial(int, base=8) - >>> print(r.repr(callback_1)) - functools.partial(, ...) - - Indentation (Python 3.12+): - - >>> if version_info < (3, 12): - ... from pytest import skip - ... - ... skip( - ... '`Repr.indent` not available on {}.{},{}' - ... .format(*sys.version_info) - ... ) - - >>> r = MyRepr(maxenv=2, maxargs=4) - >>> r.indent = 2 - >>> callback_1 = partial(int, base=8) - >>> print(r.repr(callback_1)) - functools.partial( - , - base=8, - ) - - >>> callback_2 = partial(min, 5, 4, 3, 2, 1) - >>> r.indent = '----' - >>> print(r.repr(callback_2)) - functools.partial( - ----, - ----5, - ----4, - ----3, - ----2, - ----..., - ) - - >>> r.indent = ' ' - >>> r.maxenv = 2 - >>> print(r.repr(my_env.some_method)) - - """ - def __init__( - self, - *, - maxargs: int = 5, - maxenv: int = 3, - **kwargs: Unpack[_ReprAttributes] - ) -> None: - super().__init__() # kwargs are 3.12+ - for k, v in kwargs.items(): - setattr(self, k, v) - self.maxargs = maxargs - self.maxenv = maxenv - - def repr__Environ(self, env: os._Environ[AnyStr], level: int) -> str: - get: Callable[[AnyStr], str] = partial(self.repr1, level=level-1) - # Truncate envvar values, but not their names - envvars = ['{!r}: {}'.format(k, get(v)) for k, v in env.items()] - return self._format_items(envvars, ('environ({', '})'), self.maxenv) - - def repr_method(self, method: MethodType, level: int) -> str: - instance = self.repr1(method.__self__, level-1) - func = getattr(method.__func__, '__qualname__', '?') - prefix, suffix = f'' - # Take care of possible multi-line reprs - return block_indent(instance, prefix) + suffix - - def repr_partial(self, ptl: partial, level: int) -> str: - get: Callable[[Any], str] = partial(self.repr1, level=level-1) - args = [get(arg) for arg in ptl.args] - args.extend('{}={}'.format(k, get(v)) for k, v in ptl.keywords.items()) - args.insert(0, get(ptl.func)) - name = '{0.__module__}.{0.__qualname__}'.format(type(ptl)) - # The +1 is to account for `ptl.func` - return self._format_items(args, (name + '(', ')'), self.maxargs + 1) - - def _format_items( - self, - items: Collection[str], - delims: tuple[str, str], - maxlen: int | None = None, - ) -> str: - start, end = delims - if maxlen is not None and len(items) > maxlen: - items = list(items)[:maxlen] + ['...'] - indent_prefix: str | None = self._get_indent() - if indent_prefix is None or not items: - return '{}{}{}'.format(start, ', '.join(items), end) - return '\n'.join([ - start, *(indent(item + ',', indent_prefix) for item in items), end, - ]) - - if sys.version_info >= (3, 12): - # Note: `.indent` only available since 3.12 - def _get_indent(self) -> str | None: - indent = self.indent - if indent is None or isinstance(indent, str): - return indent - return ' ' * indent - else: - @staticmethod - def _get_indent() -> None: - return None - - -_CALLBACK_REPR = _CallbackRepr(maxother=cast(int, float('inf'))).repr - - +@final @dataclasses.dataclass -class LineProfilingCache: +class LineProfilingCache(Cleanup): """ Helper object for coordinating a line-profiling session, caching the info required to make profiling persist into child processes. @@ -261,65 +80,10 @@ class LineProfilingCache: _cleanup_stacks: dict[float, list[Callable[[], Any]]] = dataclasses.field( default_factory=dict, init=False, repr=False, ) - _loaded_instance: ClassVar[Self | None] = None + _loaded_instance: ClassVar[LineProfilingCache | None] = None - def cleanup(self) -> None: - """ - Pop all the cleanup callbacks from the internal stack added via - :py:meth:`~.add_cleanup` and call them in order. - """ - stacks = self._cleanup_stacks - ncallbacks_total = sum(len(stack) for stack in stacks.values()) - if not ncallbacks_total: - self._debug_output('Cleanup aborted (no registered callbacks)') - return - # Bookend the cleanup loop with log messages to help detect if - # child processes are prematurely terminated - self._debug_output( - f'Starting cleanup ({ncallbacks_total} callback(s))...', - ) - ncallbacks_run = 0 - for priority in sorted(stacks): - callbacks = stacks.pop(priority) - while callbacks: - callback = callbacks.pop() - callback_repr = _CALLBACK_REPR(callback) - ncallbacks_run += 1 - try: - callback() - except Exception as e: - state = 'failed' - msg = f'{callback_repr}: {type(e).__name__}: {e}' - else: - state, msg = 'succeeded', f'{callback_repr}' - self._debug_output( - f'- Cleanup {state} ' - f'({ncallbacks_run}/{ncallbacks_total}): {msg}', - ) - self._debug_output( - f'... cleanup completed ({ncallbacks_total} callback(s))', - ) - - def add_cleanup( - self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, - ) -> None: - """ - Add a cleanup callback to the internal stack; they can be later - called by :py:meth:`~.cleanup`. - """ - self._add_cleanup(callback, 0, *args, **kwargs) - - def _add_cleanup( - self, callback: Callable[PS, Any], priority: float, - *args: PS.args, **kwargs: PS.kwargs, - ) -> None: - if args or kwargs: - callback = partial(callback, *args, **kwargs) - self._cleanup_stacks.setdefault(priority, []).append(callback) - header = 'Cleanup callback added' - if priority: - header = f'{header} (priority: {priority})' - self._debug_output(f'{header}: {_CALLBACK_REPR(callback)}') + def __post_init__(self) -> None: + super().__init__() def copy( self, *, @@ -369,7 +133,9 @@ def load(cls) -> Self: If a previously :py:meth:`.~.load`-ed instance exists, it is returned instead of a new instance. """ - instance = cls._loaded_instance + # `ty` needs some help here, evenif we've marked the class to be + # `@final` + instance = cast(Self | None, cls._loaded_instance) if instance is None: pid = os.environ[INHERITED_PID_ENV_VARNAME] cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}' @@ -464,32 +230,28 @@ def _get_debug_logfiles(self) -> Iterable[Path]: return Path(self.cache_dir).glob(pattern) def inject_env_vars( - self, env: dict[str, str] | None = None, + self, env: MutableMapping[str, str] | None = None, ) -> None: """ Inject the :py:attr:`~.environ` variables into ``env`` and add cleanup callbacks to reverse them. Args: - env (dict[str, str] | None): + env (MutableMapping[str, str] | None): Dictionary in the format of :py:data:`os.environ`; default is to use that """ - if env is None: - env = cast(dict[str, str], os.environ) - for name, value in self.environ.items(): - try: - old = env[name] - except KeyError: - self.add_cleanup(env.pop, name, None) - change = f'{value!r} (new)' - else: - self.add_cleanup(setitem, env, name, old) - change = f'{old!r} -> {value!r}' - self._debug_output(f'Injecting env var ${{{name}}}: {change}') - env[name] = value + self.update_mapping( + os.environ if env is None else env, + self.environ, + _format_debug_msg='Injecting env var ${{{1}}}: {2}'.format, + ) def _debug_output(self, msg: str) -> None: + """ + Beside writing to the logger, also write to the + :py:attr:`~._debug_log`. + """ try: self._make_debug_entry(msg).write(self._debug_log) except OSError: # Cache dir may have been rm-ed during cleanup @@ -592,8 +354,9 @@ def _setup_in_child_process( prefix='child-prof-output-{}-{}-{:#x}-' .format(self.main_pid, os.getpid(), id(prof)), suffix='.lprof', + delete=False, ) - self._add_cleanup(prof.dump_stats, -1, prof_outfile) + self.add_cleanup_with_priority(prof.dump_stats, 1, prof_outfile) # Various setups self._setup_common(wrap_os_fork, reboot_forkserver=False) @@ -651,28 +414,6 @@ def wrapper() -> int: self.patch(os, 'fork', wrapper, name='os') - def patch( - self, obj: Any, attr: str, value: Any, *, - name: str | None = None, cleanup: bool = True, - ) -> None: - """ - Patch ``attr`` on ``obj`` with ``value``. If ``cleanup`` is - true, register a cleanup callback to either reset or delete the - attribute. - """ - add_cleanup = self.add_cleanup if cleanup else (lambda *_, **__: None) - try: - old = getattr(obj, attr) - except AttributeError: - add_cleanup(delattr, obj, attr) - else: - add_cleanup(setattr, obj, attr, old) - setattr(obj, attr, value) - if name is None: - name = repr(obj) - msg = 'Patched `{}.{}` -> `{}`'.format(name, attr, value) - self._debug_output(msg) - def make_tempfile(self, **kwargs) -> Path: """ Create a fresh tempfile under :py:attr:`~.cache_dir`. The other @@ -682,17 +423,16 @@ def make_tempfile(self, **kwargs) -> Path: path (Path): Path to the created file. """ - path = make_tempfile(dir=self.cache_dir, **kwargs) - self._debug_output(f'Created tempfile: {path.name!r}') - return path + kwargs.setdefault('dir', self.cache_dir) + kwargs.setdefault( + '_format_debug_msg', 'Created tempfile: {0.name!r}'.format, + ) + return super().make_tempfile(**kwargs) def _replace_loaded_instance(self, force: bool = False) -> bool: cls = type(self) if force or self._consistent_with_loaded_instance: - # Note: `ty` REALLY hates assigning an instance to - # `ClassVar[Self]` (#3274); no choice but to ignore it for - # the time being... - cls._loaded_instance = self # type: ignore + cls._loaded_instance = self return True return False diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 5a19cbce..fa3ee9dd 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -51,13 +51,12 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path """ import os from sysconfig import get_path - from .misc_utils import make_tempfile if not os.path.exists(cache.filename): cache.dump() assert os.path.exists(cache.filename) - fpath = make_tempfile( + fpath = cache.make_tempfile( prefix='_line_profiler_profiling_hook_', suffix='.pth', dir=get_path('purelib'), @@ -67,7 +66,6 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path (lambda: None).__module__, cache.main_pid, ) fpath.write_text(pth_content) - cache.add_cleanup(fpath.unlink, missing_ok=True) except Exception: fpath.unlink(missing_ok=True) raise diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index 6794e203..97b6b05a 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -436,7 +436,7 @@ def make_tempfile( delete: bool = True, priority: float = 0, _format_debug_msg: Callable[[Path], str] = ( - lambda path: f'Created tempfile: {path.name!r}' + 'Created tempfile: {}'.format ), **kwargs ) -> Path: From e906f6b452eaf879ffc43416bbeb6cb77638f852 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 27 Apr 2026 01:31:12 +0200 Subject: [PATCH 071/165] Finish migration kernprof.py _touch_tempfile() Now an imported alias for `line_profiler.line_profiler_utils.make_tempfile()` _manage_profiler.__enter__(), .__exit__() Simplified implementations making use of `CuratedProfilerContext.[add_]cleanup()` line_profiler/_child_process_profiling/misc_utils.py Deleted because its functionalities have been migrated to `line_profiler.line_profiler_utils` tests/test_child_procs.py test_cache_cleanup_order() Removed test because it is covered by `Cleanup.add_cleanup_with_priority()`'s doctest test_cache_setup_main_process() No longer checking the patching of `threading` because it is no longer handled by `LineProfilingCache._setup_in_main_process()` test_cache_setup_child() No longer protecting against the patching of `threading` because it is no longer handled by `LineProfilingCache._setup_in_main_process()` --- kernprof.py | 51 ++++++++----------- .../_child_process_profiling/misc_utils.py | 40 --------------- tests/test_child_procs.py | 34 ------------- 3 files changed, 21 insertions(+), 104 deletions(-) delete mode 100644 line_profiler/_child_process_profiling/misc_utils.py diff --git a/kernprof.py b/kernprof.py index 96888468..0e1a4b77 100755 --- a/kernprof.py +++ b/kernprof.py @@ -230,6 +230,9 @@ def main(): positive_float, short_string_path, ) +from line_profiler.line_profiler_utils import ( + make_tempfile as _touch_tempfile, # Compatibility +) from line_profiler.profiler_mixin import ByCountProfilerMixin from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._logger import Logger @@ -237,9 +240,9 @@ def main(): DIAGNOSITICS_VERBOSITY = 2 -CLEANUP_PRIORITIES = { # Bigger number -> more delayed - 'rm_cache_dir': 1024, - 'gather_logs': 1, +CLEANUP_PRIORITIES = { # More negative number -> more delayed + 'rm_cache_dir': -1024, + 'gather_logs': -1, } @@ -1012,21 +1015,6 @@ def main(args=None, *, exit_on_error=True): cleanup() -def _touch_tempfile(*args, **kwargs): - """ - Wrapper around :py:func:`tempfile.mkstemp()` which drops and closes - the integer handle (which we don't need and may cause issues on some - platforms). - """ - handle, path = tempfile.mkstemp(*args, **kwargs) - try: - os.close(handle) - except Exception: - os.remove(path) - raise - return path - - def _write_tempfile(source, content, options): """ Called by :py:func:`main()` to handle :command:`kernprof -c` and @@ -1218,13 +1206,20 @@ def __enter__(self): keep_preimports_file=self.set_up_child_profiling, ) if self.set_up_child_profiling: - self.cache = _prepare_child_profiling_cache( + self.cache = cache = _prepare_child_profiling_cache( self.options, self.prof, preimports_file, script_file, ) - # Add a deferred callback for gathering debug logfiles - # (should run right before `.cache.cache_dir` is wiped) + self._ctx.add_cleanup(cache.cleanup) + # Add deferred callbacks for gathering debug logfiles + # (should run right before `.cache.cache_dir` is wiped): + # - Write the debug logs to the `._diagnostics` logger + if cache.debug: + cache.add_cleanup_with_priority( + cache._dump_debug_logs, CLEANUP_PRIORITIES['gather_logs'], + ) + # - Write the debug logs to a specific file if self.options.debug_log: - self.cache._add_cleanup( + cache.add_cleanup_with_priority( self._gather_debug_log, CLEANUP_PRIORITIES['gather_logs'], self.options.debug_log, @@ -1235,15 +1230,11 @@ def __exit__(self, *_, **__): try: extra_stats = None if self.set_up_child_profiling: - try: - if self.cache.debug: - # Recover debug output from child processes - self.cache._dump_debug_logs() - extra_stats = self.cache.gather_stats() - finally: - self.cache.cleanup() + extra_stats = self.cache.gather_stats() _post_profile(self.options, self.prof, extra_stats) finally: + # This also calls its `.cleanup()`, and in turn that of + # `.cache`'s self._ctx.uninstall() def _gather_debug_log(self, logfile): @@ -1374,7 +1365,7 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): clean_up = functools.partial(cache.add_cleanup, _remove, missing_ok=True) if not diagnostics.KEEP_TEMPDIRS: # Defer the scrubbing of the cache dir - cache._add_cleanup( + cache.add_cleanup_with_priority( _remove, CLEANUP_PRIORITIES['rm_cache_dir'], cache.cache_dir, recursive=True, ) diff --git a/line_profiler/_child_process_profiling/misc_utils.py b/line_profiler/_child_process_profiling/misc_utils.py deleted file mode 100644 index 050380bf..00000000 --- a/line_profiler/_child_process_profiling/misc_utils.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Misc. utility functions used by the subpackage. -""" -import os -from pathlib import Path -from tempfile import mkstemp -from textwrap import indent - - -__all__ = ('block_indent', 'make_tempfile') - - -def block_indent(string: str, prefix: str, fill_char: str = ' ') -> str: - r""" - Example: - >>> string = 'foo\nbar\nbaz' - >>> print(string) - foo - bar - baz - >>> print(block_indent(string, '++++', '-')) - ++++foo - ----bar - ----baz - """ - width = len(prefix) - return prefix + indent(string, fill_char * width)[width:] - - -def make_tempfile(**kwargs) -> Path: - """ - Convenience wrapper around :py:func:`tempfile.mkstemp`, discarding - and closing the integer handle (which if left unattended causes - problems on some platforms). - """ - handle, fname = mkstemp(**kwargs) - try: - return Path(fname) - finally: - os.close(handle) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index e03e41d2..5330535a 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -37,9 +37,6 @@ from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) -from line_profiler._child_process_profiling.threading_patches import ( - SHOULD_PATCH_THREADING, -) from line_profiler._child_process_profiling.multiprocessing_patches import ( _Poller, ) @@ -1227,8 +1224,6 @@ def _check_output(output: str, tag: str, nhits: int) -> None: 'multiprocessing.spawn': frozenset({'runpy'}), 'os': frozenset({'fork'}), } -if SHOULD_PATCH_THREADING: - _GLOBAL_PATCHES['threading.Thread'] = frozenset({'__init__'}) # NOTE: we need a function which isn't used by the codebase itself # (esp. during cache cleanup); otherwise the profiling results may @@ -1374,35 +1369,6 @@ def test_cache_dump_load( assert set(os.environ) == envvars -@pytest.mark.parametrize(('debug', 'label'), - [(True, 'with-debug'), (False, 'no-debug')]) -def test_cache_cleanup_order( - create_cache: Callable[..., LineProfilingCache], debug: bool, label: str, -) -> None: - """ - Test that :py:meth`LineProfilingCache.cleanup` executes the cleanup - callback stacks in order. - """ - strings: list[str] = [] - cache = create_cache(debug=debug) - - cache.add_cleanup(strings.append, 'first') - # Decreased priority - cache._add_cleanup(strings.append, 1, 'second') - # Increased priority - cache._add_cleanup(strings.append, -1, 'third') - cache.add_cleanup(strings.append, 'fourth') - - cache.cleanup() - assert strings == ['third', 'fourth', 'first', 'second'] - - pattern = '\n'.join( - rf'.*succeeded \({i + 1}/{len(strings)}\): .*append.*{string!r}.*' - for i, string in enumerate(strings) - ) - _search_cache_logs(cache, debug, [pattern]) - - @pytest.mark.parametrize(('wrap_os_fork', 'label1'), [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) @pytest.mark.parametrize(('debug', 'label2'), From e02d8979c6c8b59e91b5b9404d86d4364888f172 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 27 Apr 2026 02:26:37 +0200 Subject: [PATCH 072/165] Tweak config layout line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes] New subtable for options related to child-processing profiling: - [pth_files] New subtable for options for .pth-file creation - [multiprocessing] Migrated from `[tool.line_profiler.multiprocessing]` line_profiler/_child_process_profiling/pth_hook.py::write_pth_hook() Now loading affixes to the .pth-filename stem from `cache.config` line_profiler/_child_process_profiling/multiprocessing_patches.py _get_config_cached() Updated load location of config options _apply_mp_patches() Renamed (currently not explicitly used) argument `debug` -> `intercept_mp_logs` tests/test_child_procs.py::test_apply_mp_patches() Updated .toml file written to align with the new config locations --- .../multiprocessing_patches.py | 19 ++++++-------- .../_child_process_profiling/pth_hook.py | 10 ++++++-- line_profiler/rc/line_profiler.toml | 25 +++++++++++++------ tests/test_child_procs.py | 2 +- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 97dd803c..b2822f49 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -243,7 +243,7 @@ def _get_config_cached( ) -> Mapping[str, Any]: cd = dict( ConfigSource.from_config(config) - .get_subconfig('multiprocessing', copy=True) + .get_subconfig('child_processes', 'multiprocessing', copy=True) .conf_dict ) assert isinstance(cd.get('polling'), Mapping) @@ -474,7 +474,7 @@ def _apply_patches_generic( def _apply_mp_patches( lp_cache: LineProfilingCache, reboot_forkserver: bool = True, - debug: bool | None = None, + intercept_mp_logs: bool | None = None, ) -> None: # In a child process, we don't care about polluting the # `multiprocessing` namespace, so don't bother with cleanup @@ -494,14 +494,11 @@ def _apply_mp_patches( pass else: if hasattr(spawn, 'runpy'): - lp_cache.patch( - spawn, 'runpy', create_runpy_wrapper(lp_cache), - name='multiprocessing.spawn', - ) + lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) # Intercept `multiprocessing` debug messages - if debug is None: - debug = _get_config(lp_cache.config)['intercept_logs'] - if debug: + if intercept_mp_logs is None: + intercept_mp_logs = _get_config(lp_cache.config)['intercept_logs'] + if intercept_mp_logs: lfuncs = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] lpatches = {func: partial(partial, tee_log, func) for func in lfuncs} apply_patches('util', {'': lpatches}) @@ -524,9 +521,7 @@ def _apply_mp_patches( stop() lp_cache.add_cleanup(stop) # Mark `multiprocessing` as having been patched - lp_cache.patch( - multiprocessing, _PATCHED_MARKER, True, name='multiprocessing', - ) + lp_cache.patch(multiprocessing, _PATCHED_MARKER, True) def _no_op(*_, **__) -> None: diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index fa3ee9dd..b6197600 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -51,14 +51,20 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path """ import os from sysconfig import get_path + from ..toml_config import ConfigSource if not os.path.exists(cache.filename): cache.dump() assert os.path.exists(cache.filename) + pth_config = ( + ConfigSource.from_config(cache.config) + .get_subconfig('child_processes', 'pth_files') + .conf_dict + ) fpath = cache.make_tempfile( - prefix='_line_profiler_profiling_hook_', - suffix='.pth', + prefix=pth_config['prefix'], + suffix=pth_config['suffix'] + '.pth', dir=get_path('purelib'), ) try: diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index a420aad8..5c53aadd 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -211,29 +211,38 @@ perhit = 8 percent = 8 # XXX: --- Start of implementation details --- -# `line_profiler._child_process_profiling.multiprocessing_patches` -# settings +# `line_profiler._child_process_profiling` settings -[tool.line_profiler.multiprocessing] +[tool.line_profiler.child_processes.pth_files] -# - `intercept_logs` (bool): +# - `pth_files.prefix`, `.suffix` (str): +# Affixes to use for the stem of the name of the .pth file created +# Note: may be useful to tweak this in case of issues with .pth file +# precedence. +prefix = '_line_profiler-profiling-hook-' +suffix = '' + +[tool.line_profiler.child_processes.multiprocessing] + +# - `multiprocessing.intercept_logs` (bool): # Whether to patch the `multiprocessing.util` logging funcions so that # the internal log messages are captured and writtent to the debug # logs intercept_logs = false -[tool.line_profiler.multiprocessing.polling] +[tool.line_profiler.child_processes.multiprocessing.polling] -# - `polling.cooldown` (float): +# - `multiprocessing.polling.cooldown` (float): # Cooldown time (seconds) before successive polls on lock files (set # to <= 0 to disable cooldowns) cooldown = 0.03125 # 1/32-nd of a second -# - `polling.timeout` (float): +# - `multiprocessing.polling.timeout` (float): # Time (seconds) before the main process disregards the existence of # lock file and unblocks calls to `.terminate()` a (most probably # errored-out) child process anyway (set to <= 0 to disable timeouts) timeout = 0.25 -# - `polling.on_timeout` (Literal['error', 'warn', 'ignore']) +# - `multiprocessing.polling.on_timeout` +# (Literal['error', 'warn', 'ignore']): # What to do when the above timeout is exhausted, before actually # `.terminate()`-ing the child process: # - 'error': raise an error diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 5330535a..6a2ed112 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1562,7 +1562,7 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: if debug: config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' config.write_text( - '[tool.line_profiler.multiprocessing]\n' + '[tool.line_profiler.child_processes.multiprocessing]\n' 'intercept_logs = true' ) From bdbb683f0d9743a36fc6cad42ded071d3e021651 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 27 Apr 2026 10:22:41 +0200 Subject: [PATCH 073/165] WIP: increase test coverage line_profiler/_child_process_profiling/pth_hook.py::load_pth_hook() - Added `# nocover` directive to unlikely `except` clause - Now patching `load_pth_hook.called` with the `cache` where possible tests/test_child_procs.py ext_module_object New fixture for actually loading the module object represented by the `ext_module` fixture another_pid Convenience fixture for a dummy PID _preserve_attributes .fetch_current_values(), .compare_with_current_values() New convenience methods for comparing the preserved vs. current attributes of multiple objects test_cache_setup_main_process() Simplified implementation with the `_preserve_attributes` methods test_cache_setup_child() Now using the new `ext_module_object` fixture for the profiled function test_load_pth_hook() New unit test for `load_pth_hook()` to boost coverage of `pth_hook.py` test_apply_mp_patches() Refactored to: - Use the new `ext_module_object` fixture for the profiled function - Test cases where the parallel workload fails TODO: `@LineProfilingCache._method_wrapper` and `wrap_terminate()` still not covered; try to tweak`test_apply_mp_patches()` to cover them --- .../_child_process_profiling/pth_hook.py | 5 +- tests/test_child_procs.py | 291 +++++++++++++----- 2 files changed, 214 insertions(+), 82 deletions(-) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index b6197600..4fa6b9ef 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -110,10 +110,11 @@ def load_pth_hook(ppid): # type: (int) -> None try: cache = LineProfilingCache.load() cache._setup_in_child_process(True, 'pth') - except Exception as e: + except Exception as e: # nocover if DEBUG: msg = f'{type(e)}: {e}' warnings.warn(msg) log.warning(msg) - finally: load_pth_hook.called = True # type: ignore + else: + cache.patch(load_pth_hook, 'called', True) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 6a2ed112..cc3c3f0e 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1,6 +1,5 @@ from __future__ import annotations -import ast import dataclasses import enum import inspect @@ -16,8 +15,8 @@ from collections.abc import ( Callable, Collection, Generator, Iterable, Mapping, Sequence, ) -from contextlib import ExitStack -from functools import lru_cache, partial, wraps +from contextlib import AbstractContextManager, ExitStack, nullcontext +from functools import partial, wraps from io import StringIO from importlib import import_module from pathlib import Path @@ -25,7 +24,7 @@ from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic -from types import TracebackType +from types import ModuleType, TracebackType from typing import Any, Generic, Literal, TypeVar, cast, final, overload from typing_extensions import Self, ParamSpec from uuid import uuid4 @@ -37,6 +36,7 @@ from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) +from line_profiler._child_process_profiling.pth_hook import load_pth_hook from line_profiler._child_process_profiling.multiprocessing_patches import ( _Poller, ) @@ -200,6 +200,21 @@ def install( if children: self.monkeypatch.setenv('PYTHONPATH', path, prepend=os.pathsep) + def _import_module_helper(self) -> Generator[ModuleType, None, None]: + def iter_module_names( + module: _ModuleFixture, + ) -> Generator[str, None, None]: + yield module.name + for dep in module.dependencies: + yield from iter_module_names(dep) + + self.install(local=True, children=True) + try: + yield import_module(self.name) + finally: + for name in set(iter_module_names(self)): + sys.modules.pop(name, None) + @staticmethod def propose_name(prefix: str) -> Generator[str, None, None]: """ @@ -291,6 +306,19 @@ def test_module_clone( yield _ModuleFixture(path, monkeypatch, [ext_module]) +@pytest.fixture +def ext_module_object( + ext_module: _ModuleFixture, +) -> Generator[ModuleType, None, None]: + """ + Yields: + :py:class:`ModuleType` object containing the code at + :py:data:`EXTERNAL_MODULE_BODY`, and is torn down at the end of + the test + """ + yield from ext_module._import_module_helper() + + @pytest.fixture def create_cache( tmp_path_factory: pytest.TempPathFactory, @@ -439,6 +467,17 @@ def curated_profiler() -> Generator[LineProfiler, None, None]: yield prof +@pytest.fixture +def another_pid() -> int: + """ + Get a PID which is distinct from the current one. + """ + curr_pid = os.getpid() + pid = (curr_pid - 42) % (2 * 16) + assert pid != curr_pid + return pid + + @pytest.fixture(autouse=True) def _trim_mismatch_traceback(pytestconfig: pytest.Config) -> None: """ @@ -866,6 +905,43 @@ def __enter__(self) -> dict[str, dict[str, Any]]: def __exit__(self, *_, **__) -> None: self._stacks.pop().close() + @staticmethod + def fetch_current_values( + targets: Mapping[str, Collection[str]], + ) -> dict[str, dict[str, Any]]: + result: dict[str, dict[str, Any]] = {} + na = _NotSupplied.NOT_SUPPLIED + for target, attrs in targets.items(): + obj = _import_target(target) + result[target] = {attr: getattr(obj, attr, na) for attr in attrs} + return result + + @classmethod + def compare_with_current_values( + cls, + old: Mapping[str, Mapping[str, Any]], + comparator: Callable[[Any, Any], bool] = operator.is_, + assert_true: bool = True, + ) -> dict[str, dict[str, bool]]: + result: dict[str, dict[str, bool]] = {} + new = cls.fetch_current_values(old) + for target, old_values in old.items(): + new_values = new[target] + cmp_results = result[target] = {} + for attr, old_value in old_values.items(): + print(f'Checking: {target}.{attr}') + new_value = new_values[attr] + if assert_true: + assert cmp_results.setdefault( + attr, comparator(new_value, old_value), + ) + else: + # There's probably a more concise way to write this, + # but we want more info to be available in the + # traceback should the above assertion fail... + cmp_results[attr] = comparator(new_value, old_value) + return result + class _preserve_pth_files(_CallableContextManager[frozenset[str]]): def __init__(self, debug: bool = _DEBUG) -> None: @@ -937,25 +1013,6 @@ def _search_cache_logs( ) -@lru_cache() -def _find_return_lines(func: str) -> list[int]: - class FindReturns(ast.NodeVisitor): - def __init__(self) -> None: - self.found: set[int] = set() - - def visit_Return(self, node: ast.Return) -> None: - self.found.add(node.lineno) - self.generic_visit(node) - - func_obj = _import_target(func) - assert inspect.isfunction(func_obj) - lines, start = inspect.getsourcelines(func_obj) - tree = ast.parse(''.join(lines)) - finder = FindReturns() - finder.visit(tree) - return sorted(lineno + start - 1 for lineno in finder.found) - - # `shlex.join()` doesn't work properly on Windows, so use # `subprocess.list2cmdline()` instead; # though an "intentionally" undocumented API (cpython issue #10308), @@ -1225,16 +1282,6 @@ def _check_output(output: str, tag: str, nhits: int) -> None: 'os': frozenset({'fork'}), } -# NOTE: we need a function which isn't used by the codebase itself -# (esp. during cache cleanup); otherwise the profiling results may -# be skewed -_SAFE_TARGET = 'calendar.weekday' -_SAFE_TARGET_ARGS = [ - (1970, 1, 1), - (2000, 12, 31), - (2008, 9, 16), # Where the repo started -] - @pytest.mark.parametrize(('run_profiled_code', 'label1'), [(True, 'run-profiled'), (False, 'run-unrelated')]) @@ -1389,33 +1436,21 @@ def test_cache_setup_main_process( for target, attrs in _GLOBAL_PATCHES.items() } patches['os']['fork'] = wrap_os_fork and (sys.platform != 'win32') - targets: dict[str, Any] = { - target: _import_target(target) for target in patches - } with ExitStack() as stack: patched = stack.enter_context(_preserve_attributes(patches)) + compare_patched = partial( + _preserve_attributes.compare_with_current_values, patched, + ) original_pths = stack.enter_context(_preserve_pth_files()) cache._setup_in_main_process(wrap_os_fork=wrap_os_fork) # There should be exactly one extra `.pth` file new_pth_hook, = _preserve_pth_files.get_pth_files() - original_pths # Check whether the patches are applied - for target, maybe_patches in patches.items(): - obj = targets[target] - for attr, is_patched in maybe_patches.items(): - orig_value = patched[target][attr] - if orig_value is _NotSupplied.NOT_SUPPLIED: - assert not hasattr(obj, attr) - else: - assert (getattr(obj, attr) is orig_value) != is_patched + patch_summary = compare_patched(operator.is_not, assert_true=False) + assert patch_summary == patches # Check whether the patches are reversed cache.cleanup() - for target, orig_attrs in patched.items(): - obj = targets[target] - for attr, orig_value in orig_attrs.items(): - if orig_value is _NotSupplied.NOT_SUPPLIED: - assert not hasattr(obj, attr) - else: - assert getattr(obj, attr) is orig_value + compare_patched() # Check that the instance is set as the `.load()`-ed one assert cache is cache.load() @@ -1444,14 +1479,18 @@ def test_cache_setup_main_process( [(True, 'no-profiler'), (False, 'with-profiler')]) @pytest.mark.parametrize(('debug', 'label4'), [(True, 'with-debug'), (False, 'no-debug')]) +@pytest.mark.parametrize('n', [100]) @_preserve_attributes(_GLOBAL_PATCHES) def test_cache_setup_child( create_cache: Callable[..., LineProfilingCache], curated_profiler: LineProfiler, + ext_module_object: ModuleType, + another_pid: int, wrap_os_fork: bool, preimports: bool, new_profiler: bool, debug: bool, + n: int, label1: str, label2: str, label3: str, label4: str, ) -> None: """ @@ -1464,16 +1503,12 @@ def list_profiled_funcs() -> list[str]: for func in getattr(cache.profiler, 'functions', []) ] - # Make sure we get a different PID from the current process - curr_pid = os.getpid() - main_pid = (curr_pid - 42) % (2 * 16) - assert main_pid != curr_pid - + func = ext_module_object.my_external_sum cache = create_cache( - profiling_targets=[_SAFE_TARGET], + profiling_targets=[f'{func.__module__}.{func.__qualname__}'], preimports_module=preimports, _use_curated_profiler=not new_profiler, - main_pid=main_pid, + main_pid=another_pid, debug=debug, ) assert (cache.profiler is None) == new_profiler @@ -1502,7 +1537,7 @@ def list_profiled_funcs() -> list[str]: # Check that on cache cleanup: # - Profiling data is collected # - `os.fork()` is restored - _import_target(_SAFE_TARGET)(*_SAFE_TARGET_ARGS[0]) + assert func(range(1, n + 1)) == n * (n + 1) // 2 stats = cache.profiler.get_stats() for callback, has_prof_data, fork_patched in [ (lambda: None, False, wrap_os_fork), @@ -1532,19 +1567,91 @@ def list_profiled_funcs() -> list[str]: _search_cache_logs(cache, debug, patterns) +@_preserve_attributes({ + **_GLOBAL_PATCHES, + 'line_profiler._child_process_profiling.pth_hook.load_pth_hook': + frozenset({'called'}), +}) +@pytest.mark.parametrize('ppid_should_match', [True, False, None]) +def test_load_pth_hook( + create_cache: Callable[..., LineProfilingCache], + another_pid: int, + ppid_should_match: bool | None, +) -> None: + """ + Simulate calling :py:func:`line_profiler._child_process_profiling\ +.pth_hook.load_pth_hook()` in a child process. + + Notes: + + - The function is CALLED in the .pth file, but we don't actually + NEED a .pth file to call and test it. + + - The counterpart :py:func:`line_profiler\ +._child_process_profiling.pth_hook.write_pth_hook()` + is implicitly tested in + :py:func:`test_cache_setup_main_process()`. + """ + # This test is mostly here to hack coverage; since the function is + # only to be called in child processes, `coverage` seems to have + # trouble getting data on it... + + # We basically only need this cache instance to set up the + # environment variables and the requisite files... + cache = create_cache(main_pid=another_pid) + if ppid_should_match is not None: + cache.inject_env_vars() + if ppid_should_match: + call_ppid = another_pid + else: # On a PPID mismatch, the function bails after checking + call_ppid = another_pid + 10 + else: + # Without the requisite envvars, the hook should bail very + # quickly (due to the `environ` lookup erroring out), regardless + # of the provided PPID + call_ppid = 0 + cache.dump() + + compare = _preserve_attributes.compare_with_current_values + with _preserve_attributes(_GLOBAL_PATCHES) as patched: + load_pth_hook(call_ppid) + # Check that the patches are applied where appropriate + assert ( + getattr(load_pth_hook, 'called', False) == bool(ppid_should_match) + ) + if ppid_should_match: + compare(patched, operator.is_not) + else: # no-op + compare(patched) + return + # Check that calling `load_pth_hook()` again is a no-op + with _preserve_attributes(_GLOBAL_PATCHES) as re_patched: + load_pth_hook(call_ppid) + compare(re_patched) + # Check that the patches are reversed + LineProfilingCache.load().cleanup() # 'Current' instance + compare(patched) + + @pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) -@pytest.mark.parametrize(('debug', 'label'), +@pytest.mark.parametrize(('debug', 'label1'), [(True, 'with-debug'), (False, 'no-debug')]) +@pytest.mark.parametrize(('fail', 'label2'), + [(True, 'failure'), (False, 'success')]) +@pytest.mark.parametrize('n', [[100, 200, 1000]]) @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) def test_apply_mp_patches( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], + ext_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], debug: bool, - label: str, + fail: bool, + n: list[int], + label1: str, label2: str, ) -> None: """ Test that :py:func:`line_profiler._child_process_profiling\ @@ -1558,6 +1665,15 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: return False return True + def get_lineno(path: os.PathLike[str] | str, query: str) -> int: + with Path(path).open() as fobj: + for i, line in enumerate(fobj): + if query in line: + return 1 + i + raise RuntimeError( + f'Did not find line containing {query!r} in {path!r}', + ) + config: Path | None = None if debug: config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' @@ -1566,8 +1682,10 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: 'intercept_logs = true' ) + func = ext_module_object.my_external_sum + func_name = f'{func.__module__}.{func.__qualname__}' cache = create_cache( - profiling_targets=[_SAFE_TARGET], + profiling_targets=[func_name], preimports_module=True, config=config, debug=True, @@ -1584,14 +1702,24 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: assert cache.preimports_module is not None run_path(str(cache.preimports_module), {'profile': cache.profiler}) - func = _import_target(_SAFE_TARGET) - return_lines = _find_return_lines(_SAFE_TARGET) + ranges = [range(1, x + 1) for x in n] + expected_nloops = sum(n) + timing_key = ( + inspect.getfile(func), + inspect.getsourcelines(func)[1], + func.__qualname__, + ) + assert ext_module_object.__file__ + loop_line = get_lineno(ext_module_object.__file__, 'EXT-LOOP') Pool: Callable[..., multiprocessing.pool.Pool] + + if fail: + xc_context: AbstractContextManager[Any] = pytest.raises(RuntimeError) + else: + xc_context = nullcontext() + if start_method == 'dummy': Pool = _import_target('multiprocessing.dummy.Pool') - # Twice the counted calls because we're also collecting the - # checking calls in this process - expected_ncalls = len(_SAFE_TARGET_ARGS) * 2 get_stats: Callable[[], LineStats] = cache.profiler.get_stats elif start_method not in START_METHODS: pytest.skip( @@ -1600,23 +1728,26 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: ) else: Pool = multiprocessing.get_context(start_method).Pool - expected_ncalls = len(_SAFE_TARGET_ARGS) get_stats = cache.gather_stats - with Pool(2) as pool: - par_result = pool.starmap(func, _SAFE_TARGET_ARGS) - pool.close() - pool.join() - assert par_result == [func(*args) for args in _SAFE_TARGET_ARGS] + with ExitStack() as stack: + pool = stack.enter_context(Pool(2)) + # Check for the expected error (if `fail`) + stack.enter_context(xc_context) + try: + par_result = pool.starmap(func, [(r, fail) for r in ranges]) + finally: + pool.close() + pool.join() + + # Check correctness of the results + if not fail: + assert par_result == [x * (x + 1) // 2 for x in n] # Check that calls in children are traced - line_entries = get_stats().timings[ - inspect.getfile(func), inspect.getsourcelines(func)[1], func.__name__, - ] - num_returns = sum( - nhits for lineno, nhits, _ in line_entries if lineno in return_lines - ) - ResultMismatch.compare(expected_ncalls, num_returns) + entries = get_stats().timings[timing_key] + nloops = sum(nhits for ln, nhits, _ in entries if ln == loop_line) + ResultMismatch.compare(expected_nloops, nloops) # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests From 2c875eee72507197afbc013726a883542e7afe54 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 27 Apr 2026 22:12:48 +0200 Subject: [PATCH 074/165] Updated test code path for coverage tests/test_child_procs.py TEST_MODULE_TEMPLATE, test_module Updated code to allow for calling `sum_in_child_procs()` with the 'dummy' argument and switching to `multiprocessing.dummy.Pool` test_module_object `ModuleType` fixture representing the content of the above test_apply_mp_patches() - New parametrization `nprocs` for the number of child processes/threads created - Updated implementation to use `test_module_object.sum_in_child_procs()` After the changes, `coverage` seem to fare much better in tracing `multiprocessing_patches.py` and `cache.py`. --- tests/test_child_procs.py | 95 +++++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 38 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index cc3c3f0e..340dd963 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -88,7 +88,7 @@ def my_external_sum(x: list[int], fail: bool = False) -> int: from argparse import ArgumentParser from collections.abc import Callable -from multiprocessing import get_context, Pool +from multiprocessing import dummy, get_context, Pool from typing import Literal from {EXT_MODULE} import my_external_sum @@ -107,7 +107,9 @@ def my_local_sum(x: list[int], fail: bool = False) -> int: def sum_in_child_procs( length: int, n: int, my_sum: Callable[[list[int]], int], - start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, + start_method: Literal[ + 'fork', 'forkserver', 'spawn', 'dummy' + ] | None = None, fail: bool = False, ) -> int: my_list: list[int] = list(range(1, length + 1)) @@ -119,7 +121,9 @@ def sum_in_child_procs( while my_list: sublist, my_list = my_list[:sublength], my_list[sublength:] sublists.append(sublist) - if start_method: + if start_method == 'dummy': + pool = dummy.Pool(n) + elif start_method: pool = get_context(start_method).Pool(n) else: pool = Pool(n) @@ -319,6 +323,19 @@ def ext_module_object( yield from ext_module._import_module_helper() +@pytest.fixture +def test_module_object( + test_module: _ModuleFixture, ext_module_object: ModuleType, +) -> Generator[ModuleType, None, None]: + """ + Yields: + :py:class:`ModuleType` object containing the code at + :py:data:`TEST_MODULE_TEMPLATE`, and is torn down at the end of + the test + """ + yield from test_module._import_module_helper() + + @pytest.fixture def create_cache( tmp_path_factory: pytest.TempPathFactory, @@ -1640,17 +1657,19 @@ def test_load_pth_hook( [(True, 'with-debug'), (False, 'no-debug')]) @pytest.mark.parametrize(('fail', 'label2'), [(True, 'failure'), (False, 'success')]) -@pytest.mark.parametrize('n', [[100, 200, 1000]]) +@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) def test_apply_mp_patches( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, + test_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], debug: bool, fail: bool, - n: list[int], + n: int, + nprocs: int, label1: str, label2: str, ) -> None: """ @@ -1674,16 +1693,29 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: f'Did not find line containing {query!r} in {path!r}', ) - config: Path | None = None + config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' + cfg_chunks: list[str] = [] if debug: - config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' - config.write_text( + cfg_chunks.append( '[tool.line_profiler.child_processes.multiprocessing]\n' 'intercept_logs = true' ) + cfg_chunks.append( + '[tool.line_profiler.child_processes.multiprocessing.polling]\n' + 'on_timeout = "error"' + ) + config.write_text('\n\n'.join(cfg_chunks)) - func = ext_module_object.my_external_sum - func_name = f'{func.__module__}.{func.__qualname__}' + profiled_func = ext_module_object.my_external_sum + called_func = partial( + test_module_object.sum_in_child_procs, + n=nprocs, + my_sum=profiled_func, + start_method=start_method, + fail=fail, + ) + + func_name = f'{profiled_func.__module__}.{profiled_func.__qualname__}' cache = create_cache( profiling_targets=[func_name], preimports_module=True, @@ -1702,52 +1734,39 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: assert cache.preimports_module is not None run_path(str(cache.preimports_module), {'profile': cache.profiler}) - ranges = [range(1, x + 1) for x in n] - expected_nloops = sum(n) timing_key = ( - inspect.getfile(func), - inspect.getsourcelines(func)[1], - func.__qualname__, + inspect.getfile(profiled_func), + inspect.getsourcelines(profiled_func)[1], + profiled_func.__qualname__, ) assert ext_module_object.__file__ loop_line = get_lineno(ext_module_object.__file__, 'EXT-LOOP') - Pool: Callable[..., multiprocessing.pool.Pool] if fail: xc_context: AbstractContextManager[Any] = pytest.raises(RuntimeError) + nloops_expected = n else: xc_context = nullcontext() + nloops_expected = n + nprocs - if start_method == 'dummy': - Pool = _import_target('multiprocessing.dummy.Pool') - get_stats: Callable[[], LineStats] = cache.profiler.get_stats - elif start_method not in START_METHODS: + if start_method not in ('dummy', *START_METHODS): pytest.skip( f'`multiprocessing` start method {start_method!r} ' 'not available on the platform' ) - else: - Pool = multiprocessing.get_context(start_method).Pool - get_stats = cache.gather_stats - - with ExitStack() as stack: - pool = stack.enter_context(Pool(2)) - # Check for the expected error (if `fail`) - stack.enter_context(xc_context) - try: - par_result = pool.starmap(func, [(r, fail) for r in ranges]) - finally: - pool.close() - pool.join() - # Check correctness of the results - if not fail: - assert par_result == [x * (x + 1) // 2 for x in n] + with xc_context: + result = called_func(n) + # Check correctness of the results + if not fail: + assert result == n * (n + 1) // 2 # Check that calls in children are traced - entries = get_stats().timings[timing_key] + stats = cache.profiler.get_stats() + stats += cache.gather_stats() + entries = stats.timings[timing_key] nloops = sum(nhits for ln, nhits, _ in entries if ln == loop_line) - ResultMismatch.compare(expected_nloops, nloops) + ResultMismatch.compare(nloops_expected, nloops) # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests From f0e17433da2281bec6732306da663f9e3a11659b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 28 Apr 2026 23:27:12 +0200 Subject: [PATCH 075/165] Fixed test pollution line_profiler/_child_process_profiling/multiprocessing_patches.py ::_apply_mp_patches() Refactored internals; existence of `multiprocessing.spawn` and `multiprocessing.forkserver` now checked at import time tests/test_child_procs.py _preserve_attributes.compare_with_current_values() - Argument `assert_true` can now take a mapping to which the result is compared - Now explicitly constructing an error message when the assertion is false, instead of relying on `pytest`'s assertion rewriting _GLOBAL_PATCHES Now protecting against changes to additional attributes patched_attributes Wrapper fixture around `_GLOBAL_PATCHES` which excludes nonexistent names (e.g. `os.fork()` on Windows) test_cache_dump_load() test_load_pth_hook() Now making sure that all cache instances created are cleaned up, not just those created by the `create_cache()` fixture test_cache_setup_main_process() Minor refactoring --- .../multiprocessing_patches.py | 54 +++++--- tests/test_child_procs.py | 127 +++++++++++++----- 2 files changed, 127 insertions(+), 54 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index b2822f49..c9631ad2 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -28,6 +28,21 @@ ) from typing_extensions import Concatenate, ParamSpec, Self +try: + from multiprocessing import spawn +except ImportError: + _CAN_USE_SPAWN = False +else: + _CAN_USE_SPAWN = True +try: + from multiprocessing import forkserver +except ImportError: + _CAN_USE_FORKSERVER = False +else: + _CAN_USE_FORKSERVER = ( + 'forkserver' in multiprocessing.get_all_start_methods() + ) + from .. import _diagnostics as diagnostics from ..toml_config import ConfigSource from .cache import LineProfilingCache @@ -488,13 +503,8 @@ def _apply_mp_patches( '_bootstrap': wrap_bootstrap}}, ) # Patch `multiprocessing.spawn` - try: - from multiprocessing import spawn - except ImportError: - pass - else: - if hasattr(spawn, 'runpy'): - lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) + if _CAN_USE_SPAWN and hasattr(spawn, 'runpy'): + lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) # Intercept `multiprocessing` debug messages if intercept_mp_logs is None: intercept_mp_logs = _get_config(lp_cache.config)['intercept_logs'] @@ -506,23 +516,25 @@ def _apply_mp_patches( # - Now, so that the (rebooted) fork-server process has profiling # set up; and # - Also as a part of cache cleanup - # (this uses `ForkServer._stop()` which is private API, but it's the - # same hack used in Python's own test suite -- see the comment to - # said method) - if reboot_forkserver: - try: - from multiprocessing import forkserver - except ImportError: # Incompatible platform # nocover - pass - else: - server_instance: forkserver.ForkServer = forkserver._forkserver - stop = getattr(server_instance, '_stop', None) - assert callable(stop) # Appease the type checker - stop() - lp_cache.add_cleanup(stop) + if _CAN_USE_FORKSERVER and reboot_forkserver: + _stop_forkserver() + lp_cache.add_cleanup(_stop_forkserver) # Mark `multiprocessing` as having been patched lp_cache.patch(multiprocessing, _PATCHED_MARKER, True) +def _stop_forkserver() -> None: + """ + Note: + This uses `ForkServer._stop()` which is private API, but it's + the same hack used in Python's own test suite -- see the comment + to said method + """ + # Appease the type-checker since `._stop()` is not public API + stop = getattr(forkserver._forkserver, '_stop', None) + assert callable(stop) + stop() + + def _no_op(*_, **__) -> None: pass diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 340dd963..38119dd3 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -24,8 +24,10 @@ from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic -from types import ModuleType, TracebackType -from typing import Any, Generic, Literal, TypeVar, cast, final, overload +from types import MappingProxyType, ModuleType, TracebackType +from typing import ( + TYPE_CHECKING, Any, Generic, Literal, TypeVar, cast, final, overload, +) from typing_extensions import Self, ParamSpec from uuid import uuid4 @@ -38,7 +40,7 @@ ) from line_profiler._child_process_profiling.pth_hook import load_pth_hook from line_profiler._child_process_profiling.multiprocessing_patches import ( - _Poller, + _Poller, _PATCHED_MARKER, ) from line_profiler.curated_profiling import ( CuratedProfilerContext, ClassifiedPreimportTargets, @@ -938,8 +940,21 @@ def compare_with_current_values( cls, old: Mapping[str, Mapping[str, Any]], comparator: Callable[[Any, Any], bool] = operator.is_, - assert_true: bool = True, + assert_true: bool | Mapping[str, Mapping[str, bool]] = True, ) -> dict[str, dict[str, bool]]: + def get_from_mapping(target: str, attr: str) -> bool: + if TYPE_CHECKING: + assert isinstance(assert_true, Mapping) + return assert_true[target][attr] + + def get_from_boolean(*_, **__) -> bool: + return True + + if isinstance(assert_true, Mapping): + get_expected: Callable[[str, str], bool] = get_from_mapping + else: + get_expected = get_from_boolean + result: dict[str, dict[str, bool]] = {} new = cls.fetch_current_values(old) for target, old_values in old.items(): @@ -948,15 +963,18 @@ def compare_with_current_values( for attr, old_value in old_values.items(): print(f'Checking: {target}.{attr}') new_value = new_values[attr] - if assert_true: - assert cmp_results.setdefault( - attr, comparator(new_value, old_value), + cmp_results[attr] = cmp_result = comparator( + new_value, old_value, + ) + expected_result = get_expected(target, attr) + if assert_true and (cmp_result != expected_result): + assert False, ( + f'Comparing `{target}.{attr}` ' + f'(old: {old_value!r} @ {id(old_value):#x}; ' + f'new: {new_value!r} @ {id(new_value):#x}): ' + f'expected comparison with {comparator!r} to return ' + f'{expected_result}, got {cmp_result}' ) - else: - # There's probably a more concise way to write this, - # but we want more info to be available in the - # traceback should the above assertion fail... - cmp_results[attr] = comparator(new_value, old_value) return result @@ -1292,14 +1310,33 @@ def _check_output(output: str, tag: str, nhits: int) -> None: # tested APIs and behaviors MUST NOT be relied upon by end-users. _GLOBAL_PATCHES = { + f'{load_pth_hook.__module__}.{load_pth_hook.__qualname__}': frozenset({ + 'called', + }), + 'multiprocessing': frozenset({_PATCHED_MARKER}), 'multiprocessing.process.BaseProcess': frozenset({ '_bootstrap', 'terminate', }), 'multiprocessing.spawn': frozenset({'runpy'}), + 'multiprocessing.util': frozenset({ + 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', + }), 'os': frozenset({'fork'}), } +@pytest.fixture(scope='module') +def patched_attributes() -> MappingProxyType[str, frozenset[str]]: + result: dict[str, frozenset[str]] = {} + for target, attrs in _GLOBAL_PATCHES.items(): + try: + obj = _import_target(target) + except ImportError: + continue + result[target] = frozenset(a for a in attrs if hasattr(obj, a)) + return MappingProxyType(result) + + @pytest.mark.parametrize(('run_profiled_code', 'label1'), [(True, 'run-profiled'), (False, 'run-unrelated')]) @pytest.mark.parametrize(('as_module', 'label2'), @@ -1409,6 +1446,7 @@ def test_cache_dump_load( original = create_cache( profiling_targets=['foo', 'bar', 'baz'], main_pid=123456, ) + cache_instances: list[LineProfilingCache] = [original] envvars: set[str] = set(os.environ) try: original.inject_env_vars() # Needed for `.load()` @@ -1417,7 +1455,9 @@ def test_cache_dump_load( assert set(os.environ) == envvars.union(original.environ) > envvars original.dump() loaded = original.load() + cache_instances.append(loaded) reloaded = original.load() + cache_instances.append(reloaded) assert original is not loaded is reloaded # Compare init fields for field in dataclasses.fields(LineProfilingCache): @@ -1428,7 +1468,8 @@ def test_cache_dump_load( == getattr(loaded, field.name) ) finally: # Explicitly cleanup - original.cleanup() + for cache in cache_instances: + cache.cleanup() finally: # Env vars restored after cleanup assert set(os.environ) == envvars @@ -1439,6 +1480,7 @@ def test_cache_dump_load( [(True, 'with-debug'), (False, 'no-debug')]) def test_cache_setup_main_process( create_cache: Callable[..., LineProfilingCache], + patched_attributes: MappingProxyType[str, frozenset[str]], wrap_os_fork: bool, debug: bool, label1: str, label2: str, @@ -1448,11 +1490,17 @@ def test_cache_setup_main_process( as expected. """ cache = create_cache(debug=debug) + # By default, we don't patch the `multiprocessing.util` logging + # facilities patches: dict[str, dict[str, bool]] = { - target: dict.fromkeys(attrs, True) - for target, attrs in _GLOBAL_PATCHES.items() + target: dict.fromkeys(attrs, target != 'multiprocessing.util') + for target, attrs in patched_attributes.items() } - patches['os']['fork'] = wrap_os_fork and (sys.platform != 'win32') + try: + patches['os']['fork'] = wrap_os_fork + except KeyError: + # `os.fork()` pruned because it doesn't exist on e.g. Windows + assert not hasattr(os, 'fork') with ExitStack() as stack: patched = stack.enter_context(_preserve_attributes(patches)) compare_patched = partial( @@ -1463,8 +1511,7 @@ def test_cache_setup_main_process( # There should be exactly one extra `.pth` file new_pth_hook, = _preserve_pth_files.get_pth_files() - original_pths # Check whether the patches are applied - patch_summary = compare_patched(operator.is_not, assert_true=False) - assert patch_summary == patches + compare_patched(operator.is_not, assert_true=patches) # Check whether the patches are reversed cache.cleanup() compare_patched() @@ -1592,6 +1639,7 @@ def list_profiled_funcs() -> list[str]: @pytest.mark.parametrize('ppid_should_match', [True, False, None]) def test_load_pth_hook( create_cache: Callable[..., LineProfilingCache], + patched_attributes: MappingProxyType[str, frozenset[str]], another_pid: int, ppid_should_match: bool | None, ) -> None: @@ -1630,23 +1678,36 @@ def test_load_pth_hook( cache.dump() compare = _preserve_attributes.compare_with_current_values - with _preserve_attributes(_GLOBAL_PATCHES) as patched: - load_pth_hook(call_ppid) - # Check that the patches are applied where appropriate - assert ( - getattr(load_pth_hook, 'called', False) == bool(ppid_should_match) - ) - if ppid_should_match: - compare(patched, operator.is_not) - else: # no-op - compare(patched) - return - # Check that calling `load_pth_hook()` again is a no-op - with _preserve_attributes(_GLOBAL_PATCHES) as re_patched: + patches = patched_attributes.copy() + del patches['multiprocessing.util'] # Not patched by default + with _preserve_attributes(patches) as patched: + try: + # NOTE: this creates a cache instance that isn't + # automatically cleaned up by the `create_cache()` + # fixture!!! Hence the try-finally load_pth_hook(call_ppid) - compare(re_patched) + # Check that the patches are applied where appropriate + assert ( + getattr(load_pth_hook, 'called', False) + == bool(ppid_should_match) + ) + if ppid_should_match: + compare(patched, operator.is_not) + else: # no-op + compare(patched) + return + # Check that calling `load_pth_hook()` again is a no-op + with _preserve_attributes(patches) as re_patched: + load_pth_hook(call_ppid) + compare(re_patched) + finally: + try: + current_cache = LineProfilingCache.load() + except Exception: + pass + else: + current_cache.cleanup() # Check that the patches are reversed - LineProfilingCache.load().cleanup() # 'Current' instance compare(patched) From ec6c6e8ed725bb0453a423d2b069f510887faadb Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 29 Apr 2026 00:55:37 +0200 Subject: [PATCH 076/165] Bugfix + SIGTERM handling line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes.multiprocessing] New key-value pair `catch_sigterm` for whether to catch SIGTERM in child processes started by `multiprocessing` line_profiler/_child_process_profiling/cache.py::LineProfilingCache _setup_common() Updated call signature _handle_sigterm(), _wrap_sigterm() New helper methods for dealing with `SIGTERM` (esp. in child processes created by `multiprocessing`) line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_terminate() Now writing debug messages to the log if the poller timed out and raised an error wrap_bootstrap() Now setting the SIGTERM handler before running the workload tests/test_child_procs.py::patched_attributes Fixed bug where objects with no attributes to be patched remain in the returned mapping --- .../_child_process_profiling/cache.py | 47 +++++++++++++++++-- .../multiprocessing_patches.py | 10 +++- line_profiler/rc/line_profiler.toml | 4 ++ tests/test_child_procs.py | 5 +- 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index d7978f84..8ae702ee 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -7,6 +7,7 @@ import atexit import dataclasses import os +import signal try: import _pickle as pickle except ImportError: @@ -17,6 +18,7 @@ from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent +from threading import current_thread, main_thread from types import ModuleType from typing import Any, ClassVar, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self @@ -292,7 +294,7 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: self.dump() self.inject_env_vars() _import_sibling('pth_hook').write_pth_hook(self) - self._setup_common(wrap_os_fork, reboot_forkserver=True) + self._setup_common(wrap_os_fork, {'reboot_forkserver': True}) self._replace_loaded_instance() def _setup_in_child_process( @@ -359,7 +361,7 @@ def _setup_in_child_process( self.add_cleanup_with_priority(prof.dump_stats, 1, prof_outfile) # Various setups - self._setup_common(wrap_os_fork, reboot_forkserver=False) + self._setup_common(wrap_os_fork, {'reboot_forkserver': False}) # Set `.cleanup()` as an atexit hook to handle everything when # the child process is about to terminate @@ -369,22 +371,59 @@ def _setup_in_child_process( return True def _setup_common( - self, wrap_os_fork: bool, reboot_forkserver: bool, + self, + wrap_os_fork: bool, + mp_apply_kwargs: dict[str, Any] | None = None, ) -> None: if wrap_os_fork: self._wrap_os_fork() _import_sibling('multiprocessing_patches').apply( - self, reboot_forkserver, + self, **(mp_apply_kwargs or {}), ) + def _handle_sigterm(self, signum: int, _) -> None: # nocover + """ + See also: + :py:meth:`coverage.control.Converage._on_sigterm` + """ + try: + name = signal.Signals(signum).name + self._debug_output( + f'Got signal {signum} ({name}), ' + 'cleaning up before passing it on...', + ) + # Also restores handler overwritten by `._wrap_sigterm()` + self.cleanup() + finally: + os.kill(os.getpid(), signum) + + def _wrap_sigterm(self) -> None: # nocover + """ + Side effects: + If on the main thread: + + - :py:func:`signal.signal` called to set + :py:meth:`~._handle_sigterm` as the ``SIGTERM`` handler + + - :py:meth:`~.cleanup` callback registered undoing that + """ + if current_thread() != main_thread(): + return + set_handler = signal.signal + self._debug_output('Adding `SIGTERM` handler...') + handler = set_handler(signal.SIGTERM, self._handle_sigterm) + self.add_cleanup_with_priority(set_handler, 1, signal.SIGTERM, handler) + def _wrap_os_fork(self) -> None: """ Create a wrapper around :py:func:`os.fork` which handles profiling. Side effects: + - :py:func:`os.fork` (if available) replaced with the wrapper + - :py:meth:`~.cleanup` callback registered undoing that """ try: diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index c9631ad2..bfca6643 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -26,6 +26,7 @@ from typing import ( Any, Generic, Literal, Protocol, TypeVar, Union, NoReturn, cast, ) + from typing_extensions import Concatenate, ParamSpec, Self try: @@ -331,6 +332,9 @@ def wait_for_return( try: with wait_for_return(cache.config): pass + except (_Poller.Timeout, _Poller.TimeoutWarning) as e: + cache._debug_output(f'{type(e).__qualname__}: {e}') + raise finally: # Always call `Process.terminate()` to avoid orphans vanilla_impl(self) @@ -355,6 +359,10 @@ def wrap_bootstrap( :py:mod:`line_profiler` to be loaded before it. Hence the ``# nocover``. """ + # Set a signal handler for SIGTERM to help child processes with + # consistently cleaning up + if _get_config(cache.config)['catch_sigterm']: + cache._wrap_sigterm() try: return vanilla_impl(self, *args, **kwargs) finally: @@ -454,7 +462,7 @@ def apply( .. _GH-126631: https://github.com/python/cpython/issues/126631 """ if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, reboot_forkserver) + _apply_mp_patches(lp_cache, reboot_forkserver=reboot_forkserver) def _apply_patches_generic( diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 5c53aadd..529a6783 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -229,6 +229,10 @@ suffix = '' # the internal log messages are captured and writtent to the debug # logs intercept_logs = false +# - `multiprocessing.catch_sigterm` (bool): +# Whether to set a hook for the child-process session caches to +# `.cleanup()` on SIGTERM +catch_sigterm = true [tool.line_profiler.child_processes.multiprocessing.polling] diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 38119dd3..e30309fe 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1333,7 +1333,10 @@ def patched_attributes() -> MappingProxyType[str, frozenset[str]]: obj = _import_target(target) except ImportError: continue - result[target] = frozenset(a for a in attrs if hasattr(obj, a)) + present_attrs = frozenset(a for a in attrs if hasattr(obj, a)) + # Drop if none of the attributes is present + if present_attrs: + result[target] = present_attrs return MappingProxyType(result) From 04e3b8d7ef8eca871af439aaf9fd60e082be9eb3 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 29 Apr 2026 03:39:32 +0200 Subject: [PATCH 077/165] Fix signal handling line_profiler/_child_process_profiling/cache.py::LineProfilingCache cleanup() Now thread-locked to ensure that the signal handler doesn't call `Cleanup.cleanup()` in the middle of another call thereto _handle_signal() Refactored from `._handle_sigterm()` _add_signal_handler() - Refactored from `._wrap_sigterm()` - Now a no-op on Windows line_profiler/_child_process_profiling/multiprocessing_patches.py Updated correspondingly --- .../_child_process_profiling/cache.py | 89 ++++++++++++++----- .../multiprocessing_patches.py | 22 +++-- 2 files changed, 81 insertions(+), 30 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 8ae702ee..7174f3af 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -8,6 +8,7 @@ import dataclasses import os import signal +import sys try: import _pickle as pickle except ImportError: @@ -18,8 +19,8 @@ from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent -from threading import current_thread, main_thread -from types import ModuleType +from threading import current_thread, main_thread, RLock, Thread +from types import FrameType, ModuleType from typing import Any, ClassVar, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self @@ -42,6 +43,7 @@ PS = ParamSpec('PS') # Note: `typing.AnyStr` deprecated since 3.13 AnyStr = TypeVar('AnyStr', str, bytes) +_SignalHandler = Callable[[int, FrameType | None], Any] _THIS_SUBPACKAGE, *_ = (lambda: None).__module__.rpartition('.') INHERITED_CACHE_ENV_VARNAME_PREFIX = ( @@ -55,6 +57,9 @@ def _import_sibling(submodule: str) -> ModuleType: return import_module(f'{_THIS_SUBPACKAGE}.{submodule}') +_private_field = partial(dataclasses.field, init=False, repr=False) + + @final @dataclasses.dataclass class LineProfilingCache(Cleanup): @@ -76,12 +81,15 @@ class LineProfilingCache(Cleanup): insert_builtin: bool = True debug: bool = diagnostics.DEBUG - profiler: LineProfiler | None = dataclasses.field( - default=None, init=False, repr=False, + profiler: LineProfiler | None = _private_field(default=None) + _cleanup_stacks: dict[float, list[Callable[[], Any]]] = _private_field( + default_factory=dict, ) - _cleanup_stacks: dict[float, list[Callable[[], Any]]] = dataclasses.field( - default_factory=dict, init=False, repr=False, + _sighandlers: dict[int, _SignalHandler | int | None] = ( + _private_field(default_factory=dict) ) + _rlock: RLock = _private_field(default_factory=RLock) + _loaded_instance: ClassVar[LineProfilingCache | None] = None def __post_init__(self) -> None: @@ -166,6 +174,30 @@ def dump(self) -> None: with open(self.filename, mode='wb') as fobj: pickle.dump(content, fobj, protocol=HIGHEST_PROTOCOL) + def cleanup(self, *args, **kwargs) -> None: + """ + Perform cleanup. + + Args: + *args, **kwargs + Passed to :py:meth:`Cleanup.cleanup` + + Note: + In child processes we set a ``SIGTERM`` handler to always + call :py:meth:`~.cleanup`. However, this may happen when + we're in the middle of a cleanup call, which results in + undefined behavior. To prevent this, each + :py:meth:`~.cleanup` call is handled by a separate thread + which acquires an instance-specific lock. + """ + thread = Thread(target=self._cleanup_worker, args=args, kwargs=kwargs) + thread.start() + thread.join() + + def _cleanup_worker(self, *args, **kwargs) -> None: + with self._rlock: + super().cleanup(*args, **kwargs) + def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: """ Gather the profiling output files matching ``glob_pattern`` from @@ -381,38 +413,51 @@ def _setup_common( self, **(mp_apply_kwargs or {}), ) - def _handle_sigterm(self, signum: int, _) -> None: # nocover + def _handle_signal(self, signum: int, *_) -> None: # nocover """ See also: :py:meth:`coverage.control.Converage._on_sigterm` """ + name = self._get_signal_name(signum) + msg = f'Cleaning up before passing `{name}` ({signum}) on...' + self._debug_output(msg) try: - name = signal.Signals(signum).name - self._debug_output( - f'Got signal {signum} ({name}), ' - 'cleaning up before passing it on...', - ) - # Also restores handler overwritten by `._wrap_sigterm()` self.cleanup() finally: - os.kill(os.getpid(), signum) + handler = self._sighandlers.pop(signum, None) + if handler is not None: + signal.signal(signum, handler) + signal.raise_signal(signum) - def _wrap_sigterm(self) -> None: # nocover + def _add_signal_handler( + self, signum: int = signal.SIGTERM, + ) -> None: # nocover """ Side effects: - If on the main thread: + If on the main thread and not on Windows: - :py:func:`signal.signal` called to set - :py:meth:`~._handle_sigterm` as the ``SIGTERM`` handler + :py:meth:`~._handle_signal` as the ``SIGTERM`` handler - :py:meth:`~.cleanup` callback registered undoing that + + Note: + ``SIGTERM`` handling is known to be faulty on Windows; see + previous discussions at (examples `1`_, `2`_). + + .. _1: https://github.com/coveragepy/coveragepy/blob/main/\ +coverage/control.py + .. _2: https://stackoverflow.com/questions/35772001/ """ - if current_thread() != main_thread(): + if current_thread() != main_thread() or sys.platform == 'win32': return - set_handler = signal.signal - self._debug_output('Adding `SIGTERM` handler...') - handler = set_handler(signal.SIGTERM, self._handle_sigterm) - self.add_cleanup_with_priority(set_handler, 1, signal.SIGTERM, handler) + name = self._get_signal_name(signum) + self._debug_output(f'Adding `{name}` handler...') + self._sighandlers[signum] = signal.signal(signum, self._handle_signal) + + @staticmethod + def _get_signal_name(signum: int) -> str: + return signal.Signals(signum).name def _wrap_os_fork(self) -> None: """ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index bfca6643..d156581e 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -332,7 +332,7 @@ def wait_for_return( try: with wait_for_return(cache.config): pass - except (_Poller.Timeout, _Poller.TimeoutWarning) as e: + except _Poller.Timeout as e: # Also handles `~.TimeoutWarning` cache._debug_output(f'{type(e).__qualname__}: {e}') raise finally: # Always call `Process.terminate()` to avoid orphans @@ -352,17 +352,23 @@ def wrap_bootstrap( ``LineProfilingCache.load().cleanup()`` so that profiling results can be gathered. - Note: - This is only invoked in child processes, and :py:mod:`coverage` - seem to be having trouble with them in the current setup, - probably due to issues with .pth file precendence causing - :py:mod:`line_profiler` to be loaded before it. Hence the - ``# nocover``. + Notes: + + - This is only invoked in child processes, and + :py:mod:`coverage` seems to be having trouble with them in the + current setup, probably due to issues with .pth file + precendence causing :py:mod:`line_profiler` to be loaded + before it. Hence the ``# nocover``. + + - ``SIGTERM`` handling is not consistent on Windows, so we made + :py:meth:`LineProfilingCache._add_signal_handler` a no-op + there. Hence :py:func:`wrap_terminate` remains necessary in + mitigating unclean exits. """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up if _get_config(cache.config)['catch_sigterm']: - cache._wrap_sigterm() + cache._add_signal_handler() try: return vanilla_impl(self, *args, **kwargs) finally: From f88577d2930bc35718907746e4e3d3c775ba17f2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 29 Apr 2026 06:25:44 +0200 Subject: [PATCH 078/165] Fix for profiler cleanup line_profiler/curated_profiling.py::CuratedProfilerContext install() Now handling profiler disabling as a regular cleanup callback uninstall() - Simplifed implementation - Now always calling `.cleanup()` _disable_profiler() New helper method for disabling the profiler (code previosuly lived in `.uninstall()`) line_profiler/_child_process_profiling/cache.py::LineProfilingCache _cleanup_stacks Removed unused attribute (superseded by `._contexts`) cleanup() Added argument `new_thread` for controlling whether to spin up a new thread for do cleanup (note that the profiler has a different `.enable_count` on each thread, so cleaning up on a separate thread may impact proper profiler cleanup) copy() - Simplified implementation - Removed unused arguments `inherit_cleanups` and `inherit_profiler` _setup_in_child_process() Now intercepting the debug-log messages of the created `CuratedProfilerContext` _handle_signal() Now calling `.cleanup()` with `new_thread=True` tests/test_child_procs.py create_cache() Fixed typo in doc test_cache_setup_main_process() - Removed reference to unused/indirectly-used `curated_profiler` fixture - Now explicitly checking for the disabling of the profiler --- .../_child_process_profiling/cache.py | 74 ++++++++++--------- line_profiler/curated_profiling.py | 13 ++-- tests/test_child_procs.py | 5 +- 3 files changed, 50 insertions(+), 42 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 7174f3af..82bc1ef2 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -76,15 +76,12 @@ class LineProfilingCache(Cleanup): profile_imports: bool = False preimports_module: os.PathLike[str] | str | None = None main_pid: int = dataclasses.field(default_factory=os.getpid) - # Note: if we're using the line profiler, `kernprof` always set + # Note: if we're using the line profiler, `kernprof` always sets # `builtin` to true insert_builtin: bool = True debug: bool = diagnostics.DEBUG profiler: LineProfiler | None = _private_field(default=None) - _cleanup_stacks: dict[float, list[Callable[[], Any]]] = _private_field( - default_factory=dict, - ) _sighandlers: dict[int, _SignalHandler | int | None] = ( _private_field(default_factory=dict) ) @@ -95,22 +92,11 @@ class LineProfilingCache(Cleanup): def __post_init__(self) -> None: super().__init__() - def copy( - self, *, - inherit_cleanups: bool = False, - inherit_profiler: bool = False, - **replacements - ) -> Self: + def copy(self, /, **replacements) -> Self: """ Make a copy with optionally replaced fields. Args: - inherit_cleanups (bool): - If true, the copy also makes a (shallow) copy of the - cleanup-callback stack. - inherit_profiler (bool): - If true, the copy also gets a reference to - :py:attr:`~.profiler` **replacements (Any): Optional fields to replace @@ -121,15 +107,7 @@ def copy( init_args: dict[str, Any] = {} for field, value in self._get_init_args().items(): init_args[field] = replacements.get(field, value) - copy = type(self)(**init_args) - if inherit_cleanups: - copy._cleanup_stacks = { - priority: list(callbacks) - for priority, callbacks in self._cleanup_stacks.items() - } - if inherit_profiler: - copy.profiler = self.profiler - return copy + return type(self)(**init_args) @classmethod def load(cls) -> Self: @@ -174,22 +152,35 @@ def dump(self) -> None: with open(self.filename, mode='wb') as fobj: pickle.dump(content, fobj, protocol=HIGHEST_PROTOCOL) - def cleanup(self, *args, **kwargs) -> None: + def cleanup(self, *args, new_thread: bool = False, **kwargs) -> None: """ Perform cleanup. Args: - *args, **kwargs + new_thread (bool): + Whether to relegate the call to + :py:meth:`Cleanup,cleanup` to a new thread (see Notes) + *args, **kwargs: Passed to :py:meth:`Cleanup.cleanup` Note: - In child processes we set a ``SIGTERM`` handler to always - call :py:meth:`~.cleanup`. However, this may happen when - we're in the middle of a cleanup call, which results in - undefined behavior. To prevent this, each - :py:meth:`~.cleanup` call is handled by a separate thread - which acquires an instance-specific lock. - """ + - In child processes we set a ``SIGTERM`` handler to always + call :py:meth:`~.cleanup`. However, this may happen when + we're already in the middle of a cleanup call, which + results in undefined behavior. To prevent this, we can + supply ``new_thread=True`` so that the + :py:meth:`Cleanup.cleanup` call is handled by a separate + thread which acquires an instance-specific lock. + + - However, this method is supposed to clean up the session + profiler by completely disabling it, and that part must + happen on the main thread or deallocation will be botched. + Hence ``new_thread=True`` is made an option and not the + default. + """ + if not new_thread: + self._cleanup_worker(*args, **kwargs) + return thread = Thread(target=self._cleanup_worker, args=args, kwargs=kwargs) thread.start() thread.join() @@ -356,6 +347,11 @@ def _setup_in_child_process( False the instance has already been set up prior to calling this function, true otherwise """ + def wrap_ctx_debug( + ctx: CuratedProfilerContext, msg: str, + ) -> None: + self._debug_output(f' Context {id(ctx):#x}: {msg}') + if not context: context = '...' self._debug_output(f'Setting up ({context})...') @@ -369,6 +365,8 @@ def _setup_in_child_process( prof = LineProfiler() self.profiler = prof ctx = CuratedProfilerContext(prof, insert_builtin=self.insert_builtin) + if self.debug: + self.patch(ctx, '_debug_output', wrap_ctx_debug.__get__(ctx)) ctx.install() self.add_cleanup(ctx.uninstall) self._debug_output(f'Set up `.profiler` at {id(prof):#x}') @@ -422,7 +420,13 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover msg = f'Cleaning up before passing `{name}` ({signum}) on...' self._debug_output(msg) try: - self.cleanup() + # We don't care about profiler state and such if we're + # already being SIGTERM-ed, so just handle cleanup on a + # separate thread; + # this prevents a duplicate call to `.cleanup()` when we're + # already inside one (e.g. when `Process._bootstrap()` is + # exiting, or the `atexit` hook is triggered) + self.cleanup(new_thread=True) finally: handler = self._sighandlers.pop(signum, None) if handler is not None: diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index 3b4c8ac4..80513daa 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -193,6 +193,11 @@ def _global_install(self, prof: ByCountProfilerMixin | None) -> None: # objects to it should be the exception, not the norm. self._kpo(cast(LineProfiler, prof)) + @staticmethod + def _disable_profiler(prof: ByCountProfilerMixin) -> None: + for _ in range(getattr(prof, 'enable_count', 0)): + prof.disable_by_count() + def install(self) -> None: if self._installed: return @@ -208,14 +213,12 @@ def install(self) -> None: # Set up hooks to deal with inserting `.prof` as a builtin name if self.insert_builtin: self.patch(builtins, self.builtin_loc, self.prof) + # Disable the profiler + self.add_cleanup(self._disable_profiler, self.prof) + self.patch(self, '_installed', True) def uninstall(self) -> None: - if not self._installed: - return - # Fully disable the profiler - for _i in range(getattr(self.prof, 'enable_count', 0)): - self.prof.disable_by_count() self.cleanup() def __enter__(self) -> Self: diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index e30309fe..486b6720 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -360,7 +360,7 @@ def create_cache( - ``False``: equivalent to ``None``. - Unless the argument ``_use_curated_profiler: bool = True`` is set - to :py:const`True`, automatically creates an instance of + to :py:const:`False`, automatically creates an instance of :py:class:`LineProfiler` that is curated by a :py:class:`CuratedProfilerContext` and provides it as the :py:attr:`LineProfilingCache.profiler`, and @@ -1550,7 +1550,6 @@ def test_cache_setup_main_process( @_preserve_attributes(_GLOBAL_PATCHES) def test_cache_setup_child( create_cache: Callable[..., LineProfilingCache], - curated_profiler: LineProfiler, ext_module_object: ModuleType, another_pid: int, wrap_os_fork: bool, @@ -1617,6 +1616,8 @@ def list_profiled_funcs() -> list[str]: assert (os.fork is not old_fork) == fork_patched else: # E.g. Windows assert old_fork == _NotSupplied.NOT_SUPPLIED + # Check that after cleaning up the profiler has been disabled + assert not getattr(cache.profiler, 'enable_count', 0) # Check that profiling results have been written to the cache # directory From bf586405723276bd012af0bef8b214d4e4333729 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 29 Apr 2026 07:44:17 +0200 Subject: [PATCH 079/165] Fix test failing corner case --- tests/test_retry_tests.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py index b18df4e0..873c7681 100644 --- a/tests/test_retry_tests.py +++ b/tests/test_retry_tests.py @@ -678,18 +678,21 @@ def parse_line(cls, line: str) -> Self: # that created them went obsolete, before the next rerun; # we can verify that by checking that the ids of the `makefile()` # fixtures appear in contiguous blocks + + # Note: there seems to be a weird corner case where neighboring tests + # may reuse the same fixture id (see failing job 73520441960 in + # pipeline 25091142386); probably has to do with object lifetime. + # So instead of just checking the `fixture_id`, also consult + # `test`; it suffices to see that WITHIN THE SAME TEST we don't have + # fixture values stepping over one another with log.open() as fobj: entries = [LogEntry.parse_line(line.rstrip('\n')) for line in fobj] pprint.pprint(entries) - field: tuple[str, ...] | str - for field in ('test', 'stage'), 'fixture_id': - if isinstance(field, str): - getter: Callable[[LogEntry], Any] = attrgetter(field) - else: - getter = attrgetter(*field) + for fields in ('test', 'stage'), ('test', 'fixture_id'): + getter = attrgetter(*fields) values = [getter(entry) for entry in entries] assert _identical_items_are_adjacent(values), ( - f'Inconsistency in {field} order: {values!r}' + f'Inconsistency in {fields} order: {values!r}' ) From e05d57000b70da0313af24eb447aff5af0489d75 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 2 May 2026 01:23:12 +0200 Subject: [PATCH 080/165] Streamlined tests line_profiler/_child_process_profiling/multiprocessing_patches.py ::_Poller.__enter__() If `on_timeout='warn'`, the log message is now written BEFORE issuing the warning (so as to avoid having the latter preventing the latter via raising the warning) tests/test_child_procs.py _Param.sorted() New method for sorting the parametrizations of instances test_apply_mp_patches() - Renamed parameter `intercept_logs` from `debug` - Removed several redundant subtests (no need to test `intercept_logs` with all `multiprocessing` start methods) test_multiproc_script_sanity_check() Removed several redundant subtests (tests with `preimports=True` overlapping with `test_apply_mp_patches`) --- .../multiprocessing_patches.py | 4 +- tests/test_child_procs.py | 153 ++++++++++++------ 2 files changed, 106 insertions(+), 51 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index d156581e..26d01c49 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -201,8 +201,10 @@ def error(msg: str) -> NoReturn: raise type(self).Timeout(msg) def warn(msg: str) -> None: - warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) + # Write log before issuing the warning because that may be + # promoted to an exception diagnostics.log.warning(msg) + warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) def ignore(_): pass diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 486b6720..dcd55e92 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -19,6 +19,7 @@ from functools import partial, wraps from io import StringIO from importlib import import_module +from numbers import Real from pathlib import Path from runpy import run_path from tempfile import TemporaryDirectory @@ -692,12 +693,57 @@ def __add__(self, other: Self) -> Self: ) return type(self)(new_params, new_values, new_defaults_tuple) + def sorted( + self, + *, + sort_by: Sequence[str] | None = None, + sortable_types: type[Any] | tuple[type[Any], ...] = (Real, str, bytes), + ) -> Self: + """ + Sort by parametrization values. + + Args: + sort_by (Sequence[str] | None): + Column names to sort by; default is to sort by all + sortable params. + sortable_types (type[Any] | tuple[type[Any], ...]): + Type(s) where if a param has all its values being + instances thereof (excl. :py:const:`None`s), said param + is considered sortable. + + Returns: + New instance + """ + def sort_key(obj: Any) -> tuple[bool, str, Any]: + type_name = '{0.__module__}.{0.__qualname__}'.format(type(obj)) + return (obj is None), type_name, obj + + if sort_by is None: + sort_by = self.params + sortable_columns: set[str] = { + param for param, *values in zip(self.params, *self.values) + if all(isinstance(v, sortable_types) or v is None for v in values) + } + sorted_column_indices: tuple[int, ...] = tuple( + i for i, param in enumerate(sort_by) if param in sortable_columns + ) + + if sorted_column_indices: + new_values = sorted( + self.values, key=lambda vtuple: tuple( + sort_key(vtuple[i]) for i in sorted_column_indices + ), + ) + else: # Fallback + new_values = self.values.copy() + return type(self)(self.params, new_values, self.defaults) + def __call__(self, func: C) -> C: """ Mark a callable as with :py:func:`pytest.mark.parametrize`. """ # Note: `pytest` automatically assumes single-param values to - # be unpackes, so comply here + # be unpacked, so comply here if len(self.params) == 1: marker = pytest.mark.parametrize( self.params[0], [v[0] for v in self.values], @@ -1716,12 +1762,17 @@ def test_load_pth_hook( @pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) -@pytest.mark.parametrize('start_method', - ['fork', 'forkserver', 'spawn', 'dummy']) -@pytest.mark.parametrize(('debug', 'label1'), - [(True, 'with-debug'), (False, 'no-debug')]) -@pytest.mark.parametrize(('fail', 'label2'), - [(True, 'failure'), (False, 'success')]) +@(_Params.new('start_method', + ['fork', 'forkserver', 'spawn', 'dummy'], + defaults='dummy') + * _Params.new(('fail', 'label2'), + [(True, 'failure'), (False, 'success')], + defaults=(False, 'success')) + # We only need to check if `intercept_logs` work, the other + # parametrizations don't matter + + _Params.new(('intercept_logs', 'label1'), + [(True, 'with-intercept-logs'), (False, 'no-intercept-logs')], + defaults=(False, 'no-intercept-logs'))).sorted() @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) @@ -1731,7 +1782,7 @@ def test_apply_mp_patches( ext_module_object: ModuleType, test_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], - debug: bool, + intercept_logs: bool, fail: bool, n: int, nprocs: int, @@ -1760,17 +1811,21 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' cfg_chunks: list[str] = [] - if debug: + if intercept_logs: cfg_chunks.append( '[tool.line_profiler.child_processes.multiprocessing]\n' 'intercept_logs = true' ) + # This is easier to debug than `ResultMismatch` cfg_chunks.append( '[tool.line_profiler.child_processes.multiprocessing.polling]\n' 'on_timeout = "error"' ) config.write_text('\n\n'.join(cfg_chunks)) + # Note: no need to test the case for `my_local_sum()` separately, + # with `preimports_module=True`, both are just imported and added + # to the profiler, so the code paths are the same profiled_func = ext_module_object.my_external_sum called_func = partial( test_module_object.sum_in_child_procs, @@ -1840,7 +1895,7 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: for path in Path(cache.cache_dir).glob('*.lprof') if is_valid_stats_file(path) } - patterns[re.escape('`multiprocessing` logging (debug)')] = debug + patterns[re.escape('`multiprocessing` logging (debug)')] = intercept_logs _search_cache_logs(cache, True, patterns) @@ -1864,24 +1919,19 @@ def _get_mp_start_method_fuzzer(label_name: str) -> _Params: return fuzz_fail * fuzz_start -_fuzz_sanity = ( - _Params.new(('run_func', 'label1'), - [(run_module, 'module'), (run_script, 'script')]) - * _Params.new(('use_local_func', 'label2'), - [(True, 'local'), (False, 'ext')]) - # Python can't pickle things unless they resided in a retrievable - # location (so not the script supplied by `python -c`) - + _Params.new(('run_func', 'label1', 'use_local_func', 'label2'), - [(run_literal_code, 'literal-code', False, 'ext')]) - # Also fuzz the parallelization-related stuff, esp. check what - # happens if an exception is raised inside the parallelly-run func - + _get_mp_start_method_fuzzer('label3') - + _Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], - defaults=(None, None)) -) - - -@_fuzz_sanity +@(_Params.new(('run_func', 'label1'), + [(run_module, 'module'), (run_script, 'script')]) + * _Params.new(('use_local_func', 'label2'), + [(True, 'local'), (False, 'ext')]) + # Python can't pickle things unless they resided in a retrievable + # location (so not the script supplied by `python -c`) + + _Params.new(('run_func', 'label1', 'use_local_func', 'label2'), + [(run_literal_code, 'literal-code', False, 'ext')]) + # Also fuzz the parallelization-related stuff, esp. check what + # happens if an exception is raised inside the parallelly-run func + + _get_mp_start_method_fuzzer('label3') + + _Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], + defaults=(None, None))).sorted() def test_multiproc_script_sanity_check( run_func: Callable[..., subprocess.CompletedProcess], test_module: _ModuleFixture, @@ -1951,28 +2001,31 @@ def test_running_multiproc_script( run_func(test_module, tmp_path_factory, runner, outfile, profile) -_fuzz_prof_mp_1 = ( - _Params.new(('run_func', 'label1'), - [(run_module, 'module'), - (run_script, 'script'), - (run_literal_code, 'literal-code')], - defaults=(run_script, 'script')) - + _Params.new(('prof_child_procs', 'label2'), - [(True, 'with-child-prof'), (False, 'no-child-prof')]) - + _get_mp_start_method_fuzzer('label3') -) -_fuzz_prof_mp_2 = ( - _Params.new(('preimports', 'label4'), - [(True, 'with-preimports'), (False, 'no-preimports')], - defaults=(False, 'no-preimports')) - + _Params.new(('use_local_func', 'label5'), - [(True, 'local'), (False, 'external')], - defaults=(False, 'external')) -) - - -@_fuzz_prof_mp_1 -@_fuzz_prof_mp_2 +_fuzz_prof_mp_run_func = _Params.new(('run_func', 'label1'), + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')], + defaults=(run_script, 'script')) + + +@( + (_fuzz_prof_mp_run_func + + _Params.new(('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')]) + + _get_mp_start_method_fuzzer('label3')) + # Test all `multiproc` start methods with both locally- and + # externally-defined profiling targets + * (_Params.new(('preimports', 'label4'), [(False, 'no-preimports')]) + + _Params.new(('use_local_func', 'label5'), + [(True, 'local'), (False, 'external')], + defaults=(False, 'external'))) + # The 'with-preimports' case is already tested rather thoroughly in + # `test_apply_mp_patches()`, so exclude these from the above "main" + # param matrix and just test the different `kernprof` modes via the + # `run_func()`s + + (_fuzz_prof_mp_run_func + + _Params.new(('preimports', 'label4'), [(True, 'with-preimports')])) +).sorted() @pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, subprocess.TimeoutExpired)) @pytest.mark.parametrize( From 09fb005f176f3585959d59c0834bc779cf5c31f9 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 4 May 2026 05:39:32 +0200 Subject: [PATCH 081/165] Update `line_profiler.line_profiler.LineStats` line_profiler/line_profiler.py Remapped some `typing` imports and annotations because we're now on 3.10+ LineStats.__eq__(), .__[i]add__(), from_stats_objects() Fixed misrepresentation of the data structure in docstrings (filenames and function names swapped in `.timings` keys) LineStats.from_files() Added `on_empty` argument to handle empty files (that we know `pickle.load()` will fail on) and special handling therefor, so that error/warning messages are more informative --- line_profiler/line_profiler.py | 114 +++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 48 deletions(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 50ef0d88..98845f3a 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -20,20 +20,10 @@ import tokenize import warnings from argparse import ArgumentParser +from collections.abc import Callable, Collection, Mapping, Sequence from datetime import datetime from os import PathLike -from typing import ( - TYPE_CHECKING, - IO, - Callable, - Literal, - Mapping, - Protocol, - Sequence, - TypeVar, - cast, - Tuple, -) +from typing import TYPE_CHECKING, IO, Any, Literal, Protocol, TypeVar, cast try: from ._line_profiler import ( @@ -63,7 +53,7 @@ class _IPythonLike(Protocol): def register_magics(self, magics: type) -> None: ... PS = ParamSpec('PS') - _TimingsMap = Mapping[Tuple[str, int, str], list[Tuple[int, int, int]]] + _TimingsMap = Mapping[tuple[str, int, str], list[tuple[int, int, int]]] T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) @@ -227,6 +217,15 @@ def tokeneater( return super().tokeneater(type, token, srowcol, erowcol, line) +class _EmptyFileError(OSError): + """ + Error raised when trying to read profiling data from an empty file. + """ + def __init__(self, file: PathLike[str] | str) -> None: + super().__init__(str(file)) + self.file = file + + class _WrapperInfo: """ Helper object for holding the state of a wrapper function. @@ -265,8 +264,8 @@ def __eq__(self, other: object) -> bool: Example: >>> from copy import deepcopy >>> stats1 = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 2, 1000), (12, 1, 500)]}, ... 1E-6) >>> stats2 = deepcopy(stats1) @@ -275,7 +274,7 @@ def __eq__(self, other: object) -> bool: >>> assert stats2 != stats1 >>> stats3 = deepcopy(stats1) >>> assert stats1 == stats3 is not stats1 - >>> stats3.timings['foo', 1, 'spam.py'][:] = [(2, 11, 330)] + >>> stats3.timings['spam.py', 1, 'foo'][:] = [(2, 11, 330)] >>> assert stats3 != stats1 """ for attr in 'timings', 'unit': @@ -291,20 +290,20 @@ def __add__(self, other: _StatsLike) -> Self: """ Example: >>> stats1 = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 2, 1000), (12, 1, 500)]}, ... 1E-6) >>> stats2 = LineStats( - ... {('bar', 10, 'spam.py'): + ... {('spam.py', 10, 'bar'): ... [(11, 10, 20000), (12, 5, 1000)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 5000)]}, + ... ('eggs.py', 5, 'baz'): [(5, 2, 5000)]}, ... 1E-7) >>> stats_sum = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 12, 3000), (12, 6, 600)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 500)]}, + ... ('eggs.py', 5, 'baz'): [(5, 2, 500)]}, ... 1E-6) >>> assert stats1 + stats2 == stats2 + stats1 == stats_sum """ @@ -315,20 +314,20 @@ def __iadd__(self, other: _StatsLike) -> Self: """ Example: >>> stats1 = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 2, 1000), (12, 1, 500)]}, ... 1E-6) >>> stats2 = LineStats( - ... {('bar', 10, 'spam.py'): + ... {('spam.py', 10, 'bar'): ... [(11, 10, 20000), (12, 5, 1000)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 5000)]}, + ... ('eggs.py', 5, 'baz'): [(5, 2, 5000)]}, ... 1E-7) >>> stats_sum = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 12, 3000), (12, 6, 600)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 500)]}, + ... ('eggs.py', 5, 'baz'): [(5, 2, 500)]}, ... 1E-6) >>> address = id(stats2) >>> stats2 += stats1 @@ -386,6 +385,7 @@ def from_files( file: PathLike[str] | str, /, *files: PathLike[str] | str, + on_empty: Literal['ignore', 'warn', 'error'] = 'warn', on_defective: Literal['ignore', 'warn', 'error'] = 'error', ) -> Self: """ @@ -396,10 +396,11 @@ def from_files( File to load profiling data from *files (PathLike[str] | str): Ditto above - on_defective (Literal['ignore', 'warn', 'error']): - What to do if some files fail to load: ``'ignore'`` - those files, skip them but with a ``'warn'``-ing, or - raise the ``'error'`` as soon as one is encountered + on_empty, on_defective (Literal['ignore', 'warn', 'error']): + What to do if some files are empty (resp. otherwise fail + to load): ``'ignore'`` those files, skip them but with a + ``'warn'``-ing, or raise the ``'error'`` as soon as one + is encountered Returns: instance (LineStats): @@ -407,25 +408,42 @@ def from_files( """ stats_objs = [] failures: dict[str, str] = {} + empty_files: set[str] = set() all_files = [file, *files] + for file in all_files: with open(file, 'rb') as f: try: + if not os.stat(file).st_size: + raise _EmptyFileError(file) stats_objs.append(pickle.load(f)) + except _EmptyFileError as e: + if on_empty == 'error': + raise + empty_files.add(str(e.file)) except Exception as e: if on_defective == 'error': raise failures[str(file)] = f'{type(e).__name__}: {e}' - if failures: - msg = ( - '{} file(s) out of {} failed to load and are skipped: {!r}' - .format(len(failures), len(all_files), failures) + + problems: Collection[Any] + for problems, description, behavior in [ + (list(empty_files), 'is/are empty and thus skipped:', on_empty), + (failures, 'failed to load and is/are skipped', on_defective), + ]: + if not problems: + continue + msg = '{} file(s) out of {} {}: {!r}'.format( + len(problems), len(all_files), description, problems, ) - if on_defective == 'warn': - warnings.warn(msg) + if behavior == 'warn': + # Log before warning because warnings may be promoted to + # errors diagnostics.log.warning(msg) + warnings.warn(msg) else: # 'ignore' diagnostics.log.debug(msg) + if not stats_objs: return cls.get_empty_instance() return cls.from_stats_objects(*stats_objs) @@ -437,23 +455,23 @@ def from_stats_objects( """ Example: >>> stats1 = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 2, 1000), (12, 1, 500)]}, ... 1E-6) >>> stats2 = LineStats( - ... {('bar', 10, 'spam.py'): + ... {('spam.py', 10, 'bar'): ... [(11, 10, 20000), (12, 5, 1000)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 5000)]}, + ... ('eggs.py', 5, 'baz'): [(5, 2, 5000)]}, ... 1E-7) >>> stats_combined = LineStats.from_stats_objects( ... stats1, stats2) >>> assert stats_combined.unit == 1E-6 >>> assert stats_combined.timings == { - ... ('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... ('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 12, 3000), (12, 6, 600)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 500)]} + ... ('eggs.py', 5, 'baz'): [(5, 2, 500)]} """ timings, unit = cls._get_aggregated_timings([stats, *more_stats]) return cls(timings, unit) @@ -890,7 +908,7 @@ def show_func( func_name (str): name of profiled function - timings (List[Tuple[int, int, float]]): + timings (list[tuple[int, int, float]]): Measurements for each line (lineno, nhits, time). unit (float): From c1a2b0efe8c80e3200a0e0aa612767068d29d475 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 4 May 2026 05:51:20 +0200 Subject: [PATCH 082/165] Various cleanup line_profiler/_child_process_profiling/cache.py ::LineProfilingCache.gather_stats() Now emitting warnings for all anomalous cases (empty and defective files) line_profiler/cleanup.py::Cleanup.patch() Fixed bug where the `priority` argument is unused, instead of passed to `.add_cleanup_with_priority()` as advertised --- line_profiler/_child_process_profiling/cache.py | 4 +++- line_profiler/cleanup.py | 15 +++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 82bc1ef2..39f72297 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -202,7 +202,9 @@ def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: ) if not fnames: return LineStats.get_empty_instance() - return LineStats.from_files(*fnames, on_defective='ignore') + return LineStats.from_files( + *fnames, on_empty='warn', on_defective='warn', + ) def _dump_debug_logs(self) -> None: """ diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index 97b6b05a..ac866128 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -15,7 +15,7 @@ from pathlib import Path from types import MethodType from typing import Any, TypeVar, TypedDict, cast -from typing_extensions import ParamSpec, Self, Unpack +from typing_extensions import Concatenate, ParamSpec, Self, Unpack from .line_profiler_utils import block_indent, make_tempfile from . import _diagnostics as diagnostics @@ -517,13 +517,20 @@ def patch( >>> assert obj.foo == 1 >>> assert not hasattr(obj, 'bar') """ - add_cleanup = self.add_cleanup if cleanup else (lambda *_, **__: None) + if cleanup: + add_cleanup: Callable[ + Concatenate[Callable[..., Any], float, ...], Any + ] = self.add_cleanup_with_priority + else: + # ... yeah gotta disagree with flake8, a lambda makes + # perfect sense here + add_cleanup = lambda *_, **__: None # noqa: E731 try: old = getattr(obj, attr) except AttributeError: - add_cleanup(delattr, obj, attr) + add_cleanup(delattr, priority, obj, attr) else: - add_cleanup(setattr, obj, attr, old) + add_cleanup(setattr, priority, obj, attr, old) setattr(obj, attr, value) if name is None: name = self._get_name(obj) From b0df0287762c4f00dcf8bdd70d3549ac00361b6e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 4 May 2026 18:58:21 +0200 Subject: [PATCH 083/165] Curb stray warnings for profiling outfiles line_profiler/_child_process_profiling/cache.py::LineProfilingCache gather_stats() - Updated call signature - Now excluding the profiling-stats file from the `multiprocessing` resource-tracker server process, which is often empty because of process longevity and would have resulted in a stray warning _get_profiling_outfiles() New helper method for finding specific profiling-stats files line_profiler/_child_process_profiling/pth_hook.py::write_pth_hook() The .pth file written is now cleaned up with increased priority so as to be less disruptive to other Python processes; most importantly, it is removed before the `multiprocessing` fork-server process is rebooted, so that the new server process will not be polluted by profiling stuff kernprof.py _manage_profiler.__enter__(), .__exit__() Reworked cleanup-task priorities: - Writing of debug logs now managed by the `CuratedProfilerContext` instead of the `LineProfilingCache` - `LineProfilingCache.cleanup()` now called before the gathering of child-process profiling results, to ensure that results from the `multiprocessing` fork-server process is properly handled _prepare_child_profiling_cache() - Updated call signature, taking also a `CuratedProfilerContext` - Said context (instead of the cache) now manages the lifetime of the tempdir --- kernprof.py | 27 ++++--- .../_child_process_profiling/cache.py | 78 +++++++++++++++++-- .../_child_process_profiling/pth_hook.py | 2 + 3 files changed, 91 insertions(+), 16 deletions(-) diff --git a/kernprof.py b/kernprof.py index 0e1a4b77..e9612fad 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1207,19 +1207,19 @@ def __enter__(self): ) if self.set_up_child_profiling: self.cache = cache = _prepare_child_profiling_cache( - self.options, self.prof, preimports_file, script_file, + self.options, self._ctx, self.prof, + preimports_file, script_file, ) - self._ctx.add_cleanup(cache.cleanup) # Add deferred callbacks for gathering debug logfiles # (should run right before `.cache.cache_dir` is wiped): # - Write the debug logs to the `._diagnostics` logger if cache.debug: - cache.add_cleanup_with_priority( + self._ctx.add_cleanup_with_priority( cache._dump_debug_logs, CLEANUP_PRIORITIES['gather_logs'], ) # - Write the debug logs to a specific file if self.options.debug_log: - cache.add_cleanup_with_priority( + self._ctx.add_cleanup_with_priority( self._gather_debug_log, CLEANUP_PRIORITIES['gather_logs'], self.options.debug_log, @@ -1230,11 +1230,13 @@ def __exit__(self, *_, **__): try: extra_stats = None if self.set_up_child_profiling: + # Cleaning up here ensures the `multiprocessing` + # fork-server process is rebooted, thus any profiling + # data on it will be properly collected + self.cache.cleanup() extra_stats = self.cache.gather_stats() _post_profile(self.options, self.prof, extra_stats) finally: - # This also calls its `.cleanup()`, and in turn that of - # `.cache`'s self._ctx.uninstall() def _gather_debug_log(self, logfile): @@ -1344,7 +1346,9 @@ def _prepare_exec_script( return script_file, preimports_file -def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): +def _prepare_child_profiling_cache( + options, ctx, prof, preimports_file, script_file +): """ Handle the (line-)profiling of spawned/forked child Python processes. @@ -1364,8 +1368,10 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): ) clean_up = functools.partial(cache.add_cleanup, _remove, missing_ok=True) if not diagnostics.KEEP_TEMPDIRS: - # Defer the scrubbing of the cache dir - cache.add_cleanup_with_priority( + # Defer the scrubbing of the cache dir and let the context + # handle it, ideally speaking the cache dir should survive the + # cache object + ctx.add_cleanup_with_priority( _remove, CLEANUP_PRIORITIES['rm_cache_dir'], cache.cache_dir, recursive=True, ) @@ -1383,6 +1389,9 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): cache._setup_in_main_process() cache.profiler = prof + # Have the context clean up the cache as a failsafe + ctx.add_cleanup(cache.cleanup) + return cache diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 39f72297..2ff902a7 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -16,12 +16,13 @@ from collections.abc import Collection, Callable, MutableMapping, Iterable from functools import partial, cached_property, wraps from importlib import import_module +from operator import attrgetter from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent from threading import current_thread, main_thread, RLock, Thread from types import FrameType, ModuleType -from typing import Any, ClassVar, TypeVar, cast, final, overload +from typing import Any, ClassVar, Literal, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self from .. import _diagnostics as diagnostics @@ -51,6 +52,21 @@ ) CACHE_FILENAME = 'line_profiler_cache.pkl' _DEBUG_LOG_FILENAME_PATTERN = 'debug_log_{main_pid}_{current_pid}.log' +_PROFILING_OUTPUT_PREFIX_PATTERN = ( + 'child-prof-output-{main_pid}-{current_pid}-{prof}-' +) +_DEFAULT_GATHER_STATS_EXCLUDES: set[tuple[str, str]] = { + # Note: the `ResourceTracker` server process is spawned when the + # first `multiprocessing` child process is created via the `spawn` + # or `forkserver` start method. While this server process does not + # meaningfully contribute to the profiling result either way, since + # it can be created with profiling set up, its longevity means that + # `LineProfilingCache.gather_stats()` often catches an empty .lprof + # file which it has occupied but not written to. To reduce noise + # while keeping the warning for other zero-length files, just + # explictly exclude said process + ('multiprocessing.resource_tracker', '_resource_tracker._pid'), +} def _import_sibling(submodule: str) -> ModuleType: @@ -189,13 +205,48 @@ def _cleanup_worker(self, *args, **kwargs) -> None: with self._rlock: super().cleanup(*args, **kwargs) - def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: + def gather_stats( + self, + exclude_pids: Collection[int] | None = None, + *, + on_empty: Literal['error', 'warn', 'ignore'] = 'warn', + on_defective: Literal['error', 'warn', 'ignore'] = 'warn', + ) -> LineStats: """ Gather the profiling output files matching ``glob_pattern`` from :py:attr:`~.cache_dir`, consolidating them into a single :py:class:`LineStats` object. + + Args: + exclude_pids (Collection[int] | None): + Exclude output from child processes with these PIDs; + the default value :py:const:`None` fetches relevant + PIDs dynamically. + on_empty, on_defective (Literal['error', 'warn', 'ignore']): + Passed to :py:meth:`LineStats.from_files`. + + Returns: + :py:class:`LineStats` instance """ - fnames = list(Path(self.cache_dir).glob(glob_pattern)) + if exclude_pids is None: + exclude_pids = set() + for import_target, attr in _DEFAULT_GATHER_STATS_EXCLUDES: + try: + module = import_module(import_target) + except ImportError: + continue + try: + maybe_pid = attrgetter(attr)(module) + except AttributeError: + maybe_pid = None + if maybe_pid is None: + continue + exclude_pids.add(cast(int, maybe_pid)) + + fnames_ = set(self._get_profiling_outfiles()) + for pid in exclude_pids: + fnames_.difference_update(self._get_profiling_outfiles(pid)) + fnames = sorted(fnames_) self._debug_output( 'Loading results from {} child profiling file(s): {!r}' .format(len(fnames), fnames) @@ -203,7 +254,7 @@ def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: if not fnames: return LineStats.get_empty_instance() return LineStats.from_files( - *fnames, on_empty='warn', on_defective='warn', + *fnames, on_empty=on_empty, on_defective=on_defective, ) def _dump_debug_logs(self) -> None: @@ -252,10 +303,20 @@ def _gather_debug_log_entries( def _get_debug_logfiles(self) -> Iterable[Path]: pattern = _DEBUG_LOG_FILENAME_PATTERN.format( - main_pid=self.main_pid, current_pid='*', + main_pid=self.main_pid, current_pid='?*', ) return Path(self.cache_dir).glob(pattern) + def _get_profiling_outfiles(self, pid: Any = '?*') -> Iterable[Path]: + prefix = _PROFILING_OUTPUT_PREFIX_PATTERN.format( + main_pid=self.main_pid, + current_pid=pid, + # We always format the profiler ID with `hex()`, see + # `._setup_in_child_process()` + prof='0x?*', + ) + return Path(self.cache_dir).glob(prefix + '?*.lprof') + def inject_env_vars( self, env: MutableMapping[str, str] | None = None, ) -> None: @@ -385,8 +446,11 @@ def wrap_ctx_debug( # up to write thereto when the process terminates (with high # priority) prof_outfile = self.make_tempfile( - prefix='child-prof-output-{}-{}-{:#x}-' - .format(self.main_pid, os.getpid(), id(prof)), + prefix=_PROFILING_OUTPUT_PREFIX_PATTERN.format( + main_pid=self.main_pid, + current_pid=os.getpid(), + prof=hex(id(prof)), + ), suffix='.lprof', delete=False, ) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 4fa6b9ef..416c3d76 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -66,6 +66,8 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path prefix=pth_config['prefix'], suffix=pth_config['suffix'] + '.pth', dir=get_path('purelib'), + # Get rid of the .pth file ASAP so as to be the least disruptive + priority=1, ) try: pth_content = 'import {0}; {0}.load_pth_hook({1})'.format( From e9255b767498daa2746e0741662142a1000a2a3c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 4 May 2026 21:02:02 +0200 Subject: [PATCH 084/165] WIP: migrate .pth-related code line_profiler/_child_process_profiling/cache.py::LineProfilingCache write_pth_hook() Refactored from `~~.pth_hook.write_pth_hook` ._setup_in_main_process() Now using `.write_pth_hook()` ._config_source Convenience property for accessing the `ConfigSource` object loaded from `.config` _line_profiler_hooks.py Moved from `line_profiler/_child_process_profiling/pth_hook.py` setup.py Now installing `_line_profiler_hooks` --- .../pth_hook.py => _line_profiler_hooks.py | 0 .../_child_process_profiling/cache.py | 86 +++++++++++++++++-- setup.py | 4 +- 3 files changed, 80 insertions(+), 10 deletions(-) rename line_profiler/_child_process_profiling/pth_hook.py => _line_profiler_hooks.py (100%) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/_line_profiler_hooks.py similarity index 100% rename from line_profiler/_child_process_profiling/pth_hook.py rename to _line_profiler_hooks.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 2ff902a7..f04ea971 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -9,11 +9,14 @@ import os import signal import sys +import sysconfig try: import _pickle as pickle except ImportError: import pickle # type: ignore[assignment,no-redef] -from collections.abc import Collection, Callable, MutableMapping, Iterable +from collections.abc import ( + Collection, Callable, Mapping, MutableMapping, Iterable, +) from functools import partial, cached_property, wraps from importlib import import_module from operator import attrgetter @@ -25,16 +28,13 @@ from typing import Any, ClassVar, Literal, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self +from _line_profiler_hooks import INHERITED_PID_ENV_VARNAME, load_pth_hook from .. import _diagnostics as diagnostics from ..cleanup import Cleanup from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats +from ..toml_config import ConfigSource from ._cache_logging import CacheLoggingEntry -# Note: this should have been defined here in this file, but we moved it -# over to `~._child_process_hook` because that module contains the .pth -# hook, which must run with minimal overhead when a Python process isn't -# associated with a profiled process -from .pth_hook import INHERITED_PID_ENV_VARNAME __all__ = ('LineProfilingCache',) @@ -335,6 +335,67 @@ def inject_env_vars( _format_debug_msg='Injecting env var ${{{1}}}: {2}'.format, ) + def write_pth_hook( + self, *, + prefix: str | None = None, + suffix: str | None = None, + dir: os.PathLike[str] | str | None = None, + # Get rid of the .pth file ASAP so as to be the least disruptive + priority: float = 1, + **kwargs + ) -> Path: + """ + Write a .pth file which allows for setting up profiling in child + Python processes. + + Args: + prefix, suffix (str | None): + Optional filename-stem affixes of the .pth file; default + is to use default values loaded from :py:attr:`.config` + dir (os.PathLike[str] | str | None): + Optional directory to create the .pth file in; default + is to use ``sysconfig.get_path('purelib')`` + priority, **kwargs: + Passed to :py:meth:`.make_tempfile`. + + Returns: + fpath (Path): + Path to the written .pth file + """ + def get_pth_config() -> Mapping[str, Any]: + # Note: the only keys in it should be `prefix` and `suffix` + return ( + self._config_source # Cached + .get_subconfig('child_processes', 'pth_files') + .conf_dict + ) + + if not os.path.exists(self.filename): + self.dump() + assert os.path.exists(self.filename) + + # The string casts are failsafes in case inappropriate values + # (e.g. numbers and booleans) are supplied + if prefix is None: + prefix = str(get_pth_config()['prefix']) + if suffix is None: + suffix = str(get_pth_config()['suffix']) + if dir is None: + dir = sysconfig.get_path('purelib') + + template = 'import {0.__module__}; {0.__module__}.{0.__name__}({1})' + fpath = self.make_tempfile( + prefix=prefix, suffix=suffix + '.pth', dir=dir, priority=priority, + **kwargs, + ) + try: + fpath.write_text(template.format(load_pth_hook, self.main_pid)) + except Exception: + fpath.unlink(missing_ok=True) + raise + + return fpath + def _debug_output(self, msg: str) -> None: """ Beside writing to the logger, also write to the @@ -364,8 +425,7 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: - A ``.pth`` file written so that child processes automaticaly runs setup code (see - :py:func:`line_profiler._child_process_hook.pth_hook.\ -write_pth_hook`) + :py:meth:`.write_pth_hook`) - :py:func:`os.fork` wrapped so that profiling set up in forked processes is properly handled (if @@ -379,7 +439,7 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: """ self.dump() self.inject_env_vars() - _import_sibling('pth_hook').write_pth_hook(self) + self.write_pth_hook() self._setup_common(wrap_os_fork, {'reboot_forkserver': True}) self._replace_loaded_instance() @@ -747,3 +807,11 @@ def _make_debug_entry(self) -> Callable[[str], CacheLoggingEntry]: @cached_property def _consistent_with_loaded_instance(self) -> bool: return type(self).load()._get_init_args() == self._get_init_args() + + @cached_property + def _config_source(self) -> ConfigSource: + if self.config is None: + config: str | None = None + else: + config = str(self.config) + return ConfigSource.from_config(config) diff --git a/setup.py b/setup.py index 739d5d96..a0810cc6 100755 --- a/setup.py +++ b/setup.py @@ -314,7 +314,9 @@ def run_cythonize(force=False): setupkw['long_description_content_type'] = 'text/x-rst' setupkw['license'] = 'BSD' setupkw['packages'] = list(setuptools.find_packages()) - setupkw['py_modules'] = ['kernprof', 'line_profiler'] + setupkw['py_modules'] = [ + 'kernprof', 'line_profiler', '_line_profiler_hooks', + ] setupkw['python_requires'] = '>=3.10' setupkw['license_files'] = ['LICENSE.txt', 'LICENSE_Python.txt'] setupkw['package_data'] = {'line_profiler': ['py.typed', '*.pyi', '*.toml']} From 8d862bfb2e99a809ea2f6dd4dc5967f50f5603d8 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 4 May 2026 21:29:15 +0200 Subject: [PATCH 085/165] Finish .pth code migration _line_profiler_hooks.py write_pth_hook() Removed (superseded by `LineProfilingCache.write_pth_hook()`) load_pth_hook() Fixed broken relative imports from `line_profiler` tests/test_child_procs.py Fixed import location of `load_pth_hook()` --- _line_profiler_hooks.py | 94 +++++++++------------------------------ tests/test_child_procs.py | 20 ++++----- 2 files changed, 30 insertions(+), 84 deletions(-) diff --git a/_line_profiler_hooks.py b/_line_profiler_hooks.py index 416c3d76..5b5c888a 100644 --- a/_line_profiler_hooks.py +++ b/_line_profiler_hooks.py @@ -1,96 +1,44 @@ """ -Hooks to set up shop in a child Python process and extend profiling -to therein. +Additional hooks installed by :py:mod:`line_profiler`. -Note: - - The current implementation writes temporary .pth files to the +Notes: + - This file and its content should be considered an implmentation + detail of :py:mod:`line_profiler`; currently we just use this to + set up shop in a child Python process, and extend profiling to + therein. + + - This current implementation writes temporary .pth files to the site-packages directory, which are executed for all Python processes referring to the same :path:`lib/`. However, only processes originating from a parent which set the requisite environment variables will execute to the profiling code. + - Said .pth file always import this module; hence, this file is kept - intentionally lean to reduce overhead: - - Imports in this file are deferred to being as late as possible. - - Type annotations are replaced with type comments. - - Non-essential functionalities are split into small separate - submodules (e.g. :py:mod:`~.cache`). + intentionally lean and separate from the main + :py:mod:`line_profiler` package to reduce overhead; e.g. + imports in this file are deferred to being as late as possible. + - Inspired by similar code in :py:mod:`coverage.control` and :py:mod:`pytest_autoprofile.startup_hook`. """ -from typing import TYPE_CHECKING +import os -if TYPE_CHECKING: - from pathlib import Path # noqa: F401 - from .cache import LineProfilingCache # noqa: F401 - -__all__ = ('write_pth_hook', 'load_pth_hook') +__all__ = ('load_pth_hook',) INHERITED_PID_ENV_VARNAME = ( 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID' ) -def write_pth_hook(cache): # type: (LineProfilingCache) -> Path - """ - Write a .pth file which allows for setting up profiling in child - Python processes. - - Args: - cache (:py:class:`~.LineProfilingCache`): - Cache object - - Returns: - fpath (Path): - Path to the written .pth file - - Note: - - To be called in the main process. - - The ``cache`` is responsible for deleting the written .pth - file via the registered cleanup callback. - """ - import os - from sysconfig import get_path - from ..toml_config import ConfigSource - - if not os.path.exists(cache.filename): - cache.dump() - assert os.path.exists(cache.filename) - - pth_config = ( - ConfigSource.from_config(cache.config) - .get_subconfig('child_processes', 'pth_files') - .conf_dict - ) - fpath = cache.make_tempfile( - prefix=pth_config['prefix'], - suffix=pth_config['suffix'] + '.pth', - dir=get_path('purelib'), - # Get rid of the .pth file ASAP so as to be the least disruptive - priority=1, - ) - try: - pth_content = 'import {0}; {0}.load_pth_hook({1})'.format( - (lambda: None).__module__, cache.main_pid, - ) - fpath.write_text(pth_content) - except Exception: - fpath.unlink(missing_ok=True) - raise - - return fpath - - -def load_pth_hook(ppid): # type: (int) -> None +def load_pth_hook(ppid: int) -> None: """ Function imported and called by the written .pth file; to reduce overhead, we immediately return if ``ppid`` doesn't match :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID`. """ - from os import environ - try: - env_ppid = int(environ[INHERITED_PID_ENV_VARNAME]) + env_ppid = int(os.environ[INHERITED_PID_ENV_VARNAME]) except (KeyError, ValueError): return if env_ppid != ppid: @@ -100,8 +48,8 @@ def load_pth_hook(ppid): # type: (int) -> None # profiled Python process, so we can be more liberal with the # imports without worrying about overhead import warnings - from .._diagnostics import DEBUG, log - from .cache import LineProfilingCache # noqa: F811 + from line_profiler._diagnostics import DEBUG, log + from line_profiler._child_process_profiling.cache import LineProfilingCache # Note: .pth files may be double-loaded in a virtual environment # (see https://stackoverflow.com/questions/58807569), so work around @@ -115,8 +63,10 @@ def load_pth_hook(ppid): # type: (int) -> None except Exception as e: # nocover if DEBUG: msg = f'{type(e)}: {e}' - warnings.warn(msg) + # Write log befor issuing warning, in case the warning is + # promoted to an exception log.warning(msg) + warnings.warn(msg) load_pth_hook.called = True # type: ignore else: cache.patch(load_pth_hook, 'called', True) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index dcd55e92..ec182b12 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -35,11 +35,11 @@ import pytest import ubelt as ub +from _line_profiler_hooks import load_pth_hook from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) -from line_profiler._child_process_profiling.pth_hook import load_pth_hook from line_profiler._child_process_profiling.multiprocessing_patches import ( _Poller, _PATCHED_MARKER, ) @@ -1298,8 +1298,8 @@ def _run_test_module( expected = nnums * (nnums + 1) // 2 output_lines = proc.stdout.splitlines() ResultMismatch.compare(str(expected), output_lines[0]) - # - Temporary `.pth` file(s) created by `~~.pth_hook` has - # been cleaned up + # - Temporary `.pth` file(s) created by + # `LineProfilingCache.write_pth_hook()` has been cleaned up assert _preserve_pth_files.get_pth_files() == old_pth_files # - Profiling results are written to the specified file prof_result: LineStats | None = None @@ -1681,12 +1681,8 @@ def list_profiled_funcs() -> list[str]: _search_cache_logs(cache, debug, patterns) -@_preserve_attributes({ - **_GLOBAL_PATCHES, - 'line_profiler._child_process_profiling.pth_hook.load_pth_hook': - frozenset({'called'}), -}) @pytest.mark.parametrize('ppid_should_match', [True, False, None]) +@_preserve_attributes(_GLOBAL_PATCHES) def test_load_pth_hook( create_cache: Callable[..., LineProfilingCache], patched_attributes: MappingProxyType[str, frozenset[str]], @@ -1694,16 +1690,16 @@ def test_load_pth_hook( ppid_should_match: bool | None, ) -> None: """ - Simulate calling :py:func:`line_profiler._child_process_profiling\ -.pth_hook.load_pth_hook()` in a child process. + Simulate calling :py:func:`_line_profiler_hooks.load_pth_hook()` in + a child process. Notes: - The function is CALLED in the .pth file, but we don't actually NEED a .pth file to call and test it. - - The counterpart :py:func:`line_profiler\ -._child_process_profiling.pth_hook.write_pth_hook()` + - The counterpart :py:meth:`line_profiler\ +._child_process_profiling.cache.LineProfilingCache.write_pth_hook()` is implicitly tested in :py:func:`test_cache_setup_main_process()`. """ From b037c119a3b4a7c73a67194372889268a40a48cd Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 5 May 2026 01:47:33 +0200 Subject: [PATCH 086/165] Fixes for cache management line_profiler/_child_process_profiling/cache.py ::LineProfilingCache.gather_stats() Now only excluding profiling-stats files from the "auto-PIDs" if they are empty, to catch the corner case where a properly-profiled child process happens to share a PID with a non-contemporary process that we exclude (currently only the `multiprocessing.resoure_tracker.ResourceTracker` process) tests/test_child_procs.py::test_apply_mp_patches() Added call to `LineProfilingCache.cleanup()` before `.gather_stats()` and the inspection of the cache-dir contents to better replicate the `kernprof` workflow; in particular this makes sure the fork-server process is properly terminated and its profiling-stats file is written to --- line_profiler/_child_process_profiling/cache.py | 16 ++++++++++++++-- tests/test_child_procs.py | 1 + 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index f04ea971..0e1ced85 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -15,7 +15,7 @@ except ImportError: import pickle # type: ignore[assignment,no-redef] from collections.abc import ( - Collection, Callable, Mapping, MutableMapping, Iterable, + Collection, Callable, Iterable, Mapping, MutableMapping, ) from functools import partial, cached_property, wraps from importlib import import_module @@ -228,6 +228,10 @@ def gather_stats( Returns: :py:class:`LineStats` instance """ + def is_empty(path: Path) -> bool: + return not path.stat().st_size + + filter_excludes: Callable[[Iterable[Path]], Iterable[Path]] if exclude_pids is None: exclude_pids = set() for import_target, attr in _DEFAULT_GATHER_STATS_EXCLUDES: @@ -242,10 +246,18 @@ def gather_stats( if maybe_pid is None: continue exclude_pids.add(cast(int, maybe_pid)) + # NOTE: there is no guarantee that the PID hasn't previously + # been used for another child process that we DID properly + # profile and SHOULD include, so we only filter out empty + # files + filter_excludes = partial(filter, is_empty) + else: # User-provided values, who are we to object? + filter_excludes = iter fnames_ = set(self._get_profiling_outfiles()) for pid in exclude_pids: - fnames_.difference_update(self._get_profiling_outfiles(pid)) + excludes = filter_excludes(self._get_profiling_outfiles(pid)) + fnames_.difference_update(excludes) fnames = sorted(fnames_) self._debug_output( 'Loading results from {} child profiling file(s): {!r}' diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index ec182b12..b1aa90be 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1878,6 +1878,7 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: assert result == n * (n + 1) // 2 # Check that calls in children are traced + cache.cleanup() stats = cache.profiler.get_stats() stats += cache.gather_stats() entries = stats.timings[timing_key] From e1c4085aecece6e05cd707243f4876fcbb05d6ff Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 5 May 2026 01:50:37 +0200 Subject: [PATCH 087/165] Refactor `multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py _PollerArgs New helper class for getting `_Poller` args from `LineProfilingCache.config` _get_config() Instead of taking an argument from which a `ConfigSource` can be loaded, now taking the `ConfigSource` itself wrap_terminate() Refactored internals into separate functions wrap_bootstrap(), _apply_mp_patches() Updated calls to `_get_config()` --- .../multiprocessing_patches.py | 148 +++++++++--------- 1 file changed, 72 insertions(+), 76 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 26d01c49..80b94de8 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -2,14 +2,14 @@ Patch :py:mod:`multiprocessing` so that profiling extends into processes it creates. -Notes ------ -- Based on the implementations in :py:mod:`coverage.multiproc` and - :py:mod:`pytest_autoprofile._multiprocessing`. -- Results may vary if the process pool is not properly - :py:meth:`multiprocessing.pool.Pool.close`-d and - :py:meth:`multiprocessing.pool.Pool.join`-ed; - see `this caveat `__. """ from __future__ import annotations @@ -17,14 +17,14 @@ import multiprocessing import warnings from collections.abc import Callable, Mapping -from functools import lru_cache, partial +from functools import partial from importlib import import_module from multiprocessing.process import BaseProcess -from os import PathLike from time import sleep, monotonic from types import MappingProxyType from typing import ( - Any, Generic, Literal, Protocol, TypeVar, Union, NoReturn, cast, + Any, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, + cast, final, ) from typing_extensions import Concatenate, ParamSpec, Self @@ -206,14 +206,11 @@ def warn(msg: str) -> None: diagnostics.log.warning(msg) warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) - def ignore(_): - pass - timeout = self._timeout callback = self._func handle_timeout: Callable[[str], Any] = { - 'error': error, 'warn': warn, 'ignore': ignore, + 'error': error, 'warn': warn, 'ignore': _no_op, }[self._on_timeout] fmt = '.3g' timeout_msg_header = f'{type(self).__name__} at {id(self):#x}' @@ -247,27 +244,64 @@ class TimeoutWarning(Timeout, UserWarning): pass -def _get_config( - config: PathLike[str] | str | bool | None = None, -) -> Mapping[str, Any]: - if config not in (True, False, None): - config = str(config) - return _get_config_cached(cast(Union[str, bool, None], config)) +@final +class _PollerArgs(NamedTuple): + cooldown: float + timeout: float + on_timeout: str | None + @classmethod + def from_config(cls, config: ConfigSource) -> Self: + values = _get_config(config)['polling'] + try: + cooldown = max(float(values['cooldown']), 0) + except (TypeError, ValueError): + cooldown = 0 + try: + timeout = max(float(values['timeout']), 0) + except (TypeError, ValueError): + timeout = 0 + try: + on_timeout: str | None = values['on_timeout'].lower() + except Exception: # Fallback (use `_Poller`'s default) + on_timeout = None + return cls(cooldown, timeout, on_timeout) -@lru_cache() -def _get_config_cached( - config: PathLike[str] | str | bool | None = None, -) -> Mapping[str, Any]: + @classmethod + def get_defaults(cls) -> Self: + namespace = globals() + try: + return namespace['_DEFAULT_POLLER_ARGS'] + except KeyError: + defaults = cls.from_config(ConfigSource.from_default(copy=False)) + return namespace.setdefault('_DEFAULT_POLLER_ARGS', defaults) + + +def _get_config(config: ConfigSource) -> Mapping[str, Any]: cd = dict( - ConfigSource.from_config(config) - .get_subconfig('child_processes', 'multiprocessing', copy=True) + config.get_subconfig('child_processes', 'multiprocessing', copy=True) .conf_dict ) assert isinstance(cd.get('polling'), Mapping) return MappingProxyType({**cd, 'polling': MappingProxyType(cd['polling'])}) +def _process_has_returned( + proc: BaseProcess, cache: LineProfilingCache, timeout: float, +) -> bool: + popen = getattr(proc, '_popen', None) + if popen is None: + msg, result = 'No associated process', True + else: + result = popen.wait(timeout) is not None + if result: + msg = f'Process {popen.pid} has returned' + else: + msg = f'Waiting for process {popen.pid} to return...' + cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') + return result + + @LineProfilingCache._method_wrapper def wrap_terminate( cache: LineProfilingCache, @@ -285,54 +319,15 @@ def wrap_terminate( on the bad path (e.g. the parallel workload errored out), and after the performance-critical part of the code (said workload). """ - # XXX: why can `coverage` get away with not doing all these - # lock-file hijinks and just patching `BaseProcess._bootstrap()`? - def get_poller_args( - config: PathLike[str] | str | bool | None = None, - ) -> tuple[float, float, str | None]: - values = _get_config(config)['polling'] - try: - cooldown = max(float(values['cooldown']), 0) - except (TypeError, ValueError): - cooldown = 0 - try: - timeout = max(float(values['timeout']), 0) - except (TypeError, ValueError): - timeout = 0 - try: - on_timeout: str | None = values['on_timeout'].lower() - except Exception: # Fallback (use `_Poller`'s default) - on_timeout = None - return cooldown, timeout, on_timeout - - def process_has_returned(proc: BaseProcess, timeout: float) -> bool: - popen = getattr(proc, '_popen', None) - if popen is None: - msg, result = 'No associated process', True - else: - result = popen.wait(timeout) is not None - if result: - msg = f'Process {popen.pid} has returned' - else: - msg = f'Waiting for process {popen.pid} to return...' - cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') - return result - - def wait_for_return( - config: PathLike[str] | str | None = None, - ) -> _Poller: - cooldown, timeout, on_timeout = get_poller_args(config) - # `False` -> no resolution, force loading the vanilla file - *_, default_on_timeout = get_poller_args(False) - if on_timeout not in ('ignore', 'warn', 'error'): - on_timeout = default_on_timeout - return ( - _Poller.poll_until(process_has_returned, self, cooldown) - .with_timeout(timeout, cast(_OnTimeout, on_timeout)) - ) - try: - with wait_for_return(cache.config): + cd, timeout, on_timeout = _PollerArgs.from_config(cache._config_source) + if on_timeout not in ('ignore', 'warn', 'error'): + on_timeout = _PollerArgs.get_defaults().on_timeout + # `_process_has_returned()` takes a `timeout` which it passes to + # `popen.wait()`; said timeout is essentially a limit as to how + # often the function is called, hence our cooldown + poller = _Poller.poll_until(_process_has_returned, self, cache, cd) + with poller.with_timeout(timeout, cast(_OnTimeout, on_timeout)): pass except _Poller.Timeout as e: # Also handles `~.TimeoutWarning` cache._debug_output(f'{type(e).__qualname__}: {e}') @@ -369,7 +364,7 @@ def wrap_bootstrap( """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up - if _get_config(cache.config)['catch_sigterm']: + if _get_config(cache._config_source)['catch_sigterm']: cache._add_signal_handler() try: return vanilla_impl(self, *args, **kwargs) @@ -523,7 +518,8 @@ def _apply_mp_patches( lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) # Intercept `multiprocessing` debug messages if intercept_mp_logs is None: - intercept_mp_logs = _get_config(lp_cache.config)['intercept_logs'] + config = lp_cache._config_source + intercept_mp_logs = _get_config(config)['intercept_logs'] if intercept_mp_logs: lfuncs = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] lpatches = {func: partial(partial, tee_log, func) for func in lfuncs} From 0263b0ec618f4747a29ef8d9bf2cf37786278c75 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 5 May 2026 05:29:14 +0200 Subject: [PATCH 088/165] WIP: refactor flaky tests tests/test_child_procs.py _Params.drop_params(), .split_on_params() New helper methods for constructing instances based on existing ones _check_warnings New context manager for helping with checking the absence and presence of warnings at exit time _run_test_module() (This function underlies these tests: - `test_multiproc_script_sanity_check()` - `test_running_multiproc_script` - `test_profiling_multiproc_script_{success,failure}()`) Added protection against stray .pth file created when running `kernprof` test_cache_setup_child() Added check that warnings for empty profiling files are emitted iff expected test_apply_mp_patches_{success,failure}() - Split parametrized test into separate tests, and only allow retries on the known flaky cases - Added check that no stray warnings are emitted test_profiling_multiproc_script_{success,failure}() test_profiling_bare_python_{success,failure}() Split parametrized tests into separate tests, and only allow retries on the known flaky cases --- tests/test_child_procs.py | 791 +++++++++++++++++++++++++++++++++----- 1 file changed, 690 insertions(+), 101 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index b1aa90be..5092b220 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -11,9 +11,10 @@ import subprocess import sys import sysconfig +import warnings from abc import ABC, abstractmethod from collections.abc import ( - Callable, Collection, Generator, Iterable, Mapping, Sequence, + Callable, Collection, Generator, Iterable, Iterator, Mapping, Sequence, ) from contextlib import AbstractContextManager, ExitStack, nullcontext from functools import partial, wraps @@ -27,7 +28,8 @@ from time import monotonic from types import MappingProxyType, ModuleType, TracebackType from typing import ( - TYPE_CHECKING, Any, Generic, Literal, TypeVar, cast, final, overload, + TYPE_CHECKING, Any, Generic, Literal, Protocol, TypeVar, + cast, final, overload, ) from typing_extensions import Self, ParamSpec from uuid import uuid4 @@ -36,6 +38,7 @@ import ubelt as ub from _line_profiler_hooks import load_pth_hook +from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, @@ -525,9 +528,10 @@ def __init__( actual: Any | _NotSupplied = _NotSupplied.NOT_SUPPLIED, _trunc_tb: int = 0, ) -> None: - msg = f'expected: {expected}' - if actual != _NotSupplied.NOT_SUPPLIED: - msg = f'{msg}, got {actual}' + if actual == _NotSupplied.NOT_SUPPLIED: + msg = f'expected: {expected}' + else: + msg = f'expected {expected}, got {actual}' super().__init__(msg) self.expected = expected self.actual = actual @@ -738,6 +742,119 @@ def sort_key(obj: Any) -> tuple[bool, str, Any]: new_values = self.values.copy() return type(self)(self.params, new_values, self.defaults) + def drop_params(self, params: Collection[str] | str) -> Self: + """ + Return a new instance with the named ``params`` dropped; params + that don't match :py:attr:`.params` are ignored. + + Example: + >>> p = _Params.new(('a', 'b'), [(1, 2), (3, 4)]) + >>> p.drop_params('a') + _Params(params=('b',), values=[(2,), (4,)], defaults=(2,)) + >>> assert p.drop_params(['c', 'd']) == p + """ + def drop(t: tuple[T, ...]) -> tuple[T, ...]: + return tuple(item for i, item in enumerate(t) if i not in dropped) + + if isinstance(params, str): + params = params, + dropped = {i for i, p in enumerate(self.params) if p in params} + return type(self)( + drop(self.params), + [drop(pvalues) for pvalues in self.values], + drop(self.defaults), + ) + + @overload + def split_on_params( + self, params: tuple[str, ...], *, drop_split_params: bool = True, + ) -> dict[tuple[Any, ...], Self]: + ... + + @overload + def split_on_params( + self, params: str, *, drop_split_params: bool = True, + ) -> dict[Any, Self]: + ... + + def split_on_params( + self, params: tuple[str, ...] | str, *, drop_split_params: bool = True, + ) -> dict[tuple[Any, ...], Self] | dict[Any, Self]: + """ + Return new instances splitting on the values of the named + ``params``; params that don't match :py:attr:`.params` results + in an error. + + Example: + >>> p = _Params.new(('a', 'b', 'c'), + ... [(1, 2, True), + ... (1, 2, False), + ... (3, 4, True)]) + + >>> p.split_on_params('a') # doctest: +NORMALIZE_WHITESPACE + {1: _Params(params=('b', 'c'), + values=[(2, True), (2, False)], + defaults=(2, True)), + 3: _Params(params=('b', 'c'), + values=[(4, True)], + defaults=(2, True))} + + >>> p.split_on_params( # doctest: +NORMALIZE_WHITESPACE + ... ('a', 'b'), + ... ) + {(1, 2): _Params(params=('c',), + values=[(True,), (False,)], + defaults=(True,)), + (3, 4): _Params(params=('c',), + values=[(True,)], + defaults=(True,))} + + >>> p.split_on_params( # doctest: +NORMALIZE_WHITESPACE + ... 'a', drop_split_params=False, + ... ) + {1: _Params(params=('a', 'b', 'c'), + values=[(1, 2, True), (1, 2, False)], + defaults=(1, 2, True)), + 3: _Params(params=('a', 'b', 'c'), + values=[(3, 4, True)], + defaults=(1, 2, True))} + + >>> p.split_on_params( # doctest: +NORMALIZE_WHITESPACE + ... ('c', 'd'), + ... ) + Traceback (most recent call last): + ... + ValueError: params = ('c', 'd'): + these params not found: ['d'] + """ + if isinstance(params, str): + params = params, + unpack = True + else: + unpack = False + nonexistent = sorted(set(params) - set(self.params)) + if nonexistent: + raise ValueError( + f'params = {params!r}: these params not found: {nonexistent!r}' + ) + split_params: dict[tuple[Any, ...], list[tuple[Any, ...]]] = {} + indices = tuple(self.params.index(p) for i, p in enumerate(params)) + for pvalues in self.values: + key = tuple(pvalues[i] for i in indices) + split_params.setdefault(key, []).append(pvalues) + new = partial(type(self), params=self.params, defaults=self.defaults) + instances: dict[tuple[Any, ...], Self] = { + key: new(values=values) for key, values in split_params.items() + } + if drop_split_params: + instances = { + key: instance.drop_params(params) + for key, instance in instances.items() + } + if not unpack: + return instances + return {key[0]: instance for key, instance in instances.items()} + def __call__(self, func: C) -> C: """ Mark a callable as with :py:func:`pytest.mark.parametrize`. @@ -1050,6 +1167,278 @@ def _get_path() -> Path: return Path(sysconfig.get_path('purelib')) +class _WarningInfo(Protocol): + @property + def message(self) -> str | Warning: + ... + + @property + def category(self) -> type[Warning]: + ... + + @property + def filename(self) -> str: + ... + + @property + def lineno(self) -> int: + ... + + @property + def line(self) -> str | None: + ... + + +@dataclasses.dataclass +class _WarningMatcher: + message: str | None = None + category: type[Warning] | None = None + module: str | None = None + lineno: int | None = None + _filters: dict[str, Callable[[Any], Any]] = dataclasses.field( + repr=False, init=False, default_factory=dict, + ) + + def __post_init__(self) -> None: + if self.message is not None: + self._filters['message'] = partial( + self._check_message, re.compile(self.message), + ) + if self.category is not None: + self._filters['category'] = partial( + self._check_category, self.category, + ) + if self.module is not None: + self._filters['filename'] = partial( + self._check_module, re.compile(self.module), + ) + if self.lineno is not None: + self._filters['lineno'] = partial(operator.eq, self.lineno) + + def __repr__(self) -> str: + fields: dict[str, Any] = { + field.name: getattr(self, field.name, None) + for field in dataclasses.fields(self) + if field.repr + } + return '{}({})'.format( + type(self).__name__, + ', '.join( + f'{k}={v!r}' for k, v in fields.items() if v is not None + ), + ) + + def match(self, info: _WarningInfo) -> bool: + for field, check in self._filters.items(): + if not check(getattr(info, field)): + return False + return True + + @staticmethod + def _check_message( + msg_regex: re.Pattern, msg: str | Warning, + ) -> re.Match | None: + if not isinstance(msg, str): + msg = str(msg) + return msg_regex.match(msg) + + @staticmethod + def _check_category(parent: type[Any], maybe_child: type[Any]) -> bool: + try: + return issubclass(maybe_child, parent) + except Exception: + return False + + @staticmethod + def _check_module( + module_regex: re.Pattern, filename: str, + ) -> re.Match | None: + module = modpath_to_modname(filename, hide_main=False, hide_init=False) + return module_regex.match(module) + + +@dataclasses.dataclass +class _WarningContext: + catch_warnings: warnings.catch_warnings = dataclasses.field( + default_factory=partial(warnings.catch_warnings, record=True) + ) + checks: list[tuple[_WarningMatcher, bool]] = dataclasses.field( + default_factory=list, + ) + + def forbid_warnings( + self, + message: str | None = None, + category: type[Warning] | None = Warning, + module: str | None = None, + lineno: int | None = None, + ) -> None: + matcher = _WarningMatcher( + message=message, category=category, module=module, lineno=lineno, + ) + self.checks.append((matcher, False)) + + def expect_warnings( + self, + message: str | None = None, + category: type[Warning] | None = Warning, + module: str | None = None, + lineno: int | None = None, + ) -> None: + matcher = _WarningMatcher( + message=message, category=category, module=module, lineno=lineno, + ) + self.checks.append((matcher, True)) + + def check(self, warnings: Sequence[_WarningInfo]) -> None: + for matcher, allowed_or_required in self.checks: + matches = [info for info in warnings if matcher.match(info)] + if matches and not allowed_or_required: + raise ResultMismatch( + expected=f'no warnings matching {matcher!r}', + actual=f'{len(matches)} ({matches!r})', + ) + if not matches and allowed_or_required: + raise ResultMismatch( + expected=f'warnings matching {matcher!r}', + actual=f'none out of {len(warnings)} ({warnings!r})', + ) + + @classmethod + def new(cls, **kwargs) -> Self: + kwargs['record'] = True + return cls(warnings.catch_warnings(**kwargs)) + + +class _check_warnings(Sequence[_WarningInfo]): + """ + Helper context for deferring the checking of warnings to until + context exit. + + Example: + >>> import warnings + + >>> cw = _check_warnings() + + >>> with cw: # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... cw.forbid_warnings('foo', UserWarning) + ... warnings.warn('foobar') + ... print('This is printed before the error') + ... + This is printed before the error + Traceback (most recent call last): + ... + test_child_procs.ResultMismatch: expected no warnings matching + _WarningMatcher(message='foo', + category=), + got 1 ([...]) + + >>> with cw: # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... cw.expect_warnings(category=UserWarning) + ... warnings.warn('foobar', Warning) + ... print('This is printed before the error') + ... + This is printed before the error + Traceback (most recent call last): + ... + test_child_procs.ResultMismatch: expected warnings matching + _WarningMatcher(category=), + got none out of 1 ([...]) + >>> assert len(cw) == 1 + >>> assert str(cw[0].message) == 'foobar' + """ + def __init__(self, **kwargs) -> None: + self._new_context: Callable[[], _WarningContext] = partial( + _WarningContext.new, **kwargs, + ) + self._contexts: list[ + tuple[_WarningContext, Sequence[_WarningInfo]] + ] = [] + self._last_captured: Sequence[_WarningInfo] = [] + + def forbid_warnings(self, *args, **kwargs) -> None: + """ + Equivalent to calling + ``filterwarnings('error', *args, **kwargs)``; + at context exit, if ANY matching warning has been issued, an + error will be raised. + """ + ctx, _ = self._current_context + ctx.forbid_warnings(*args, **kwargs) + + def expect_warnings(self, *args, **kwargs) -> None: + """ + Equivalent to calling + ``filterwarnings('always', *args, **kwargs)``; + at context exit, if NO matching warnings have been issued, an + error will be raised. + """ + ctx, _ = self._current_context + ctx.expect_warnings(*args, **kwargs) + + def __enter__(self) -> Self: + ctx = self._new_context() + infos: Sequence[_WarningInfo] | None + infos = ctx.catch_warnings.__enter__() + assert infos is not None + self._contexts.append((ctx, infos)) + return self + + def __exit__(self, *args, **kwargs) -> None: + ctx, infos = self._contexts.pop() + try: + ctx.check(infos) + finally: + self._last_captured = infos + ctx.catch_warnings.__exit__(*args, **kwargs) + + @overload + def __getitem__(self, i: int, /) -> _WarningInfo: + ... + + @overload + def __getitem__(self, i: slice, /) -> list[_WarningInfo]: + ... + + def __getitem__( + self, i: int | slice, /, + ) -> _WarningInfo | Sequence[_WarningInfo]: + return self._current_warnings[i] + + def __len__(self) -> int: + return len(self._current_warnings) + + def __iter__(self) -> Iterator[_WarningInfo]: + return iter(self._current_warnings) + + def __reversed__(self) -> Iterator[_WarningInfo]: + return iter(reversed(self._current_warnings)) + + def __contains__(self, item: Any, /) -> bool: + return item in self._current_warnings + + def index(self, *args, **kwargs) -> int: + return self._current_warnings.index(*args, **kwargs) + + def count(self, *args, **kwargs) -> int: + return self._current_warnings.count(*args, **kwargs) + + @property + def _current_context( + self, + ) -> tuple[_WarningContext, Sequence[_WarningInfo]]: + return self._contexts[-1] + + @property + def _current_warnings(self) -> Sequence[_WarningInfo]: + try: + return self._current_context[1] + except IndexError: + # Outside of contexts, just provide the last captured values + # for convenience + return self._last_captured + + def _import_target(target: str) -> Any: try: return import_module(target) @@ -1215,6 +1604,7 @@ def _run_subproc( ) +@_preserve_pth_files() def _run_test_module( run_helper: Callable[..., subprocess.CompletedProcess], test_module: _ModuleFixture, @@ -1361,7 +1751,7 @@ def _check_output(output: str, tag: str, nhits: int) -> None: }), 'multiprocessing': frozenset({_PATCHED_MARKER}), 'multiprocessing.process.BaseProcess': frozenset({ - '_bootstrap', 'terminate', + '_bootstrap', # 'terminate', }), 'multiprocessing.spawn': frozenset({'runpy'}), 'multiprocessing.util': frozenset({ @@ -1649,15 +2039,23 @@ def list_profiled_funcs() -> list[str]: # Check that on cache cleanup: # - Profiling data is collected # - `os.fork()` is restored + # - The warning for empty profiling files is only issued when + # expected assert func(range(1, n + 1)) == n * (n + 1) // 2 stats = cache.profiler.get_stats() - for callback, has_prof_data, fork_patched in [ - (lambda: None, False, wrap_os_fork), - (cache.cleanup, preimports, False), + for callback, has_nonempty_file, has_stats, fork_patched in [ + (lambda: None, False, False, wrap_os_fork), + (cache.cleanup, True, preimports, False), ]: callback() - gathered = cache.gather_stats() - assert any(gathered.timings.values()) == has_prof_data, gathered + with _check_warnings() as cw: + if has_nonempty_file: + check_warning = cw.forbid_warnings + else: + check_warning = cw.expect_warnings + check_warning(r'.* file\(s\) .* empty', module='line_profiler') + gathered = cache.gather_stats() + assert any(gathered.timings.values()) == has_stats, gathered if hasattr(os, 'fork'): assert (os.fork is not old_fork) == fork_patched else: # E.g. Windows @@ -1757,22 +2155,9 @@ def test_load_pth_hook( compare(patched) -@pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) -@(_Params.new('start_method', - ['fork', 'forkserver', 'spawn', 'dummy'], - defaults='dummy') - * _Params.new(('fail', 'label2'), - [(True, 'failure'), (False, 'success')], - defaults=(False, 'success')) - # We only need to check if `intercept_logs` work, the other - # parametrizations don't matter - + _Params.new(('intercept_logs', 'label1'), - [(True, 'with-intercept-logs'), (False, 'no-intercept-logs')], - defaults=(False, 'no-intercept-logs'))).sorted() -@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) -def test_apply_mp_patches( +def _test_apply_mp_patches( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, @@ -1782,13 +2167,7 @@ def test_apply_mp_patches( fail: bool, n: int, nprocs: int, - label1: str, label2: str, ) -> None: - """ - Test that :py:func:`line_profiler._child_process_profiling\ -.multiprocessing_patches.apply` - works as expected. - """ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: try: LineStats.from_files(path, on_defective='error') @@ -1896,21 +2275,108 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: _search_cache_logs(cache, True, patterns) +@(_Params.new('start_method', + ['fork', 'forkserver', 'spawn', 'dummy'], + defaults='dummy') + # We only need to check if `intercept_logs` work, the other + # parametrizations don't matter + + _Params.new(('intercept_logs', 'label'), + [(True, 'with-intercept-logs'), (False, 'no-intercept-logs')], + defaults=(False, 'no-intercept-logs'))).sorted() +@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) +def test_apply_mp_patches_success( + tmp_path_factory: pytest.TempPathFactory, + create_cache: Callable[..., LineProfilingCache], + ext_module_object: ModuleType, + test_module_object: ModuleType, + start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], + intercept_logs: bool, + n: int, + nprocs: int, + label: str, +) -> None: + """ + Test that :py:func:`line_profiler._child_process_profiling\ +.multiprocessing_patches.apply` + works as expected when the parallel workload does not error out. + + See also: + :py:func:`test_apply_mp_patches_failure` + """ + with _check_warnings() as cw: + cw.forbid_warnings(category=UserWarning, module='line_profiler') + cw.forbid_warnings(module='multiprocessing') + _test_apply_mp_patches( + tmp_path_factory=tmp_path_factory, + create_cache=create_cache, + ext_module_object=ext_module_object, + test_module_object=test_module_object, + start_method=start_method, + intercept_logs=intercept_logs, + fail=False, + n=n, + nprocs=nprocs, + ) + + +@pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) +@pytest.mark.parametrize('start_method', + ['fork', 'forkserver', 'spawn', 'dummy']) +@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) +def test_apply_mp_patches_failure( + tmp_path_factory: pytest.TempPathFactory, + create_cache: Callable[..., LineProfilingCache], + ext_module_object: ModuleType, + test_module_object: ModuleType, + start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], + n: int, + nprocs: int, +) -> None: + """ + Test that :py:func:`line_profiler._child_process_profiling\ +.multiprocessing_patches.apply` + works as expected when the parallel workload errors out. + + See also: + :py:func:`test_apply_mp_patches_success` + """ + with _check_warnings() as cw: + cw.forbid_warnings(category=UserWarning, module='line_profiler') + cw.forbid_warnings(module='multiprocessing') + _test_apply_mp_patches( + tmp_path_factory=tmp_path_factory, + create_cache=create_cache, + ext_module_object=ext_module_object, + test_module_object=test_module_object, + start_method=start_method, + intercept_logs=False, + fail=True, + n=n, + nprocs=nprocs, + ) + + # XXX: End of tests for implementation details # ========================= Integration tests ========================== -def _get_mp_start_method_fuzzer(label_name: str) -> _Params: +def _get_mp_start_method_fuzzer(label_name: str | None) -> _Params: """ Returns: :py:class:`_Params` object which does a full Cartesian-product fuzz between ``fail`` (true or false) and ``start_method`` ('fork', 'forkserver', and 'spawn'; default :py:const:`None`) """ + if label_name is None: + label_name, drop_label = '_', True + else: + drop_label = False fuzz_fail = _Params.new(('fail', label_name), [(True, 'failure'), (False, 'success')], defaults=(False, 'success')) + if drop_label: + fuzz_fail = fuzz_fail.drop_params(label_name) fuzz_start = _Params.new('start_method', ['fork', 'forkserver', 'spawn'], defaults=None) return fuzz_fail * fuzz_start @@ -2003,17 +2469,15 @@ def test_running_multiproc_script( (run_script, 'script'), (run_literal_code, 'literal-code')], defaults=(run_script, 'script')) - - -@( +_fuzz_prof_mp_markers = ( (_fuzz_prof_mp_run_func + _Params.new(('prof_child_procs', 'label2'), [(True, 'with-child-prof'), (False, 'no-child-prof')]) - + _get_mp_start_method_fuzzer('label3')) + + _get_mp_start_method_fuzzer(None)) # Test all `multiproc` start methods with both locally- and # externally-defined profiling targets - * (_Params.new(('preimports', 'label4'), [(False, 'no-preimports')]) - + _Params.new(('use_local_func', 'label5'), + * (_Params.new(('preimports', 'label3'), [(False, 'no-preimports')]) + + _Params.new(('use_local_func', 'label4'), [(True, 'local'), (False, 'external')], defaults=(False, 'external'))) # The 'with-preimports' case is already tested rather thoroughly in @@ -2021,16 +2485,77 @@ def test_running_multiproc_script( # param matrix and just test the different `kernprof` modes via the # `run_func()`s + (_fuzz_prof_mp_run_func - + _Params.new(('preimports', 'label4'), [(True, 'with-preimports')])) -).sorted() -@pytest.mark.retry(_NUM_RETRIES, - exceptions=(ResultMismatch, subprocess.TimeoutExpired)) + + _Params.new(('preimports', 'label3'), [(True, 'with-preimports')])) +).sorted().split_on_params('fail') + + +def _test_profiling_multiproc_script( + run_func: Callable[..., subprocess.CompletedProcess], + test_module: _ModuleFixture, + ext_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + prof_child_procs: bool, + preimports: bool, + use_local_func: bool, + fail: bool, + start_method: Literal['fork', 'forkserver', 'spawn'] | None, + nnums: int, + nprocs: int, +) -> None: + # How many calls do we expect? + nhits = dict.fromkeys( + ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, + ) + # Make sure we're profiling the right function + tag = 'LOCAL' if use_local_func else 'EXT' + tag_call = tag + '-INVOCATION' + tag_loop = tag + '-LOOP' + if not fail: + # The final sum in the parent process should always be profiled + # unless the child processes failed and we never returned from + # `Pool.starmap()` + nhits[tag_call] += 1 + nhits[tag_loop] += nprocs + if prof_child_procs: + # When profiling extends into child processes, each of them + # invokes the sum function once and when combined they loop thru + # all the items + nhits[tag_call] += nprocs + nhits[tag_loop] += nnums + + runner = ['kernprof', '-l'] + runner.extend([ + '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), + '--{}preimports'.format('' if preimports else 'no-'), + ]) + if not use_local_func: + # Also make sure to include the external module in `--prof-mod` + runner.append(f'--prof-mod={ext_module.name}') + run_func( + test_module, tmp_path_factory, + runner=runner, + outfile='out.lprof', + profile=True, + use_local_func=use_local_func, + fail=fail, + start_method=start_method, + nhits=nhits, + nnums=nnums, + nprocs=nprocs, + timeout=_SUBPROC_TIMEOUT, + debug_log=( + 'debug.log' if prof_child_procs and _DEBUG else None + ), + ) + + +@(_fuzz_prof_mp_markers[False]) @pytest.mark.parametrize( # XXX: should we explicitly test the single-proc case? We already # have quite a lot of subtests tho... ('nnums', 'nprocs'), [(2000, 3)], ) -def test_profiling_multiproc_script( +def test_profiling_multiproc_script_success( run_func: Callable[..., subprocess.CompletedProcess], test_module: _ModuleFixture, ext_module: _ModuleFixture, @@ -2038,16 +2563,16 @@ def test_profiling_multiproc_script( prof_child_procs: bool, preimports: bool, use_local_func: bool, - fail: bool, start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int, nprocs: int, # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, label3: str, label4: str, label5: str, + label1: str, label2: str, label3: str, label4: str, ) -> None: """ Check that `kernprof` can PROFILE the test module in various - contexts, optionally extending profiling into child processes. + contexts when the parallel workload runs without errors, optionally + extending profiling into child processes. Note: This test function is heavily parametrized. Here is why that is @@ -2085,78 +2610,83 @@ def test_profiling_multiproc_script( - ``prof_child_procs`` of course toggles whether to do the patches to set up profiling in child processes. - """ - # How many calls do we expect? - nhits = dict.fromkeys( - ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, - ) - # Make sure we're profiling the right function - tag = 'LOCAL' if use_local_func else 'EXT' - tag_call = tag + '-INVOCATION' - tag_loop = tag + '-LOOP' - if not fail: - # The final sum in the parent process should always be profiled - # unless the child processes failed and we never returned from - # `Pool.starmap()` - nhits[tag_call] += 1 - nhits[tag_loop] += nprocs - if prof_child_procs: - # When profiling extends into child processes, each of them - # invokes the sum function once and when combined they loop thru - # all the items - nhits[tag_call] += nprocs - nhits[tag_loop] += nnums - runner = ['kernprof', '-l'] - runner.extend([ - '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), - '--{}preimports'.format('' if preimports else 'no-'), - ]) - if not use_local_func: - # Also make sure to include the external module in `--prof-mod` - runner.append(f'--prof-mod={ext_module.name}') - run_func( - test_module, tmp_path_factory, - runner=runner, - outfile='out.lprof', - profile=True, + See also: + :py:func:`test_profiling_multiproc_script_failure` + """ + _test_profiling_multiproc_script( + run_func=run_func, + test_module=test_module, + ext_module=ext_module, + tmp_path_factory=tmp_path_factory, + prof_child_procs=prof_child_procs, + preimports=preimports, use_local_func=use_local_func, - fail=fail, + fail=False, start_method=start_method, - nhits=nhits, nnums=nnums, nprocs=nprocs, - timeout=_SUBPROC_TIMEOUT, - debug_log=( - 'debug.log' if prof_child_procs and _DEBUG else None - ), ) @pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, subprocess.TimeoutExpired)) -@pytest.mark.parametrize(('use_subprocess', 'label1'), - [(True, 'subprocess.run'), (False, 'os.system')]) -@pytest.mark.parametrize(('prof_child_procs', 'label2'), - [(True, 'with-child-prof'), (False, 'no-child-prof')]) -@pytest.mark.parametrize(('fail', 'label3'), - [(True, 'failure'), (False, 'success')]) -@pytest.mark.parametrize('n', [200]) -def test_profiling_bare_python( +@(_fuzz_prof_mp_markers[True]) +@pytest.mark.parametrize(('nnums', 'nprocs'), [(2000, 3)]) +def test_profiling_multiproc_script_failure( + run_func: Callable[..., subprocess.CompletedProcess], + test_module: _ModuleFixture, + ext_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + prof_child_procs: bool, + preimports: bool, + use_local_func: bool, + start_method: Literal['fork', 'forkserver', 'spawn'] | None, + nnums: int, + nprocs: int, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, label3: str, label4: str, +) -> None: + """ + Check that `kernprof` can PROFILE the test module in various + contexts when the parallel workload errors out, optionally + extending profiling into child processes. + + See also: + :py:func:`test_profiling_multiproc_script_success` + """ + _test_profiling_multiproc_script( + run_func=run_func, + test_module=test_module, + ext_module=ext_module, + tmp_path_factory=tmp_path_factory, + prof_child_procs=prof_child_procs, + preimports=preimports, + use_local_func=use_local_func, + fail=True, + start_method=start_method, + nnums=nnums, + nprocs=nprocs, + ) + + +_fuzz_bare = ( + _Params.new(('use_subprocess', 'label1'), + [(True, 'subprocess.run'), (False, 'os.system')]) + * _Params.new(('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')]) + * _Params.new('n', [200]) +) + + +def _test_profiling_bare_python( tmp_path_factory: pytest.TempPathFactory, ext_module: _ModuleFixture, use_subprocess: bool, prof_child_procs: bool, fail: bool, n: int, - # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, label3: str, ) -> None: - """ - Check that `kernprof` can profile the target functions if the code - invokes another bare Python process (via either :py:func:`os.system` - or :py:func:`subprocess.run`) that calls them. - """ ext_module.install(children=True) temp_dir = tmp_path_factory.mktemp('mytemp') @@ -2226,3 +2756,62 @@ def test_profiling_bare_python( end='', file=sys.stderr, ) print('-- End of debug logs --', file=sys.stderr) + + +@_fuzz_bare +def test_profiling_bare_python_success( + tmp_path_factory: pytest.TempPathFactory, + ext_module: _ModuleFixture, + use_subprocess: bool, + prof_child_procs: bool, + n: int, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, +) -> None: + """ + Check that `kernprof` can profile the target functions if the code + invokes another bare Python process (via either :py:func:`os.system` + or :py:func:`subprocess.run`) that calls them and exits without + errors. + + See also: + :py:func:`test_profiling_bare_python_failure` + """ + _test_profiling_bare_python( + tmp_path_factory=tmp_path_factory, + ext_module=ext_module, + use_subprocess=use_subprocess, + prof_child_procs=prof_child_procs, + fail=False, + n=n, + ) + + +@pytest.mark.retry(_NUM_RETRIES, + exceptions=(ResultMismatch, subprocess.TimeoutExpired)) +@_fuzz_bare +def test_profiling_bare_python_failure( + tmp_path_factory: pytest.TempPathFactory, + ext_module: _ModuleFixture, + use_subprocess: bool, + prof_child_procs: bool, + n: int, + label1: str, + label2: str, +) -> None: + """ + Check that `kernprof` can profile the target functions if the code + invokes another bare Python process (via either :py:func:`os.system` + or :py:func:`subprocess.run`) that calls them and exits with errors. + + See also: + :py:func:`test_profiling_bare_python_success` + """ + _test_profiling_bare_python( + tmp_path_factory=tmp_path_factory, + ext_module=ext_module, + use_subprocess=use_subprocess, + prof_child_procs=prof_child_procs, + fail=True, + n=n, + ) From 03640712e676a628daeda302dfa510d617341b24 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 6 May 2026 04:26:44 +0200 Subject: [PATCH 089/165] Fixup signal handling and child bootstrapping line_profiler/_child_process_profiling cache.py::LineProfilingCache._handle_signal() Added more output and handling for the outcome of the `.cleanup()` call multiprocessing_patches.py::wrap_bootstrap() Now calling cleanup with the `new_thread=True` argument to prevent deadlocks when SIGTERM arrives at an inopportune time -- fellow child threads seem to play well together with the lock, but not the main thread and a child thread line_profiler curated_profiling.py::ClassifiedPreimportTargets.write_preimport_module() line_profiler.py::LineStats.from_files() Fixed calls to `warnings.warn()` so that the warnings are attributed to more informative frames --- line_profiler/_child_process_profiling/cache.py | 12 +++++++++++- .../multiprocessing_patches.py | 6 +++++- line_profiler/curated_profiling.py | 3 ++- line_profiler/line_profiler.py | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 0e1ced85..e814edb4 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -555,8 +555,9 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover :py:meth:`coverage.control.Converage._on_sigterm` """ name = self._get_signal_name(signum) - msg = f'Cleaning up before passing `{name}` ({signum}) on...' + msg = f'Caught `{name}` ({signum}), cleaning up...' self._debug_output(msg) + state = 'succeeded' try: # We don't care about profiler state and such if we're # already being SIGTERM-ed, so just handle cleanup on a @@ -565,8 +566,17 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover # already inside one (e.g. when `Process._bootstrap()` is # exiting, or the `atexit` hook is triggered) self.cleanup(new_thread=True) + except BaseException as e: + xc = f'{type(e).__name__}' + msg = str(e) + if msg: + xc = f'{xc}: {msg}' + state = f'failed ({xc})' + raise e finally: handler = self._sighandlers.pop(signum, None) + msg = f'Cleanup {state}, passing `{name}` onto {handler!r}...' + self._debug_output(msg) if handler is not None: signal.signal(signum, handler) signal.raise_signal(signum) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 80b94de8..52ddbd78 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -371,7 +371,11 @@ def wrap_bootstrap( finally: msg = 'Calling cleanup hook via `BaseProcess._bootstrap`' cache._debug_output(msg) - cache.cleanup() + # Execute cleanup in a separate thread so as to avoid deadlocks, + # in case when `LineProfilingCache._handle_signal()` caught a + # signal as we're in the middle of this and initiated another + # `.cleanup()` call + cache.cleanup(new_thread=True) def _cache_hook( diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index 80513daa..a120824f 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -71,8 +71,9 @@ def write_preimport_module( invalid_targets, ) ) - warnings.warn(msg) + # Log before warn in case the warning is raised diagnostics.log.warning(msg) + warnings.warn(msg, stacklevel=2) if not self: return None diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 98845f3a..539f8703 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -440,7 +440,7 @@ def from_files( # Log before warning because warnings may be promoted to # errors diagnostics.log.warning(msg) - warnings.warn(msg) + warnings.warn(msg, stacklevel=2) else: # 'ignore' diagnostics.log.debug(msg) From e69894a1e06e1cbd42c41601dccb473304851b0f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 6 May 2026 06:09:11 +0200 Subject: [PATCH 090/165] Fix stats file detection in `test_apply_mp_patches_*` --- tests/test_child_procs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 5092b220..126c83d5 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -2170,7 +2170,7 @@ def _test_apply_mp_patches( ) -> None: def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: try: - LineStats.from_files(path, on_defective='error') + LineStats.from_files(path, on_empty='error', on_defective='error') except Exception: return False return True From 7943588dc82834fbee173a810279f57bdf7266ab Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 6 May 2026 08:58:15 +0200 Subject: [PATCH 091/165] Moved code for formatting calls line_profiler/line_profiler_utils.py::CallbackRepr Migrated from `line_profiler/cleanup.py::_CallbackRepr` --- line_profiler/cleanup.py | 199 +---------------------- line_profiler/line_profiler_utils.py | 232 ++++++++++++++++++++++++++- 2 files changed, 235 insertions(+), 196 deletions(-) diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index ac866128..e9e17359 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -3,21 +3,16 @@ """ from __future__ import annotations -import sys -import os from collections.abc import ( - Callable, Generator, Iterable, Mapping, MutableMapping, Collection, + Callable, Generator, Iterable, Mapping, MutableMapping, ) from functools import partial from operator import setitem -from reprlib import Repr -from textwrap import indent from pathlib import Path -from types import MethodType -from typing import Any, TypeVar, TypedDict, cast -from typing_extensions import Concatenate, ParamSpec, Self, Unpack +from typing import Any, TypeVar, cast +from typing_extensions import Concatenate, ParamSpec, Self -from .line_profiler_utils import block_indent, make_tempfile +from .line_profiler_utils import CallbackRepr, make_tempfile from . import _diagnostics as diagnostics @@ -26,194 +21,12 @@ PS = ParamSpec('PS') K = TypeVar('K') V = TypeVar('V') -# Note: `typing.AnyStr` deprecated since 3.13 -AnyStr = TypeVar('AnyStr', str, bytes) _Stacks = dict[float, list[Callable[[], Any]]] _StackContexts = list[_Stacks] -class _ReprAttributes(TypedDict, total=False): - """ - Note: - We use this typed dict instead of directly supplying them in the - :py:meth:`_CallbackRepr.__init__()` signature, because we don't - want to bother with the default values there. - """ - maxlevel: int - maxtuple: int - maxlist: int - maxarray: int - maxdict: int - maxset: int - maxfrozenset: int - maxdeque: int - maxstring: int - maxlog: int - maxother: int - fillvalue: str - indent: str | int | None - - -class _CallbackRepr(Repr): - """ - :py:class:`reprlib.Repr` subclass to help with representing cleanup - callbacks, special-casing certain relevant object types (see - examples below). - - Example: - >>> from functools import partial - >>> from sys import version_info - - >>> class MyEnviron(dict): - ... def some_method(self) -> None: - ... ... - ... - >>> - >>> class MyRepr(_CallbackRepr): - ... # Since we can't instantiate a new `os._Environ`, test - ... # the relevant method with a mock - ... repr_MyEnviron = _CallbackRepr.repr__Environ - ... - >>> - >>> r = MyRepr(maxenv=3, maxargs=4, maxstring=15) - - Environ-dict formatting: - - >>> my_env = MyEnviron( - ... foo='1', - ... bar='2', - ... this_varname_is_long_but_isnt_truncated=( - ... "THIS VALUE IS TRUNCATED BECAUSE IT'S TOO LONG" - ... ), - ... baz='4', - ... ) - >>> print(r.repr(my_env)) - environ({'foo': '1', 'bar': '2', \ -'this_varname_is_long_but_isnt_truncated': 'THIS ... LONG', ...}) - - Partial-object formatting: - - >>> r.maxenv = 0 - >>> print(r.repr(my_env.some_method)) - - - Bound-method formatting: - - >>> r.maxargs = 0 - >>> callback_1 = partial(int, base=8) - >>> print(r.repr(callback_1)) - functools.partial(, ...) - - Indentation (Python 3.12+): - - >>> if version_info < (3, 12): - ... from pytest import skip - ... - ... skip( - ... '`Repr.indent` not available on {}.{},{}' - ... .format(*sys.version_info) - ... ) - - >>> r = MyRepr(maxenv=2, maxargs=4) - >>> r.indent = 2 - >>> callback_1 = partial(int, base=8) - >>> print(r.repr(callback_1)) - functools.partial( - , - base=8, - ) - - >>> callback_2 = partial(min, 5, 4, 3, 2, 1) - >>> r.indent = '----' - >>> print(r.repr(callback_2)) - functools.partial( - ----, - ----5, - ----4, - ----3, - ----2, - ----..., - ) - - >>> r.indent = ' ' - >>> r.maxenv = 2 - >>> print(r.repr(my_env.some_method)) - - """ - def __init__( - self, - *, - maxargs: int = 5, - maxenv: int = 3, - **kwargs: Unpack[_ReprAttributes] - ) -> None: - super().__init__() # kwargs are 3.12+ - valid_kwargs = ( - _ReprAttributes.__optional_keys__ - | _ReprAttributes.__required_keys__ - ) - for k, v in kwargs.items(): - if k in valid_kwargs: - setattr(self, k, v) - self.maxargs = maxargs - self.maxenv = maxenv - - def repr__Environ(self, env: os._Environ[AnyStr], level: int) -> str: - get: Callable[[AnyStr], str] = partial(self.repr1, level=level-1) - # Truncate envvar values, but not their names - envvars = ['{!r}: {}'.format(k, get(v)) for k, v in env.items()] - return self._format_items(envvars, ('environ({', '})'), self.maxenv) - - def repr_method(self, method: MethodType, level: int) -> str: - instance = self.repr1(method.__self__, level-1) - func = getattr(method.__func__, '__qualname__', '?') - prefix, suffix = f'' - # Take care of possible multi-line reprs - return block_indent(instance, prefix) + suffix - - def repr_partial(self, ptl: partial, level: int) -> str: - get: Callable[[Any], str] = partial(self.repr1, level=level-1) - args = [get(arg) for arg in ptl.args] - args.extend('{}={}'.format(k, get(v)) for k, v in ptl.keywords.items()) - args.insert(0, get(ptl.func)) - name = '{0.__module__}.{0.__qualname__}'.format(type(ptl)) - # The +1 is to account for `ptl.func` - return self._format_items(args, (name + '(', ')'), self.maxargs + 1) - - def _format_items( - self, - items: Collection[str], - delims: tuple[str, str], - maxlen: int | None = None, - ) -> str: - start, end = delims - if maxlen is not None and len(items) > maxlen: - items = list(items)[:maxlen] + ['...'] - indent_prefix: str | None = self._get_indent() - if indent_prefix is None or not items: - return '{}{}{}'.format(start, ', '.join(items), end) - return '\n'.join([ - start, *(indent(item + ',', indent_prefix) for item in items), end, - ]) - - if sys.version_info >= (3, 12): - # Note: `.indent` only available since 3.12 - def _get_indent(self) -> str | None: - indent = self.indent - if indent is None or isinstance(indent, str): - return indent - return ' ' * indent - else: - @staticmethod - def _get_indent() -> None: - return None - - -_CALLBACK_REPR = _CallbackRepr(maxother=cast(int, float('inf'))).repr +_CALLBACK_REPR_HELPER = CallbackRepr(maxother=cast(int, float('inf'))) +_CALLBACK_REPR = _CALLBACK_REPR_HELPER.repr class Cleanup: diff --git a/line_profiler/line_profiler_utils.py b/line_profiler/line_profiler_utils.py index ea86695e..eae5d27f 100644 --- a/line_profiler/line_profiler_utils.py +++ b/line_profiler/line_profiler_utils.py @@ -6,14 +6,22 @@ import enum import os +import sys +from collections.abc import Callable, Collection, Mapping, Sequence +from functools import partial from pathlib import Path +from reprlib import Repr from tempfile import mkstemp from textwrap import indent -from typing import TYPE_CHECKING -from typing_extensions import Self +from types import MethodType +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar +from typing_extensions import Self, Unpack -__all__ = ('StringEnum', 'block_indent', 'make_tempfile') +__all__ = ('StringEnum', 'CallbackRepr', 'block_indent', 'make_tempfile') + +# Note: `typing.AnyStr` deprecated since 3.13 +AnyStr = TypeVar('AnyStr', str, bytes) class _StrEnumBase(str, enum.Enum): @@ -96,6 +104,224 @@ def _missing_(cls, value: object) -> Self | None: return members.get(value.casefold()) +class _ReprAttributes(TypedDict, total=False): + """ + Note: + We use this typed dict instead of directly supplying them in the + :py:meth:`CallbackRepr.__init__()` signature, because we don't + want to bother with the default values there. + """ + maxlevel: int + maxtuple: int + maxlist: int + maxarray: int + maxdict: int + maxset: int + maxfrozenset: int + maxdeque: int + maxstring: int + maxlog: int + maxother: int + fillvalue: str + indent: str | int | None + + +class CallbackRepr(Repr): + """ + :py:class:`reprlib.Repr` subclass to help with representing cleanup + callbacks, special-casing certain relevant object types (see + examples below). + + Example: + >>> from functools import partial + >>> from sys import version_info + + >>> class MyEnviron(dict): + ... def some_method(self) -> None: + ... ... + ... + >>> + >>> class MyRepr(CallbackRepr): + ... # Since we can't instantiate a new `os._Environ`, test + ... # the relevant method with a mock + ... repr_MyEnviron = CallbackRepr.repr__Environ + ... + >>> + >>> r = MyRepr(maxenv=3, maxargs=4, maxstring=15) + + Environ-dict formatting: + + >>> my_env = MyEnviron( + ... foo='1', + ... bar='2', + ... this_varname_is_long_but_isnt_truncated=( + ... "THIS VALUE IS TRUNCATED BECAUSE IT'S TOO LONG" + ... ), + ... baz='4', + ... ) + >>> print(r.repr(my_env)) + environ({'foo': '1', 'bar': '2', \ +'this_varname_is_long_but_isnt_truncated': 'THIS ... LONG', ...}) + + Partial-object formatting: + + >>> r.maxenv = 0 + >>> print(r.repr(my_env.some_method)) + + + Bound-method formatting: + + >>> r.maxargs = 0 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial(, ...) + + Indentation (Python 3.12+): + + >>> if version_info < (3, 12): + ... from pytest import skip + ... + ... skip( + ... '`Repr.indent` not available on {}.{},{}' + ... .format(*sys.version_info) + ... ) + + >>> r = MyRepr(maxenv=2, maxargs=4) + >>> r.indent = 2 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial( + , + base=8, + ) + + >>> callback_2 = partial(min, 5, 4, 3, 2, 1) + >>> r.indent = '----' + >>> print(r.repr(callback_2)) + functools.partial( + ----, + ----5, + ----4, + ----3, + ----2, + ----..., + ) + + >>> r.indent = ' ' + >>> r.maxenv = 2 + >>> print(r.repr(my_env.some_method)) + + """ + def __init__( + self, + *, + maxargs: int = 5, + maxenv: int = 3, + **kwargs: Unpack[_ReprAttributes] + ) -> None: + super().__init__() # kwargs are 3.12+ + valid_kwargs = ( + _ReprAttributes.__optional_keys__ + | _ReprAttributes.__required_keys__ + ) + for k, v in kwargs.items(): + if k in valid_kwargs: + setattr(self, k, v) + self.maxargs = maxargs + self.maxenv = maxenv + + def repr__Environ(self, env: os._Environ[AnyStr], level: int) -> str: + """ + Format :py:data:`os.environ` or :py:data:`os.environb`. + """ + get: Callable[[AnyStr], str] = partial(self.repr1, level=level-1) + # Truncate envvar values, but not their names + envvars = ['{!r}: {}'.format(k, get(v)) for k, v in env.items()] + return self._format_items(envvars, ('environ({', '})'), self.maxenv) + + def repr_method(self, method: MethodType, level: int) -> str: + """ + Format a :py:class:`types.MethodType`. + """ + instance = self.repr1(method.__self__, level-1) + func = getattr(method.__func__, '__qualname__', '?') + prefix, suffix = f'' + # Take care of possible multi-line reprs + return block_indent(instance, prefix) + suffix + + def repr_partial(self, ptl: partial, level: int) -> str: + """ + Format a :py:func:`functools.partial`. + """ + name = '{0.__module__}.{0.__qualname__}'.format(type(ptl)) + # The +1 is to account for `ptl.func` + return self._format_call( + level, (name + '(', ')'), self.maxargs + 1, + [ptl.func, *ptl.args], ptl.keywords, + ) + + def format_call(self, /, *args, **kwargs) -> str: + """ + Convenience method for Formating a call a la + :py:meth:`inspect.BoundArguments.__str__`. + + Example: + >>> r = CallbackRepr(maxargs=3, maxlist=3) + >>> print(r.format_call( + ... [1, 2, 3, 4, 5], 'foo', spam=1, ham=2, + ... )) + ([1, 2, 3, ...], 'foo', spam=1, ...) + """ + return self._format_call( + self.maxlevel, ('(', ')'), self.maxargs, args, kwargs, + ) + + def _format_call( + self, + level: int, + delims: tuple[str, str], + maxargs: int, + args: Sequence[Any], + kwargs: Mapping[str, Any], + ) -> str: + get: Callable[[Any], str] = partial(self.repr1, level=level-1) + args = [get(arg) for arg in args] + args.extend('{}={}'.format(k, get(v)) for k, v in kwargs.items()) + return self._format_items(args, delims, maxargs) + + def _format_items( + self, + items: Collection[str], + delims: tuple[str, str], + maxlen: int | None = None, + ) -> str: + start, end = delims + if maxlen is not None and len(items) > maxlen: + items = list(items)[:maxlen] + ['...'] + indent_prefix: str | None = self._get_indent() + if indent_prefix is None or not items: + return '{}{}{}'.format(start, ', '.join(items), end) + return '\n'.join([ + start, *(indent(item + ',', indent_prefix) for item in items), end, + ]) + + if sys.version_info >= (3, 12): + # Note: `.indent` only available since 3.12 + def _get_indent(self) -> str | None: + indent = self.indent + if indent is None or isinstance(indent, str): + return indent + return ' ' * indent + else: + @staticmethod + def _get_indent() -> None: + return None + + def block_indent(string: str, prefix: str, fill_char: str = ' ') -> str: r""" Example: From ea018128a08cb7035e2755a3b440faaea8ee81ea Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 6 May 2026 09:44:45 +0200 Subject: [PATCH 092/165] Refactored call formatting line_profiler/_child_process_profiling/ cache.py::LineProfilingCache _dump_stats() New pseudo-method set by `._setup_in_child_process()` for writing profiling output @_method_wrapper Refactored implementation to use `._format_call()` for formatting calls in debug-log messages _format_call() New convenience method for formatting a call runpy_patches.py::_run() Refactored implementation to use: - `cache._format_call()` for formatting calls in debug-log messages - `Cleanup.patch()` for monkeypatching `exec()` --- .../_child_process_profiling/cache.py | 67 ++++++++++--------- .../_child_process_profiling/runpy_patches.py | 30 +++------ 2 files changed, 45 insertions(+), 52 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index e814edb4..f6a79edb 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -30,7 +30,7 @@ from _line_profiler_hooks import INHERITED_PID_ENV_VARNAME, load_pth_hook from .. import _diagnostics as diagnostics -from ..cleanup import Cleanup +from ..cleanup import Cleanup, _CALLBACK_REPR_HELPER from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats from ..toml_config import ConfigSource @@ -102,6 +102,7 @@ class LineProfilingCache(Cleanup): _private_field(default_factory=dict) ) _rlock: RLock = _private_field(default_factory=RLock) + _dump_stats: Callable[..., None] | None = _private_field(default=None) _loaded_instance: ClassVar[LineProfilingCache | None] = None @@ -526,7 +527,8 @@ def wrap_ctx_debug( suffix='.lprof', delete=False, ) - self.add_cleanup_with_priority(prof.dump_stats, 1, prof_outfile) + dump_stats = self._dump_stats = partial(prof.dump_stats, prof_outfile) + self.add_cleanup_with_priority(dump_stats, 1) # Various setups self._setup_common(wrap_os_fork, {'reboot_forkserver': False}) @@ -755,39 +757,31 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: cache = cls.load() write = cache._debug_output debug_: bool | None = debug + call = partial(wrapper, cache, vanilla_impl, *args, **kwargs) + if debug_ is None: debug_ = cache.debug - if debug_: - arg_reprs: list[str] = [repr(arg) for arg in args] - arg_reprs.extend(f'{k}={v!r}' for k, v in kwargs.items()) - formatted_call = f'{name}({", ".join(arg_reprs)})' - write(f'Wrapped call made: {formatted_call}...') - try: - result = wrapper(cache, vanilla_impl, *args, **kwargs) - except Exception as e: - if debug_: - write( - 'Wrapped call failed: ' - f'{formatted_call} -> {type(e).__name__}: {e}', - ) - raise + call_fmt = cache._format_call(name, *args, **kwargs) + write(f'Wrapped call made: {call_fmt}...') + state = 'succeeded' + try: + result = call() + except Exception as e: + state = 'failed' + outcome = f'{type(e).__name__}' + if str(e): + outcome = f'{outcome}: {e}' + raise e + else: + outcome = _CALLBACK_REPR_HELPER.repr(result) + return result + finally: + write(f'Wrapped call {state}: {call_fmt} -> {outcome}') else: - if debug_: - write( - 'Wrapped call succeeded: ' - f'{formatted_call} -> {result!r}', - ) - return result - - if ( - hasattr(vanilla_impl, '__module__') - and hasattr(vanilla_impl, '__qualname__') - ): - name = '{0.__module__}.{0.__qualname__}'.format(vanilla_impl) - else: - name = f'' + return call() + name = cls._get_name(vanilla_impl) return wrapped_impl for field in 'name', 'qualname', 'doc': @@ -797,6 +791,19 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: setattr(inner_wrapper, dunder, value) return inner_wrapper + @classmethod + def _format_call( + cls, func: Callable[..., Any] | str, /, *args, **kwargs, + ) -> str: + if isinstance(func, partial): + return cls._format_call( + func.func, [*func.args, *args], {**func.keywords, **kwargs}, + ) + call = _CALLBACK_REPR_HELPER.format_call(*args, **kwargs) + if not isinstance(func, str): + func = cls._get_name(func) + return func + call + @property def environ(self) -> dict[str, str]: """ diff --git a/line_profiler/_child_process_profiling/runpy_patches.py b/line_profiler/_child_process_profiling/runpy_patches.py index 659f4de8..22a600ea 100644 --- a/line_profiler/_child_process_profiling/runpy_patches.py +++ b/line_profiler/_child_process_profiling/runpy_patches.py @@ -16,6 +16,7 @@ from ..autoprofile.ast_tree_profiler import AstTreeProfiler from ..autoprofile.run_module import AstTreeModuleProfiler from ..autoprofile.util_static import modname_to_modpath +from ..cleanup import Cleanup from .cache import LineProfilingCache @@ -67,13 +68,8 @@ def _exec( is always executed. """ assert cache.rewrite_module - cache._debug_output('Calling via {}: `exec({})`'.format( - THIS_MODULE, - ', '.join( - [repr(a) for a in (_code, *args)] - + [f'{k}={v!r}' for k, v in kwargs.items()] - ), - )) + call = cache._format_call('exec', _code, *args, **kwargs) + cache._debug_output(f'Calling via {THIS_MODULE}: `{call}`') fname = str(cache.rewrite_module) code_writer = CodeWriter( fname, @@ -95,14 +91,8 @@ def _run( /, *args: PS.args, **kwargs: PS.kwargs ) -> T: - cache._debug_output('Calling via {}: `runpy.{}({})`'.format( - THIS_MODULE, - name, - ', '.join( - [repr(a) for a in (target, *args)] - + [f'{k}={v!r}' for k, v in kwargs.items()] - ), - )) + call = cache._format_call('runpy.' + name, target, *args, **kwargs) + cache._debug_output(f'Calling via {THIS_MODULE}: `{call}`') if cache.rewrite_module: try: filename = resolve_target_to_path(target) @@ -119,14 +109,10 @@ def _run( # If we are about to run the code to be autoprofiled, monkey-patch # `exec()` into the `runpy` namespace which just rewrites # `cache.rewrite_module` and executes it - if profile: - # Dodge attr-defined errors and their ilk - vars(runpy)['exec'] = partial(_exec, cache, CodeWriter) - try: + with Cleanup() as cleanup: + if profile: + cleanup.patch(runpy, 'exec', partial(_exec, cache, CodeWriter)) return func(target, *args, **kwargs) - finally: - if hasattr(runpy, 'exec'): - del runpy.exec def create_runpy_wrapper(cache: LineProfilingCache) -> ModuleType: From c1b35e05149dec6bc0deca7566b82ef1852f7636 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 6 May 2026 23:56:05 +0200 Subject: [PATCH 093/165] Extended `@pytest.mark.retry` tests/conftest.py _RetryFailure New exception class for when a retry condition fails evaluation _RetryHelper condition New attribute for text condition manage_call() Added handling for evaluating `.condition` before each retry using the test function's globals locals get_helper() Added handling for the `condition=...` parameter of the marker: - If `False`, returns `None` - If a string, returns an instance with `.condition` set pytest_configure() Updated help text for the `@pytest.mark.retry` marker tests/test_retry_tests.py _TestModule run() New params `additional_std{out,err}_lines` for additional patterns to match the output streams to check_verbose_summary(), check_concise_summary() New params `std{out,err}` for additional patterns to match the output streams to conditions_module New `_TestModule` fixture for a test module using `@pytest.mark.retry(condition=...)` test_retry_conditions() New test using `conditions_module` to check the behavior of `@pytest.mark.retry(condition=...)` --- tests/conftest.py | 83 ++++++++++++++++++- tests/test_retry_tests.py | 166 +++++++++++++++++++++++++++++++++++--- 2 files changed, 236 insertions(+), 13 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a11bb3aa..cfbf0c33 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,6 +41,38 @@ def __call__(self, *, pyfuncitem: pytest.Function) -> Any: ... +class _RetryFailure(RuntimeError): + def __init__( + self, + previous_error: Exception, + condition_error: Exception, + condition: str | None = None + ) -> None: + self.previous_error = previous_error + self.condition_error = condition_error + self.condition = condition + super().__init__(self._format_message()) + + condition_error.__cause__ = previous_error + self.__cause__ = condition_error + + def _format_message(self) -> str: + prev = self._format_exception(self.previous_error) + if len(prev.split()) > 1: + prev = f'({prev})' + condition = self._format_exception(self.condition_error) + if self.condition: + condition = f'(condition: {self.condition!r} -> {condition})' + return '{} -> {}'.format(prev, condition) + + @staticmethod + def _format_exception(xc: Exception) -> str: + msg = type(xc).__name__ + if str(xc): + return f'{msg}: {xc}' + return msg + + @final @dataclasses.dataclass class _RetryEntry: @@ -118,6 +150,7 @@ class _RetryHelper: retries: int = 1 exceptions: type[Exception] | tuple[type[Exception], ...] = () reset_fixtures: bool | Collection[str] = True + condition: str | None = None name: ClassVar[str] = 'retry' def __post_init__(self) -> None: @@ -171,6 +204,21 @@ def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: for i in range(1 + self.retries): if i: reset_fixtures(func) + if self.condition: + cond, error = self._check_condition(self.condition, func) + if error: # Bail + # XXX: would be nice if we can directly force an + # internal error, but that doesn't seem to be + # possible from within `pytest_pyfunc_call()`; + # directly calling `pytest_internalerror()` + # results in botched teardown and weird + # tracebacks, and leaves the test session in a + # bad state... + assert xc is not None + raise _RetryFailure(xc, cond, self.condition) + if not cond: + i -= 1 + break try: result = impl(pyfuncitem=func) except self.exceptions as e: @@ -195,6 +243,21 @@ def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: else: raise xc + @staticmethod + def _check_condition( + condition: str, func: pytest.Function, + ) -> tuple[Any, Literal[False]] | tuple[Exception, Literal[True]]: + global_ns: dict[str, Any] | None = None + try: + global_ns = func.obj.__globals__ + except AttributeError: # Not a `types.FunctionType` + pass + local_ns = func.funcargs + try: + return (eval(condition, global_ns, local_ns), False) + except Exception as e: + return (e, True) + @staticmethod def _reset_between_retries( func: pytest.Function, @@ -297,11 +360,13 @@ def get_helper(cls, pyfuncitem: pytest.Function) -> _RetryHelper | None: retries: int = 0 xc: set[type[Exception]] = set() reset_fixtures: bool | set[str] = True + condition: bool | str | None = None for mark in pyfuncitem.iter_markers(): if mark.name != cls.name: continue instance = cls(*mark.args, **mark.kwargs) retries += instance.retries + condition = instance.condition if isinstance(instance.exceptions, tuple): xc.update(instance.exceptions) else: @@ -322,9 +387,14 @@ def get_helper(cls, pyfuncitem: pytest.Function) -> _RetryHelper | None: reset_fixtures = bool(instance.reset_fixtures) if not retries: return None + if not (condition is None or isinstance(condition, str)): + if condition: + condition = None + else: + return None if not xc: xc = {Exception} - return cls(retries, tuple(xc), reset_fixtures) + return cls(retries, tuple(xc), reset_fixtures, condition) def _pluralize(noun: str, count: int, plural: str | None = None) -> str: @@ -347,7 +417,8 @@ def pytest_configure(config: pytest.Config) -> None: Register the :py:deco:`pytest.mark.retry` marker. """ help_text = ' '.join(""" - retry(retries=1, exceptions=Exception, reset_fixtures=True): + retry(retries=1, exceptions=Exception, \ +reset_fixtures=True, condition=None): mark the test for retrying upon failure. @@ -359,7 +430,13 @@ def pytest_configure(config: pytest.Config) -> None: reset_fixtures (bool | Collection[str]): Names of function-scoped fixtures to reset between retries, `True` reset all such fixtures, - `False` none thereof + `False` none thereof; + condition (bool | str | None): + Optional condition for retry: + if a boolean, only retry if true; + if a string, only retry if it `eval()`s true + (w/globals of the test module and locals from the fixtures + and parametrizations) """.split()) config.addinivalue_line('markers', help_text) diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py index 873c7681..52338e78 100644 --- a/tests/test_retry_tests.py +++ b/tests/test_retry_tests.py @@ -7,7 +7,7 @@ import re import pprint import textwrap -from collections.abc import Callable, Generator, Iterable +from collections.abc import Callable, Collection, Generator, Iterable from dataclasses import dataclass from functools import partial from operator import attrgetter @@ -280,6 +280,67 @@ def test_three_xc_types(items: Iterable[Any]) -> None: ''' assert int(next(items)) < 0 """ +TEST_CONDITIONS = """ +from __future__ import annotations + +from sys import version_info + +import pytest + + +@pytest.mark.retry(2, condition=(11 % 2)) +def test_concrete_positive_condition() -> None: + ''' + This should fail after 2 retries because its condition is true. + ''' + raise RuntimeError + + +@pytest.mark.retry(condition=('a' in 'foo')) +def test_concrete_negative_condition() -> None: + ''' + This should fail without retries because its condition is false. + ''' + raise RuntimeError + + +@pytest.mark.retry(condition='version_info.major >= 3') +def test_dynamic_positive_condition_test_module_globals() -> None: + ''' + This should fail after 1 retry because the condition evaluates to + true on the test module's ``globals()``. + ''' + raise RuntimeError + + +@pytest.mark.retry(condition='version_info.major < 3') +def test_dynamic_negative_condition_test_module_globals() -> None: + ''' + This should fail without retries because the condition evaluates to + false on the test module's ``globals()``. + ''' + raise RuntimeError + + +@pytest.mark.retry(condition='foo == 1') +def test_bad_dynamic_condition() -> None: + ''' + This should fail without retries because the condition cannot be + evaluated (``NameError: name 'foo' is not defined``). + ''' + raise RuntimeError('bar') + + +@pytest.mark.retry(condition='n % 2') +@pytest.mark.parametrize('n', [0, 1, 2]) +def test_dynamic_condition_test_params(n: int) -> None: + ''' + Subtests `[0]` and `[2]` (resp. subtest `[1]`) should fail without + retries (resp. with 1 retry) because the condition evaluates to + false (resp. true) on the test's parametrization. + ''' + raise RuntimeError +""" @dataclass @@ -325,6 +386,8 @@ def run( check_summary: Literal['verbose', 'concise'] | None = None, check_warnings: int | None = None, runner: _RunPytest_Method = 'runpytest', + additional_stdout_lines: Collection[str] = (), + additional_stderr_lines: Collection[str] = (), ) -> pytest.RunResult: """ Args: @@ -344,6 +407,11 @@ def run( 'runpytest_subprocess']): The :py:class:`pytest.Pytester` method used to run the test module + additional_stdout_lines, additional_stderr_lines \ +(Collection[str]): + Additional regex patterns (other than the + automatically-generated ones) to match against the + output streams Returns: :py:class:`pytest.RunResult` object returned by the @@ -374,9 +442,14 @@ def run( self.check_results(result, check_warnings) if check_summary is not None: if check_summary == 'verbose': - self.check_verbose_summary(result) + checker = self.check_verbose_summary else: - self.check_concise_summary(result) + checker = self.check_concise_summary + checker( + result, + stdout=additional_stdout_lines, + stderr=additional_stderr_lines, + ) return result finally: for path in tempfiles: @@ -403,7 +476,12 @@ def check_results( counts[outcome.status] = counts.get(outcome.status, 0) + 1 result.assert_outcomes(warnings=warnings, **counts) - def check_verbose_summary(self, result: pytest.RunResult) -> None: + def check_verbose_summary( + self, + result: pytest.RunResult, + stdout: Collection[str] = (), + stderr: Collection[str] = (), + ) -> None: lines: list[str] = [] counts: dict[_Status, int] = {} for outcomes in self.expected_outcomes.values(): @@ -422,11 +500,14 @@ def check_verbose_summary(self, result: pytest.RunResult) -> None: lines.extend( self._format_header(status, n) for status, n in counts.items() ) + self._check_lines(result, [*lines, *stdout], stderr) - print(f'Expecting these lines in the output: {lines!r}...') - result.stdout.re_match_lines_random(lines) - - def check_concise_summary(self, result: pytest.RunResult) -> None: + def check_concise_summary( + self, + result: pytest.RunResult, + stdout: Collection[str] = (), + stderr: Collection[str] = (), + ) -> None: lines: list[str] = [] counts: dict[_Status, int] = {} test_names: dict[_Status, dict[str, set[str]]] = {} @@ -459,8 +540,7 @@ def check_concise_summary(self, result: pytest.RunResult) -> None: parent_test, n, '' if n == 1 else 's', )) - print(f'Expecting these lines in the output: {lines!r}...') - result.stdout.re_match_lines_random(lines) + self._check_lines(result, [*lines, *stdout], stderr) for status, n in counts.items(): header = self._format_header(status, n) @@ -471,6 +551,20 @@ def check_concise_summary(self, result: pytest.RunResult) -> None: for test_name in names: assert test_name in line + @staticmethod + def _check_lines( + result: pytest.RunResult, + stdout: Collection[str], + stderr: Collection[str], + ) -> None: + for stream, lines in { + 'stdout': list(stdout), 'stderr': list(stderr), + }.items(): + if not lines: + continue + print(f'Expecting these lines in the {stream}: {lines!r}...') + getattr(result, stream).re_match_lines_random(lines) + @staticmethod def _find_line(pattern: str, text: str) -> str: pattern = f'^.*{pattern}.*' @@ -616,6 +710,37 @@ def exceptions_module( ) +@pytest.fixture +def conditions_module( + pytester: pytest.Pytester, +) -> Generator[_TestModule, None, None]: + test = partial(_TestOutcome, status='failed') + param_test_name = 'test_dynamic_condition_test_params' + param_test = partial(test(param_test_name).subtest, status='failed') + yield _TestModule( + 'test_conditions', + TEST_CONDITIONS, + { + 'test_concrete_positive_condition': + [test('test_concrete_positive_condition', retries=2)], + 'test_concrete_negative_condition': + [test('test_concrete_negative_condition')], + 'test_dynamic_positive_condition_test_module_globals': + [test( + 'test_dynamic_positive_condition_test_module_globals', + retries=1, + )], + 'test_dynamic_negative_condition_test_module_globals': + [test('test_dynamic_negative_condition_test_module_globals')], + 'test_bad_dynamic_condition': + [test('test_bad_dynamic_condition')], + param_test_name: + [param_test('0'), param_test('1', retries=1), param_test('2')], + }, + pytester, + ) + + @pytest.mark.parametrize('verbose', [True, False]) def test_fixture_scoping(counters_module: _TestModule, verbose: bool) -> None: """ @@ -705,3 +830,24 @@ def test_exception_restrictions(exceptions_module: _TestModule) -> None: '--verbose', check_results=True, check_summary='verbose', check_warnings=0, ) + + +def test_retry_conditions(conditions_module: _TestModule) -> None: + """ + Test that the decorator correctly handles retry conditions. + """ + conditions_module.run( + '--verbose', + check_results=True, + check_summary='verbose', + check_warnings=0, + # `test_bad_dynamic_condition()` should have failed with a + # `_RetryFailure`, listing the error encountered in the last + # trial and the error encountered when `eval()`-ing the + # condition + additional_stdout_lines=[ + 'FAILED +.*::test_bad_dynamic_condition - ' + r".*_RetryFailure: +\(RuntimeError: bar\) +" + r"-> +\(condition: +'foo == 1' +-> +NameError: .*'foo'.*\)" + ], + ) From 5a25d667a611a53b2e251771d1a5360fb509c854 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 7 May 2026 03:37:47 +0200 Subject: [PATCH 094/165] Reworked `multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py TaskWrapper New pickleable wrapper object around a callable to be sent as tasks to child processes, pking sure that rofiling output is written before the call result is pushed back to the parent (and thus leaving the child prone to termination) Patch New helper object for declaratively constructing patches for subcomponents, reworked from `_apply_patches_generic()`; the patches are: - `pool`: Patch `multiprocessing.pool.Pool`, so that each tasks write profiling output before pushing result back to the parent process - `process`: Patch `multiprocessing.process.BaseProcess`, so that each child process write profiling output before exiting - `logging`: Patch logging functions in `multiprocessing.util`, so that the internal logs of `multiprocessing` are teed to the session cache's debug logs wrap_get_tasks(), wrap_guarded_task_generation() New wrappers for the eponymous methods in `multiprocessing.pool.Pool` which wraps the input callables in a `TaskWrapper` apply() - Merged with `_apply_patches_generic()` which it was a thin wrapper for - Added param `patches` for explicitly supplying which patches to apply - Refactored internals to work with `Patch` objects line_profiler/rc/line_profiler.toml [tool.line_profiler.child_processes.multiprocessing] Removed key `intercept_logs` (superseded by `patches.logging`) [tool.line_profiler.child_processes.multiprocessing.patches] New subtable of boolean switches for whether to apply each of the aforementioned patches [toll.line_profiler.child_processes.multiprocessing.polling] Updated obsolete comments --- .../multiprocessing_patches.py | 402 ++++++++++++++---- line_profiler/rc/line_profiler.toml | 36 +- 2 files changed, 341 insertions(+), 97 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 52ddbd78..efecb2f7 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -14,16 +14,21 @@ """ from __future__ import annotations +import dataclasses import multiprocessing import warnings -from collections.abc import Callable, Mapping +from collections.abc import Callable, Collection, Mapping, Sequence from functools import partial from importlib import import_module +from inspect import getattr_static, signature from multiprocessing.process import BaseProcess +from multiprocessing.pool import Pool +from operator import attrgetter from time import sleep, monotonic -from types import MappingProxyType +from types import MappingProxyType, ModuleType from typing import ( - Any, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, + TYPE_CHECKING, + Any, ClassVar, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, cast, final, ) @@ -54,10 +59,18 @@ T = TypeVar('T') +T1 = TypeVar('T1') +T2 = TypeVar('T2') PS = ParamSpec('PS') +PS1 = ParamSpec('PS1') +PS2 = ParamSpec('PS2') _OnTimeout = Literal['ignore', 'warn', 'error'] +_PatchName = Literal['pool', 'process', 'logging'] _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' +_LOGGERS = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] + +# ------------------------------ Helpers ------------------------------- class _Wrapper(Protocol, Generic[PS, T]): @@ -277,13 +290,185 @@ def get_defaults(cls) -> Self: return namespace.setdefault('_DEFAULT_POLLER_ARGS', defaults) +class TaskWrapper(Generic[PS, T]): + """ + Pickle-able wrapper around the supplied task callable, which writes + to the session's profiling-stats file on exit. + """ + def __init__(self, func: Callable[PS, T]) -> None: + self.func = func + try: + self.__signature__ = signature(func) + except Exception: # nocover + # Can happen with e.g. certain builin/c-based callables + pass + + def __call__(self, *args, **kwargs) -> T: + dump_stats = LineProfilingCache.load()._dump_stats + try: + return self.func(*args, **kwargs) + finally: + if dump_stats is not None: + dump_stats() + + +@dataclasses.dataclass +class Patch: + """ + Patch to apply to a component in :py:mod:`multiprocessing`. + + Attributes: + submodule (str): + Name of the :py:mod:`multiprocessing` submodule. + targets (dict[str,\ +dict[str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]]]]): + Dictionary mapping (dot-chained) names in said submodule to + a dictionary of patches; said patches dictionary should have + the format of + ``dict[simple_attribute, wrapper | [wrapper1, ...]]``. See + Example for details. + + Example: + Consider + ``Patch('foo', {'bar.baz': {'foobar': foofoo},\ +'': {'spam': [ham, eggs]}})``. + This instance would perform the following patches on the module + ``multiprocessing.foo``: + + - Replace ``multiprocessing.foo.bar.baz.foobar`` with + ``foofoo(multiprocessing.foo.bar.baz.foobar)`` + + - Replace ``multiprocessing.foo.spam`` with + ``eggs(ham(multiprocessing.foo.spam))``; + note that the two wrappers are applied in order to the + original attribute. + """ + submodule: str + targets: dict[ + str, dict[str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]]] + ] = dataclasses.field(default_factory=dict) + package: ClassVar[str] = 'multiprocessing' + + def add_target( + self, + target: str, + patches: Mapping[ + str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]] + ], + ) -> Self: + """ + Convenience method for gradually constructing the patch with a + fluent interface. + + Returns: + This instance + """ + self.targets.setdefault(target, {}).update(patches) + return self + + def apply( + self, + cache: LineProfilingCache, + *, + cleanup: bool = True, + static: bool = False, + ) -> list[str]: + """ + Apply the patch. + + Args: + cache (LineProfilingCache): + Session cache + cleanup (bool): + Whether ``cache.cleanup()`` should reverse the patches + static (bool): + Whether to use :py:func:`inspect.getattr_static` to + retrieve to the attributes to be patched on the patch + targets + + Returns: + replacements (list[str]): + Names of entities replaced + """ + submod_name = f'{self.package}.{self.submodule}' + get_attribute = getattr_static if static else getattr + result: list[str] = [] + try: + mod = self.load_module() + except ImportError: # nocover + return [] + + for target in sorted(self.targets, key=len, reverse=True): + if TYPE_CHECKING: + # See `ty` issue #2572 + assert isinstance(target, str) + if target: + try: + obj: Any = attrgetter(target)(mod) + except AttributeError: # nocover + continue + name = f'{submod_name}.{target}' + else: + obj, name = mod, submod_name + replace = partial(cache.patch, obj, cleanup=cleanup, name=name) + for method, method_wrappers in self.targets[target].items(): + if callable(method_wrappers): + method_wrappers = cast( + Sequence[Callable[[Any], Any]], (method_wrappers,), + ) + try: + impl = get_attribute(obj, method) + except AttributeError: + continue + for wrapper in method_wrappers: + impl = wrapper(impl) + replace(method, impl) + result.append(f'{name}.{method}') + return result + + def load_module(self) -> ModuleType: + """ + Returns: + Module object :py:attr:`.module` points to + """ + return import_module(self.module) + + @staticmethod + def _join(s: str, *strs: str, sep: str = '.') -> str: + return sep.join(string for string in (s, *strs) if string) + + @property + def module(self) -> str: + """ + Module where the patches are applied + """ + return self._join(self.package, self.submodule) + + @property + def summary(self) -> MappingProxyType[str, frozenset[str]]: + """ + Summary of the dotted paths to the patched objects and their + patched attributes + """ + add_prefix = partial(self._join, self.module) + return MappingProxyType({ + add_prefix(target): frozenset(patches) + for target, patches in self.targets.items() + }) + + def _get_config(config: ConfigSource) -> Mapping[str, Any]: cd = dict( config.get_subconfig('child_processes', 'multiprocessing', copy=True) .conf_dict ) + assert isinstance(cd.get('patches'), Mapping) assert isinstance(cd.get('polling'), Mapping) - return MappingProxyType({**cd, 'polling': MappingProxyType(cd['polling'])}) + return MappingProxyType({ + **cd, + 'patches': MappingProxyType(cd['patches']), + 'polling': MappingProxyType(cd['polling']), + }) def _process_has_returned( @@ -302,6 +487,53 @@ def _process_has_returned( return result +def _no_op(*_, **__) -> None: + pass + + +# ---------------- `multiprocessing.pool.Pool` patches ----------------- + + +@LineProfilingCache._method_wrapper +def wrap_get_tasks( + _, # No need to use the cache, but `_method_wrapper` expects it + vanilla_impl: Callable[Concatenate[Callable[PS1, T1], PS2], T2], + func: Callable[PS1, T1], + *args: PS2.args, + **kwargs: PS2.kwargs +) -> T2: + """ + Wrap around :py:meth:`.Pool._get_tasks` so that the writing of + profiling stats is handled within the callables sent to the child + processes before the parent process assumes control. + + Note: + :py:meth:`Pool._get_tasks` is a static method. + """ + return vanilla_impl(TaskWrapper(func), *args, **kwargs) + + +@LineProfilingCache._method_wrapper +def wrap_guarded_task_generation( + _, # No need to use the cache, but `_method_wrapper` expects it + vanilla_impl: Callable[Concatenate[Pool, int, Callable[PS1, T1], PS2], T2], + self: Pool, + result_job: int, + func: Callable[PS1, T1], + *args: PS2.args, + **kwargs: PS2.kwargs +) -> T2: + """ + Wrap around :py:meth:`.Pool._guarded_task_generation` so that the + writing of profiling stats is handled within the callables sent to + the child processes before the parent process assumes control. + """ + return vanilla_impl(self, result_job, TaskWrapper(func), *args, **kwargs) + + +# ----------- `multiprocessing.process.BaseProcess` patches ------------ + + @LineProfilingCache._method_wrapper def wrap_terminate( cache: LineProfilingCache, @@ -378,6 +610,9 @@ def wrap_bootstrap( cache.cleanup(new_thread=True) +# --------------- `multiprocessing.util` logging patches --------------- + + def _cache_hook( vanilla_impl: Callable[PS, T], get_logging_message: Callable[PS, str], @@ -412,8 +647,47 @@ def get_msg(msg: str, *_, **__) -> str: ) +# -------------------------- Applying patches -------------------------- + + +_PATCHES: dict[_PatchName, Patch] = { + 'process': Patch('process').add_target( + 'BaseProcess', + {'terminate': wrap_terminate, '_bootstrap': wrap_bootstrap}, + ), + 'pool': Patch('pool').add_target( + 'Pool', { + # `._get_task()` is a static method, so the wrapper function + # needs additional wrapping + '_get_tasks': [wrap_get_tasks, staticmethod], + '_guarded_task_generation': wrap_guarded_task_generation, + }, + ), + 'logging': Patch('util').add_target( + # The logging functions exists directly in the module namespace + # so no further attribute access is needed + '', {func: partial(partial, tee_log, func) for func in _LOGGERS}, + ), +} + + +def _stop_forkserver() -> None: + """ + Note: + This uses `ForkServer._stop()` which is private API, but it's + the same hack used in Python's own test suite -- see the comment + to said method + """ + # Appease the type-checker since `._stop()` is not public API + stop = getattr(forkserver._forkserver, '_stop', None) + assert callable(stop) + stop() + + def apply( - lp_cache: LineProfilingCache, reboot_forkserver: bool = True, + lp_cache: LineProfilingCache, + reboot_forkserver: bool = True, + patches: Collection[_PatchName] | None = None, ) -> None: """ Set up profiling in :py:mod:`multiprocessing` child processes by @@ -421,21 +695,35 @@ def apply( Args: lp_cache (LineProfilingCache): - Cache instance governing the profiling run + Cache instance governing the profiling run. reboot_forkserver (bool): Whether to reboot the global :py:class`multiprocessing.forkserver.ForkServer` instance so as to ensure that profiling happens on processes forked - therefrom (see Note) + therefrom (see Note). + patches \ +(Collection[Literal['pool', 'process', 'logging'] | None]): + Patches to apply to :py:mod:`multiprocessing`; see the + following section for a description of each; + the default is taken from the TOML config file. + + Patches: + ``'pool'``: + Patch :py:class:`multiprocessing.pool.Pool`'s + ``._get_tasks()`` and ``._guarded_task_generation()`` + methods so that parallel tasks write profiling output. + ``'process'``: + Patch :py:class:`multiprocessing.process.BaseProcess`'s + ``.terminate()`` and ``._bootstrap()`` methods so that child + processes write profiling output on exit and are given + enough time for that. + ``'logging'``: + Patch :py:mod:`multiprocessing.util`'s logging methods (e.g. + ``debug()`` and ``info()``) so that their messages are teed + to the cache's debug log. Side effects: - - :py:mod:`multiprocessing` marked as having been set up - - - The following methods and functions patched: - - - :py:meth:`multiprocessing.process.BaseProcess.terminate` - - - :py:meth:`multiprocessing.process.BaseProcess._bootstrap` + - The aforementioned patches applied - If ``reboot_forkserver=True``, fork-server process rebooted: @@ -468,66 +756,23 @@ def apply( .. _GH-126631: https://github.com/python/cpython/issues/126631 """ - if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, reboot_forkserver=reboot_forkserver) - - -def _apply_patches_generic( - lp_cache: LineProfilingCache, - submodule: str, - targets: Mapping[str, Mapping[str, Callable[[Any], Any]]], - cleanup: bool = True, -) -> None: - submod_name = 'multiprocessing.' + submodule - try: - mod = import_module(submod_name) - except ImportError: # nocover + if getattr(multiprocessing, _PATCHED_MARKER, False): return - for target, patches in targets.items(): - if target: - try: - obj: Any = getattr(mod, target) - except AttributeError: # nocover - continue - name = f'{submod_name}.{target}' - else: - obj, name = mod, submod_name - replace = partial(lp_cache.patch, obj, cleanup=cleanup, name=name) - for method, method_wrapper in patches.items(): - try: - vanilla = getattr(obj, method) - except AttributeError: - continue - replace(method, method_wrapper(vanilla)) - - -def _apply_mp_patches( - lp_cache: LineProfilingCache, - reboot_forkserver: bool = True, - intercept_mp_logs: bool | None = None, -) -> None: - # In a child process, we don't care about polluting the - # `multiprocessing` namespace, so don't bother with cleanup - apply_patches = partial(_apply_patches_generic, lp_cache) - # Patch `multiprocessing.process.BaseProcess` methods - # Note: the type checkers seem to need some help figuring the - # `patches` out... so do explicit `cast()`s - apply_patches( - 'process', - {'BaseProcess': {'terminate': wrap_terminate, - '_bootstrap': wrap_bootstrap}}, - ) + if patches is None: + config = _get_config(lp_cache._config_source)['patches'] + patches_ = {patch for patch, applied in config.items() if applied} + else: + patches_ = {p.lower() for p in patches} # Patch `multiprocessing.spawn` if _CAN_USE_SPAWN and hasattr(spawn, 'runpy'): lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) - # Intercept `multiprocessing` debug messages - if intercept_mp_logs is None: - config = lp_cache._config_source - intercept_mp_logs = _get_config(config)['intercept_logs'] - if intercept_mp_logs: - lfuncs = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] - lpatches = {func: partial(partial, tee_log, func) for func in lfuncs} - apply_patches('util', {'': lpatches}) + # Patch methods/functions in these entities: + # - `multiprocessing.pool.Pool` + # - `multiprocessing.process.BaseProcess` + # - `multiprocessing.util` + for name, patch in _PATCHES.items(): + if name in patches_: + patch.apply(lp_cache) # Stop the current `ForkServer` server process: # - Now, so that the (rebooted) fork-server process has profiling # set up; and @@ -537,20 +782,3 @@ def _apply_mp_patches( lp_cache.add_cleanup(_stop_forkserver) # Mark `multiprocessing` as having been patched lp_cache.patch(multiprocessing, _PATCHED_MARKER, True) - - -def _stop_forkserver() -> None: - """ - Note: - This uses `ForkServer._stop()` which is private API, but it's - the same hack used in Python's own test suite -- see the comment - to said method - """ - # Appease the type-checker since `._stop()` is not public API - stop = getattr(forkserver._forkserver, '_stop', None) - assert callable(stop) - stop() - - -def _no_op(*_, **__) -> None: - pass diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 529a6783..20eb2dd7 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -224,26 +224,42 @@ suffix = '' [tool.line_profiler.child_processes.multiprocessing] -# - `multiprocessing.intercept_logs` (bool): -# Whether to patch the `multiprocessing.util` logging funcions so that -# the internal log messages are captured and writtent to the debug -# logs -intercept_logs = false # - `multiprocessing.catch_sigterm` (bool): # Whether to set a hook for the child-process session caches to # `.cleanup()` on SIGTERM catch_sigterm = true +# List of individual patches to apply to `multiprocessing` + +[tool.line_profiler.child_processes.multiprocessing.patches] + +# - `multiprocessing.patches.pool` (bool): +# Whether to patch `multiprocessing.pool.Pool`, so that each tasks +# write profiling output before pushing result back to the parent +# process +pool = true +# - `multiprocessing.patches.process` (bool): +# Whether to patch `multiprocessing.process.BaseProcess`, so that each +# child process write profiling output before exiting +process = true +# - `multiprocessing.patches.logging` (bool): +# Whether to patch logging functions in `multiprocessing.util`, so +# that the internal logs of `multiprocessing` are teed to the session +# cache's debug logs +logging = false + +# Polling controls, used for the `pool` patch + [tool.line_profiler.child_processes.multiprocessing.polling] # - `multiprocessing.polling.cooldown` (float): -# Cooldown time (seconds) before successive polls on lock files (set -# to <= 0 to disable cooldowns) +# Cooldown time (seconds) before successive polls on child processes +# (set to <= 0 to disable cooldowns) cooldown = 0.03125 # 1/32-nd of a second # - `multiprocessing.polling.timeout` (float): -# Time (seconds) before the main process disregards the existence of -# lock file and unblocks calls to `.terminate()` a (most probably -# errored-out) child process anyway (set to <= 0 to disable timeouts) +# Time (seconds) before the main process disregards the alive-ness of +# child processes, and unblocks calls to `.terminate()` a (most +# probably errored-out) child anyway (set to <= 0 to disable timeouts) timeout = 0.25 # - `multiprocessing.polling.on_timeout` # (Literal['error', 'warn', 'ignore']): From cd87aea0b3901190badb12545b13e33563360daf Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 7 May 2026 07:00:13 +0200 Subject: [PATCH 095/165] Fix: `getattr_static` line_profiler/_child_process_profiling/multiprocessing_patches.py Patch.apply(static=...) Changed default to true _PATCHES['pool'].targets['Pool']['_get_tasks'] Updated to work with the new default line_profiler/cleanup.py::Cleanup.patch() Added new parameter `static` to toggle whether to use `inspect.getattr_static()` to get the pre-patch value; now defaulting to true, while the old behavior is equivalent to false --- .../_child_process_profiling/multiprocessing_patches.py | 6 ++++-- line_profiler/cleanup.py | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index efecb2f7..0e3cc190 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -371,7 +371,7 @@ def apply( cache: LineProfilingCache, *, cleanup: bool = True, - static: bool = False, + static: bool = True, ) -> list[str]: """ Apply the patch. @@ -659,7 +659,9 @@ def get_msg(msg: str, *_, **__) -> str: 'Pool', { # `._get_task()` is a static method, so the wrapper function # needs additional wrapping - '_get_tasks': [wrap_get_tasks, staticmethod], + '_get_tasks': [ + attrgetter('__func__'), wrap_get_tasks, staticmethod, + ], '_guarded_task_generation': wrap_guarded_task_generation, }, ), diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index e9e17359..d4bd55cf 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -7,6 +7,7 @@ Callable, Generator, Iterable, Mapping, MutableMapping, ) from functools import partial +from inspect import getattr_static from operator import setitem from pathlib import Path from typing import Any, TypeVar, cast @@ -292,6 +293,7 @@ def make_tempfile( def patch( self, obj: Any, attr: str, value: Any, *, name: str | None = None, + static: bool = True, cleanup: bool = True, priority: float = 0, ) -> None: @@ -307,6 +309,9 @@ def patch( Value to be assigned to said attribute of ``obj`` name (str | None): Optional name for ``obj`` to be used in debug messages + static (bool): + Whether to use :py:func:`inspect.getattr_static` to + get the current value of the attribute cleanup (bool): Whether to reverse the patch (by resetting or deleting the attribute) on cleanup @@ -338,8 +343,10 @@ def patch( # ... yeah gotta disagree with flake8, a lambda makes # perfect sense here add_cleanup = lambda *_, **__: None # noqa: E731 + get_attribute = getattr_static if static else getattr + try: - old = getattr(obj, attr) + old = get_attribute(obj, attr) except AttributeError: add_cleanup(delattr, priority, obj, attr) else: From 7ad0a70ea8030b1f28634384e8c6fd40b6f7d690 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 7 May 2026 07:16:15 +0200 Subject: [PATCH 096/165] WIP: test update tests/test_child_procs.py _preserve_obj_attributes _preserve_attributes[.fetch_current_values()] New param `static` for toggling whether to use `inspect.getattr_static()` _preserve_attributes.compare_with_current_values() - New param `static` for toggling whether to use `inspect.getattr_static()` - Refactored to give more useful output and to only raise the `AssertionError` after all comparisons have been made get_patched_attributes() New convenience function superseding the `patched_attributes` fixture test_cache_setup_main_process() Refactored to test the various combinations of applying `multiprocessing` patches test_apply_mp_patches_{success,failure}() - Refactored to test the various combinations of applying `multiprocessing` patches - Further narrowed the scope of the `@pytest.mark.retry` on the failing case: retries are only attempted when the fully-deterministic `pool` patch isn't used test_profiling_{multiproc_script,bare_python}_failure() No longer using the `@pytest.mark.retry` decorator because the 'pool' patch is used by default test_load_pth_hook() Updated to check (against) the correct patched attributes --- tests/test_child_procs.py | 348 +++++++++++++++++++++++++++++--------- 1 file changed, 272 insertions(+), 76 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 126c83d5..6216d3be 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -14,10 +14,11 @@ import warnings from abc import ABC, abstractmethod from collections.abc import ( - Callable, Collection, Generator, Iterable, Iterator, Mapping, Sequence, + Callable, + Collection, Generator, Iterable, Iterator, Mapping, Sequence, Set, ) from contextlib import AbstractContextManager, ExitStack, nullcontext -from functools import partial, wraps +from functools import lru_cache, partial, wraps from io import StringIO from importlib import import_module from numbers import Real @@ -38,18 +39,19 @@ import ubelt as ub from _line_profiler_hooks import load_pth_hook -from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) from line_profiler._child_process_profiling.multiprocessing_patches import ( - _Poller, _PATCHED_MARKER, + _Poller, _PATCHED_MARKER, _PATCHES as MP_PATCHES, ) +from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler.curated_profiling import ( CuratedProfilerContext, ClassifiedPreimportTargets, ) from line_profiler.line_profiler import LineProfiler, LineStats +from line_profiler.toml_config import ConfigSource T = TypeVar('T') @@ -520,6 +522,23 @@ class _NotSupplied(enum.Enum): NOT_SUPPLIED = enum.auto() +class _GetAttr(Protocol): + """ + Function signature for functions that behave like + :py:func:`getattr``. + """ + @overload + def __call__(self, obj: Any, attr: str, /) -> Any: + ... + + @overload + def __call__(self, obj: Any, attr: str, default: Any, /) -> Any: + ... + + def __call__(self, *args): + ... + + @final class ResultMismatch(ValueError): def __init__( @@ -969,17 +988,19 @@ def _debug(self, msg: str, **kwargs) -> None: class _preserve_obj_attributes(_CallableContextManager[dict[str, Any]]): def __init__( - self, obj: Any, attrs: Collection[str], debug: bool = _DEBUG, + self, obj: Any, attrs: Collection[str], *, + static: bool = True, debug: bool = _DEBUG, ) -> None: self.obj = obj self.attrs = set(attrs) self._callbacks: list[Callable[[], None]] = [] self.debug = debug + self.static = static def __enter__(self) -> dict[str, Any]: def get_repr(attr: str) -> str: try: - value = getattr(self.obj, attr) + value = get_attribute(self.obj, attr) except ValueError: return '' else: @@ -1000,9 +1021,14 @@ def reset(attr: str, value: Any) -> None: )) setattr(self.obj, attr, value) + if self.static: + get_attribute: _GetAttr = inspect.getattr_static + else: + get_attribute = getattr + result: dict[str, Any] = {} for attr in self.attrs: - old = getattr(self.obj, attr, _NotSupplied.NOT_SUPPLIED) + old = get_attribute(self.obj, attr, _NotSupplied.NOT_SUPPLIED) if old is _NotSupplied.NOT_SUPPLIED: callback = partial(delete, attr) else: @@ -1066,12 +1092,14 @@ class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): >>> assert main is not line_profiler.main """ def __init__( - self, targets: Mapping[str, Collection[str]], debug: bool = _DEBUG, + self, targets: Mapping[str, Collection[str]], *, + static: bool = True, debug: bool = _DEBUG, ) -> None: self.targets = { target: set(attrs) for target, attrs in targets.items() } self._stacks: list[ExitStack] = [] + self.static = static self.debug = debug def __enter__(self) -> dict[str, dict[str, Any]]: @@ -1080,7 +1108,8 @@ def __enter__(self) -> dict[str, dict[str, Any]]: result: dict[str, Any] = {} for target, attrs in self.targets.items(): result[target] = stack.enter_context(_preserve_obj_attributes( - _import_target(target), attrs, debug=self.debug, + _import_target(target), attrs, + debug=self.debug, static=self.static, )) return result @@ -1089,13 +1118,17 @@ def __exit__(self, *_, **__) -> None: @staticmethod def fetch_current_values( - targets: Mapping[str, Collection[str]], + targets: Mapping[str, Collection[str]], static: bool = True, ) -> dict[str, dict[str, Any]]: result: dict[str, dict[str, Any]] = {} na = _NotSupplied.NOT_SUPPLIED + if static: + get: _GetAttr = inspect.getattr_static + else: + get = getattr for target, attrs in targets.items(): obj = _import_target(target) - result[target] = {attr: getattr(obj, attr, na) for attr in attrs} + result[target] = {attr: get(obj, attr, na) for attr in attrs} return result @classmethod @@ -1104,6 +1137,7 @@ def compare_with_current_values( old: Mapping[str, Mapping[str, Any]], comparator: Callable[[Any, Any], bool] = operator.is_, assert_true: bool | Mapping[str, Mapping[str, bool]] = True, + static: bool = True, ) -> dict[str, dict[str, bool]]: def get_from_mapping(target: str, attr: str) -> bool: if TYPE_CHECKING: @@ -1119,7 +1153,8 @@ def get_from_boolean(*_, **__) -> bool: get_expected = get_from_boolean result: dict[str, dict[str, bool]] = {} - new = cls.fetch_current_values(old) + new = cls.fetch_current_values(old, static) + failures: list[str] = [] for target, old_values in old.items(): new_values = new[target] cmp_results = result[target] = {} @@ -1129,15 +1164,31 @@ def get_from_boolean(*_, **__) -> bool: cmp_results[attr] = cmp_result = comparator( new_value, old_value, ) + format_msg = partial( + '{}: {}'.format, + f'Compared `{target}.{attr}` ' + f'(old: {old_value!r} @ {id(old_value):#x}; ' + f'new: {new_value!r} @ {id(new_value):#x})', + ) expected_result = get_expected(target, attr) - if assert_true and (cmp_result != expected_result): - assert False, ( - f'Comparing `{target}.{attr}` ' - f'(old: {old_value!r} @ {id(old_value):#x}; ' - f'new: {new_value!r} @ {id(new_value):#x}): ' - f'expected comparison with {comparator!r} to return ' - f'{expected_result}, got {cmp_result}' + if assert_true: + if cmp_result == expected_result: + message = format_msg( + f'comparison result with {comparator!r} is ' + f'{cmp_result} (as expected)' + ) + else: + message = format_msg( + f'expected comparison with {comparator!r} to ' + f'return {expected_result}, got {cmp_result}' + ) + failures.append(message) + else: + message = format_msg( + f'comparison result with {comparator!r}: {cmp_result}' ) + print(message) + assert (not failures), '\n'.join(failures) return result @@ -1745,35 +1796,125 @@ def _check_output(output: str, tag: str, nhits: int) -> None: # XXX: Tests in this section concerns implementation details, and the # tested APIs and behaviors MUST NOT be relied upon by end-users. -_GLOBAL_PATCHES = { - f'{load_pth_hook.__module__}.{load_pth_hook.__qualname__}': frozenset({ - 'called', - }), +_PatchSummary = Mapping[str, Set[str]] + +_GLOBAL_MINIMAL_PATCHES = { 'multiprocessing': frozenset({_PATCHED_MARKER}), - 'multiprocessing.process.BaseProcess': frozenset({ - '_bootstrap', # 'terminate', - }), 'multiprocessing.spawn': frozenset({'runpy'}), - 'multiprocessing.util': frozenset({ - 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', - }), 'os': frozenset({'fork'}), } +if not hasattr(os, 'fork'): # E.g. Windows + _GLOBAL_MINIMAL_PATCHES.pop('os') + + +def get_patched_attributes( + applied_mp_patches: Collection[str] | None = None, +) -> MappingProxyType[str, frozenset[str]]: + if applied_mp_patches is None: + applied_mp_patches = { + patch for patch, applied in ( + ConfigSource.from_config() + .get_subconfig('child_processes', 'multiprocessing', 'patches') + .conf_dict.items() + ) if applied + } + return _get_patched_attributes(frozenset(applied_mp_patches)) + + +@lru_cache() +def _get_patched_attributes( + applied_mp_patches: frozenset[str], +) -> MappingProxyType[str, frozenset[str]]: + # Get the contents of the individual patches + patches = _GLOBAL_MINIMAL_PATCHES.copy() + for patch in applied_mp_patches: + maybe_patch = MP_PATCHES.get(patch) # type: ignore + if maybe_patch: + for target, attrs in maybe_patch.summary.items(): + patches[target] = patches.get(target, frozenset()) | attrs + return MappingProxyType({ + target: frozenset(attrs) + for target, attrs in _filter_patches(patches).items() + }) + + +def _get_toml_patches_section(mp_patches: Collection[str]) -> str: + mp_patches_as_dict = {name: name in mp_patches for name in MP_PATCHES} + return ( + '[tool.line_profiler.child_processes.multiprocessing.patches]\n' + + '\n'.join( + f'{patch} = {str(applied).lower()}' + for patch, applied in mp_patches_as_dict.items() + ) + ) + + +def _summarize_patches( + summaries: Collection[tuple[bool, _PatchSummary]] +) -> dict[str, dict[str, bool]]: + """ + Example: + >>> _summarize_patches([(False, {'foo': {'bar'}})]) + {'foo': {'bar': False}} + >>> _summarize_patches([ # doctest: +NORMALIZE_WHITESPACE + ... (False, {'foo': {'bar', 'baz'}}), + ... (True, {'foo': {'baz', 'foobar'}, 'spam': {'ham'}}), + ... (False, {'foo': {'baz'}, 'spam': {'eggs'}}) + ... ]) + {'foo': {'bar': False, 'baz': True, 'foobar': True}, + 'spam': {'eggs': False, 'ham': True}} + """ + def get_all_mentioned(s: Iterable[_PatchSummary]) -> dict[str, set[str]]: + all_items: dict[str, set[str]] = {} + for summary in s: + for target, attrs in summary.items(): + all_items.setdefault(target, set()).update(attrs) + return all_items + + all_items = get_all_mentioned(s for _, s in summaries) + all_patched = get_all_mentioned(s for applied, s in summaries if applied) + result: dict[str, dict[str, bool]] = { + target: + {attr: attr in all_patched.get(target, set()) for attr in attrs} + for target, attrs in all_items.items() + } + # Normalize the order for convenience + return { + target: dict(sorted(attrs.items())) + for target, attrs in sorted(result.items()) + } -@pytest.fixture(scope='module') -def patched_attributes() -> MappingProxyType[str, frozenset[str]]: - result: dict[str, frozenset[str]] = {} - for target, attrs in _GLOBAL_PATCHES.items(): +def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: + result: dict[str, set[str]] = {} + for target, attrs in summary.items(): try: obj = _import_target(target) except ImportError: continue - present_attrs = frozenset(a for a in attrs if hasattr(obj, a)) + present_attrs = {a for a in attrs if hasattr(obj, a)} # Drop if none of the attributes is present if present_attrs: result[target] = present_attrs - return MappingProxyType(result) + return result + + +# This is only patched if we called +# `_line_profiler_hooks.load_pth_hook()` +_HOOK_PATCHES = { + f'{load_pth_hook.__module__}.{load_pth_hook.__qualname__}': + frozenset({'called'}), +} +# Upper limit of what we could've patched +_GLOBAL_PATCHES = { + **_GLOBAL_MINIMAL_PATCHES, + **_HOOK_PATCHES, + **get_patched_attributes(MP_PATCHES), +} +# Actual patches using the default config +DEFAULT_GLOBAL_PATCHES = { + **_GLOBAL_MINIMAL_PATCHES, **get_patched_attributes(), +} @pytest.mark.parametrize(('run_profiled_code', 'label1'), @@ -1913,33 +2054,59 @@ def test_cache_dump_load( assert set(os.environ) == envvars -@pytest.mark.parametrize(('wrap_os_fork', 'label1'), - [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) -@pytest.mark.parametrize(('debug', 'label2'), - [(True, 'with-debug'), (False, 'no-debug')]) +@(_Params.new(('wrap_os_fork', 'label1'), + [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) + + _Params.new(('debug', 'label2'), + [(True, 'with-debug'), (False, 'no-debug')]) + + _Params.new(('patch_pool', 'patch_process', 'intercept_logs', 'label3'), + [(True, True, True, 'all-patches'), + (True, True, False, 'pool-and-process'), + (True, False, True, 'pool-and-logging'), + (True, False, False, 'pool-only'), + (False, True, True, 'process-and-logging'), + (False, True, False, 'process-only'), + (False, False, True, 'logging-only'), + (False, False, False, 'no-patches')])).sorted() def test_cache_setup_main_process( + tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], - patched_attributes: MappingProxyType[str, frozenset[str]], wrap_os_fork: bool, debug: bool, - label1: str, label2: str, + patch_pool: bool, + patch_process: bool, + intercept_logs: bool, + label1: str, label2: str, label3: str, ) -> None: """ Test that :py:meth:`LineProfilingCache._setup_in_main_process` works as expected. """ - cache = create_cache(debug=debug) - # By default, we don't patch the `multiprocessing.util` logging - # facilities - patches: dict[str, dict[str, bool]] = { - target: dict.fromkeys(attrs, target != 'multiprocessing.util') - for target, attrs in patched_attributes.items() - } + mp_patches: set[str] = set() + if patch_pool: + mp_patches.add('pool') + if patch_process: + mp_patches.add('process') + if intercept_logs: + mp_patches.add('logging') + + config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' + config.write_text(_get_toml_patches_section(mp_patches)) + cache = create_cache(debug=debug, config=config) + + # Check that only the requested patches are applied + patches = _summarize_patches([ + (True, _GLOBAL_MINIMAL_PATCHES), + *( + (name in mp_patches, _filter_patches(patch.summary)) + for name, patch in MP_PATCHES.items() + ), + ]) try: patches['os']['fork'] = wrap_os_fork except KeyError: # `os.fork()` pruned because it doesn't exist on e.g. Windows assert not hasattr(os, 'fork') + with ExitStack() as stack: patched = stack.enter_context(_preserve_attributes(patches)) compare_patched = partial( @@ -2083,7 +2250,6 @@ def list_profiled_funcs() -> list[str]: @_preserve_attributes(_GLOBAL_PATCHES) def test_load_pth_hook( create_cache: Callable[..., LineProfilingCache], - patched_attributes: MappingProxyType[str, frozenset[str]], another_pid: int, ppid_should_match: bool | None, ) -> None: @@ -2122,8 +2288,7 @@ def test_load_pth_hook( cache.dump() compare = _preserve_attributes.compare_with_current_values - patches = patched_attributes.copy() - del patches['multiprocessing.util'] # Not patched by default + patches = {**DEFAULT_GLOBAL_PATCHES, **_HOOK_PATCHES} with _preserve_attributes(patches) as patched: try: # NOTE: this creates a cache instance that isn't @@ -2163,7 +2328,7 @@ def _test_apply_mp_patches( ext_module_object: ModuleType, test_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], - intercept_logs: bool, + mp_patches: Collection[str], fail: bool, n: int, nprocs: int, @@ -2185,17 +2350,14 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: ) config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' - cfg_chunks: list[str] = [] - if intercept_logs: - cfg_chunks.append( - '[tool.line_profiler.child_processes.multiprocessing]\n' - 'intercept_logs = true' - ) - # This is easier to debug than `ResultMismatch` - cfg_chunks.append( + intercept_logs = 'logging' in mp_patches + patch_process = 'process' in mp_patches + cfg_chunks: list[str] = [ + _get_toml_patches_section(mp_patches), + # This is easier to debug than `ResultMismatch` '[tool.line_profiler.child_processes.multiprocessing.polling]\n' - 'on_timeout = "error"' - ) + 'on_timeout = "error"', + ] config.write_text('\n\n'.join(cfg_chunks)) # Note: no need to test the case for `my_local_sum()` separately, @@ -2266,11 +2428,16 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests - patterns: dict[str, bool] = { - 'Cleanup succeeded.*: .*dump_stats.*' + re.escape(path.name): True - for path in Path(cache.cache_dir).glob('*.lprof') - if is_valid_stats_file(path) - } + patterns: dict[str, bool] = {} + if patch_process: + # Note: if we're not using `Process`-based patch, there is no + # guaratee that the profiling result is written via cleanup + iter_stats: Iterable[Path] = Path(cache.cache_dir).glob('*.lprof') + iter_stats = filter(is_valid_stats_file, iter_stats) + pat = 'Cleanup succeeded.*: .*dump_stats.*{}' + patterns.update({ + pat.format(re.escape(path.name)): True for path in iter_stats + }) patterns[re.escape('`multiprocessing` logging (debug)')] = intercept_logs _search_cache_logs(cache, True, patterns) @@ -2280,9 +2447,13 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: defaults='dummy') # We only need to check if `intercept_logs` work, the other # parametrizations don't matter - + _Params.new(('intercept_logs', 'label'), + + _Params.new(('intercept_logs', 'label1'), [(True, 'with-intercept-logs'), (False, 'no-intercept-logs')], defaults=(False, 'no-intercept-logs'))).sorted() +@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label2'), + [(True, True, 'pool-and-process'), + (True, False, 'pool-only'), + (False, True, 'process-only')]) @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) def test_apply_mp_patches_success( tmp_path_factory: pytest.TempPathFactory, @@ -2290,10 +2461,13 @@ def test_apply_mp_patches_success( ext_module_object: ModuleType, test_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], + patch_pool: bool, + patch_process: bool, intercept_logs: bool, n: int, nprocs: int, - label: str, + label1: str, + label2: str, ) -> None: """ Test that :py:func:`line_profiler._child_process_profiling\ @@ -2303,6 +2477,13 @@ def test_apply_mp_patches_success( See also: :py:func:`test_apply_mp_patches_failure` """ + mp_patches: list[str] = [] + if patch_pool: + mp_patches.append('pool') + if patch_process: + mp_patches.append('process') + if intercept_logs: + mp_patches.append('logging') with _check_warnings() as cw: cw.forbid_warnings(category=UserWarning, module='line_profiler') cw.forbid_warnings(module='multiprocessing') @@ -2312,16 +2493,27 @@ def test_apply_mp_patches_success( ext_module_object=ext_module_object, test_module_object=test_module_object, start_method=start_method, - intercept_logs=intercept_logs, + mp_patches=mp_patches, fail=False, n=n, nprocs=nprocs, ) -@pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) +@pytest.mark.retry( + _NUM_RETRIES, + exceptions=(ResultMismatch, _Poller.Timeout), + # Patching `Pool` should be foolproof no matter the platform + # (as long as we use `Pool` for our parallelism); + # `Process` though... + condition='not patch_pool', +) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) +@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label'), + [(True, True, 'pool-and-process'), + (True, False, 'pool-only'), + (False, True, 'process-only')]) @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) def test_apply_mp_patches_failure( tmp_path_factory: pytest.TempPathFactory, @@ -2329,8 +2521,11 @@ def test_apply_mp_patches_failure( ext_module_object: ModuleType, test_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], + patch_pool: bool, + patch_process: bool, n: int, nprocs: int, + label: str, ) -> None: """ Test that :py:func:`line_profiler._child_process_profiling\ @@ -2340,6 +2535,11 @@ def test_apply_mp_patches_failure( See also: :py:func:`test_apply_mp_patches_success` """ + mp_patches: list[str] = [] + if patch_pool: + mp_patches.append('pool') + if patch_process: + mp_patches.append('process') with _check_warnings() as cw: cw.forbid_warnings(category=UserWarning, module='line_profiler') cw.forbid_warnings(module='multiprocessing') @@ -2349,7 +2549,7 @@ def test_apply_mp_patches_failure( ext_module_object=ext_module_object, test_module_object=test_module_object, start_method=start_method, - intercept_logs=False, + mp_patches=mp_patches, fail=True, n=n, nprocs=nprocs, @@ -2629,8 +2829,6 @@ def test_profiling_multiproc_script_success( ) -@pytest.mark.retry(_NUM_RETRIES, - exceptions=(ResultMismatch, subprocess.TimeoutExpired)) @(_fuzz_prof_mp_markers[True]) @pytest.mark.parametrize(('nnums', 'nprocs'), [(2000, 3)]) def test_profiling_multiproc_script_failure( @@ -2787,8 +2985,6 @@ def test_profiling_bare_python_success( ) -@pytest.mark.retry(_NUM_RETRIES, - exceptions=(ResultMismatch, subprocess.TimeoutExpired)) @_fuzz_bare def test_profiling_bare_python_failure( tmp_path_factory: pytest.TempPathFactory, From 0e989337cc0f1e904f159ea2c7f985e14a6e67c1 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 8 May 2026 03:23:15 +0200 Subject: [PATCH 097/165] Reworked `multiprocessing_patches`... again line_profiler/_child_process_profiling/cache.py::LineProfilingCache _additional_data New free-for-all field allowing for unstructured data to be `.dump()`-ed and `.load()`-ed .dump(), .load(), ._from_path() Updated implementations to handle `._additional_data` _warn_possible_lack_of_stats() New method for arbitrary (child or main) process to communicate that some PIDs are expected to not have written to their profiling-stats file ("maybe-bad PIDs"); this fixes the stray empty-file warning sometimes emitted when the 'process' patch is not applied _get_pids_possibly_lacking_stats() New method for retrieving said PIDs line_profiler/_child_process_profiling/multiprocessing_patches.py Massively refactored. _PollerArgs.new() Refactored from `.from_config()` MPConfig New object for holding the submodule-specific config options extracted from `LineProfilingCache.load().config` _Patch New protocol for objects (e/g. `Patch`) capable of `.apply()`-ing themselves via a `LineProfilingCache` and providing a `.summary` of the patched entities Patch.add_method() New convenience method for adding a single (instance, class, or static) method to be patched wrap_repopulate_pool_static() New wrapper for `Pool._repopulate_pool_static()` which alerts the session cache to "maybe-bad PIDs"; this is needed because when wrapping only `Pool` and not `Process`, processes which exited without receiving a single task would leave their profiling-stats file empty (see `LineProfilingCache._warn_possible_lack_of_stats()` above) wrap_terminate(), wrap_bootstrap() Simplified implementations using `MPConfig` RebootForkserverPatch, RunpyPatch New helper `_Patch` classes consisting of functionalities previously embedded in `apply()` ResourceTrackerPatch New helper `_Patch` class patching `multiprocessing.resource_tracker.ResourceTracker` to set up the reporting of "maybe-bad PIDs"; this is needed because the resource-tracker server, while not directly handling the profiled parallel workload, is long-lived and may not have exited (and thus written profiling stats) at stats-gathering time; supersedes `cache.py::_DEFAULT_GATHER_STATS_EXCLUDES` apply() - Renamed parameter `lp_cache` -> `cache` - Simplified implementation because the components are all outsourced to `_Patch` objects tests/test_child_procs.py test_cache_dump_load() Added check for the round-tripping of `._additional_data` _test_apply_mp_patches() Fixed minor typing issue _GLOBAL_MINIMAL_PATCHES, DEFAULT_GLOBAL_PATCHES, etc. Updated definitions so that we check for the patching and restoration of the correct attributes --- .../_child_process_profiling/cache.py | 102 +++- .../multiprocessing_patches.py | 565 ++++++++++++++---- tests/test_child_procs.py | 79 ++- 3 files changed, 572 insertions(+), 174 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index f6a79edb..ac70eb8e 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -19,7 +19,6 @@ ) from functools import partial, cached_property, wraps from importlib import import_module -from operator import attrgetter from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent @@ -55,18 +54,9 @@ _PROFILING_OUTPUT_PREFIX_PATTERN = ( 'child-prof-output-{main_pid}-{current_pid}-{prof}-' ) -_DEFAULT_GATHER_STATS_EXCLUDES: set[tuple[str, str]] = { - # Note: the `ResourceTracker` server process is spawned when the - # first `multiprocessing` child process is created via the `spawn` - # or `forkserver` start method. While this server process does not - # meaningfully contribute to the profiling result either way, since - # it can be created with profiling set up, its longevity means that - # `LineProfilingCache.gather_stats()` often catches an empty .lprof - # file which it has occupied but not written to. To reduce noise - # while keeping the warning for other zero-length files, just - # explictly exclude said process - ('multiprocessing.resource_tracker', '_resource_tracker._pid'), -} +_POSSIBLE_EMPTY_STATS_PREFIX_PATTERN = ( + 'ignore-empty-stats-file-{main_pid}-{current_pid}-' +) def _import_sibling(submodule: str) -> ModuleType: @@ -103,6 +93,10 @@ class LineProfilingCache(Cleanup): ) _rlock: RLock = _private_field(default_factory=RLock) _dump_stats: Callable[..., None] | None = _private_field(default=None) + # These are unstructured fields; other components can decide on what + # to put in them. They are also pickled by `.dump()`, and are thus + # retrievable in `.load()`-ed instances. + _additional_data: dict[str, Any] = _private_field(default_factory=dict) _loaded_instance: ClassVar[LineProfilingCache | None] = None @@ -163,7 +157,10 @@ def dump(self) -> None: Note: Cleanup callbacks are not serialized. """ - content = self._get_init_args() + content = { + 'init_args': self._get_init_args(), + 'additional_data': self._additional_data, + } msg = f'Dumping instance data to {self.filename}: {content!r}' self._debug_output(msg) with open(self.filename, mode='wb') as fobj: @@ -234,23 +231,11 @@ def is_empty(path: Path) -> bool: filter_excludes: Callable[[Iterable[Path]], Iterable[Path]] if exclude_pids is None: - exclude_pids = set() - for import_target, attr in _DEFAULT_GATHER_STATS_EXCLUDES: - try: - module = import_module(import_target) - except ImportError: - continue - try: - maybe_pid = attrgetter(attr)(module) - except AttributeError: - maybe_pid = None - if maybe_pid is None: - continue - exclude_pids.add(cast(int, maybe_pid)) # NOTE: there is no guarantee that the PID hasn't previously # been used for another child process that we DID properly # profile and SHOULD include, so we only filter out empty # files + exclude_pids = self._get_pids_possibly_lacking_stats() filter_excludes = partial(filter, is_empty) else: # User-provided values, who are we to object? filter_excludes = iter @@ -314,11 +299,13 @@ def _gather_debug_log_entries( for entry in CacheLoggingEntry.from_file(log) ) + def _glob(self, *args, **kwargs) -> Iterable[Path]: + return Path(self.cache_dir).glob(*args, **kwargs) + def _get_debug_logfiles(self) -> Iterable[Path]: - pattern = _DEBUG_LOG_FILENAME_PATTERN.format( + return self._glob(_DEBUG_LOG_FILENAME_PATTERN.format( main_pid=self.main_pid, current_pid='?*', - ) - return Path(self.cache_dir).glob(pattern) + )) def _get_profiling_outfiles(self, pid: Any = '?*') -> Iterable[Path]: prefix = _PROFILING_OUTPUT_PREFIX_PATTERN.format( @@ -328,7 +315,7 @@ def _get_profiling_outfiles(self, pid: Any = '?*') -> Iterable[Path]: # `._setup_in_child_process()` prof='0x?*', ) - return Path(self.cache_dir).glob(prefix + '?*.lprof') + return self._glob(prefix + '?*.lprof') def inject_env_vars( self, env: MutableMapping[str, str] | None = None, @@ -652,6 +639,46 @@ def wrapper() -> int: self.patch(os, 'fork', wrapper, name='os') + def _warn_possible_lack_of_stats( + self, pids: int | Collection[int], + ) -> None: + """ + Register PID(s) which may have created a profiling stats file + without writing to it; when calling :py:meth:`.gather_stats`, + empty stats files associated with those PIDs are ignored instead + of warned against or treated as an error. + """ + if not isinstance(pids, Collection): + pids = pids, + with self._empty_stats_pid_registry.open(mode='a') as fobj: + print(*pids, sep='\n', file=fobj) + + def _get_pids_possibly_lacking_stats(self) -> set[int]: + """ + See also + :py:meth:`._warn_possible_lack_of_stats` + """ + prefix = _POSSIBLE_EMPTY_STATS_PREFIX_PATTERN.format( + main_pid=self.main_pid, + current_pid='?*', # Gather from all child processes + ) + result: set[int] = set() + for registry in self._glob(prefix + '?*.dat'): + from_reg: set[int] = set() + with registry.open() as fobj: + for line in fobj: + try: + from_reg.add(int(line)) + except ValueError: + pass + if from_reg: + self._debug_output( + f'Loaded {len(from_reg)} PID(s) possibly lacking ' + f'profiling output from {registry.name!r}: {from_reg!r}' + ) + result.update(from_reg) + return result + def make_tempfile(self, **kwargs) -> Path: """ Create a fresh tempfile under :py:attr:`~.cache_dir`. The other @@ -677,7 +704,10 @@ def _replace_loaded_instance(self, force: bool = False) -> bool: @classmethod def _from_path(cls, fname: os.PathLike[str] | str) -> Self: with open(fname, mode='rb') as fobj: - return cls(**pickle.load(fobj)) + content = pickle.load(fobj) + instance = cls(**content['init_args']) + instance._additional_data.update(content.get('additional_data', {})) + return instance def _get_init_args(self) -> dict[str, Any]: init_fields = [ @@ -844,3 +874,11 @@ def _config_source(self) -> ConfigSource: else: config = str(self.config) return ConfigSource.from_config(config) + + @cached_property + def _empty_stats_pid_registry(self) -> Path: + prefix = _POSSIBLE_EMPTY_STATS_PREFIX_PATTERN.format( + main_pid=self.main_pid, + current_pid=os.getpid(), + ) + return self.make_tempfile(prefix=prefix, suffix='.dat', delete=False) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 0e3cc190..67299fdb 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,20 +16,22 @@ import dataclasses import multiprocessing +import sys import warnings -from collections.abc import Callable, Collection, Mapping, Sequence +from collections.abc import Callable, Collection, Mapping, Sequence, Set from functools import partial from importlib import import_module from inspect import getattr_static, signature +from multiprocessing.context import BaseContext from multiprocessing.process import BaseProcess from multiprocessing.pool import Pool from operator import attrgetter from time import sleep, monotonic -from types import MappingProxyType, ModuleType +from types import MappingProxyType as mappingproxy, MethodType, ModuleType from typing import ( TYPE_CHECKING, Any, ClassVar, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, - cast, final, + cast, final, overload, ) from typing_extensions import Concatenate, ParamSpec, Self @@ -48,6 +50,12 @@ _CAN_USE_FORKSERVER = ( 'forkserver' in multiprocessing.get_all_start_methods() ) +try: + from multiprocessing import resource_tracker +except ImportError: + _CAN_USE_RESOURCE_TRACKER = False +else: + _CAN_USE_RESOURCE_TRACKER = True from .. import _diagnostics as diagnostics from ..toml_config import ConfigSource @@ -61,14 +69,18 @@ T = TypeVar('T') T1 = TypeVar('T1') T2 = TypeVar('T2') +P = TypeVar('P', bound=BaseProcess) +Pt = TypeVar('Pt', bound='_Patch') PS = ParamSpec('PS') PS1 = ParamSpec('PS1') PS2 = ParamSpec('PS2') _OnTimeout = Literal['ignore', 'warn', 'error'] -_PatchName = Literal['pool', 'process', 'logging'] +PublicPatch = Literal['pool', 'process', 'logging'] _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' _LOGGERS = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] +_PATCHES: dict[str, '_Patch'] = {} + # ------------------------------ Helpers ------------------------------- @@ -264,30 +276,94 @@ class _PollerArgs(NamedTuple): on_timeout: str | None @classmethod - def from_config(cls, config: ConfigSource) -> Self: - values = _get_config(config)['polling'] + def new(cls, cooldown: Any, timeout: Any, on_timeout: Any) -> Self: try: - cooldown = max(float(values['cooldown']), 0) + cd = max(float(cooldown), 0) except (TypeError, ValueError): - cooldown = 0 + cd = 0 try: - timeout = max(float(values['timeout']), 0) + to = max(float(timeout), 0) except (TypeError, ValueError): - timeout = 0 + to = 0 try: - on_timeout: str | None = values['on_timeout'].lower() + ot: str | None = on_timeout.lower() except Exception: # Fallback (use `_Poller`'s default) - on_timeout = None - return cls(cooldown, timeout, on_timeout) + ot = None + return cls(cd, to, ot) + + +@final +@dataclasses.dataclass +class MPConfig: + """ + Consolidate the config options into a structured object. + """ + catch_sigterm: bool + patches: dict[PublicPatch, bool] + polling: _PollerArgs + + def _get_terminate_poller( + self, cache: LineProfilingCache, process: BaseProcess, + ) -> _Poller: + cd, timeout, on_timeout = self.polling + if on_timeout not in ('ignore', 'warn', 'error'): + on_timeout = self.get_defaults().polling.on_timeout + # `_process_has_returned()` takes a `timeout` which it passes to + # `popen.wait()`; said timeout is essentially a limit as to how + # often the function is called, hence our cooldown + poller = _Poller.poll_until( + self._process_has_returned, process, cache, cd, + ) + return poller.with_timeout(timeout, cast(_OnTimeout, on_timeout)) + + @classmethod + def from_config(cls, config: ConfigSource) -> Self: + loaded = ( + config + .get_subconfig('child_processes', 'multiprocessing') + .conf_dict + ) + polling = _PollerArgs.new(**loaded['polling']) + return cls( + catch_sigterm=loaded['catch_sigterm'], + patches=dict(loaded['patches']), + polling=polling, + ) + + @classmethod + def from_cache(cls, cache: LineProfilingCache) -> Self: + key = 'mp_config' + try: + return cache._additional_data[key] + except KeyError: + config = cls.from_config(cache._config_source) + return cache._additional_data.setdefault(key, config) @classmethod def get_defaults(cls) -> Self: namespace = globals() + name = '_DEFAULT_CONFIG' try: - return namespace['_DEFAULT_POLLER_ARGS'] + return namespace[name] except KeyError: defaults = cls.from_config(ConfigSource.from_default(copy=False)) - return namespace.setdefault('_DEFAULT_POLLER_ARGS', defaults) + return namespace.setdefault(name, defaults) + + @staticmethod + def _process_has_returned( + proc: BaseProcess, cache: LineProfilingCache, timeout: float, + ) -> bool: + popen = getattr(proc, '_popen', None) + if popen is None: + msg, result = 'No associated process', True + else: + result = popen.wait(timeout) is not None + if result: + msg = f'Process {popen.pid} has returned' + else: + msg = f'Waiting for process {popen.pid} to return...' + cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') + return result class TaskWrapper(Generic[PS, T]): @@ -312,6 +388,47 @@ def __call__(self, *args, **kwargs) -> T: dump_stats() +def _no_op(*_, **__) -> None: + pass + + +# ---------------------- Patching infrastructure ----------------------- + + +class _Patch(Protocol): + """ + Interface for patches. + """ + def apply( + self, + cache: LineProfilingCache, + *, + cleanup: bool = True, + **kwargs + ) -> Any: + """ + Apply the patch. + + Args: + cache (LineProfilingCache): + Session cache + cleanup (bool): + Whether ``cache.cleanup()`` should reverse the patch + **kwargs + Individual implementations should pick the ones they + need and ignore the rest. + """ + ... + + @property + def summary(self) -> Mapping[str, Set[str]]: + """ + A mapping from dotted-path names of objects to the set of + attributes patched thereon. + """ + ... + + @dataclasses.dataclass class Patch: """ @@ -360,18 +477,71 @@ def add_target( Convenience method for gradually constructing the patch with a fluent interface. + Args: + target (str): + Dotted path to the object in :py:attr:`.submodule` + patches (Mapping[str, Callable[[Any], Any] \ +| Sequence[Callable[[Any], Any]]]): + Mapping from patched attrbute names to the wrappers to + apply thereto; sequences of wrappers are applied in + order + Returns: This instance """ self.targets.setdefault(target, {}).update(patches) return self + def add_method( + self, + target: str, + method: str, + wrapper: Callable[[Any], Any], + methodtype: ( + type[classmethod] | type[staticmethod] + | Literal['class', 'static'] | None + ) = None, + ) -> Self: + """ + Convenience method for gradually constructing the patch with a + fluent interface. + + Args: + target (str): + Dotted path to the object in :py:attr:`.submodule` + method (str): + Name of the (class, static, or instance) method to patch + wrapper (Callable[[Any], Any]): + Wrapping callable which takes the method-implementaion + callable and returns a wrapper thereof + methodtype (type[classmethod] | type[staticmethod] | \ +Literal['class', 'static'] | None): + Optional type of the method if not an instance method; + the strings ``'class'`` and ``'static'`` are respective + shorthands for :py:class:`classmethod` and + :py:class:`staticmethod` + + Returns: + This instance + """ + wrappers: Callable[[Any], Any] | list[Callable[[Any], Any]] + if methodtype is None: + wrappers = wrapper + else: + if methodtype == 'class': + methodtype = classmethod + elif methodtype == 'static': + methodtype = staticmethod + wrappers = [attrgetter('__func__'), wrapper, methodtype] + return self.add_target(target, {method: wrappers}) + def apply( self, cache: LineProfilingCache, *, cleanup: bool = True, static: bool = True, + **_ ) -> list[str]: """ Apply the patch. @@ -380,7 +550,7 @@ def apply( cache (LineProfilingCache): Session cache cleanup (bool): - Whether ``cache.cleanup()`` should reverse the patches + Whether ``cache.cleanup()`` should reverse the patch static (bool): Whether to use :py:func:`inspect.getattr_static` to retrieve to the attributes to be patched on the patch @@ -445,50 +615,46 @@ def module(self) -> str: return self._join(self.package, self.submodule) @property - def summary(self) -> MappingProxyType[str, frozenset[str]]: + def summary(self) -> mappingproxy[str, frozenset[str]]: """ Summary of the dotted paths to the patched objects and their patched attributes """ add_prefix = partial(self._join, self.module) - return MappingProxyType({ + return mappingproxy({ add_prefix(target): frozenset(patches) for target, patches in self.targets.items() }) -def _get_config(config: ConfigSource) -> Mapping[str, Any]: - cd = dict( - config.get_subconfig('child_processes', 'multiprocessing', copy=True) - .conf_dict - ) - assert isinstance(cd.get('patches'), Mapping) - assert isinstance(cd.get('polling'), Mapping) - return MappingProxyType({ - **cd, - 'patches': MappingProxyType(cd['patches']), - 'polling': MappingProxyType(cd['polling']), - }) - - -def _process_has_returned( - proc: BaseProcess, cache: LineProfilingCache, timeout: float, -) -> bool: - popen = getattr(proc, '_popen', None) - if popen is None: - msg, result = 'No associated process', True - else: - result = popen.wait(timeout) is not None - if result: - msg = f'Process {popen.pid} has returned' - else: - msg = f'Waiting for process {popen.pid} to return...' - cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') - return result +@overload +def _register_patch(name: str, patch: Pt) -> Pt: + ... -def _no_op(*_, **__) -> None: - pass +@overload +def _register_patch(name: str, patch: None = None) -> _Patch: + ... + + +def _register_patch(name: str, patch: _Patch | None = None) -> _Patch: + """ + Register the ``patch`` under ``name`` and return it as-is. If + ``patch`` isn't provided, look for the existing patch registered + under the name. + + Note: + Patches named with leading double underscores are applied no + matter the user input (e.g. via ``apply(..., patches=...)`` or + the config file). + """ + if patch is not None: + if _PATCHES.setdefault(name, patch) is not patch: + raise ValueError( + f'name = {name!r}, patch = {patch!r}: ' + 'name already in use by {_PATCHES[name]}' + ) + return _PATCHES[name] # ---------------- `multiprocessing.pool.Pool` patches ----------------- @@ -508,7 +674,7 @@ def wrap_get_tasks( processes before the parent process assumes control. Note: - :py:meth:`Pool._get_tasks` is a static method. + :py:meth:`.Pool._get_tasks` is a static method. """ return vanilla_impl(TaskWrapper(func), *args, **kwargs) @@ -531,6 +697,58 @@ def wrap_guarded_task_generation( return vanilla_impl(self, result_job, TaskWrapper(func), *args, **kwargs) +@LineProfilingCache._method_wrapper +def wrap_repopulate_pool_static( + cache: LineProfilingCache, + vanilla_impl: Callable[ + Concatenate[BaseContext, type[P], int, list[P], PS], None + ], + ctx: BaseContext, + Process: type[P], + processes: int, + pool: list[P], + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:meth:`.Pool._repopulate_pool_static` so that we can + keep track of the PIDs of the created child processes. + + Note: + :py:meth:`.Pool._repopulate_pool_static` is a static method. + """ + try: + vanilla_impl(ctx, Process, processes, pool, *args, **kwargs) + finally: + patches = MPConfig.from_cache(cache).patches + if not patches.get('process', False): + # Notes: + # - if the `process` patch is active, child processes should + # always have called `.cache.profiler.dump_stats()` at + # least once before exiting, so there's no need to warn + # the cache against children possibly not having written + # profiling data. + # - Since the vanilla implementation calls `Process.start()` + # on each child, they should all have valid PIDs. However: + # - The process might have terminated for whatever reason. + # and + # - `multiprocessing.dummy.DummyProcess` doesn't have + # `.pid`. + # So we add a fallback to `None` just in case. + pids = cast( + set[int], + {getattr(process, 'pid', None) for process in pool} - {None}, + ) + cache._warn_possible_lack_of_stats(pids) + + +_patch_pool = partial( + _register_patch('pool', Patch('pool')).add_method, 'Pool', +) +_patch_pool('_get_tasks', wrap_get_tasks, 'static') +_patch_pool('_guarded_task_generation', wrap_guarded_task_generation) +_patch_pool('_repopulate_pool_static', wrap_repopulate_pool_static, 'static') + # ----------- `multiprocessing.process.BaseProcess` patches ------------ @@ -541,7 +759,7 @@ def wrap_terminate( self: BaseProcess, ) -> None: """ - Wrap around :py:meth:`BaseProcess.terminate` to make sure that we + Wrap around :py:meth:`.BaseProcess.terminate` to make sure that we don't actually kill the child (OS-level) process before it has the chance to properly clean up. @@ -552,14 +770,8 @@ def wrap_terminate( after the performance-critical part of the code (said workload). """ try: - cd, timeout, on_timeout = _PollerArgs.from_config(cache._config_source) - if on_timeout not in ('ignore', 'warn', 'error'): - on_timeout = _PollerArgs.get_defaults().on_timeout - # `_process_has_returned()` takes a `timeout` which it passes to - # `popen.wait()`; said timeout is essentially a limit as to how - # often the function is called, hence our cooldown - poller = _Poller.poll_until(_process_has_returned, self, cache, cd) - with poller.with_timeout(timeout, cast(_OnTimeout, on_timeout)): + config = MPConfig.from_cache(cache) + with config._get_terminate_poller(cache, self): pass except _Poller.Timeout as e: # Also handles `~.TimeoutWarning` cache._debug_output(f'{type(e).__qualname__}: {e}') @@ -577,7 +789,7 @@ def wrap_bootstrap( *args: PS.args, **kwargs: PS.kwargs ) -> T: """ - Wrap around :py:meth:`BaseProcess._bootstrap` to run + Wrap around :py:meth:`.BaseProcess._bootstrap` to run ``LineProfilingCache.load().cleanup()`` so that profiling results can be gathered. @@ -590,13 +802,13 @@ def wrap_bootstrap( before it. Hence the ``# nocover``. - ``SIGTERM`` handling is not consistent on Windows, so we made - :py:meth:`LineProfilingCache._add_signal_handler` a no-op + :py:meth:`.LineProfilingCache._add_signal_handler` a no-op there. Hence :py:func:`wrap_terminate` remains necessary in mitigating unclean exits. """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up - if _get_config(cache._config_source)['catch_sigterm']: + if MPConfig.from_cache(cache).catch_sigterm: cache._add_signal_handler() try: return vanilla_impl(self, *args, **kwargs) @@ -610,6 +822,15 @@ def wrap_bootstrap( cache.cleanup(new_thread=True) +_patch_process = partial( + _register_patch('process', Patch('process')).add_method, 'BaseProcess', +) +_patch_process('_bootstrap', wrap_bootstrap) +# We only need to patch `Process.terminate()` if we can't do SIGTERM +# handling, i.e. on Windows +if sys.platform == 'win32': + _patch_process('terminate', wrap_terminate) + # --------------- `multiprocessing.util` logging patches --------------- @@ -634,7 +855,7 @@ def tee_log( **kwargs: PS.kwargs ) -> None: """ - Wrap around logging functions like +Wrap around logging functions like :py:func:`multiprocessing.util.debug` so that we can tee log messages from the package to our own logs. """ @@ -647,56 +868,165 @@ def get_msg(msg: str, *_, **__) -> str: ) -# -------------------------- Applying patches -------------------------- +_register_patch('logging', Patch('util')).add_target( + # The logging functions exists directly in the module namespace so + # no further attribute access is needed + '', {func: partial(partial, tee_log, func) for func in _LOGGERS}, +) + +# --------------------------- Misc. patches ---------------------------- -_PATCHES: dict[_PatchName, Patch] = { - 'process': Patch('process').add_target( - 'BaseProcess', - {'terminate': wrap_terminate, '_bootstrap': wrap_bootstrap}, - ), - 'pool': Patch('pool').add_target( - 'Pool', { - # `._get_task()` is a static method, so the wrapper function - # needs additional wrapping - '_get_tasks': [ - attrgetter('__func__'), wrap_get_tasks, staticmethod, - ], - '_guarded_task_generation': wrap_guarded_task_generation, - }, - ), - 'logging': Patch('util').add_target( - # The logging functions exists directly in the module namespace - # so no further attribute access is needed - '', {func: partial(partial, tee_log, func) for func in _LOGGERS}, - ), -} - - -def _stop_forkserver() -> None: +class RebootForkserverPatch: """ + Reboot the process backing the global + :py:class:`multiprocessing.forkserver.ForkServer` instance: + + - When the patch is applied, so as to ensure that child processes + forked therefrom actually receives the active patches; and + + - When the session cache is cleaned up, so that child processes + forked therefrom is no longer polluted by the patches. + Note: - This uses `ForkServer._stop()` which is private API, but it's - the same hack used in Python's own test suite -- see the comment - to said method + This uses + :py:method:`multiprocessing.forkserver.ForkServer._stop()` which + is private API, but it's the same hack used in Python's own test + suite -- see the comment to said method. """ - # Appease the type-checker since `._stop()` is not public API - stop = getattr(forkserver._forkserver, '_stop', None) - assert callable(stop) - stop() + summary: ClassVar[mappingproxy[str, frozenset[str]]] = mappingproxy({}) + + @classmethod + def apply(cls, cache: LineProfilingCache, **_) -> None: + if not _CAN_USE_FORKSERVER: + return + cls.reboot() + cache.add_cleanup(cls.reboot) + + @staticmethod + def reboot() -> None: + # Appease the type-checker since `._stop()` is not public API + stop = getattr(forkserver._forkserver, '_stop', None) + assert callable(stop) + stop() + + +class ResourceTrackerPatch: + """ + Patch :py:mod:`multiprocessing.resource_tracker` so that + :py:func:`multiprocessing.resource_tracker.ensure_running` and the + eponymous method of + :py:class:`multiprocessing.resource_tracker.ResourceTracker` report + the resource-tracker server PIDs to the session cache. + + Note: + The ``ResourceTracker`` server process is spawned when the first + :py:mod:`multiprocessing` child process is created via the + ``spawn`` or ``forkserver`` start methods. While this server + process does not meaningfully contribute to the profiling result + either way, since it can be created with profiling set up, its + longevity means that :py:meth:`.LineProfilingCache.gather_stats` + often catches empty .lprof files which it has occupied but not + written to. + + To reduce noise while keeping the empty-file warning for other + output files, we report the PIDs used by the server to the + session cache so that they can be ignored if necessary. + """ + if _CAN_USE_RESOURCE_TRACKER: + summary: ClassVar[mappingproxy[str, frozenset[str]]] = mappingproxy({ + 'multiprocessing.resource_tracker': + frozenset({'ensure_running'}), + 'multiprocessing.resource_tracker.ResourceTracker': + frozenset({'ensure_running'}), + }) + else: + summary = mappingproxy({}) + + @staticmethod + @LineProfilingCache._method_wrapper + def wrap_ensure_running( + cache: LineProfilingCache, + vanilla_impl: Callable[['resource_tracker.ResourceTracker'], None], + self: 'resource_tracker.ResourceTracker', + ) -> None: + """ + Wrap around :py:meth:`multiprocessing.resource_tracker\ +.ResourceTracker.ensure_running` + so that the session cache can keep track of the PIDs used by the + resource-tracer server. + """ + maybe_pids: set[int | None] = {getattr(self, '_pid', None)} + try: + vanilla_impl(self) + finally: + maybe_pids.add(getattr(self, '_pid', None)) + pids = cast(set[int], maybe_pids - {None}) + if pids: + cache._warn_possible_lack_of_stats(pids) + + @classmethod + def apply( + cls, cache: LineProfilingCache, *, cleanup: bool = True, **_, + ) -> list[str]: + if _CAN_USE_RESOURCE_TRACKER: + patch = partial(cache.patch, cleanup=cleanup) + # Patch the method on the class + method = resource_tracker.ResourceTracker.ensure_running + method = cls.wrap_ensure_running(method) + patch(resource_tracker.ResourceTracker, 'ensure_running', method) + # Patch the preexisting bound method on the module + instance = resource_tracker._resource_tracker + bound_method = MethodType(method, instance) + patch(resource_tracker, 'ensure_running', bound_method) + return list(cls.summary) + + +class RunpyPatch: + """ + Patch the copy of :py:mod:`runpy` in the + :py:mod:`multiprocessing.spawn` namespace so that subprocesses can + perform rewrite-based profiling as with + :py:func:`line_profiler.autoprofile.autoprofile.run`. + + See also: + :py:mod:`line_profiler._child_process_profiling.runpy_patches` + """ + summary: ClassVar[mappingproxy[str, frozenset[str]]] + if _CAN_USE_SPAWN and hasattr(spawn, 'runpy'): + summary = mappingproxy({'multiprocessing.spawn': frozenset({'runpy'})}) + else: + summary = mappingproxy({}) + + @classmethod + def apply( + cls, cache: LineProfilingCache, *, cleanup: bool = True, **_, + ) -> list[str]: + if cls.summary: + patch = partial(cache.patch, cleanup=cleanup) + patch(spawn, 'runpy', create_runpy_wrapper(cache)) + return list(cls.summary) + + +# See `ty` issue #3429 for why we need the casts +_register_patch('__reboot_forkserver', cast(_Patch, RebootForkserverPatch)) +_register_patch('__resource_tracker', cast(_Patch, ResourceTrackerPatch)) +_register_patch('__spawn_runpy', cast(_Patch, RunpyPatch)) + +# -------------------------- Applying patches -------------------------- def apply( - lp_cache: LineProfilingCache, + cache: LineProfilingCache, reboot_forkserver: bool = True, - patches: Collection[_PatchName] | None = None, + patches: Collection[PublicPatch] | None = None, ) -> None: """ Set up profiling in :py:mod:`multiprocessing` child processes by applying patches to the module. Args: - lp_cache (LineProfilingCache): + cache (LineProfilingCache): Cache instance governing the profiling run. reboot_forkserver (bool): Whether to reboot the global @@ -716,9 +1046,9 @@ def apply( methods so that parallel tasks write profiling output. ``'process'``: Patch :py:class:`multiprocessing.process.BaseProcess`'s - ``.terminate()`` and ``._bootstrap()`` methods so that child - processes write profiling output on exit and are given - enough time for that. + ``._bootstrap()`` method (and ``.terminate()`` on Windows) + so that child processes write profiling output on exit and + are given enough time for that. ``'logging'``: Patch :py:mod:`multiprocessing.util`'s logging methods (e.g. ``debug()`` and ``info()``) so that their messages are teed @@ -731,9 +1061,9 @@ def apply( - Immediately - - When ``lp_cache.cleanup()`` is run + - When ``cache.cleanup()`` is run - - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` + - Cleanup callbacks registered via ``cache.add_cleanup()`` Note: Rebooting the fork server is necessary because its process @@ -761,26 +1091,21 @@ def apply( if getattr(multiprocessing, _PATCHED_MARKER, False): return if patches is None: - config = _get_config(lp_cache._config_source)['patches'] - patches_ = {patch for patch, applied in config.items() if applied} + patches_dict = MPConfig.from_cache(cache).patches + patches_: set[str] = {p for p, use in patches_dict.items() if use} else: patches_ = {p.lower() for p in patches} - # Patch `multiprocessing.spawn` - if _CAN_USE_SPAWN and hasattr(spawn, 'runpy'): - lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) - # Patch methods/functions in these entities: - # - `multiprocessing.pool.Pool` - # - `multiprocessing.process.BaseProcess` - # - `multiprocessing.util` for name, patch in _PATCHES.items(): if name in patches_: - patch.apply(lp_cache) - # Stop the current `ForkServer` server process: - # - Now, so that the (rebooted) fork-server process has profiling - # set up; and - # - Also as a part of cache cleanup - if _CAN_USE_FORKSERVER and reboot_forkserver: - _stop_forkserver() - lp_cache.add_cleanup(_stop_forkserver) + should_apply = True + elif name.startswith('__'): + should_apply = (name != '__reboot_forkserver' or reboot_forkserver) + else: + should_apply = False + if should_apply: + msg = f'applying `multiprocessing` patch {name!r}' + cache._debug_output(msg.capitalize() + '...') + patch.apply(cache) + cache._debug_output('Done with ' + msg) # Mark `multiprocessing` as having been patched - lp_cache.patch(multiprocessing, _PATCHED_MARKER, True) + cache.patch(multiprocessing, _PATCHED_MARKER, True) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 6216d3be..19c7ca0f 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -3,6 +3,7 @@ import dataclasses import enum import inspect +import itertools import multiprocessing.pool import operator import os @@ -1798,14 +1799,6 @@ def _check_output(output: str, tag: str, nhits: int) -> None: _PatchSummary = Mapping[str, Set[str]] -_GLOBAL_MINIMAL_PATCHES = { - 'multiprocessing': frozenset({_PATCHED_MARKER}), - 'multiprocessing.spawn': frozenset({'runpy'}), - 'os': frozenset({'fork'}), -} -if not hasattr(os, 'fork'): # E.g. Windows - _GLOBAL_MINIMAL_PATCHES.pop('os') - def get_patched_attributes( applied_mp_patches: Collection[str] | None = None, @@ -1827,11 +1820,11 @@ def _get_patched_attributes( ) -> MappingProxyType[str, frozenset[str]]: # Get the contents of the individual patches patches = _GLOBAL_MINIMAL_PATCHES.copy() - for patch in applied_mp_patches: - maybe_patch = MP_PATCHES.get(patch) # type: ignore - if maybe_patch: - for target, attrs in maybe_patch.summary.items(): - patches[target] = patches.get(target, frozenset()) | attrs + iter_summaries = ( + MP_PATCHES[patch].summary + for patch in applied_mp_patches if patch in MP_PATCHES + ) + patches = _get_patch_summary_union(patches, *iter_summaries) return MappingProxyType({ target: frozenset(attrs) for target, attrs in _filter_patches(patches).items() @@ -1849,6 +1842,16 @@ def _get_toml_patches_section(mp_patches: Collection[str]) -> str: ) +def _get_patch_summary_union( + *summaries: _PatchSummary, +) -> dict[str, frozenset[str]]: + result: dict[str, frozenset[str]] = {} + for summary in summaries: + for target, attrs in summary.items(): + result[target] = result.get(target, frozenset()) | frozenset(attrs) + return result + + def _summarize_patches( summaries: Collection[tuple[bool, _PatchSummary]] ) -> dict[str, dict[str, bool]]: @@ -1899,6 +1902,30 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: return result +_GLOBAL_MINIMAL_PATCHES = { + 'multiprocessing': frozenset({_PATCHED_MARKER}), +} +# Get patches that are dynamically resolved: while these patches are +# always applied, some of the patch targets are +# platform-/Pyhon-version-specific and may not always exist +_dynamically_resolved_patch_summaries: Iterable[_PatchSummary] = ( + patch.summary for name, patch in MP_PATCHES.items() + # Basic `multiprocessing` patches are always applied + if name.startswith('__') +) +_dynamically_resolved_patch_summaries = itertools.chain( + _dynamically_resolved_patch_summaries, + # some platforms e.g. Windows don't have `fork()` + [{'os': frozenset({'fork'})}], +) +_dynamically_resolved_patch_summaries = cast( # See `ty` issue #3428 + Iterable[_PatchSummary], + map(_filter_patches, _dynamically_resolved_patch_summaries), +) +_GLOBAL_MINIMAL_PATCHES = _get_patch_summary_union( + _GLOBAL_MINIMAL_PATCHES, *_dynamically_resolved_patch_summaries, +) + # This is only patched if we called # `_line_profiler_hooks.load_pth_hook()` _HOOK_PATCHES = { @@ -1906,15 +1933,17 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: frozenset({'called'}), } # Upper limit of what we could've patched -_GLOBAL_PATCHES = { - **_GLOBAL_MINIMAL_PATCHES, - **_HOOK_PATCHES, - **get_patched_attributes(MP_PATCHES), -} +_GLOBAL_PATCHES = _get_patch_summary_union( + _GLOBAL_MINIMAL_PATCHES, + _HOOK_PATCHES, + get_patched_attributes([ + name for name in MP_PATCHES if not name.startswith('__') + ]), +) # Actual patches using the default config -DEFAULT_GLOBAL_PATCHES = { - **_GLOBAL_MINIMAL_PATCHES, **get_patched_attributes(), -} +DEFAULT_GLOBAL_PATCHES = _get_patch_summary_union( + _GLOBAL_MINIMAL_PATCHES, get_patched_attributes(), +) @pytest.mark.parametrize(('run_profiled_code', 'label1'), @@ -2030,6 +2059,8 @@ def test_cache_dump_load( envvars: set[str] = set(os.environ) try: original.inject_env_vars() # Needed for `.load()` + # Also test slipping stuff into the `._additional_data` + original._additional_data['foo'] = [1, 'string', None] try: # Env vars should be inserted assert set(os.environ) == envvars.union(original.environ) > envvars @@ -2047,6 +2078,8 @@ def test_cache_dump_load( getattr(original, field.name) == getattr(loaded, field.name) ) + # Compare `._additional_data` + assert original._additional_data == loaded._additional_data finally: # Explicitly cleanup for cache in cache_instances: cache.cleanup() @@ -2433,7 +2466,9 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # Note: if we're not using `Process`-based patch, there is no # guaratee that the profiling result is written via cleanup iter_stats: Iterable[Path] = Path(cache.cache_dir).glob('*.lprof') - iter_stats = filter(is_valid_stats_file, iter_stats) + iter_stats = cast( # See `ty` issue #3428 + Iterable[Path], filter(is_valid_stats_file, iter_stats), + ) pat = 'Cleanup succeeded.*: .*dump_stats.*{}' patterns.update({ pat.format(re.escape(path.name)): True for path in iter_stats From 51ee622a59d9e7a527d88ffddf9174d7d28e4f33 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 11 May 2026 20:07:15 +0200 Subject: [PATCH 098/165] Retry tests for consistency instead of leniency tests/conftest.py RetryMarker Refactored from `_RetryHelper`: - `pytest_pyfunc_call()` now issues a warning for malformed `@pytest.mark.retry(...)` instead of erroring out - New attribute `require` for controlling whether to require the test to pass on 'all' or 'any' retries; old and default behavior equivalent to `'any'` - New attribute `is_active` for representing `@pytest.mark.retry(...)` setups where retries shouldn't be attempted - Constructor method `from_test_func()` refactored from `get_helper()`, and now always return an instance instead of `None` - New constructor method `from_arguments()` with doctest RetryMarkerWarning New warning class for malformed `@pytest.mark.retry(...)` markers RetryConditionFailure Refactored from `_RetryFailure` tests/test_retry_tests.py Simplified implementation of `_TestModule` fixtures test_bad_markers() New test that malformed `@pytest.mark.retry(...)` results in the appropriate warnings instead of test failure test_requirement() New test that `@pytest.mark.retry(require=...)` works as expected tests/test_child_procs.py::test_apply_mp_patches_failure() Now retrying with `require='all'` to test AGAINST flakiness --- tests/conftest.py | 476 ++++++++++++++++++++++++++++++-------- tests/test_child_procs.py | 1 + tests/test_retry_tests.py | 451 +++++++++++++++++++++++++----------- 3 files changed, 703 insertions(+), 225 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cfbf0c33..205ecf8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,20 +1,35 @@ """ A simple :py:deco:`pytest.mark.retry` decorator. -Function-scoped fixtures are re-fetched bewteen retries. +Function-scoped fixtures are re-fetched between retries. + +Note: + - This file is designed to also function as a standalone + ``conftest.py`` file. + + - Adapted from `pytest-mark-retry`_. + +.. _pytest-mark-retry: https://gitlab.com/TTsangSC/pytest-mark-retry """ from __future__ import annotations +import ast import dataclasses +import os +import sys import warnings from collections.abc import ( Callable, Collection, Generator, Hashable, Iterable, Mapping, ) -from functools import cached_property, partial +from functools import cached_property, lru_cache, partial +from importlib.util import find_spec +from inspect import Signature, signature from operator import contains from pathlib import Path from typing import ( - TYPE_CHECKING, Any, ClassVar, Literal, Protocol, TypeVar, cast, final, + TYPE_CHECKING, + Any, ClassVar, Literal, Protocol, TypedDict, TypeVar, cast, overload, ) +from typing_extensions import Self import pytest from _pytest.compat import NOTSET @@ -23,14 +38,24 @@ from _pytest.scope import Scope from _pytest.unittest import TestCaseFunction try: - from pytest import TerminalReporter + from pytest import TerminalReporter # type: ignore except ImportError: # pytest < ~8.4 - from _pytest.terminal import TerminalReporter + from _pytest.terminal import TerminalReporter # type: ignore +__all__ = ( + 'RetryMarker', + 'RetryMarkerWarning', + 'RetryConditionFailure', + 'pytest_addhooks', + 'pytest_configure', + 'pytest_terminal_summary', +) + _Status = Literal['passed', 'failed', 'skipped'] +_Require = Literal['any', 'all'] F = TypeVar('F', bound=TestCaseFunction) -FCls = TypeVar('FCls', bound=type[TestCaseFunction]) +FCls = TypeVar('FCls', bound='type[TestCaseFunction]') T = TypeVar('T') _FUNCTION_SCOPE = Scope.Function @@ -41,12 +66,41 @@ def __call__(self, *, pyfuncitem: pytest.Function) -> Any: ... -class _RetryFailure(RuntimeError): +class _RetryError(RuntimeError): + """ + Base class for errors associated with retrying tests. + """ + + +class RetryMarkerWarning(_RetryError, UserWarning): + """ + Warning issued when the :deco:`pytest.mark.retry` markers on a test + fail to resolve to a valid :py:class:`RetryMarker` instance. + """ + @classmethod + def warn_from_error( + cls, xc: Exception, *args, **kwargs + ) -> None: + msg = 'disregarding invalid `@pytest.mark.retry` marker' + msg = f'{msg}: ({_format_exception(xc)})' + if 'PYTEST_CURRENT_TEST' in os.environ: + msg = f'{os.environ["PYTEST_CURRENT_TEST"]}: {msg}' + if sys.version_info < (3, 12): # Compatibility + kwargs.pop('skip_file_prefixes', None) + return warnings.warn(msg, cls, *args, **kwargs) + + +class RetryConditionFailure(_RetryError): + """ + Error raised when an attempt to retry a test failed because we + can't :py:func:`eval` the condition. + """ def __init__( self, previous_error: Exception, condition_error: Exception, - condition: str | None = None + # Should't be here if the condition isn't a string, but whatever + condition: str | bool | None = None ) -> None: self.previous_error = previous_error self.condition_error = condition_error @@ -57,23 +111,15 @@ def __init__( self.__cause__ = condition_error def _format_message(self) -> str: - prev = self._format_exception(self.previous_error) + prev = _format_exception(self.previous_error) if len(prev.split()) > 1: prev = f'({prev})' - condition = self._format_exception(self.condition_error) + condition = _format_exception(self.condition_error) if self.condition: condition = f'(condition: {self.condition!r} -> {condition})' return '{} -> {}'.format(prev, condition) - @staticmethod - def _format_exception(xc: Exception) -> str: - msg = type(xc).__name__ - if str(xc): - return f'{msg}: {xc}' - return msg - -@final @dataclasses.dataclass class _RetryEntry: func: pytest.Function @@ -93,7 +139,7 @@ def _get_name(self, with_params: bool) -> str: return name @classmethod - def add_entry(cls, func: pytest.Function, *args, **kwargs) -> _RetryEntry: + def add_entry(cls, func: pytest.Function, *args, **kwargs) -> Self: assert func.config is not None entry = cls(func, *args, **kwargs) entry.get_entries(func.config).append(entry) @@ -141,32 +187,105 @@ def _name_prefixes(self) -> tuple[str, str]: _RETRY_ENTRIES_KEY = pytest.StashKey[list[_RetryEntry]]() -_NEXTITEMS_KEY = pytest.StashKey[dict[pytest.Item, pytest.Item | None]]() -@final +def _retry_marker_sig_helper( + retries=1, *, + exceptions=None, reset_fixtures=None, condition=None, require=None, +): + """ + Dummy callable helping with :py:meth:`RetryMarker.from_test_func`, + so that we can handle marker stacking in a sane way withou having to + special-case the defaults of the class constructor. + """ + pass + + +if sys.version_info >= (3, 10): + _keyword = partial(dataclasses.field, kw_only=True) +else: + _keyword = dataclasses.field + + +class _RetryMarkerArgs(TypedDict, total=False): + retries: int + exceptions: type[Exception] | tuple[type[Exception], ...] + reset_fixtures: bool | Collection[str] + condition: str | bool | None + require: _Require + + @dataclasses.dataclass -class _RetryHelper: +class RetryMarker: + """ + Object representing the :deco:`pytest.mark.retry` marks on a test + function, managing test retries. + + Attributes: + retries (int): + Number of retries to attempt; should be positive. + exceptions (type[Exception] | tuple[type[Exception], ...]): + "Allowed" exception type(s) which result in retries; + mismatching exceptions are propagated normally, resuling in + a failure + reset_fixtures (bool | Collection[str]): + Whether to reset the function-scoped fixtures when retrying; + if a collection of names, only reset matching fixtures + condition (str | bool | None): + Only attempt retries if this is true (or :py:const:`None`); + if a string, it is :py:func:`eval`-ed to the condition + before each retry using the globals of the test function, + and the fixtures and parametrizations as the locals + require (Literal['any', 'all']): + Whether 'any' or 'all' attempts to run a test function + should pass for the test to pass + + See also: + :py:meth:`.from_arguments` for examples + """ retries: int = 1 - exceptions: type[Exception] | tuple[type[Exception], ...] = () - reset_fixtures: bool | Collection[str] = True - condition: str | None = None + exceptions: type[Exception] | tuple[type[Exception], ...] = ( + _keyword(default=Exception) + ) + reset_fixtures: bool | Collection[str] = _keyword(default=True) + condition: str | bool | None = _keyword(default=None) + require: _Require = _keyword(default='any') + name: ClassVar[str] = 'retry' + _sig: ClassVar[Signature] = signature(_retry_marker_sig_helper) def __post_init__(self) -> None: - if not (int(self.retries) == self.retries > 0): - raise TypeError( - f'.entries = {self.retries!r}: expected a positive integer' - ) + # Normalize `.retries` + try: + self.retries = max(0, int(self.retries)) + except Exception as e: + msg = f'.retries = {self.retries!r}' + msg = f'{msg}: not a valid number {_format_exception(e)}' + raise TypeError(msg).with_traceback(e.__traceback__) + # Check `.exceptions` if isinstance(self.exceptions, tuple): xc: tuple[type[Exception], ...] = self.exceptions else: xc = self.exceptions, - if not all(issubclass(X, Exception) for X in xc): + if not all( + isinstance(X, type) and issubclass(X, Exception) for X in xc + ): raise TypeError( f'.exceptions = {self.exceptions!r}: ' 'expected an exception type or a tuple thereof' ) + # Check `.condition` + if isinstance(self.condition, str): + try: + ast.parse(self.condition, mode='eval') + except Exception as e: + msg = f'.condition = {self.condition!r}' + msg = f'{msg}: not a valid expression ({_format_exception(e)})' + raise ValueError(msg).with_traceback(e.__traceback__) + # Check `.require` + if self.require not in ('all', 'any'): + msg = f'.require = {self.require!r}: expected \'any\' or \'all\'' + raise TypeError(msg) def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: """ @@ -204,32 +323,34 @@ def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: for i in range(1 + self.retries): if i: reset_fixtures(func) - if self.condition: - cond, error = self._check_condition(self.condition, func) - if error: # Bail - # XXX: would be nice if we can directly force an - # internal error, but that doesn't seem to be - # possible from within `pytest_pyfunc_call()`; - # directly calling `pytest_internalerror()` - # results in botched teardown and weird - # tracebacks, and leaves the test session in a - # bad state... - assert xc is not None - raise _RetryFailure(xc, cond, self.condition) - if not cond: - i -= 1 - break + cond, error = self._check_condition(self.condition, func) + if error: # Bail + # XXX: would be nice if we can directly force an + # internal error, but that doesn't seem to be + # possible from within `pytest_pyfunc_call()`; + # directly calling `pytest_internalerror()` + # results in botched teardown and weird + # tracebacks, and leaves the test session in a + # bad state... + assert xc is not None + raise RetryConditionFailure(xc, cond, self.condition) + if not cond: + i -= 1 + break try: result = impl(pyfuncitem=func) except self.exceptions as e: - # `ty` doesn't agree that `e` is an exception... + # `ty` doesn't agree that `e` is an exception (#3432)... xc = cast(Exception, e) + if self.require == 'all': + break except Exception as e: # Uncaught exc. -> break to raise xc = e break else: # Correct execution -> break to return xc = None - break + if self.require == 'any': + break if i: if xc is None: status: _Status = 'passed' @@ -245,8 +366,15 @@ def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: @staticmethod def _check_condition( - condition: str, func: pytest.Function, + condition: str | bool | None, func: pytest.Function, ) -> tuple[Any, Literal[False]] | tuple[Exception, Literal[True]]: + if condition in (True, None): # Always retry + return (True, False) + if condition in (False,): # Never retry + return (False, False) + + if TYPE_CHECKING: # Help narrowing + assert isinstance(condition, str) global_ns: dict[str, Any] | None = None try: global_ns = func.obj.__globals__ @@ -347,54 +475,192 @@ def pytest_pyfunc_call(cls, pyfuncitem: pytest.Function) -> Any: number of retries if necessary. """ pm = pyfuncitem.config.pluginmanager - impl: _PyfuncCallImpl = pm.subset_hook_caller( + pytest_pyfunc_call: _PyfuncCallImpl = pm.subset_hook_caller( 'pytest_pyfunc_call', [cls], ) - helper = cls.get_helper(pyfuncitem) - if helper: - return helper.manage_call(impl, pyfuncitem) - return impl(pyfuncitem=pyfuncitem) + try: + helper = cls.from_test_func(pyfuncitem) + except Exception as e: + # Level 1 is the `.warn_from_error()` frame, 2 is here, 3 is + # where the error actually happened + warn = RetryMarkerWarning.warn_from_error + skip_ = {_find_module_path('_pytest'), _find_module_path('pluggy')} + skip = cast(tuple[str, ...], tuple(skip_ - {None})) + warn(e, stacklevel=3, skip_file_prefixes=skip) + else: + if helper.is_active: + return helper.manage_call(pytest_pyfunc_call, pyfuncitem) + return pytest_pyfunc_call(pyfuncitem=pyfuncitem) + + @classmethod + def from_test_func(cls, func: pytest.Function, /) -> Self: + """ + Returns: + Instance combining the stack of :deco:`pytest.mark.retry` + decorators on the :py:class:`pytest.Function`. + """ + marks = (m for m in func.iter_markers() if m.name == cls.name) + return cls.from_arguments(cls._get_marker_args(mark) for mark in marks) + + @classmethod + @overload + def from_arguments(cls, args: Iterable[_RetryMarkerArgs] = (), /) -> Self: + ... + + @classmethod + @overload + def from_arguments(cls, *args: _RetryMarkerArgs) -> Self: + ... @classmethod - def get_helper(cls, pyfuncitem: pytest.Function) -> _RetryHelper | None: + def from_arguments(cls, *args) -> Self: + """ + Invocations: + (, ...) -> RetryMarker + ([, ...]) -> RetryMarker + + Examples: + >>> empty = RetryMarker.from_arguments() + >>> assert not empty.retries + >>> assert empty == RetryMarker.from_arguments([]) + + >>> default = RetryMarker.from_arguments({}) + >>> assert default.retries == 1 + >>> assert default.exceptions in ((Exception,), Exception) + >>> assert default.reset_fixtures == True + >>> assert default.condition is None + >>> assert default.require == 'any' + >>> assert default.is_active + >>> assert default == RetryMarker.from_arguments([{}]) + + Some arguments result in inactive instances (i.e. no + retries): + + >>> bad_xc = RetryMarker.from_arguments({'exceptions': ()}) + >>> assert not bad_xc.exceptions + >>> assert not bad_xc.is_active + + >>> bad_retries = RetryMarker.from_arguments( + ... {}, {}, {'retries': -5}, + ... ) + >>> assert not bad_retries.retries + >>> assert not bad_retries.is_active + + >>> bad_cond = RetryMarker.from_arguments( + ... {'condition': False}, + ... ) + >>> assert bad_cond.condition == False + >>> assert not bad_cond.is_active + + Congruent values are unioned: + + >>> stacked_xcs = RetryMarker.from_arguments([ + ... {'exceptions': ()}, + ... {'exceptions': ValueError}, + ... {'retries': 3, 'exceptions': (TypeError, OSError)}, + ... ]) + >>> assert stacked_xcs.retries == 5 + >>> assert set(stacked_xcs.exceptions) == { + ... ValueError, TypeError, OSError, + ... } + + >>> stacked_resets_1 = RetryMarker.from_arguments( + ... {'reset_fixtures': ['foo', 'bar']}, + ... {'reset_fixtures': ['baz']}, + ... ) + >>> sorted(stacked_resets_1.reset_fixtures) + ['bar', 'baz', 'foo'] + + Incongruent values override one another: + + >>> stacked_resets_2 = RetryMarker.from_arguments( + ... {'reset_fixtures': ['foo', 'bar']}, + ... {'reset_fixtures': False}, + ... ) + >>> assert stacked_resets_2.reset_fixtures == False + + >>> stacked_conditions = RetryMarker.from_arguments( + ... {'condition': 'foo==bar'}, + ... {'condition': False}, + ... ) + >>> assert stacked_conditions.condition == False + >>> assert not stacked_conditions.is_active + + >>> stacked_requires = RetryMarker.from_arguments( + ... {'require': 'any'}, + ... {'require': 'all'}, + ... ) + >>> assert stacked_requires.require == 'all' + """ retries: int = 0 - xc: set[type[Exception]] = set() + xc: set[type[Exception]] | None = None reset_fixtures: bool | set[str] = True condition: bool | str | None = None - for mark in pyfuncitem.iter_markers(): - if mark.name != cls.name: - continue - instance = cls(*mark.args, **mark.kwargs) - retries += instance.retries - condition = instance.condition - if isinstance(instance.exceptions, tuple): - xc.update(instance.exceptions) - else: - xc.add(instance.exceptions) - if ( - reset_fixtures not in (True, False) - and instance.reset_fixtures not in (True, False) - ): # Both collections of fixture names - if TYPE_CHECKING: - assert not isinstance(reset_fixtures, bool) - assert not isinstance(instance.reset_fixtures, bool) - reset_fixtures.update(instance.reset_fixtures) - elif instance.reset_fixtures not in (True, False): - if TYPE_CHECKING: - assert not isinstance(instance.reset_fixtures, bool) - reset_fixtures = set(instance.reset_fixtures) - else: - reset_fixtures = bool(instance.reset_fixtures) - if not retries: - return None - if not (condition is None or isinstance(condition, str)): - if condition: - condition = None + require: _Require | None = None + + if args: + if isinstance(args[0], Mapping): + iter_args = args else: - return None - if not xc: - xc = {Exception} - return cls(retries, tuple(xc), reset_fixtures, condition) + assert len(args) == 1 + iter_args = args[0] + else: + iter_args = args + + # `ty` needs some help here... hence the `cast()` + for bound_args in cast(Iterable[_RetryMarkerArgs], iter_args): + retries += bound_args.get('retries', 1) + if 'exceptions' in bound_args: + xc_new = bound_args['exceptions'] + if xc is None: + xc = set() + if isinstance(xc_new, type): + xc.add(xc_new) + else: + xc.update(xc_new) + if 'reset_fixtures' in bound_args: + rf_new = bound_args['reset_fixtures'] + if isinstance(rf_new, Collection): + if isinstance(reset_fixtures, Collection): + reset_fixtures.update(rf_new) + else: + reset_fixtures = set(rf_new) + else: + reset_fixtures = bool(rf_new) + if 'condition' in bound_args: + condition = bound_args['condition'] + if 'require' in bound_args: + require = bound_args['require'] + + kwargs: _RetryMarkerArgs = { + 'retries': retries, 'reset_fixtures': reset_fixtures, + } + if xc is not None: + kwargs['exceptions'] = tuple(xc) + if condition is not None: + kwargs['condition'] = condition + if require is not None: + kwargs['require'] = require + return cls(**kwargs) + + @classmethod + def _get_marker_args(cls, mark: pytest.Mark) -> _RetryMarkerArgs: + args = cls._sig.bind(*mark.args, **mark.kwargs).arguments + return cast(_RetryMarkerArgs, args) + + @property + def is_active(self) -> bool: + """ + Whether the instance should possibly attempt retries in any + condition + """ + if not self.retries: + return False + if self.exceptions == (): + return False + if self.condition in (False,): + return False + return True def _pluralize(noun: str, count: int, plural: str | None = None) -> str: @@ -403,13 +669,33 @@ def _pluralize(noun: str, count: int, plural: str | None = None) -> str: return f'{count} {noun if count == 1 else plural}' +def _format_exception(xc: Exception) -> str: + msg = type(xc).__name__ + if str(xc): + return f'{msg}: {xc}' + return msg + + +@lru_cache() +def _find_module_path(module: str) -> str | None: + spec = find_spec(module) + if spec is None or spec.origin is None: + return None + file = Path(spec.origin) + if not file.exists(): + return None + if file.name == '__init__.py': # Package + file = file.parent + return str(file) + + def pytest_addhooks(pluginmanager: pytest.PytestPluginManager) -> None: """ - Register :py:class:`_RetryHelper` as a plugin so that its + Register :py:class:`RetryMarker` as a plugin so that its :py:meth:`pytest_pyfunc_call` method can safely call other implementations without recursing to itself. """ - pluginmanager.register(_RetryHelper) + pluginmanager.register(RetryMarker) def pytest_configure(config: pytest.Config) -> None: @@ -417,8 +703,8 @@ def pytest_configure(config: pytest.Config) -> None: Register the :py:deco:`pytest.mark.retry` marker. """ help_text = ' '.join(""" - retry(retries=1, exceptions=Exception, \ -reset_fixtures=True, condition=None): + retry(retries=1, *, exceptions=Exception, \ +reset_fixtures=True, condition=None, require='any'): mark the test for retrying upon failure. @@ -436,7 +722,11 @@ def pytest_configure(config: pytest.Config) -> None: if a boolean, only retry if true; if a string, only retry if it `eval()`s true (w/globals of the test module and locals from the fixtures - and parametrizations) + and parametrizations); + require (Literal['any', 'all']): + If 'any', stop retrying and record a test pass if ANY + attempt passes; + if 'all', only record a test pass if ALL attempts pass """.split()) config.addinivalue_line('markers', help_text) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 19c7ca0f..5c2e2669 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -2542,6 +2542,7 @@ def test_apply_mp_patches_success( # (as long as we use `Pool` for our parallelism); # `Process` though... condition='not patch_pool', + require='all', # Check consistency ) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py index 52338e78..16340003 100644 --- a/tests/test_retry_tests.py +++ b/tests/test_retry_tests.py @@ -1,15 +1,31 @@ """ Tests to make sure that our :py:deco:`pytest.mark.retry` decorator works. + +Notes: + This test module is written to work both: + + - When :py:mod:`pytest_mark_retry` (`link`_) is installed from + source along with this file and the rest of the test suite, or + + - In a test directory containing (among other things): + + - This file as a standalone test module, and + + - A ``conftest.py`` containing the content of single-file module + ``pytest_mark_retry.py``. + +.. _link: https://gitlab.com/TTsangSC/pytest-mark-retry """ from __future__ import annotations import re import pprint import textwrap -from collections.abc import Callable, Collection, Generator, Iterable +from collections.abc import Collection, Iterable, Sequence from dataclasses import dataclass -from functools import partial +from functools import cached_property, partial +from importlib.util import find_spec from operator import attrgetter from pathlib import Path from shutil import rmtree @@ -22,12 +38,11 @@ pytest_plugins = ('pytester',) _Status = Literal['passed', 'failed', 'skipped'] -_RunPytest_Method = Literal[ +_RunPytestMethod = Literal[ 'runpytest', 'runpytest_inprocess', 'runpytest_subprocess', ] -_RunPytest = Callable[..., pytest.RunResult] -_RunnerGetter = Callable[[str, str], _RunPytest] +PROJECT_MODULE = 'pytest_mark_retry' TEST_COUNTERS = """ from __future__ import annotations @@ -335,12 +350,95 @@ def test_bad_dynamic_condition() -> None: @pytest.mark.parametrize('n', [0, 1, 2]) def test_dynamic_condition_test_params(n: int) -> None: ''' - Subtests `[0]` and `[2]` (resp. subtest `[1]`) should fail without - retries (resp. with 1 retry) because the condition evaluates to - false (resp. true) on the test's parametrization. + Subtests ``[0]`` and ``[2]`` (resp. subtest ``[1]``) should fail + without retries (resp. with 1 retry) because the condition evaluates + to false (resp. true) on the test's parametrization. ''' raise RuntimeError """ +TEST_BAD_MARKERS = """ +from __future__ import annotations + +import pytest + + +@pytest.mark.retry(1, 2) # `exceptions` cannot be 2 +def test_passing_bad_exceptions() -> None: + ''' + This test passes with a warning because its retry marker has an + invalid :py:attr:`RetryMarker.exceptions`. + ''' + pass + + +@pytest.mark.retry(foo=1) # No argument named `foo` +def test_passing_stray_arg() -> None: + ''' + This test also passes with a warning because its retry marker has am + stray argument ``foo`` + ''' + pass + + +@pytest.mark.retry(condition='') # Syntax error +def test_failing_bad_condition() -> None: + ''' + This test fails with a warning and without retries, because its + retry marker got a bad :py:attr:`RetryMarker.condition`. + ''' + assert False +""" +TEST_REQUIRE = """ +from __future__ import annotations + +import itertools + +import pytest + + +@pytest.fixture(scope='module') +def counter() -> itertools.count: + return itertools.count() + + +@pytest.fixture +def index(counter: itertools.count) -> int: + return next(counter) + + +@pytest.mark.retry(3) +def test_passing_retry_require_any(index: int) -> None: + ''' + This passes with two retries and leave ``index`` at 2. + ''' + assert index >= 2 + + +@pytest.mark.retry(3, require='any') +def test_failing_retry_require_any(index: int) -> None: + ''' + This fails with three retries and leave ``index`` at 6. + ''' + assert index < 3 + + +@pytest.mark.retry(3, require='all') +def test_failing_retry_require_all(index: int) -> None: + ''' + This fails with zero retries and leave ``index`` at 7. + ''' + # Fails right out the gate, no need to continue retrying + assert index > 7 + + +@pytest.mark.retry(3, require='all') +def test_passing_retry_require_all(index: int) -> None: + ''' + This passes with three retries and leave ``index`` at 11. + ''' + # All attempts pass, but we are instructed to exhaust the retries + assert index > 0 +""" @dataclass @@ -385,7 +483,7 @@ def run( check_results: bool = False, check_summary: Literal['verbose', 'concise'] | None = None, check_warnings: int | None = None, - runner: _RunPytest_Method = 'runpytest', + runner: _RunPytestMethod = 'runpytest', additional_stdout_lines: Collection[str] = (), additional_stderr_lines: Collection[str] = (), ) -> pytest.RunResult: @@ -420,7 +518,11 @@ def run( tempfiles: list[Path] = [] tempdirs: list[Path] = [] try: - conftests: list[str] = [self._get_proj_conftest().read_text()] + conftests: list[str] = [] + if not self.marker_plugin_globally_installed: + # If we don't do this the project will be loaded twice + # as a plugin, leading to a clash + conftests.append(self.marker_plugin_path.read_text()) if self.conftest: conftests.append(self.conftest) # Create separate conftest.py in nested subdirs to avoid @@ -474,7 +576,9 @@ def check_results( for outcomes in self.expected_outcomes.values(): for outcome in outcomes: counts[outcome.status] = counts.get(outcome.status, 0) + 1 - result.assert_outcomes(warnings=warnings, **counts) + result.assert_outcomes( + warnings=warnings, **cast(dict[str, int], counts), + ) def check_verbose_summary( self, @@ -551,6 +655,32 @@ def check_concise_summary( for test_name in names: assert test_name in line + @property + def marker_plugin_path(self) -> Path: + return self._source[0] + + @property + def marker_plugin_globally_installed(self) -> bool: + return self._source[1] + + @cached_property + def _source(self) -> tuple[Path, bool]: + sources = { + f'module `{PROJECT_MODULE}`': (self._get_proj_module_path, True), + repr('conftest.py'): (self._get_proj_conftest, False), + } + for src, (get_path, retry_globally_installed) in sources.items(): + try: + path = get_path() + assert 'class RetryMarker' in path.read_text() + print(f'Loaded project source from {src}: {str(path)!r}') + return path, retry_globally_installed + except Exception: + pass + raise RuntimeError( + f'Failed to load the project source from any of: {sources!r}', + ) + @staticmethod def _check_lines( result: pytest.RunResult, @@ -580,9 +710,15 @@ def _format_header(status: _Status, n: int) -> str: ) @staticmethod - def _get_proj_conftest() -> Path: + def _get_proj_conftest() -> Path: # If installed as the conftest return Path(__file__).parent / 'conftest.py' + @staticmethod + def _get_proj_module_path() -> Path: # If installed as a module + spec = find_spec(PROJECT_MODULE) + assert spec and spec.origin + return Path(spec.origin) + @staticmethod def _strip(text: str) -> str: return textwrap.dedent(text).strip('\n') @@ -616,129 +752,131 @@ def _identical_items_are_adjacent(items: Iterable[Any]) -> bool: return True +def _outcomes_to_outcome_dict( + outcomes: Iterable[_TestOutcome], +) -> dict[str, list[_TestOutcome]]: + """ + Example: + >>> o0 = _TestOutcome('foo', 'passed', 0) + >>> o1 = _TestOutcome('bar[1-2-3]', 'failed', 1) + >>> o2 = _TestOutcome('bar[4-5-6]', 'passed', 2) + >>> outcomes = {'foo': [o0], 'bar': [o1, o2]} + >>> assert _outcomes_to_outcome_dict([o1, o0, o2]) == outcomes + """ + result: dict[str, list[_TestOutcome]] = {} + for outcome in outcomes: + name = outcome.name + if name.endswith(']') and '[' in name: # Subtest + base_name, *_ = name.partition('[') + else: + base_name = name + result.setdefault(base_name, []).append(outcome) + return result + + @pytest.fixture -def counters_module( - pytester: pytest.Pytester, -) -> Generator[_TestModule, None, None]: +def counters_module(pytester: pytest.Pytester) -> _TestModule: dynamic_p = _TestOutcome('test_dynamic_fixtures_persisted').subtest static_p = _TestOutcome('test_static_fixtures_persisted').subtest dynamic_r = _TestOutcome('test_dynamic_fixtures_reset').subtest static_r = _TestOutcome('test_static_fixtures_reset').subtest - outcomes = { - 'test_dynamic_fixtures_persisted': [ - dynamic_p('func-0'), - dynamic_p('func-2', retries=2), - dynamic_p('func-6', status='failed', retries=3), - dynamic_p('module-4', status='failed', retries=3), - dynamic_p('module-5', retries=1), - ], - 'test_static_fixtures_persisted': [ - static_p('func-3', retries=3), - static_p('func-4', status='failed', retries=3), - static_p('module-4'), - static_p('module-9', retries=2), - ], - 'test_dynamic_fixtures_reset': [ - dynamic_r('func-0'), - dynamic_r('func-1', status='failed', retries=1), - dynamic_r('module-11', retries=1), - ], - 'test_static_fixtures_reset': [ - static_r('func-0'), - static_r('func-1', status='failed', retries=2), - static_r('module-14', retries=2), - ], - } - yield _TestModule('test_counters', TEST_COUNTERS, outcomes, pytester) + outcomes = _outcomes_to_outcome_dict([ + dynamic_p('func-0'), + dynamic_p('func-2', retries=2), + dynamic_p('func-6', status='failed', retries=3), + dynamic_p('module-4', status='failed', retries=3), + dynamic_p('module-5', retries=1), + static_p('func-3', retries=3), + static_p('func-4', status='failed', retries=3), + static_p('module-4'), + static_p('module-9', retries=2), + dynamic_r('func-0'), + dynamic_r('func-1', status='failed', retries=1), + dynamic_r('module-11', retries=1), + static_r('func-0'), + static_r('func-1', status='failed', retries=2), + static_r('module-14', retries=2), + ]) + return _TestModule('test_counters', TEST_COUNTERS, outcomes, pytester) @pytest.fixture -def teardown_module( - pytester: pytest.Pytester, -) -> Generator[_TestModule, None, None]: - yield _TestModule( - 'test_teardown', - TEST_TEARDOWN, - { - 'test_no_fixture_reset': - [_TestOutcome('test_no_fixture_reset', 'failed', 2)], - 'test_with_fixture_reset': - [_TestOutcome('test_with_fixture_reset', 'failed', 1)], - }, - pytester, - conftest=""" - from __future__ import annotations - - from pathlib import Path - - import pytest - - - def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption( - '--my-temp-dir', - type=Path, - help=f'persisted tempdir location for {__file__!r}', - ) - parser.addoption( - '--my-log', - type=Path, - help=f'log file location for tempfile creation/deletion', - ) - """, - ) +def teardown_module(pytester: pytest.Pytester) -> _TestModule: + outcomes = _outcomes_to_outcome_dict([ + _TestOutcome('test_no_fixture_reset', 'failed', 2), + _TestOutcome('test_with_fixture_reset', 'failed', 1), + ]) + cf = """ + from __future__ import annotations + + from pathlib import Path + + import pytest + + + def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + '--my-temp-dir', + type=Path, + help=f'persisted tempdir location for {__file__!r}', + ) + parser.addoption( + '--my-log', + type=Path, + help=f'log file location for tempfile creation/deletion', + ) + """ + return _TestModule('test_teardown', TEST_TEARDOWN, outcomes, pytester, cf) @pytest.fixture -def exceptions_module( - pytester: pytest.Pytester, -) -> Generator[_TestModule, None, None]: - yield _TestModule( - 'test_exceptions', - TEST_EXCEPTIONS, - { - 'test_all_xc_types': - [_TestOutcome('test_all_xc_types', retries=3)], - 'test_one_xc_type': - [_TestOutcome('test_one_xc_type', 'failed', 1)], - 'test_two_xc_types': - [_TestOutcome('test_two_xc_types', 'failed', 2)], - 'test_three_xc_types': - [_TestOutcome('test_three_xc_types', retries=3)], - }, - pytester, - ) +def exceptions_module(pytester: pytest.Pytester) -> _TestModule: + outcomes = _outcomes_to_outcome_dict([ + _TestOutcome('test_all_xc_types', retries=3), + _TestOutcome('test_one_xc_type', 'failed', 1), + _TestOutcome('test_two_xc_types', 'failed', 2), + _TestOutcome('test_three_xc_types', retries=3), + ]) + return _TestModule('test_exceptions', TEST_EXCEPTIONS, outcomes, pytester) @pytest.fixture -def conditions_module( - pytester: pytest.Pytester, -) -> Generator[_TestModule, None, None]: +def conditions_module(pytester: pytest.Pytester) -> _TestModule: test = partial(_TestOutcome, status='failed') param_test_name = 'test_dynamic_condition_test_params' param_test = partial(test(param_test_name).subtest, status='failed') - yield _TestModule( - 'test_conditions', - TEST_CONDITIONS, - { - 'test_concrete_positive_condition': - [test('test_concrete_positive_condition', retries=2)], - 'test_concrete_negative_condition': - [test('test_concrete_negative_condition')], - 'test_dynamic_positive_condition_test_module_globals': - [test( - 'test_dynamic_positive_condition_test_module_globals', - retries=1, - )], - 'test_dynamic_negative_condition_test_module_globals': - [test('test_dynamic_negative_condition_test_module_globals')], - 'test_bad_dynamic_condition': - [test('test_bad_dynamic_condition')], - param_test_name: - [param_test('0'), param_test('1', retries=1), param_test('2')], - }, - pytester, - ) + outcomes = _outcomes_to_outcome_dict([ + test('test_concrete_positive_condition', retries=2), + test('test_concrete_negative_condition'), + test('test_dynamic_positive_condition_test_module_globals', retries=1), + test('test_dynamic_negative_condition_test_module_globals'), + test('test_bad_dynamic_condition'), + param_test('0'), + param_test('1', retries=1), + param_test('2'), + ]) + return _TestModule('test_conditions', TEST_CONDITIONS, outcomes, pytester) + + +@pytest.fixture +def bad_markers_module(pytester: pytest.Pytester) -> _TestModule: + outcomes = _outcomes_to_outcome_dict([ + _TestOutcome('test_passing_bad_exceptions'), + _TestOutcome('test_passing_stray_arg'), + _TestOutcome('test_failing_bad_condition', 'failed'), + ]) + return _TestModule('test_bad', TEST_BAD_MARKERS, outcomes, pytester) + + +@pytest.fixture +def require_module(pytester: pytest.Pytester) -> _TestModule: + outcomes = _outcomes_to_outcome_dict([ + _TestOutcome('test_passing_retry_require_any', retries=2), + _TestOutcome('test_failing_retry_require_any', 'failed', 3), + _TestOutcome('test_failing_retry_require_all', 'failed'), + _TestOutcome('test_passing_retry_require_all', retries=3), + ]) + return _TestModule('test_require', TEST_REQUIRE, outcomes, pytester) @pytest.mark.parametrize('verbose', [True, False]) @@ -804,9 +942,10 @@ def parse_line(cls, line: str) -> Self: # we can verify that by checking that the ids of the `makefile()` # fixtures appear in contiguous blocks - # Note: there seems to be a weird corner case where neighboring tests - # may reuse the same fixture id (see failing job 73520441960 in - # pipeline 25091142386); probably has to do with object lifetime. + # Note: there seems to be a weird corner case where neighboring + # tests may reuse the same fixture id (see `line_profiler` failing + # job 73520441960 in pipeline 25091142386); probably has to do with + # object lifetime. # So instead of just checking the `fixture_id`, also consult # `test`; it suffices to see that WITHIN THE SAME TEST we don't have # fixture values stepping over one another @@ -836,18 +975,66 @@ def test_retry_conditions(conditions_module: _TestModule) -> None: """ Test that the decorator correctly handles retry conditions. """ + # `test_bad_dynamic_condition()` should have failed with a + # `RetryConditionFailure`, listing the error encountered in the last + # trial and the error encountered when `eval()`-ing the condition + # (Note: grepping for the entire error message in the short test + # summary is fragile since it may be elided; so we just use a + # separate pattern to grep it from the tracebacks) + lines = [ + 'FAILED .*::test_bad_dynamic_condition - .*RetryConditionFailure', + r'.*RetryConditionFailure: \(RuntimeError: bar\) ' + r"-> \(condition: 'foo == 1' -> NameError: .*'foo'.*\)", + ] conditions_module.run( '--verbose', - check_results=True, - check_summary='verbose', - check_warnings=0, - # `test_bad_dynamic_condition()` should have failed with a - # `_RetryFailure`, listing the error encountered in the last - # trial and the error encountered when `eval()`-ing the - # condition - additional_stdout_lines=[ - 'FAILED +.*::test_bad_dynamic_condition - ' - r".*_RetryFailure: +\(RuntimeError: bar\) +" - r"-> +\(condition: +'foo == 1' +-> +NameError: .*'foo'.*\)" + check_results=True, check_summary='verbose', check_warnings=0, + additional_stdout_lines=lines, + ) + + +def test_bad_markers(bad_markers_module: _TestModule) -> None: + """ + Test that the decorator gracefully handles incorrect constructions. + """ + stdout = bad_markers_module.run( + '--verbose', + check_results=True, check_summary='verbose', check_warnings=3, + ).stdout + # Check the warnings emitted + # (Since we want to match across multiple lines we can't use + # `additional_stdout_lines`) + errors: str | Sequence[str] + pattern = ( + '{0}\n' + r'.*RetryMarkerWarning: .*{0}.*: disregarding .* marker: \(.*{1}.*\)' + ) + for test, errors in { + 'test_passing_bad_exceptions': [ + r'TypeError: \.exceptions = .*2.*: expected .*exception', + r'TypeError: .*not iterable', + r'TypeError: too many positional arguments', + r'TypeError: .*takes 1 positional argument but' ], + 'test_passing_stray_arg': + r'TypeError: .*unexpected keyword argument \'foo\'', + 'test_failing_bad_condition': + r'ValueError: \.condition = \'\': not a valid expression ' + r'\(SyntaxError.*\)', + }.items(): + if isinstance(errors, str): + errors = [errors] + messages = [pattern.format(re.escape(test), error) for error in errors] + if not any(re.search(msg, str(stdout)) for msg in messages): + msg = f'none of the patterns {messages!r} matched {stdout!r}' + raise AssertionError(msg) + + +def test_requirement(require_module: _TestModule) -> None: + """ + Test that the decorator correctly handles requirements that all + trials should pass (via ``require='all'``). + """ + require_module.run( + '--verbose', check_results=True, check_summary='verbose', ) From 0cf7989b8262459e83bf7582d9d800b5b3fed5cb Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 12 May 2026 23:18:15 +0200 Subject: [PATCH 099/165] Better debugging output line_profiler/cleanup.py::Cleanup.cleanup() Now taking an optional `reason` argument which is reflected in log messages line_profiler/_child_process_profiling/cache.py ::LineProfilingCache._handle_signal() - Updated the log message written to (hopefully) make it easier to catch corner cases where the `.cleanup()` call somehow didn't happen - Added a `reason` to the `.cleanup()` call line_profiler/_child_process_profiling/multiprocessing_patches.py ::wrap_bootstrap() line_profiler/curated_profiling.py::CuratedProfilerContext.uninstall() Added `reason`s to the `.cleanup()` calls --- .../_child_process_profiling/cache.py | 12 ++++++---- .../multiprocessing_patches.py | 7 +++--- line_profiler/cleanup.py | 23 +++++++++++++------ line_profiler/curated_profiling.py | 2 +- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index ac70eb8e..ece8edc2 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -522,7 +522,7 @@ def wrap_ctx_debug( # Set `.cleanup()` as an atexit hook to handle everything when # the child process is about to terminate - atexit.register(self.cleanup) + atexit.register(partial(self.cleanup, reason='`atexit` callback')) self._debug_output(f'Setup successful ({context})') return True @@ -544,9 +544,9 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover :py:meth:`coverage.control.Converage._on_sigterm` """ name = self._get_signal_name(signum) - msg = f'Caught `{name}` ({signum}), cleaning up...' - self._debug_output(msg) - state = 'succeeded' + # Shouldn't happen, but all kinds of weird things happen at the + # interpreter's EoL... + state = 'not initiated?!' try: # We don't care about profiler state and such if we're # already being SIGTERM-ed, so just handle cleanup on a @@ -554,7 +554,7 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover # this prevents a duplicate call to `.cleanup()` when we're # already inside one (e.g. when `Process._bootstrap()` is # exiting, or the `atexit` hook is triggered) - self.cleanup(new_thread=True) + self.cleanup(new_thread=True, reason=f'caught `{name}` ({signum})') except BaseException as e: xc = f'{type(e).__name__}' msg = str(e) @@ -562,6 +562,8 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover xc = f'{xc}: {msg}' state = f'failed ({xc})' raise e + else: + state = 'succeeded' finally: handler = self._sighandlers.pop(signum, None) msg = f'Cleanup {state}, passing `{name}` onto {handler!r}...' diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 67299fdb..61834172 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -813,13 +813,14 @@ def wrap_bootstrap( try: return vanilla_impl(self, *args, **kwargs) finally: - msg = 'Calling cleanup hook via `BaseProcess._bootstrap`' - cache._debug_output(msg) # Execute cleanup in a separate thread so as to avoid deadlocks, # in case when `LineProfilingCache._handle_signal()` caught a # signal as we're in the middle of this and initiated another # `.cleanup()` call - cache.cleanup(new_thread=True) + cache.cleanup( + new_thread=True, + reason='exiting `multiprocessing.Process._bootstrap`', + ) _patch_process = partial( diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index d4bd55cf..11d21890 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -80,11 +80,13 @@ def __exit__(self, *_, **__) -> Any: Call ``~.cleanup(1)``, clearing the level of cleanup stacks we previously :py:meth:`~.__enter__`-ed into. """ - self.cleanup(1) + self.cleanup(1, reason='context exit') # Cleanup methods - def cleanup(self, levels: int | None = None) -> None: + def cleanup( + self, levels: int | None = None, *, reason: str | None = None, + ) -> None: """ Pop cleanup callbacks from the internal stacks added via :py:meth:`~.add_cleanup` etc. and call them in order. @@ -93,6 +95,8 @@ def cleanup(self, levels: int | None = None) -> None: levels (int | None): Number of stack levels to clear; passing :py:const`None` clears the entire stack of callback stacks + reason (str | None): + Optional description of the reason for cleaning up """ def pop_all_contexts( contexts: _StackContexts, @@ -114,19 +118,24 @@ def pop_n_levels_of_contexts( pop_contexts = pop_all_contexts(self._contexts) else: pop_contexts = pop_n_levels_of_contexts(self._contexts, levels) - cleanup = partial(self._cleanup, self._debug_output) + cleanup = partial(self._cleanup, self._debug_output, reason=reason) for stacks in pop_contexts: cleanup(stacks) @staticmethod - def _cleanup(log: Callable[[str], Any], stacks: _Stacks) -> None: + def _cleanup( + log: Callable[[str], Any], stacks: _Stacks, reason: str | None, + ) -> None: ncallbacks_total = sum(len(stack) for stack in stacks.values()) + note = f'{ncallbacks_total} callback(s)' + if reason: + note = f'{reason}; {note}' if not ncallbacks_total: - log('Cleanup aborted (no registered callbacks)') + log(f'Cleanup aborted ({note})') return # Bookend the cleanup loop with log messages to help detect if # child processes are prematurely terminated - log(f'Starting cleanup ({ncallbacks_total} callback(s))...') + log(f'Starting cleanup ({note})...') ncallbacks_run = 0 for priority in sorted(stacks, reverse=True): callbacks = stacks.pop(priority) @@ -145,7 +154,7 @@ def _cleanup(log: Callable[[str], Any], stacks: _Stacks) -> None: f'- Cleanup {state} ' f'({ncallbacks_run}/{ncallbacks_total}): {msg}', ) - log(f'... cleanup completed ({ncallbacks_total} callback(s))') + log(f'... cleanup completed ({note})') def add_cleanup( self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index a120824f..e87dec60 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -220,7 +220,7 @@ def install(self) -> None: self.patch(self, '_installed', True) def uninstall(self) -> None: - self.cleanup() + self.cleanup(reason='uninstalling profiling context') def __enter__(self) -> Self: self.install() From 39e91c1cfcddba99bc365baafd1fb0a99e178a8a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 13 May 2026 01:00:33 +0200 Subject: [PATCH 100/165] Reworked `multiprocessing.pool` patch line_profiler/_child_process_profiling/multiprocessing_patches.py TaskWrapper, wrap_get_tasks(), wrap_guarded_task_generation() Removed wrap_worker() New wrapper function for wrapping `multiprocessing.pool.worker()` and writing profiling output ASAP as it catches the sentinel value --- .../multiprocessing_patches.py | 179 +++++++----------- line_profiler/rc/line_profiler.toml | 7 +- 2 files changed, 75 insertions(+), 111 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 61834172..a1c40c1c 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,15 +16,15 @@ import dataclasses import multiprocessing +import os import sys import warnings from collections.abc import Callable, Collection, Mapping, Sequence, Set from functools import partial from importlib import import_module -from inspect import getattr_static, signature -from multiprocessing.context import BaseContext +from inspect import getattr_static from multiprocessing.process import BaseProcess -from multiprocessing.pool import Pool +from multiprocessing.queues import SimpleQueue from operator import attrgetter from time import sleep, monotonic from types import MappingProxyType as mappingproxy, MethodType, ModuleType @@ -58,6 +58,7 @@ _CAN_USE_RESOURCE_TRACKER = True from .. import _diagnostics as diagnostics +from ..cleanup import Cleanup from ..toml_config import ConfigSource from .cache import LineProfilingCache from .runpy_patches import create_runpy_wrapper @@ -366,32 +367,22 @@ def _process_has_returned( return result -class TaskWrapper(Generic[PS, T]): - """ - Pickle-able wrapper around the supplied task callable, which writes - to the session's profiling-stats file on exit. - """ - def __init__(self, func: Callable[PS, T]) -> None: - self.func = func - try: - self.__signature__ = signature(func) - except Exception: # nocover - # Can happen with e.g. certain builin/c-based callables - pass - - def __call__(self, *args, **kwargs) -> T: - dump_stats = LineProfilingCache.load()._dump_stats - try: - return self.func(*args, **kwargs) - finally: - if dump_stats is not None: - dump_stats() - - def _no_op(*_, **__) -> None: pass +def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: + key = 'mp_added_sigterm_handler' + if not MPConfig.from_cache(cache).catch_sigterm: + return + if cache.main_pid == os.getpid(): # Not in a child process + return + if cache._additional_data.get(key, False): + # Already added (e.g. by another plugin) + return + cache._add_signal_handler() + cache._additional_data[key] = True + # ---------------------- Patching infrastructure ----------------------- @@ -660,94 +651,67 @@ def _register_patch(name: str, patch: _Patch | None = None) -> _Patch: # ---------------- `multiprocessing.pool.Pool` patches ----------------- -@LineProfilingCache._method_wrapper -def wrap_get_tasks( - _, # No need to use the cache, but `_method_wrapper` expects it - vanilla_impl: Callable[Concatenate[Callable[PS1, T1], PS2], T2], - func: Callable[PS1, T1], - *args: PS2.args, - **kwargs: PS2.kwargs -) -> T2: - """ - Wrap around :py:meth:`.Pool._get_tasks` so that the writing of - profiling stats is handled within the callables sent to the child - processes before the parent process assumes control. - - Note: - :py:meth:`.Pool._get_tasks` is a static method. - """ - return vanilla_impl(TaskWrapper(func), *args, **kwargs) - - -@LineProfilingCache._method_wrapper -def wrap_guarded_task_generation( - _, # No need to use the cache, but `_method_wrapper` expects it - vanilla_impl: Callable[Concatenate[Pool, int, Callable[PS1, T1], PS2], T2], - self: Pool, - result_job: int, - func: Callable[PS1, T1], - *args: PS2.args, - **kwargs: PS2.kwargs -) -> T2: - """ - Wrap around :py:meth:`.Pool._guarded_task_generation` so that the - writing of profiling stats is handled within the callables sent to - the child processes before the parent process assumes control. - """ - return vanilla_impl(self, result_job, TaskWrapper(func), *args, **kwargs) - - -@LineProfilingCache._method_wrapper -def wrap_repopulate_pool_static( +@LineProfilingCache._method_wrapper # nocover +def wrap_worker( cache: LineProfilingCache, - vanilla_impl: Callable[ - Concatenate[BaseContext, type[P], int, list[P], PS], None - ], - ctx: BaseContext, - Process: type[P], - processes: int, - pool: list[P], + vanilla_impl: Callable[Concatenate[SimpleQueue, PS], None], + inqueue: SimpleQueue, *args: PS.args, **kwargs: PS.kwargs ) -> None: """ - Wrap around :py:meth:`.Pool._repopulate_pool_static` so that we can - keep track of the PIDs of the created child processes. + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes can write profiling output as soon as the pool runs out of + tasks. Note: - :py:meth:`.Pool._repopulate_pool_static` is a static method. + This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. """ - try: - vanilla_impl(ctx, Process, processes, pool, *args, **kwargs) - finally: - patches = MPConfig.from_cache(cache).patches - if not patches.get('process', False): - # Notes: - # - if the `process` patch is active, child processes should - # always have called `.cache.profiler.dump_stats()` at - # least once before exiting, so there's no need to warn - # the cache against children possibly not having written - # profiling data. - # - Since the vanilla implementation calls `Process.start()` - # on each child, they should all have valid PIDs. However: - # - The process might have terminated for whatever reason. - # and - # - `multiprocessing.dummy.DummyProcess` doesn't have - # `.pid`. - # So we add a fallback to `None` just in case. - pids = cast( - set[int], - {getattr(process, 'pid', None) for process in pool} - {None}, + # Set a signal handler for SIGTERM to help child processes with + # consistently cleaning up + _add_sigterm_handler_in_child(cache) + # Note: using the `cache` itself as the context manager is prone to + # deadlock + with Cleanup() as cleanup: + if isinstance(inqueue.get, MethodType): + get = partial(_wrap_queue_get, cache, inqueue.get.__func__) + cleanup.patch( + inqueue, 'get', MethodType(get, inqueue), + name='._inqueue', ) - cache._warn_possible_lack_of_stats(pids) + return vanilla_impl(inqueue, *args, **kwargs) -_patch_pool = partial( - _register_patch('pool', Patch('pool')).add_method, 'Pool', -) -_patch_pool('_get_tasks', wrap_get_tasks, 'static') -_patch_pool('_guarded_task_generation', wrap_guarded_task_generation) -_patch_pool('_repopulate_pool_static', wrap_repopulate_pool_static, 'static') +def _wrap_queue_get( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[SimpleQueue, PS], T], + self: SimpleQueue, + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> T: + result = vanilla_impl(self, *args, **kwargs) + ntasks: dict[int, int] + ntasks = cache._additional_data.setdefault('mp_queue_ntasks', {}) + queue_id = id(self) + if result is None: + n = ntasks.pop(queue_id, 0) + msg = f'`multiprocessing.pool.worker`: recieved {n} task(s) in total' + cache._debug_output(msg) + # Got sentinel value, process is about to exit + cache.cleanup( + new_thread=True, + reason='ran out of tasks in `multiprocessing.process.worker()`', + ) + else: + ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 + return result + + +_patch_pool = _register_patch('pool', Patch('pool')).add_method +_patch_pool('', 'worker', wrap_worker) # ----------- `multiprocessing.process.BaseProcess` patches ------------ @@ -808,8 +772,7 @@ def wrap_bootstrap( """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up - if MPConfig.from_cache(cache).catch_sigterm: - cache._add_signal_handler() + _add_sigterm_handler_in_child(cache) try: return vanilla_impl(self, *args, **kwargs) finally: @@ -819,7 +782,7 @@ def wrap_bootstrap( # `.cleanup()` call cache.cleanup( new_thread=True, - reason='exiting `multiprocessing.Process._bootstrap`', + reason='exiting `multiprocessing.Process._bootstrap()`', ) @@ -1042,9 +1005,9 @@ def apply( Patches: ``'pool'``: - Patch :py:class:`multiprocessing.pool.Pool`'s - ``._get_tasks()`` and ``._guarded_task_generation()`` - methods so that parallel tasks write profiling output. + Patch :py:func:`multiprocessing.pool.worker` so that + profiling output is written as each child process runs out + of task. ``'process'``: Patch :py:class:`multiprocessing.process.BaseProcess`'s ``._bootstrap()`` method (and ``.terminate()`` on Windows) diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 20eb2dd7..493d9ab8 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -234,14 +234,15 @@ catch_sigterm = true [tool.line_profiler.child_processes.multiprocessing.patches] # - `multiprocessing.patches.pool` (bool): -# Whether to patch `multiprocessing.pool.Pool`, so that each tasks -# write profiling output before pushing result back to the parent -# process +# Whether to patch `multiprocessing.pool.worker()`, so that each child +# process write profiling output as it runs out of tasks pool = true # - `multiprocessing.patches.process` (bool): # Whether to patch `multiprocessing.process.BaseProcess`, so that each # child process write profiling output before exiting process = true +# NOTE: for the best result, stick to the default and have both applied + # - `multiprocessing.patches.logging` (bool): # Whether to patch logging functions in `multiprocessing.util`, so # that the internal logs of `multiprocessing` are teed to the session From d1db7bacf3cd2edfc211c9035bdcf080a25c9cd8 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 13 May 2026 05:28:17 +0200 Subject: [PATCH 101/165] Fix signal-handling edge cases line_profiler/_child_process_profiling/cache.py::LineProfilingCache _stats_dumper New private attribute superseding `._dump_stats` cleanup() No longer overriding the base-class method _setup_in_child_process() Now setting up the `._stats_dumper` attribute instead of `._dump_stats` _handle_signal() - No longer calling `.cleanup()` to avoid deadlocks - Added handling for when the signal handler was originally set from outside of Python line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_runner(), wrap_bootstrap() No longer calling `LineProfilingCache.load().cleanup()` with the `new_thread` argument --- .../_child_process_profiling/cache.py | 74 ++++++------------- .../multiprocessing_patches.py | 12 +-- 2 files changed, 27 insertions(+), 59 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index ece8edc2..060e3cd6 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -22,7 +22,7 @@ from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent -from threading import current_thread, main_thread, RLock, Thread +from threading import current_thread, main_thread from types import FrameType, ModuleType from typing import Any, ClassVar, Literal, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self @@ -91,8 +91,7 @@ class LineProfilingCache(Cleanup): _sighandlers: dict[int, _SignalHandler | int | None] = ( _private_field(default_factory=dict) ) - _rlock: RLock = _private_field(default_factory=RLock) - _dump_stats: Callable[..., None] | None = _private_field(default=None) + _stats_dumper: Cleanup | None = _private_field(default=None) # These are unstructured fields; other components can decide on what # to put in them. They are also pickled by `.dump()`, and are thus # retrievable in `.load()`-ed instances. @@ -166,43 +165,6 @@ def dump(self) -> None: with open(self.filename, mode='wb') as fobj: pickle.dump(content, fobj, protocol=HIGHEST_PROTOCOL) - def cleanup(self, *args, new_thread: bool = False, **kwargs) -> None: - """ - Perform cleanup. - - Args: - new_thread (bool): - Whether to relegate the call to - :py:meth:`Cleanup,cleanup` to a new thread (see Notes) - *args, **kwargs: - Passed to :py:meth:`Cleanup.cleanup` - - Note: - - In child processes we set a ``SIGTERM`` handler to always - call :py:meth:`~.cleanup`. However, this may happen when - we're already in the middle of a cleanup call, which - results in undefined behavior. To prevent this, we can - supply ``new_thread=True`` so that the - :py:meth:`Cleanup.cleanup` call is handled by a separate - thread which acquires an instance-specific lock. - - - However, this method is supposed to clean up the session - profiler by completely disabling it, and that part must - happen on the main thread or deallocation will be botched. - Hence ``new_thread=True`` is made an option and not the - default. - """ - if not new_thread: - self._cleanup_worker(*args, **kwargs) - return - thread = Thread(target=self._cleanup_worker, args=args, kwargs=kwargs) - thread.start() - thread.join() - - def _cleanup_worker(self, *args, **kwargs) -> None: - with self._rlock: - super().cleanup(*args, **kwargs) - def gather_stats( self, exclude_pids: Collection[int] | None = None, @@ -514,9 +476,18 @@ def wrap_ctx_debug( suffix='.lprof', delete=False, ) - dump_stats = self._dump_stats = partial(prof.dump_stats, prof_outfile) + dump_stats = partial(prof.dump_stats, prof_outfile) self.add_cleanup_with_priority(dump_stats, 1) + # Create a cleanup object for the express purpose of dumping + # stats in an emergency (e.g. when a signal is caught) + self._stats_dumper = dumper = Cleanup() + self.patch( + dumper, '_debug_output', self._debug_output, + cleanup=False, name=f'{self!r}._stats_dumper', + ) + dumper.add_cleanup(dump_stats) + # Various setups self._setup_common(wrap_os_fork, {'reboot_forkserver': False}) @@ -548,13 +519,11 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover # interpreter's EoL... state = 'not initiated?!' try: - # We don't care about profiler state and such if we're - # already being SIGTERM-ed, so just handle cleanup on a - # separate thread; - # this prevents a duplicate call to `.cleanup()` when we're - # already inside one (e.g. when `Process._bootstrap()` is - # exiting, or the `atexit` hook is triggered) - self.cleanup(new_thread=True, reason=f'caught `{name}` ({signum})') + # Just dump the stats ASAP without running `.cleanup()` to + # avoid deadlocks + self._debug_output(f'Caught `{name}` ({signum}), dumping stats...') + if self._stats_dumper is not None: + self._stats_dumper.cleanup() except BaseException as e: xc = f'{type(e).__name__}' msg = str(e) @@ -566,11 +535,14 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover state = 'succeeded' finally: handler = self._sighandlers.pop(signum, None) - msg = f'Cleanup {state}, passing `{name}` onto {handler!r}...' + msg = f'Stat-dumping {state}, passing `{name}` onto {handler!r}...' self._debug_output(msg) - if handler is not None: + if handler is None: + msg = 'original handler set from outside of Python' + raise RuntimeError(msg) + else: signal.signal(signum, handler) - signal.raise_signal(signum) + signal.raise_signal(signum) def _add_signal_handler( self, signum: int = signal.SIGTERM, diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index a1c40c1c..c493def0 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -701,10 +701,8 @@ def _wrap_queue_get( msg = f'`multiprocessing.pool.worker`: recieved {n} task(s) in total' cache._debug_output(msg) # Got sentinel value, process is about to exit - cache.cleanup( - new_thread=True, - reason='ran out of tasks in `multiprocessing.process.worker()`', - ) + reason = 'ran out of tasks in `multiprocessing.process.worker()`' + cache.cleanup(reason=reason) else: ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 return result @@ -780,10 +778,8 @@ def wrap_bootstrap( # in case when `LineProfilingCache._handle_signal()` caught a # signal as we're in the middle of this and initiated another # `.cleanup()` call - cache.cleanup( - new_thread=True, - reason='exiting `multiprocessing.Process._bootstrap()`', - ) + reason = 'exiting `multiprocessing.Process._bootstrap()`' + cache.cleanup(reason=reason) _patch_process = partial( From 1edc9d7c6988aea17e89d1c2f0468b9021b92f00 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 13 May 2026 08:02:05 +0200 Subject: [PATCH 102/165] Partial rollback of 4a22dba line_profiler/_child_process_profiling/cache.py::LineProfilingCache _stats_dumper Now a convenience subtype of `Cleanup` which can be directly called _setup_in_child_process(), _handle_signal() Updated setup and handling for `._stats_dumper` line_profiler/_child_process_profiling/multiprocessing_patches.py TaskWrapper, wrap_get_tasks(), wrap_guarded_task_generation() Partially reverted deletion because we need these on Windows wrap_worker() Patch now only applied when on POSIX (Windows uses the above instead) --- .../_child_process_profiling/cache.py | 54 +++++--- .../multiprocessing_patches.py | 115 ++++++++++++++++-- line_profiler/rc/line_profiler.toml | 10 +- 3 files changed, 150 insertions(+), 29 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 060e3cd6..c4653388 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -66,6 +66,23 @@ def _import_sibling(submodule: str) -> ModuleType: _private_field = partial(dataclasses.field, init=False, repr=False) +class _DumpStatsHelper(Cleanup): + def __init__( + self, prof: LineProfiler, outfile: os.PathLike[str] | str, + ) -> None: + super().__init__() + callback = self._callback = partial(prof.dump_stats, outfile) + self.add_cleanup(callback) + + def __repr__(self) -> str: + name = type(self).__name__ + get_repr = _CALLBACK_REPR_HELPER.repr + return f'<{name} @ {hex(id(self))}: {get_repr(self._callback)}>' + + def __call__(self) -> None: + self._callback() + + @final @dataclasses.dataclass class LineProfilingCache(Cleanup): @@ -91,7 +108,7 @@ class LineProfilingCache(Cleanup): _sighandlers: dict[int, _SignalHandler | int | None] = ( _private_field(default_factory=dict) ) - _stats_dumper: Cleanup | None = _private_field(default=None) + _stats_dumper: _DumpStatsHelper | None = _private_field(default=None) # These are unstructured fields; other components can decide on what # to put in them. They are also pickled by `.dump()`, and are thus # retrievable in `.load()`-ed instances. @@ -464,9 +481,11 @@ def wrap_ctx_debug( code = compile(fobj.read(), self.preimports_module, 'exec') exec(code, {}) # Use a fresh, empty namespace - # Occupy a tempfile slot in `.cache_dir` and set the profiler - # up to write thereto when the process terminates (with high - # priority) + # - Occupy a tempfile slot in `.cache_dir` + # - Set the profiler up to write thereto when the process + # terminates (with high priority) + # (Also keep a separate reference to the callback for e.g. + # dumping stats ASAP when a signal is caught) prof_outfile = self.make_tempfile( prefix=_PROFILING_OUTPUT_PREFIX_PATTERN.format( main_pid=self.main_pid, @@ -476,17 +495,15 @@ def wrap_ctx_debug( suffix='.lprof', delete=False, ) - dump_stats = partial(prof.dump_stats, prof_outfile) - self.add_cleanup_with_priority(dump_stats, 1) - - # Create a cleanup object for the express purpose of dumping - # stats in an emergency (e.g. when a signal is caught) - self._stats_dumper = dumper = Cleanup() + self._stats_dumper = dumper = _DumpStatsHelper(prof, prof_outfile) self.patch( + # If we call `dumper.cleanup()` instead of `dumper` (e.g. + # in `._handle_signal()`), the subsequent debug-log messages + # are attributed to and handled by this cache instance dumper, '_debug_output', self._debug_output, - cleanup=False, name=f'{self!r}._stats_dumper', + cleanup=False, name='._stats_dumper', ) - dumper.add_cleanup(dump_stats) + self.add_cleanup_with_priority(self._stats_dumper, 1) # Various setups self._setup_common(wrap_os_fork, {'reboot_forkserver': False}) @@ -519,11 +536,14 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover # interpreter's EoL... state = 'not initiated?!' try: - # Just dump the stats ASAP without running `.cleanup()` to - # avoid deadlocks - self._debug_output(f'Caught `{name}` ({signum}), dumping stats...') - if self._stats_dumper is not None: - self._stats_dumper.cleanup() + # Just use the `._stats_dumper` to dump the stats ASAP + # without running this cache's `.cleanup()` to avoid + # deadlocks + if self._stats_dumper is None: + state = 'unavailable' + else: + reason = f'caught `{name}` ({signum})' + self._stats_dumper.cleanup(reason=reason) except BaseException as e: xc = f'{type(e).__name__}' msg = str(e) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index c493def0..9a3c775d 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -22,7 +22,8 @@ from collections.abc import Callable, Collection, Mapping, Sequence, Set from functools import partial from importlib import import_module -from inspect import getattr_static +from inspect import getattr_static, signature +from multiprocessing.pool import Pool from multiprocessing.process import BaseProcess from multiprocessing.queues import SimpleQueue from operator import attrgetter @@ -78,6 +79,7 @@ _OnTimeout = Literal['ignore', 'warn', 'error'] PublicPatch = Literal['pool', 'process', 'logging'] +_CAN_CATCH_SIGTERM = sys.platform != 'win32' _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' _LOGGERS = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] _PATCHES: dict[str, '_Patch'] = {} @@ -367,6 +369,42 @@ def _process_has_returned( return result +class TaskWrapper(Generic[PS, T]): + """ + Pickle-able wrapper around the supplied task callable, which writes + to the session's profiling-stats file on exit. + + Note: + Since this produces extra overhead for each invocation of the + callable, it is only used when we can't reliably do + end-of-process cleanup. This mainly happens on Windows, where + we can't catch and handle ``SIGTERM``. + """ + def __init__(self, func: Callable[PS, T]) -> None: + self.func = func + try: + self.__signature__ = signature(func) + except Exception: # nocover + # Can happen with e.g. certain builin/c-based callables + pass + + def __call__(self, *args, **kwargs) -> T: + stats_dumper: Callable[[], None] | None = None + try: + stats_dumper = LineProfilingCache.load()._stats_dumper + except Exception: + pass + + try: + return self.func(*args, **kwargs) + finally: + if stats_dumper is not None: + # Calling the `_DumpStatsHelper` instead of using its + # `.cleanup()` prevents excessive per-task debugging + # output (and extra *extra* overhead) + stats_dumper() + + def _no_op(*_, **__) -> None: pass @@ -651,6 +689,43 @@ def _register_patch(name: str, patch: _Patch | None = None) -> _Patch: # ---------------- `multiprocessing.pool.Pool` patches ----------------- +@LineProfilingCache._method_wrapper +def wrap_get_tasks( + _, # No need to use the cache, but `_method_wrapper` expects it + vanilla_impl: Callable[Concatenate[Callable[PS1, T1], PS2], T2], + func: Callable[PS1, T1], + *args: PS2.args, + **kwargs: PS2.kwargs +) -> T2: + """ + Wrap around :py:meth:`.Pool._get_tasks` so that the writing of + profiling stats is handled within the callables sent to the child + processes before the parent process assumes control. + + Note: + :py:meth:`.Pool._get_tasks` is a static method. + """ + return vanilla_impl(TaskWrapper(func), *args, **kwargs) + + +@LineProfilingCache._method_wrapper +def wrap_guarded_task_generation( + _, # No need to use the cache, but `_method_wrapper` expects it + vanilla_impl: Callable[Concatenate[Pool, int, Callable[PS1, T1], PS2], T2], + self: Pool, + result_job: int, + func: Callable[PS1, T1], + *args: PS2.args, + **kwargs: PS2.kwargs +) -> T2: + """ + Wrap around :py:meth:`.Pool._guarded_task_generation` so that the + writing of profiling stats is handled within the callables sent to + the child processes before the parent process assumes control. + """ + return vanilla_impl(self, result_job, TaskWrapper(func), *args, **kwargs) + + @LineProfilingCache._method_wrapper # nocover def wrap_worker( cache: LineProfilingCache, @@ -664,10 +739,14 @@ def wrap_worker( processes can write profiling output as soon as the pool runs out of tasks. - Note: - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. + Notes: + - This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. + + - This only works reliably for POSIX because we can handle + ``SIGTERM`` on child processes and ensure that they aren't + prematurely terminated. """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up @@ -709,7 +788,18 @@ def _wrap_queue_get( _patch_pool = _register_patch('pool', Patch('pool')).add_method -_patch_pool('', 'worker', wrap_worker) +if _CAN_CATCH_SIGTERM: + # Only write profiling output once per process if it can be helped + _patch_pool('', 'worker', wrap_worker) +else: + # Don't have a choice on platform like Windows, the only reliable + # way to ensure that the child survives until profiling output is + # written is to write it for every task before control is returned + # to the parent + _patch_pool('Pool', '_get_tasks', wrap_get_tasks, 'static') + _patch_pool( + 'Pool', '_guarded_task_generation', wrap_guarded_task_generation, + ) # ----------- `multiprocessing.process.BaseProcess` patches ------------ @@ -788,7 +878,7 @@ def wrap_bootstrap( _patch_process('_bootstrap', wrap_bootstrap) # We only need to patch `Process.terminate()` if we can't do SIGTERM # handling, i.e. on Windows -if sys.platform == 'win32': +if not _CAN_CATCH_SIGTERM: _patch_process('terminate', wrap_terminate) # --------------- `multiprocessing.util` logging patches --------------- @@ -1001,9 +1091,14 @@ def apply( Patches: ``'pool'``: - Patch :py:func:`multiprocessing.pool.worker` so that - profiling output is written as each child process runs out - of task. + On Windows + Patch :py:class:`multiprocessing.pool.Pool`'s + ``._get_tasks()`` and ``._guarded_task_generation()`` + methods so that parallel tasks write profiling output. + Else + Patch :py:func:`multiprocessing.pool.worker` so that + profiling output is written as each child process runs + out of task. ``'process'``: Patch :py:class:`multiprocessing.process.BaseProcess`'s ``._bootstrap()`` method (and ``.terminate()`` on Windows) diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 493d9ab8..592beba9 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -234,8 +234,14 @@ catch_sigterm = true [tool.line_profiler.child_processes.multiprocessing.patches] # - `multiprocessing.patches.pool` (bool): -# Whether to patch `multiprocessing.pool.worker()`, so that each child -# process write profiling output as it runs out of tasks +# Whether to patch `multiprocessing.pool`: +# - POSIX: +# Patch `multiprocessing.pool.worker()` so that each child process +# write profiling output as it runs out of tasks +# - Windows: +# Patch `multiprocessing.pool.Pool` so that each task writes +# profiling output before pushing the result back to the parent +# process pool = true # - `multiprocessing.patches.process` (bool): # Whether to patch `multiprocessing.process.BaseProcess`, so that each From e1aa599c7a34c865a831454ca586eedc1b4a716a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 13 May 2026 09:18:47 +0200 Subject: [PATCH 103/165] Updated tests tests/test_child_procs.py::test_apply_mp_patches_{success,failure}() - Updated implementations - Loosened checks against empty-file warnings - Updated retrying rules for `test_apply_mp_patches_failure()`: - Only retrying when on Windows and `patch_pool` is false - Reverted back to `require='any'` --- tests/test_child_procs.py | 117 +++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 53 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 5c2e2669..8c7e495c 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -66,13 +66,9 @@ NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) -# XXX: owing to the shenanigans in -# `line_profiler._child_process_profiling.multiprocessing_patches`, -# there is a risk that failing child processes are not properly -# `.terminate()`-ed. So just put in a timeout... -_NUM_RETRIES = 2 _SUBPROC_TIMEOUT = 5 # Seconds _DEBUG = True +_WINDOWS = sys.platform == 'win32' def strip(s: str) -> str: @@ -1540,7 +1536,7 @@ def _search_cache_logs( # though an "intentionally" undocumented API (cpython issue #10308), # it's been around since 2.4, seems stable enough, and does exactly what # is needed -if sys.platform == 'win32': +if _WINDOWS: concat_command_line: Callable[ [Sequence[str]], str ] = subprocess.list2cmdline @@ -2355,7 +2351,7 @@ def test_load_pth_hook( @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) -def _test_apply_mp_patches( +def _test_apply_mp_patches_inner( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, @@ -2477,6 +2473,27 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: _search_cache_logs(cache, True, patterns) +def _test_apply_mp_patches( + patch_pool: bool, patch_process: bool, intercept_logs: bool, **kwargs +) -> None: + mp_patches: list[str] = [] + if patch_pool: + mp_patches.append('pool') + if patch_process: + mp_patches.append('process') + if intercept_logs: + mp_patches.append('logging') + with _check_warnings() as cw: + # Note: we can't guarantee that everything is set up early + # enough for cleanup to occur properly in child processes which + # don't receive any tasks, unless both the `process` and `pool` + # patches are applied + if patch_pool and patch_process: + cw.forbid_warnings(category=UserWarning, module='line_profiler') + cw.forbid_warnings(module='multiprocessing') + _test_apply_mp_patches_inner(mp_patches=mp_patches, **kwargs) + + @(_Params.new('start_method', ['fork', 'forkserver', 'spawn', 'dummy'], defaults='dummy') @@ -2512,37 +2529,37 @@ def test_apply_mp_patches_success( See also: :py:func:`test_apply_mp_patches_failure` """ - mp_patches: list[str] = [] - if patch_pool: - mp_patches.append('pool') - if patch_process: - mp_patches.append('process') - if intercept_logs: - mp_patches.append('logging') - with _check_warnings() as cw: - cw.forbid_warnings(category=UserWarning, module='line_profiler') - cw.forbid_warnings(module='multiprocessing') - _test_apply_mp_patches( - tmp_path_factory=tmp_path_factory, - create_cache=create_cache, - ext_module_object=ext_module_object, - test_module_object=test_module_object, - start_method=start_method, - mp_patches=mp_patches, - fail=False, - n=n, - nprocs=nprocs, - ) + _test_apply_mp_patches( + patch_pool, + patch_process, + intercept_logs, + tmp_path_factory=tmp_path_factory, + create_cache=create_cache, + ext_module_object=ext_module_object, + test_module_object=test_module_object, + start_method=start_method, + fail=False, + n=n, + nprocs=nprocs, + ) +# XXX: on POSIX child processes can hang around for long enough for +# profiling-stats collection to occur somewhat robustly, thanks to +# signal handling. But unfortunately on Windows: +# - When `patch_pool` is true, we wrap the task callables so that they +# always write profiling stats before returning/erroring out. This +# incurs extra overhead, but effectively prevents the reliquishing of +# control back to the parent process before the stats are ready. +# - However, when `patch_pool` is false, we can only try to block/delay +# child-process termination. A timeout is used to prevent indefinite +# waits for them to finish, and there's always the off chance that the +# end-of-process cleanup still haven't finished at the end. +# Hence the conditional need for retries... @pytest.mark.retry( - _NUM_RETRIES, + retries=2, + condition='_WINDOWS and not patch_pool', exceptions=(ResultMismatch, _Poller.Timeout), - # Patching `Pool` should be foolproof no matter the platform - # (as long as we use `Pool` for our parallelism); - # `Process` though... - condition='not patch_pool', - require='all', # Check consistency ) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) @@ -2571,25 +2588,19 @@ def test_apply_mp_patches_failure( See also: :py:func:`test_apply_mp_patches_success` """ - mp_patches: list[str] = [] - if patch_pool: - mp_patches.append('pool') - if patch_process: - mp_patches.append('process') - with _check_warnings() as cw: - cw.forbid_warnings(category=UserWarning, module='line_profiler') - cw.forbid_warnings(module='multiprocessing') - _test_apply_mp_patches( - tmp_path_factory=tmp_path_factory, - create_cache=create_cache, - ext_module_object=ext_module_object, - test_module_object=test_module_object, - start_method=start_method, - mp_patches=mp_patches, - fail=True, - n=n, - nprocs=nprocs, - ) + _test_apply_mp_patches( + patch_pool, + patch_process, + intercept_logs=False, + tmp_path_factory=tmp_path_factory, + create_cache=create_cache, + ext_module_object=ext_module_object, + test_module_object=test_module_object, + start_method=start_method, + fail=True, + n=n, + nprocs=nprocs, + ) # XXX: End of tests for implementation details From 7b32f9ae2d1064df03222e4d3b95ac1b945a0363 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 13 May 2026 10:13:15 +0200 Subject: [PATCH 104/165] Fix `tests/test_retry_tests.py` tests/test_retry_tests.py::test_retry_conditions() Fix failure when running in a narrow terminal; see `pytest-mark-retry` commit d53a515 --- tests/test_retry_tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py index 16340003..a3251671 100644 --- a/tests/test_retry_tests.py +++ b/tests/test_retry_tests.py @@ -978,11 +978,10 @@ def test_retry_conditions(conditions_module: _TestModule) -> None: # `test_bad_dynamic_condition()` should have failed with a # `RetryConditionFailure`, listing the error encountered in the last # trial and the error encountered when `eval()`-ing the condition - # (Note: grepping for the entire error message in the short test - # summary is fragile since it may be elided; so we just use a - # separate pattern to grep it from the tracebacks) + # (Note: grepping for the error details in the short test summary is + # fragile since it may be elided or even entirely omitted; so we + # just use a separate pattern to grep it from the tracebacks) lines = [ - 'FAILED .*::test_bad_dynamic_condition - .*RetryConditionFailure', r'.*RetryConditionFailure: \(RuntimeError: bar\) ' r"-> \(condition: 'foo == 1' -> NameError: .*'foo'.*\)", ] From 1994419884b18a6635ba7211b97f49a06e5829bc Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 15 May 2026 21:21:50 +0200 Subject: [PATCH 105/165] More robust PID management line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_worker_pool() Renamed from `wrap_worker()` because another function now also patches `multiprocessing.pool.worker()` wrap_handle_results(), wrap_worker_pid(), wrap_process() Parts of the new patch `child_pids` which patchs resp. - `multiprocessing.pool.Pool._handle_results()` - `multiprocessing.pool.worker()` - `multiprocessing.pool.Pool.Process()` To help the parent process keep track of which child process having completed which task, and hence be able to warn the session cache against profiling-stats file expected to be empty apply() Added documentation on the `child_pids` patch line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes.multiprocessing.patches] Added boolean key `child_pids` for whether to apply the new patch --- .../multiprocessing_patches.py | 256 +++++++++++++++++- line_profiler/rc/line_profiler.toml | 6 + 2 files changed, 248 insertions(+), 14 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 9a3c775d..0eae8cd8 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -20,13 +20,13 @@ import sys import warnings from collections.abc import Callable, Collection, Mapping, Sequence, Set -from functools import partial +from functools import partial, wraps from importlib import import_module from inspect import getattr_static, signature from multiprocessing.pool import Pool from multiprocessing.process import BaseProcess -from multiprocessing.queues import SimpleQueue from operator import attrgetter +from queue import SimpleQueue from time import sleep, monotonic from types import MappingProxyType as mappingproxy, MethodType, ModuleType from typing import ( @@ -34,7 +34,6 @@ Any, ClassVar, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, cast, final, overload, ) - from typing_extensions import Concatenate, ParamSpec, Self try: @@ -77,7 +76,7 @@ PS1 = ParamSpec('PS1') PS2 = ParamSpec('PS2') _OnTimeout = Literal['ignore', 'warn', 'error'] -PublicPatch = Literal['pool', 'process', 'logging'] +PublicPatch = Literal['pool', 'process', 'logging', 'child_pids'] _CAN_CATCH_SIGTERM = sys.platform != 'win32' _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' @@ -93,6 +92,18 @@ def __call__(self, func: Callable[PS, T], /) -> Callable[PS, T]: ... +class _Queue(Protocol): + """ + Protocol for methods common to e.g. :py:class:`queue.SimpleQueue` + and :py:class:`multiprocessing.queues.SimpleQueue`. + """ + def put(self, obj: Any) -> None: + ... + + def get(self) -> Any: + ... + + class _Poller: """ Poll a callable until it returns true-y. @@ -405,6 +416,34 @@ def __call__(self, *args, **kwargs) -> T: stats_dumper() +class _QueuePIDWrapper: + """ + Wrap around a :py:class:`queue.SimpleQueue` (used by + :py:mod:`multiprocessing.dummy`) so that the PID info is attached to + the :py:meth:`.SimpleQueue.put` tuple. + + Notes: + - Used by the ``child_pids`` patch. + + - While the PID info is useless when using + :py:mod:`multiprocessing.dummy`, it is nonetheless necessary + because :py:meth:`multiprocessing.pool.Pool._handle_results` + has also been patched to expect the queue-item-getter function + to return the tuple ``(pid, original_put_value)``. + """ + def __init__(self, queue: _Queue) -> None: + self._queue = queue + + def __getattr__(self, attr: str) -> Any: + return getattr(self._queue, attr) + + def put(self, obj: Any) -> None: + self._queue.put((os.getpid(), obj)) + + def get(self) -> Any: + return self._queue.get() + + def _no_op(*_, **__) -> None: pass @@ -727,10 +766,10 @@ def wrap_guarded_task_generation( @LineProfilingCache._method_wrapper # nocover -def wrap_worker( +def wrap_worker_pool( cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[SimpleQueue, PS], None], - inqueue: SimpleQueue, + vanilla_impl: Callable[Concatenate[_Queue, PS], None], + inqueue: _Queue, *args: PS.args, **kwargs: PS.kwargs ) -> None: @@ -755,7 +794,7 @@ def wrap_worker( # deadlock with Cleanup() as cleanup: if isinstance(inqueue.get, MethodType): - get = partial(_wrap_queue_get, cache, inqueue.get.__func__) + get = partial(_wrap_inqueue_get, cache, inqueue.get.__func__) cleanup.patch( inqueue, 'get', MethodType(get, inqueue), name='._inqueue', @@ -763,14 +802,18 @@ def wrap_worker( return vanilla_impl(inqueue, *args, **kwargs) -def _wrap_queue_get( +def _wrap_inqueue_get( cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[SimpleQueue, PS], T], - self: SimpleQueue, + vanilla_impl: Callable[Concatenate[_Queue, PS], T], + self: _Queue, /, *args: PS.args, **kwargs: PS.kwargs ) -> T: + """ + Intecept the sentinel value (:py:const:`None`) signifiying the end + of the queue and perform cleanup. + """ result = vanilla_impl(self, *args, **kwargs) ntasks: dict[int, int] ntasks = cache._additional_data.setdefault('mp_queue_ntasks', {}) @@ -790,7 +833,7 @@ def _wrap_queue_get( _patch_pool = _register_patch('pool', Patch('pool')).add_method if _CAN_CATCH_SIGTERM: # Only write profiling output once per process if it can be helped - _patch_pool('', 'worker', wrap_worker) + _patch_pool('', 'worker', wrap_worker_pool) else: # Don't have a choice on platform like Windows, the only reliable # way to ensure that the child survives until profiling output is @@ -855,7 +898,7 @@ def wrap_bootstrap( - ``SIGTERM`` handling is not consistent on Windows, so we made :py:meth:`.LineProfilingCache._add_signal_handler` a no-op - there. Hence :py:func:`wrap_terminate` remains necessary in + there. Hence :py:func:`wrap_terminate` remains necessary for mitigating unclean exits. """ # Set a signal handler for SIGTERM to help child processes with @@ -881,6 +924,178 @@ def wrap_bootstrap( if not _CAN_CATCH_SIGTERM: _patch_process('terminate', wrap_terminate) +# ---------------------- PID bookkeeping patches ----------------------- + + +@LineProfilingCache._method_wrapper +def wrap_handle_results( + cache: LineProfilingCache, + vanilla_impl: Callable[ + Concatenate[_Queue, Callable[[], tuple[Any, ...] | None], PS], + None + ], + outqueue: _Queue, + # Since we patched `outqueue.put()` in the child process, the result + # tuple pushed to the parent has an extra item (the child PID) + get: Callable[[], tuple[int, tuple[Any, ...]] | None], + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:meth:`multiprocessing.pool.Pool._handle_results` so + that it handles the extra info (PID of child process handling the + task) included by :py:func:`.wrap_worker_pid`. + + Note: + :py:meth:`.Pool._handle_results` is a static method. + """ + # Somehow this doesn't type-check with either `mypy` or `ty` when + # we use a `TypeVar` instead of `Any` with the tuple items... + # (see `ty` issue #3467) + wrapped_get = partial(_wrap_outqueue_quick_get, cache, get) + vanilla_impl(outqueue, wrapped_get, *args, **kwargs) + + +@LineProfilingCache._method_wrapper # nocover +def wrap_worker_pid( + _, # We don't need the cache instance, but `@_method_wrapper` does + vanilla_impl: Callable[Concatenate[_Queue, _Queue, PS], None], + inqueue: _Queue, + outqueue: _Queue, + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes report their PIDs as they pass the task results back to + the parent. + + Note: + This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. + """ + # Note: using the `cache` itself as the context manager is prone to + # deadlock + with Cleanup() as cleanup: + if isinstance(outqueue, SimpleQueue): + # `multiprocessing.dummy` instantiates C-based queue + # objects, which doesn't permit assigning to the instance + # (because it doesn't have an instance dict)... + # so just wrap the queue with a helper class + outqueue = _QueuePIDWrapper(outqueue) + elif isinstance(outqueue.put, MethodType): + put = partial(_wrap_outqueue_put, outqueue.put.__func__) + cleanup.patch( + outqueue, 'put', MethodType(put, outqueue), + name='._outqueue', + ) + return vanilla_impl(inqueue, outqueue, *args, **kwargs) + + +@LineProfilingCache._method_wrapper +def wrap_process( + cache: LineProfilingCache, + vanilla_impl: Callable[PS, P], + *args: PS.args, + **kwargs: PS.kwargs +) -> P: + """ + Wrap around :py:func:`multiprocessing.pool.Pool.Process` so that the + processes created can report on usage when + :py:meth:`.BaseProcess.join`-ed or + :py:meth:`.BaseProcess.terminate`-ed. + + Note: + :py:meth:`.Pool.Process` is a static method. + """ + proc = vanilla_impl(*args, **kwargs) + # Note: since we don't clean up here, there's no need to instantiate + # another `Cleanup` helper + name = f'<{type(proc).__name__} @ {hex(id(proc))}>' + patch = partial(cache.patch, cleanup=False, name=name) + for method, action in ('join', 'joining'), ('terminate', 'terminating'): + bound = getattr(proc, method) + if isinstance(bound, MethodType): + finalize = _wrap_process_finalize(cache, bound.__func__, action) + patch(proc, method, MethodType(finalize, proc)) + return proc + + +def _wrap_process_finalize( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[P, PS], None], + action: str, +) -> Callable[Concatenate[P, PS], None]: + """ + Check if the process has run any tasks; + if not, report to the cache. + + Note: + Since the process object is pickled, this method has to directly + return a function object instead of merely being + :py:func:`partial`-ed and wrapped in a + :py:class:`types.MethodType`. + """ + @wraps(vanilla_impl) + def finaize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: + pid: int | None = getattr(self, 'pid', None) + if pid is not None: + ntasks = _get_ntasks(cache).pop(pid, 0) + if not ntasks: + cache._warn_possible_lack_of_stats(pid) + msg = f'{action} process {pid} which ran {ntasks} task(s)' + cache._debug_output(msg) + vanilla_impl(self, *args, **kwargs) + + action = action.capitalize() + return finaize + + +def _wrap_outqueue_put( + vanilla_impl: Callable[ + Concatenate[_Queue, tuple[Any, ...], PS], None + ], + self: _Queue, + obj: tuple[Any, ...], + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Smuggle in the PID of the child process so that the parent can keep + track of which child completed what task. + """ + vanilla_impl(self, (os.getpid(), obj), *args, **kwargs) + + +def _wrap_outqueue_quick_get( + cache: LineProfilingCache, + vanilla_impl: Callable[PS, tuple[int, tuple[Any, ...]] | None], + *args: PS.args, + **kwargs: PS.kwargs +) -> tuple[Any, ...] | None: + """ + Take and process the PID of the child process completing the task. + """ + result = vanilla_impl(*args, **kwargs) + if result is None: + return None + pid, orig_result = result + ntasks = _get_ntasks(cache) + ntasks[pid] = ntasks.get(pid, 0) + 1 + return orig_result + + +def _get_ntasks(cache: LineProfilingCache) -> dict[int, int]: + key = 'mp_proc_ntasks' + return cache._additional_data.setdefault(key, cast(dict[int, int], {})) + + +_patch_pid = _register_patch('child_pids', Patch('pool')).add_method +_patch_pid('', 'worker', wrap_worker_pid) +_patch_pid('Pool', '_handle_results', wrap_handle_results, 'static') +_patch_pid('Pool', 'Process', wrap_process, 'static') + # --------------- `multiprocessing.util` logging patches --------------- @@ -1084,7 +1299,8 @@ def apply( so as to ensure that profiling happens on processes forked therefrom (see Note). patches \ -(Collection[Literal['pool', 'process', 'logging'] | None]): +(Collection[Literal['pool', 'process', 'logging', 'child_pids'] \ +| None]): Patches to apply to :py:mod:`multiprocessing`; see the following section for a description of each; the default is taken from the TOML config file. @@ -1108,6 +1324,18 @@ def apply( Patch :py:mod:`multiprocessing.util`'s logging methods (e.g. ``debug()`` and ``info()``) so that their messages are teed to the cache's debug log. + ``'child_pids'``: + Patch the following components of + :py:mod:`multiprocess.pool` so that the parent process keeps + track of the workload executed by each child process, + reducing stray warnings about the lack of profiling stats + reported thereby: + + - :py:func:`multiprocessing.pool.worker` + + - :py:meth:`multiprocessing.pool.Pool._handle_results` + + - :py:meth:`multiprocessing.pool.Pool.Process` Side effects: - The aforementioned patches applied diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 592beba9..bdd8549b 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -254,6 +254,12 @@ process = true # that the internal logs of `multiprocessing` are teed to the session # cache's debug logs logging = false +# - `multiprocessing.patches.child_pids` (bool): +# Whether to patch `multiprocessing.pool.worker()` and +# `multiprocessing.pool.Pool` so that the the parent process keeps +# track of the workload sent to each child, suppressing stray warnings +# about empty output files where appropriate +child_pids = true # Polling controls, used for the `pool` patch From 815de734e12a9adfad5ae929fe1eb654f1f2b688 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 15 May 2026 21:51:35 +0200 Subject: [PATCH 106/165] Updated tests tests/test_child_procs.py _test_apply_mp_patches() - Parameters `patch_pool`, `patch_process`, and `intercept_logs` now permit the value `None` for using the default-config values - New parameter `trace_pids` for toggling the `child_pids` patch - Updated the `line_profiler` warning check to only happen when the `child_pids` patch is applied (since that is when we can reliably weed out the empty-file warnings) test_apply_mp_patches_success() - Reworked most subtests to just use the default value for whether to apply the `logging` patch - Added subtest for whether to apply the `child_pids` patch test_apply_mp_patches_failure() Now using the default for whether to apply the `logging` patch --- tests/test_child_procs.py | 64 +++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 8c7e495c..26172b81 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -45,7 +45,7 @@ create_runpy_wrapper, ) from line_profiler._child_process_profiling.multiprocessing_patches import ( - _Poller, _PATCHED_MARKER, _PATCHES as MP_PATCHES, + _Poller, MPConfig, _PATCHED_MARKER, _PATCHES as MP_PATCHES, ) from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler.curated_profiling import ( @@ -1795,6 +1795,9 @@ def _check_output(output: str, tag: str, nhits: int) -> None: _PatchSummary = Mapping[str, Set[str]] +_mp_patch_is_internal: Callable[[str], bool] +_mp_patch_is_internal = operator.methodcaller('startswith', '__') + def get_patched_attributes( applied_mp_patches: Collection[str] | None = None, @@ -1828,7 +1831,10 @@ def _get_patched_attributes( def _get_toml_patches_section(mp_patches: Collection[str]) -> str: - mp_patches_as_dict = {name: name in mp_patches for name in MP_PATCHES} + mp_patches_as_dict = { + name: name in mp_patches for name in MP_PATCHES + if not _mp_patch_is_internal(name) + } return ( '[tool.line_profiler.child_processes.multiprocessing.patches]\n' + '\n'.join( @@ -1907,7 +1913,7 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: _dynamically_resolved_patch_summaries: Iterable[_PatchSummary] = ( patch.summary for name, patch in MP_PATCHES.items() # Basic `multiprocessing` patches are always applied - if name.startswith('__') + if _mp_patch_is_internal(name) ) _dynamically_resolved_patch_summaries = itertools.chain( _dynamically_resolved_patch_summaries, @@ -1933,7 +1939,7 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: _GLOBAL_MINIMAL_PATCHES, _HOOK_PATCHES, get_patched_attributes([ - name for name in MP_PATCHES if not name.startswith('__') + name for name in MP_PATCHES if not _mp_patch_is_internal(name) ]), ) # Actual patches using the default config @@ -1941,6 +1947,8 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: _GLOBAL_MINIMAL_PATCHES, get_patched_attributes(), ) +_DEFAULT_MP_CONFIG = MPConfig.from_config(ConfigSource.from_default()) + @pytest.mark.parametrize(('run_profiled_code', 'label1'), [(True, 'run-profiled'), (False, 'run-unrelated')]) @@ -2128,6 +2136,7 @@ def test_cache_setup_main_process( *( (name in mp_patches, _filter_patches(patch.summary)) for name, patch in MP_PATCHES.items() + if not _mp_patch_is_internal(name) ), ]) try: @@ -2474,21 +2483,24 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: def _test_apply_mp_patches( - patch_pool: bool, patch_process: bool, intercept_logs: bool, **kwargs + patch_pool: bool | None = None, + patch_process: bool | None = None, + intercept_logs: bool | None = None, + trace_pids: bool | None = None, + **kwargs ) -> None: - mp_patches: list[str] = [] - if patch_pool: - mp_patches.append('pool') - if patch_process: - mp_patches.append('process') - if intercept_logs: - mp_patches.append('logging') + patches = cast(dict[str, bool], _DEFAULT_MP_CONFIG.patches.copy()) + for name, applied in { + 'pool': patch_pool, 'process': patch_process, + 'logging': intercept_logs, 'child_pids': trace_pids, + }.items(): + if applied is not None: + patches[name] = applied + mp_patches = [name for name, applied in patches.items() if applied] with _check_warnings() as cw: - # Note: we can't guarantee that everything is set up early - # enough for cleanup to occur properly in child processes which - # don't receive any tasks, unless both the `process` and `pool` - # patches are applied - if patch_pool and patch_process: + if 'child_pids' in mp_patches: + # With PID bookkeeping we should be able to weed out all + # the child processes which didn't perform any work cw.forbid_warnings(category=UserWarning, module='line_profiler') cw.forbid_warnings(module='multiprocessing') _test_apply_mp_patches_inner(mp_patches=mp_patches, **kwargs) @@ -2497,12 +2509,16 @@ def _test_apply_mp_patches( @(_Params.new('start_method', ['fork', 'forkserver', 'spawn', 'dummy'], defaults='dummy') - # We only need to check if `intercept_logs` work, the other + # We only need to check if `intercept_logs = logging` work, the other # parametrizations don't matter + _Params.new(('intercept_logs', 'label1'), - [(True, 'with-intercept-logs'), (False, 'no-intercept-logs')], - defaults=(False, 'no-intercept-logs'))).sorted() -@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label2'), + [(True, 'with-logging'), (False, 'no-logging')], + defaults=(None, 'default-logging')) + # Same deal with `trace_pids = child_pids` + + _Params.new(('trace_pids', 'label2'), + [(True, 'with-child_pids'), (False, 'no-child-pids')], + defaults=(None, 'default-child-pids'))).sorted() +@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label3'), [(True, True, 'pool-and-process'), (True, False, 'pool-only'), (False, True, 'process-only')]) @@ -2515,11 +2531,13 @@ def test_apply_mp_patches_success( start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], patch_pool: bool, patch_process: bool, - intercept_logs: bool, + intercept_logs: bool | None, + trace_pids: bool | None, n: int, nprocs: int, label1: str, label2: str, + label3: str, ) -> None: """ Test that :py:func:`line_profiler._child_process_profiling\ @@ -2533,6 +2551,7 @@ def test_apply_mp_patches_success( patch_pool, patch_process, intercept_logs, + trace_pids, tmp_path_factory=tmp_path_factory, create_cache=create_cache, ext_module_object=ext_module_object, @@ -2591,7 +2610,6 @@ def test_apply_mp_patches_failure( _test_apply_mp_patches( patch_pool, patch_process, - intercept_logs=False, tmp_path_factory=tmp_path_factory, create_cache=create_cache, ext_module_object=ext_module_object, From 22cf3fe8f312d3377678d5944400e323d5ac6d8f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 16 May 2026 23:13:24 +0200 Subject: [PATCH 107/165] Timeout for `test_apply_mp_patches_*()` tests/test_child_procs.py @_timeout, _TestTimeout New decorator and error class for timing out function execution using threads _test_apply_mp_patches_inner() Now timing out the tested function (`test_module_object.sum_in_child_procs()`) which uses `multiprocessing` --- tests/test_child_procs.py | 128 +++++++++++++++++++++++++++++++++++--- 1 file changed, 121 insertions(+), 7 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 26172b81..19fce375 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -7,11 +7,13 @@ import multiprocessing.pool import operator import os +import pickle import re import shlex import subprocess import sys import sysconfig +import threading import warnings from abc import ABC, abstractmethod from collections.abc import ( @@ -20,8 +22,11 @@ ) from contextlib import AbstractContextManager, ExitStack, nullcontext from functools import lru_cache, partial, wraps -from io import StringIO +from io import BytesIO, StringIO from importlib import import_module +from multiprocessing.pool import ( # type: ignore + ExceptionWithTraceback as ExceptionHelper, +) from numbers import Real from pathlib import Path from runpy import run_path @@ -30,7 +35,7 @@ from time import monotonic from types import MappingProxyType, ModuleType, TracebackType from typing import ( - TYPE_CHECKING, Any, Generic, Literal, Protocol, TypeVar, + TYPE_CHECKING, Any, Generic, IO, Literal, Protocol, TypeVar, cast, final, overload, ) from typing_extensions import Self, ParamSpec @@ -66,7 +71,7 @@ NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) -_SUBPROC_TIMEOUT = 5 # Seconds +_TEST_TIMEOUT = 5 # Seconds _DEBUG = True _WINDOWS = sys.platform == 'win32' @@ -611,6 +616,13 @@ def rich_message(self) -> str: return msg +class _TestTimeout(RuntimeError): + """ + Error raised by the :py:func:`_timeout` decorator. + """ + pass + + @final @dataclasses.dataclass class _Params: @@ -1788,6 +1800,101 @@ def _check_output(output: str, tag: str, nhits: int) -> None: _run_test_module, _run_as_literal_code, profiled_code_is_tempfile=True, ) + +@overload +def _timeout( + func: Callable[PS, T], *, timeout: float = _TEST_TIMEOUT, +) -> Callable[PS, T]: + ... + + +@overload +def _timeout( + func: None = None, *, timeout: float = _TEST_TIMEOUT, +) -> Callable[[Callable[PS, T]], Callable[PS, T]]: + ... + + +def _timeout( + func: Callable[PS, T] | None = None, *, + timeout: float = _TEST_TIMEOUT, +) -> Callable[PS, T] | Callable[[Callable[PS, T]], Callable[PS, T]]: + """ + Decorate the test function so that it is run in another thread and + can be timed out. + + Example: + >>> from time import sleep + + >>> @_timeout(timeout=.5) + ... def my_func( + ... n: int, delay: float = 1, error: bool = False, + ... ) -> list[int]: + ... sleep(delay) + ... if error: + ... raise RuntimeError('my error message') + ... return list(range(n)) + + Normal execution: + + >>> my_func(3, 0) + [3] + [0, 1, 2, 3] + + Erroring out: + + >>> my_func(3, 0, error=True) + Traceback (most recent call last): + ... + RuntimeError: my error message + + Timing out: + + >>> my_func(4, delay=5) # doctest: +NORMALIZE_WHITESPACE + Traceback (most recent call last): + ... + test_child_procs._TestTimeout: + my_func(4, delay=5): timed out after 0.5 s + """ + if func is None: + return cast( + Callable[[Callable[PS, T]], Callable[PS, T]], + partial(_timeout, timeout=timeout), + ) + + @wraps(func) + def inner_wrapper( + fobj: IO[bytes], /, *args: PS.args, **kwargs: PS.kwargs + ) -> None: + try: + result = True, func(*args, **kwargs) + except Exception as e: + result = False, ExceptionHelper(e, e.__traceback__) + pickle.dump(result, fobj) + + @wraps(func) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + with BytesIO() as bio: + thread = new_thread(args=(bio, *args), kwargs=kwargs) + thread.start() + thread.join(timeout) + if not thread.is_alive(): + successful, result = pickle.loads(bio.getvalue()) + if successful: + return result + assert isinstance(result, Exception) + raise result + args_repr = [repr(a) for a in args] + args_repr.extend(f'{k}={v!r}' for k, v in kwargs.items()) + name = getattr(func, '__name__', repr(func)) + call_repr = f'{name}({", ".join(args_repr)})' + msg = f'{call_repr}: timed out after {timeout:.2g} s' + raise _TestTimeout(msg) + + new_thread = partial(threading.Thread, target=inner_wrapper, daemon=True) + + return wrapper + + # ============================= Unit tests ============================= # XXX: Tests in this section concerns implementation details, and the @@ -2402,8 +2509,14 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # with `preimports_module=True`, both are just imported and added # to the profiler, so the code paths are the same profiled_func = ext_module_object.my_external_sum + # Note: it would have been more intuitive to just apply `@_timeout` + # to this whole function and run it all in a new thread, but that + # seem to interact adversely with patch application and prof-data + # collection... so just timeout the function invoking + # `multiprocessing` + sum_with_timeout = _timeout(test_module_object.sum_in_child_procs) called_func = partial( - test_module_object.sum_in_child_procs, + sum_with_timeout, n=nprocs, my_sum=profiled_func, start_method=start_method, @@ -2438,7 +2551,8 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: loop_line = get_lineno(ext_module_object.__file__, 'EXT-LOOP') if fail: - xc_context: AbstractContextManager[Any] = pytest.raises(RuntimeError) + xc_context: AbstractContextManager[Any] + xc_context = pytest.raises(RuntimeError, match='^forced failure$') nloops_expected = n else: xc_context = nullcontext() @@ -2807,7 +2921,7 @@ def _test_profiling_multiproc_script( nhits=nhits, nnums=nnums, nprocs=nprocs, - timeout=_SUBPROC_TIMEOUT, + timeout=_TEST_TIMEOUT, debug_log=( 'debug.log' if prof_child_procs and _DEBUG else None ), @@ -2997,7 +3111,7 @@ def _test_profiling_bare_python( """.format(concat_command_line(sub_cmd))) cmd.extend(['-c', code]) proc = _run_subproc( - cmd, text=True, capture_output=True, timeout=_SUBPROC_TIMEOUT, + cmd, text=True, capture_output=True, timeout=_TEST_TIMEOUT, ) nhits = {'EXT-INVOCATION': 1, 'EXT-LOOP': n} From f14001e5ccf2dc653721187418daf4de9cc223e2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 17 May 2026 00:27:19 +0200 Subject: [PATCH 108/165] PID-tracking fix line_profiler/_child_process_profiling/multiprocessing_patches.py ::wrap_process() - Fixed bug where the if `Process.join()` and `Process.terminate()` are called more than once, the second call onwards will always report the process' having completed 0 tasks - Enclosed the bookkeeping in the above methods in a try-except block so that the original implementation is always run --- .../multiprocessing_patches.py | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 0eae8cd8..0004be1b 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -1038,18 +1038,23 @@ def _wrap_process_finalize( :py:class:`types.MethodType`. """ @wraps(vanilla_impl) - def finaize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: - pid: int | None = getattr(self, 'pid', None) - if pid is not None: - ntasks = _get_ntasks(cache).pop(pid, 0) - if not ntasks: - cache._warn_possible_lack_of_stats(pid) - msg = f'{action} process {pid} which ran {ntasks} task(s)' - cache._debug_output(msg) - vanilla_impl(self, *args, **kwargs) + def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: + try: + pid: int | None = getattr(self, 'pid', None) + checked_procs = _get_checked_processes(cache) + identifier = id(self), pid + if not (pid is None or identifier in checked_procs): + ntasks = _get_ntasks(cache).pop(pid, 0) + if not ntasks: + cache._warn_possible_lack_of_stats(pid) + msg = f'{action} process {pid} which ran {ntasks} task(s)' + cache._debug_output(msg) + checked_procs.add(cast(tuple[int, int], identifier)) + finally: + vanilla_impl(self, *args, **kwargs) action = action.capitalize() - return finaize + return finalize def _wrap_outqueue_put( @@ -1091,6 +1096,15 @@ def _get_ntasks(cache: LineProfilingCache) -> dict[int, int]: return cache._additional_data.setdefault(key, cast(dict[int, int], {})) +def _get_checked_processes( + cache: LineProfilingCache, +) -> set[tuple[int, int]]: + key = 'mp_proc_checked_workload' + return cache._additional_data.setdefault( + key, cast(set[tuple[int, int]], set()), + ) + + _patch_pid = _register_patch('child_pids', Patch('pool')).add_method _patch_pid('', 'worker', wrap_worker_pid) _patch_pid('Pool', '_handle_results', wrap_handle_results, 'static') From 98f8b31757affe10da0708e6fc24a4f1f2f64487 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 18 May 2026 02:05:35 +0200 Subject: [PATCH 109/165] Minor test fixes tests/test_child_procs.py @_timeout Now no longer raising an error from the new thread when the main thread has timed out, causing the file handle to be destroyed _run_test_module(), run_{module,script,literal_code}() Fixed bug where if the subprocess timed out the debug logs won't be written _test_apply_mp_patches_inner() No longer using `pytest.raises()` to check for the error when `fail=True` to allow for other `RuntimeError`s (e.g. timeouts) to be propagated --- tests/test_child_procs.py | 58 +++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 19fce375..86033b3d 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -20,7 +20,7 @@ Callable, Collection, Generator, Iterable, Iterator, Mapping, Sequence, Set, ) -from contextlib import AbstractContextManager, ExitStack, nullcontext +from contextlib import ExitStack from functools import lru_cache, partial, wraps from io import BytesIO, StringIO from importlib import import_module @@ -1732,12 +1732,12 @@ def _run_test_module( if debug_log: runner_args.extend(['--debug-log', debug_log]) old_pth_files = _preserve_pth_files.get_pth_files() - proc = run_helper( - runner_args, test_args, test_module, - text=True, capture_output=True, check=(check and not fail), - **kwargs - ) try: + proc = run_helper( + runner_args, test_args, test_module, + text=True, capture_output=True, check=(check and not fail), + **kwargs + ) # Checks: if fail: # - The process has failed as expected @@ -1769,7 +1769,7 @@ def _run_test_module( for tag, num in (nhits or {}).items(): _check_output(proc.stdout, tag, num) finally: - if debug_log is not None: + if debug_log is not None and os.path.exists(debug_log): with open(debug_log) as fobj: print('-- Combined debug logs --', file=sys.stderr) print(indent(fobj.read(), ' '), end='', file=sys.stderr) @@ -1862,14 +1862,25 @@ def _timeout( ) @wraps(func) - def inner_wrapper( + def worker( fobj: IO[bytes], /, *args: PS.args, **kwargs: PS.kwargs ) -> None: try: result = True, func(*args, **kwargs) except Exception as e: result = False, ExceptionHelper(e, e.__traceback__) - pickle.dump(result, fobj) + # Do this instead of directly using `pickle.dump(..., fobj)` so + # that pickling errors and file-handle-related errors are + # handled separately + serialized = pickle.dumps(result, protocol=pickle.HIGHEST_PROTOCOL) + try: + fobj.write(serialized) + fobj.flush() + except Exception: + # Since this is run in a daemon thread, by the time this + # write happens the main thread could've already timed out + # and destroyed `fobj`... in that case just gracefully exit + pass @wraps(func) def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: @@ -1890,7 +1901,7 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: msg = f'{call_repr}: timed out after {timeout:.2g} s' raise _TestTimeout(msg) - new_thread = partial(threading.Thread, target=inner_wrapper, daemon=True) + new_thread = partial(threading.Thread, target=worker, daemon=True) return wrapper @@ -2550,13 +2561,10 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: assert ext_module_object.__file__ loop_line = get_lineno(ext_module_object.__file__, 'EXT-LOOP') - if fail: - xc_context: AbstractContextManager[Any] - xc_context = pytest.raises(RuntimeError, match='^forced failure$') - nloops_expected = n - else: - xc_context = nullcontext() - nloops_expected = n + nprocs + nloops_expected = n + if not fail: + # Counts from the one final sum over the parallel results + nloops_expected += nprocs if start_method not in ('dummy', *START_METHODS): pytest.skip( @@ -2564,10 +2572,20 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: 'not available on the platform' ) - with xc_context: + # Note: manually handle the error here instead of using + # `pytest.raises()` since we want certain `RuntimeError`s to be + # propagated and handled by `@pytest.mark.retry` + fail_msg = 'forced failure' + try: result = called_func(n) - # Check correctness of the results - if not fail: + except RuntimeError as e: + if not (fail and str(e) == fail_msg): + raise + else: + if fail: + msg = f"expected `RuntimeError({fail_msg!r})`, no error raised" + raise ValueError(msg) + else: # Check correctness of the results assert result == n * (n + 1) // 2 # Check that calls in children are traced From 3dfe818b48d803cbbdc18a399f07fb1b0d904af4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 18 May 2026 08:00:37 +0200 Subject: [PATCH 110/165] Unreg. `atexit` hook in `multiprocessing` procs line_profiler/_child_process_profiling/cache.py _DumpStatsHelper.cleanup() New parameter `force` for forcing the writing of the profiling stats even if the cleanup stacks are empty LineProfilingCache._atexit_hook() New property for a persistent handle on the hook passed to `atexit.register()`, so that it can be unregistered when necessary line_profiler/_child_process_profiling/multiprocessing_patches.py ::wrap_{worker_pool,bootstrap}() - Now unregistering `LineProfilingCache.load()._atexit_hook()` with `atexit.unregister()` to hopefully take care of end-of-process deadlocks in child processes - Now prioritizing cleanup speed in child processes, using `cache._stats_dumper.cleanup()` instead of `cache.cleanup()` where appropriate to avoid complications --- .../_child_process_profiling/cache.py | 13 ++- .../multiprocessing_patches.py | 94 ++++++++++++++----- 2 files changed, 84 insertions(+), 23 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index c4653388..185b8076 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -82,6 +82,11 @@ def __repr__(self) -> str: def __call__(self) -> None: self._callback() + def cleanup(self, *args, force: bool = False, **kwargs) -> None: + if force and not any(self._current_context.values()): + self.add_cleanup(self._callback) + super().cleanup(*args, **kwargs) + @final @dataclasses.dataclass @@ -510,7 +515,7 @@ def wrap_ctx_debug( # Set `.cleanup()` as an atexit hook to handle everything when # the child process is about to terminate - atexit.register(partial(self.cleanup, reason='`atexit` callback')) + atexit.register(self._atexit_hook) self._debug_output(f'Setup successful ({context})') return True @@ -543,7 +548,7 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover state = 'unavailable' else: reason = f'caught `{name}` ({signum})' - self._stats_dumper.cleanup(reason=reason) + self._stats_dumper.cleanup(force=True, reason=reason) except BaseException as e: xc = f'{type(e).__name__}' msg = str(e) @@ -876,3 +881,7 @@ def _empty_stats_pid_registry(self) -> Path: current_pid=os.getpid(), ) return self.make_tempfile(prefix=prefix, suffix='.dat', delete=False) + + @cached_property + def _atexit_hook(self) -> Callable[[], None]: + return partial(self.cleanup, reason='`atexit` callback') diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 0004be1b..7c37bfe3 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -14,6 +14,7 @@ """ from __future__ import annotations +import atexit import dataclasses import multiprocessing import os @@ -400,20 +401,20 @@ def __init__(self, func: Callable[PS, T]) -> None: pass def __call__(self, *args, **kwargs) -> T: - stats_dumper: Callable[[], None] | None = None + callback: Callable[[], None] = _no_op try: - stats_dumper = LineProfilingCache.load()._stats_dumper + cache = LineProfilingCache.load() except Exception: pass + else: + # Note: this doesn't write debugging output... doing so for + # every task would be excessive + callback = partial(_dump_stats_quick, cache) try: return self.func(*args, **kwargs) finally: - if stats_dumper is not None: - # Calling the `_DumpStatsHelper` instead of using its - # `.cleanup()` prevents excessive per-task debugging - # output (and extra *extra* overhead) - stats_dumper() + callback() class _QueuePIDWrapper: @@ -448,6 +449,32 @@ def _no_op(*_, **__) -> None: pass +def _setup_in_mp_child(cache: LineProfilingCache) -> None: + """ + Perform :py:mod:`multiprocessing`-specific setup in a child process + curated by the module. Currently it does the following: + + - Set up ``cache`` to handle ``SIGTERM`` on POSIX if not already + set. + + - Unregister the :py:mod:`atexit` hook associated with ``cache`` to + avoid possible clashes with the profiling-file writing managed by + this module. + """ + xc: Exception | None = None + for setup in [_add_sigterm_handler_in_child, _unregister_atexit_hook]: + try: + setup(cache) + except Exception as e: + xc = e + if xc is not None: + xc_str = type(xc).__name__ + if str(xc): + xc_str = f'{xc_str}: {xc}' + cache._debug_output(f'Setup failed in process {os.getpid()}: {xc_str}') + raise xc + + def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: key = 'mp_added_sigterm_handler' if not MPConfig.from_cache(cache).catch_sigterm: @@ -460,6 +487,35 @@ def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: cache._add_signal_handler() cache._additional_data[key] = True + +def _unregister_atexit_hook(cache: LineProfilingCache) -> None: + atexit.unregister(cache._atexit_hook) + + +def _dump_stats_quick( + cache: LineProfilingCache, + *, + reason: str | None = None, + debug: bool = False, +) -> None: + """ + Note: + We don't really care about cleanup in the child process, so just + dump the stats and bail to reduce the chance of end-of-process + shenanigans causing a deadlock... + but do use ``._stats_dumper.cleanup()`` instead of + ``.__call__()`` so that we get debugging output (if ``debug`` is + true) + """ + stats_dumper = cache._stats_dumper + if stats_dumper is None: + return + if debug: + stats_dumper.cleanup(force=True, reason=reason) + else: + stats_dumper() + + # ---------------------- Patching infrastructure ----------------------- @@ -789,7 +845,7 @@ def wrap_worker_pool( """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up - _add_sigterm_handler_in_child(cache) + _setup_in_mp_child(cache) # Note: using the `cache` itself as the context manager is prone to # deadlock with Cleanup() as cleanup: @@ -802,7 +858,7 @@ def wrap_worker_pool( return vanilla_impl(inqueue, *args, **kwargs) -def _wrap_inqueue_get( +def _wrap_inqueue_get( # nocover cache: LineProfilingCache, vanilla_impl: Callable[Concatenate[_Queue, PS], T], self: _Queue, @@ -811,7 +867,7 @@ def _wrap_inqueue_get( **kwargs: PS.kwargs ) -> T: """ - Intecept the sentinel value (:py:const:`None`) signifiying the end + Intercept the sentinel value (:py:const:`None`) signifiying the end of the queue and perform cleanup. """ result = vanilla_impl(self, *args, **kwargs) @@ -824,7 +880,8 @@ def _wrap_inqueue_get( cache._debug_output(msg) # Got sentinel value, process is about to exit reason = 'ran out of tasks in `multiprocessing.process.worker()`' - cache.cleanup(reason=reason) + if cache.main_pid != os.getpid(): + _dump_stats_quick(cache, debug=True, reason=reason) else: ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 return result @@ -884,9 +941,8 @@ def wrap_bootstrap( *args: PS.args, **kwargs: PS.kwargs ) -> T: """ - Wrap around :py:meth:`.BaseProcess._bootstrap` to run - ``LineProfilingCache.load().cleanup()`` so that profiling results - can be gathered. + Wrap around :py:meth:`.BaseProcess._bootstrap` so that profiling + stats are written at the end. Notes: @@ -903,16 +959,12 @@ def wrap_bootstrap( """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up - _add_sigterm_handler_in_child(cache) + _setup_in_mp_child(cache) try: return vanilla_impl(self, *args, **kwargs) finally: - # Execute cleanup in a separate thread so as to avoid deadlocks, - # in case when `LineProfilingCache._handle_signal()` caught a - # signal as we're in the middle of this and initiated another - # `.cleanup()` call - reason = 'exiting `multiprocessing.Process._bootstrap()`' - cache.cleanup(reason=reason) + reason = 'exiting `multiprocessing.process.BaseProcess._bootstrap`' + _dump_stats_quick(cache, debug=True, reason=reason) _patch_process = partial( From de5583ddf5755173930c1b46b0be983262925736 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 18 May 2026 12:32:36 +0200 Subject: [PATCH 111/165] More defensive error handling line_profiler/_child_process_profiling/cache.py::LineProfilingCache @_method_wrapper Now also handling `BaseException` _format_exception() New convenience method line_profiler/_child_process_profiling/multiprocessing_patches.py ::wrap_process() Added more debugging output in `.join()` and `.terminate()` of the created `BaseProcess` --- .../_child_process_profiling/cache.py | 27 +++++++++++-------- .../multiprocessing_patches.py | 22 ++++++++++++--- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 185b8076..b658e8e0 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -550,11 +550,7 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover reason = f'caught `{name}` ({signum})' self._stats_dumper.cleanup(force=True, reason=reason) except BaseException as e: - xc = f'{type(e).__name__}' - msg = str(e) - if msg: - xc = f'{xc}: {msg}' - state = f'failed ({xc})' + state = f'failed ({self._format_exception(e)})' raise e else: state = 'succeeded' @@ -793,16 +789,18 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: if debug_: call_fmt = cache._format_call(name, *args, **kwargs) write(f'Wrapped call made: {call_fmt}...') - state = 'succeeded' try: result = call() - except Exception as e: - state = 'failed' - outcome = f'{type(e).__name__}' - if str(e): - outcome = f'{outcome}: {e}' + except BaseException as e: + # Note: be more defensive than normal and + # prepared to deal with `BaseException`; this + # decorator is often used for functions invoked + # in child processes which don't cleanly + # terminate + state, outcome = 'failed', cache._format_exception(e) raise e else: + state = 'succeeded' outcome = _CALLBACK_REPR_HELPER.repr(result) return result finally: @@ -833,6 +831,13 @@ def _format_call( func = cls._get_name(func) return func + call + @staticmethod + def _format_exception(xc: BaseException) -> str: + formatted = type(xc).__name__ + if str(xc): + formatted = f'{formatted}: {xc}' + return formatted + @property def environ(self) -> dict[str, str]: """ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 7c37bfe3..b30b1bca 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -1091,7 +1091,10 @@ def _wrap_process_finalize( """ @wraps(vanilla_impl) def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: + log = cache._debug_output + call = cache._format_call(vanilla_impl, self, *args, **kwargs) try: + log(f'Wrapped call made: {call}') pid: int | None = getattr(self, 'pid', None) checked_procs = _get_checked_processes(cache) identifier = id(self), pid @@ -1099,11 +1102,24 @@ def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: ntasks = _get_ntasks(cache).pop(pid, 0) if not ntasks: cache._warn_possible_lack_of_stats(pid) - msg = f'{action} process {pid} which ran {ntasks} task(s)' - cache._debug_output(msg) + log(f'{action} process {pid} which ran {ntasks} task(s)...') checked_procs.add(cast(tuple[int, int], identifier)) + except BaseException as e: + log( + f'Error in bookkeeping ({cache._format_exception(e)}), ' + 'invoking base implementation nonetheless...' + ) + raise e finally: - vanilla_impl(self, *args, **kwargs) + try: + vanilla_impl(self, *args, **kwargs) + except BaseException as e: + state = f'failed ({cache._format_exception(e)})' + raise e + else: + state = 'succeeded' + finally: + log(f'Wrapped call {call} {state}') action = action.capitalize() return finalize From fdf8fdd4bd49f43ee8a3622dbc5fc5558549bb32 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 21 May 2026 13:54:33 +0200 Subject: [PATCH 112/165] Added timeouts to CI job stages .github/workflows/tests.yml "Test full loose sdist", "Test wheel ${{ matrix.install-extras }}" Added 10-m timeouts "Build binary wheels" Added 60-m timeout --- .github/workflows/tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f318f5ba..49506c9b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -91,6 +91,7 @@ jobs: - name: Test full loose sdist env: COVERAGE_CORE: ctrace + timeout-minutes: 10 run: |- pwd ls -al @@ -154,6 +155,7 @@ jobs: CIBW_ARCHS_LINUX: ${{ matrix.arch }} PYTHONUTF8: '1' VSCMD_ARG_TGT_ARCH: '' + timeout-minutes: 60 - name: Show built files shell: bash run: ls -la wheelhouse @@ -426,6 +428,7 @@ jobs: env: CI_PYTHON_VERSION: py${{ matrix.python-version }} COVERAGE_CORE: ctrace + timeout-minutes: 10 run: |- echo "Creating test sandbox directory" export WORKSPACE_DNAME="testdir_${CI_PYTHON_VERSION}_${GITHUB_RUN_ID}_${RUNNER_OS}" From 0663a5877df1009e56f79825600caf115cb3145c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 22 May 2026 07:16:49 +0200 Subject: [PATCH 113/165] Test-timeout fixes tests/test_child_procs.py @_timeout Fixed bug where thread-local profiler states aren't cleaned up when used to decorate a test function _test_apply_mp_patches_inner() Moved `@_timeout` decorator from the parallel function (`test_module_object.sum_in_child_procs()`) to the test function itself --- tests/test_child_procs.py | 88 ++++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 25 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 86033b3d..4bacd4fc 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1801,6 +1801,43 @@ def _check_output(output: str, tag: str, nhits: int) -> None: ) +def _cleanup_profiling_in_current_thread() -> None: + """ + Disable all active profiler instances on the current thread, so that + we don't trip over ourselves if a new thread down the road happens + to reuse the thread ID. + + Note: + The thread-local profiler state is supposed to be handled by + :py:mod:`line_profiler._threading_patches` and + :py:class:`.CuratedProfilerContext`, but since a test function + decorated with :py:deco:`._timeout` will be isolated in a new + thread BEFORE the fixture setting up such management + (:py:func:`curated_profiler`) is invoked, we don't get that + managing and will have to deal with profilers ourselves. + """ + class Manager(Protocol): + @property + def active_instances(self) -> set[LineProfiler]: + ... + + lp_managers = cast( + dict[int, Manager], getattr(LineProfiler, '_managers', {}), + ) + thread_id = threading.get_ident() + if thread_id not in lp_managers: + return + instances = lp_managers[thread_id].active_instances + for prof in set(instances): + count = cast(int, getattr(prof, 'enable_count', 0)) + for _ in range(count): + prof.disable_by_count() + if count: + print('Disabled', prof, 'in thread', hex(thread_id)) + # Removed after the last `.disable_by_count()` + assert prof not in instances + + @overload def _timeout( func: Callable[PS, T], *, timeout: float = _TEST_TIMEOUT, @@ -1866,21 +1903,27 @@ def worker( fobj: IO[bytes], /, *args: PS.args, **kwargs: PS.kwargs ) -> None: try: - result = True, func(*args, **kwargs) - except Exception as e: - result = False, ExceptionHelper(e, e.__traceback__) - # Do this instead of directly using `pickle.dump(..., fobj)` so - # that pickling errors and file-handle-related errors are - # handled separately - serialized = pickle.dumps(result, protocol=pickle.HIGHEST_PROTOCOL) - try: - fobj.write(serialized) - fobj.flush() - except Exception: - # Since this is run in a daemon thread, by the time this - # write happens the main thread could've already timed out - # and destroyed `fobj`... in that case just gracefully exit - pass + try: + result = True, func(*args, **kwargs) + except Exception as e: + result = False, ExceptionHelper(e, e.__traceback__) + # Do this instead of directly using `pickle.dump(..., fobj)` + # so that pickling errors and file-handle-related errors are + # handled separately + serialized = pickle.dumps(result, protocol=pickle.HIGHEST_PROTOCOL) + try: + fobj.write(serialized) + fobj.flush() + except Exception: + # Since this is run in a daemon thread, by the time this + # write happens the main thread could've already timed + # out and destroyed `fobj`... in that case just + # gracefully exit + pass + finally: + # See the docstring for that function to see why this is + # needed. + _cleanup_profiling_in_current_thread() @wraps(func) def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: @@ -1908,7 +1951,7 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: # ============================= Unit tests ============================= -# XXX: Tests in this section concerns implementation details, and the +# XXX: Tests in this section concern implementation details, and the # tested APIs and behaviors MUST NOT be relied upon by end-users. _PatchSummary = Mapping[str, Set[str]] @@ -2478,6 +2521,7 @@ def test_load_pth_hook( @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) +@_timeout def _test_apply_mp_patches_inner( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], @@ -2520,14 +2564,8 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # with `preimports_module=True`, both are just imported and added # to the profiler, so the code paths are the same profiled_func = ext_module_object.my_external_sum - # Note: it would have been more intuitive to just apply `@_timeout` - # to this whole function and run it all in a new thread, but that - # seem to interact adversely with patch application and prof-data - # collection... so just timeout the function invoking - # `multiprocessing` - sum_with_timeout = _timeout(test_module_object.sum_in_child_procs) called_func = partial( - sum_with_timeout, + test_module_object.sum_in_child_procs, n=nprocs, my_sum=profiled_func, start_method=start_method, @@ -2631,8 +2669,8 @@ def _test_apply_mp_patches( mp_patches = [name for name, applied in patches.items() if applied] with _check_warnings() as cw: if 'child_pids' in mp_patches: - # With PID bookkeeping we should be able to weed out all - # the child processes which didn't perform any work + # With PID bookkeeping we should be able to weed out all the + # child processes which didn't perform any work cw.forbid_warnings(category=UserWarning, module='line_profiler') cw.forbid_warnings(module='multiprocessing') _test_apply_mp_patches_inner(mp_patches=mp_patches, **kwargs) From e8b127c9b4503482e47251c6543e8c47e6b20a53 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 25 May 2026 02:24:24 +0200 Subject: [PATCH 114/165] Recover more output on subprocess failure tests/test_child_procs.py::_run_subproc() - Simplified handling of the `check` parameter - Updated implementation so that the `CalledProcessError` or `TimeoutExpired`'s `.stdout` and `.stderr` are used for end-of-process debug printouts --- tests/test_child_procs.py | 45 +++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 4bacd4fc..0dee19af 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1587,7 +1587,6 @@ def _run_subproc( cmd: Sequence[str] | str, /, *args, - check: bool = False, env: Mapping[str, str] | None = None, **kwargs ) -> subprocess.CompletedProcess: @@ -1595,20 +1594,20 @@ def _run_subproc( Wrapper around :py:func:`subprocess.run` which writes debugging output. """ + class HasStreams(Protocol): + @property + def stdout(self) -> str | bytes | None: + ... + + @property + def stderr(self) -> str | bytes | None: + ... + if isinstance(cmd, str): cmd_str = cmd else: cmd_str = concat_command_line(cmd) - # If we're capturing outputs, it may be for the best to wait until - # we've processed the output streams to check the return code... - check_rc_in_run = check - for arg in 'stdout', 'stdin': - if kwargs.get(arg) not in {None, subprocess.DEVNULL}: - check_rc_in_run = False - if kwargs.get('capture_output'): - check_rc_in_run = False - print('Command:', cmd_str) if env is not None: diff: list[str] = [] @@ -1630,28 +1629,32 @@ def _run_subproc( # Note: somehow `mypy` doesn't agree with simply unpacking the # `*args` into `subprocess.run()`... status: int | str = '???' - proc: subprocess.CompletedProcess | None = None + result: HasStreams | None = None + subproc_errors = ( + subprocess.CalledProcessError, subprocess.TimeoutExpired, + ) time = monotonic() try: proc = subprocess.run( # type: ignore[call-overload] - cmd, *args, env=env, check=check_rc_in_run, **kwargs, + cmd, *args, env=env, **kwargs, ) - except Exception: + except Exception as e: status = 'error' + if isinstance(e, subproc_errors): + result = e + if hasattr(e, 'returncode'): # `CalledProcessError` + status = f'{status} ({e.returncode})' raise else: - assert proc is not None - if check and not check_rc_in_run: # Perform missing check - proc.check_returncode() - status = proc.returncode + result, status = proc, proc.returncode return proc finally: time = monotonic() - time - if proc is not None: + if result is not None: captured: str | bytes | None for name, captured, stream in [ - ('stdout', proc.stdout, sys.stdout), - ('stderr', proc.stderr, sys.stderr), + ('stdout', result.stdout, sys.stdout), + ('stderr', result.stderr, sys.stderr), ]: if captured is None: continue @@ -1660,7 +1663,7 @@ def _run_subproc( print(f'{name}:\n{indent(captured, " ")}', file=stream) print( f'-- Process end (time elapsed: {time:.2f} s / ' - f'return status: {status})--' + f'return status: {status}) --' ) From 5c1cf82cf1a83ec1295c3d2803389179c447d9f4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 26 May 2026 07:05:57 +0200 Subject: [PATCH 115/165] WIP: try harder to debug the flaky tests tests/test_child_procs.py _preserve_attributes Now permitting `targets=None`, which resolves to `_GLOBAL_PATCHES` at `.__enter__()` time {_run_as,run}_{module,script,literal_code}() Updated signatures: - New leading parameter `request` - New keyword-only parameter `subproc` for controlling whether to spin up a subprocess to run `kernprof` (old and default behavior) or emulate one in-process _run_kernprof_main_in_process() New function for emulating `_run_subproc(['kernprof', ...], ...)` in-process to attempt to recover more debugging output (because `capture_output` seems to completely fail when the process is timed out) test_multiproc_script_sanity_check() _test_profiling_multiproc_script() test_profiling_multiproc_script_{success,failure}() Updated signatures and implementations to pass in a `request` as required --- tests/test_child_procs.py | 234 +++++++++++++++++++++++++++++++------- 1 file changed, 194 insertions(+), 40 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 0dee19af..e1253876 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -14,8 +14,10 @@ import sys import sysconfig import threading +import traceback import warnings from abc import ABC, abstractmethod +from argparse import ArgumentError from collections.abc import ( Callable, Collection, Generator, Iterable, Iterator, Mapping, Sequence, Set, @@ -45,6 +47,7 @@ import ubelt as ub from _line_profiler_hooks import load_pth_hook +from kernprof import main as kernprof_main from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, @@ -53,6 +56,7 @@ _Poller, MPConfig, _PATCHED_MARKER, _PATCHES as MP_PATCHES, ) from line_profiler.autoprofile.util_static import modpath_to_modname +from line_profiler.cleanup import Cleanup from line_profiler.curated_profiling import ( CuratedProfilerContext, ClassifiedPreimportTargets, ) @@ -1101,12 +1105,10 @@ class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): >>> assert main is not line_profiler.main """ def __init__( - self, targets: Mapping[str, Collection[str]], *, + self, targets: Mapping[str, Collection[str]] | None = None, *, static: bool = True, debug: bool = _DEBUG, ) -> None: - self.targets = { - target: set(attrs) for target, attrs in targets.items() - } + self.targets = targets self._stacks: list[ExitStack] = [] self.static = static self.debug = debug @@ -1200,6 +1202,23 @@ def get_from_boolean(*_, **__) -> bool: assert (not failures), '\n'.join(failures) return result + @property + def targets(self) -> dict[str, set[str]]: + if self._targets is None: + # Defer resolution to context entrance + self.targets = _GLOBAL_PATCHES + assert self._targets is not None + return self._targets + + @targets.setter + def targets(self, targets: Mapping[str, Collection[str]] | None) -> None: + if targets is None: + self._targets = None + return + self._targets = { + target: set(attrs) for target, attrs in targets.items() + } + class _preserve_pth_files(_CallableContextManager[frozenset[str]]): def __init__(self, debug: bool = _DEBUG) -> None: @@ -1557,37 +1576,180 @@ def _search_cache_logs( def _run_as_script( - runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, + request: pytest.FixtureRequest, + runner_args: list[str], + test_args: list[str], + test_module: _ModuleFixture, + *, + subproc: bool = True, **kwargs ) -> subprocess.CompletedProcess: cmd = runner_args + [str(test_module.path)] + test_args - test_module.install(children=True, deps_only=True) - return _run_subproc(cmd, **kwargs) + if subproc: + run: Callable[..., subprocess.CompletedProcess] = _run_subproc + else: + run = partial(_run_kernprof_main_in_process, request) + test_module.install(children=True, local=not subproc, deps_only=True) + return run(cmd, **kwargs) def _run_as_module( - runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, + request: pytest.FixtureRequest, + runner_args: list[str], + test_args: list[str], + test_module: _ModuleFixture, + *, + subproc: bool = True, **kwargs ) -> subprocess.CompletedProcess: cmd = runner_args + ['-m', test_module.name] + test_args - test_module.install(children=True) - return _run_subproc(cmd, **kwargs) + if subproc: + run: Callable[..., subprocess.CompletedProcess] = _run_subproc + else: + run = partial(_run_kernprof_main_in_process, request) + test_module.install(children=True, local=not subproc) + return run(cmd, **kwargs) def _run_as_literal_code( - runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, + request: pytest.FixtureRequest, + runner_args: list[str], + test_args: list[str], + test_module: _ModuleFixture, + *, + subproc: bool = True, **kwargs ) -> subprocess.CompletedProcess: cmd = runner_args + ['-c', test_module.path.read_text()] + test_args - test_module.install(children=True, deps_only=True) - return _run_subproc(cmd, **kwargs) + if subproc: + run: Callable[..., subprocess.CompletedProcess] = _run_subproc + else: + run = partial(_run_kernprof_main_in_process, request) + test_module.install(children=True, local=not subproc, deps_only=True) + return run(cmd, **kwargs) + + +@_preserve_pth_files() +@_preserve_attributes() +def _run_kernprof_main_in_process( + request: pytest.FixtureRequest, cmd: Sequence[str], + *, + text: bool = False, + capture_output: bool = False, + check: bool = False, + encoding: str | None = None, + env: Mapping[str, str] | None = None, + timeout: float | None = None, + **_kwargs +) -> subprocess.CompletedProcess: + """ + Emulate running :cmd:`kernprof` in a subprocess with in-process + machineries as best as we can, so that we can retrieve more + debugging output when things do go south. + """ + def get_streams( + ) -> tuple[str, str] | tuple[bytes, bytes] | tuple[None, None]: + if cap is None: + return None, None + stdout, stderr = cap.readouterr() + if text: + return stdout, stderr + if encoding is None: + encode = operator.methodcaller('encode') + else: + encode = operator.methodcaller('encode', encoding) + return encode(stdout), encode(stderr) + + assert not _kwargs + assert cmd[0] == 'kernprof' + + cap: pytest.CaptureFixture | None = None + stdout: str | bytes | None = None + stderr: str | bytes | None = None + if capture_output: + cap = request.getfixturevalue('capsys') + print('Command:', concat_command_line(cmd)) + _print_env_deltas(env) + print('-- Emulated process start --') + get_streams() # Don't include the above in the captured outputs + + status: int | str = '???' + timed_out = False + time = monotonic() + if timeout: + main = _timeout(kernprof_main, timeout=timeout) + else: + main = kernprof_main + try: + try: + with Cleanup() as cleanup: + if env is not None: + cleanup.update_mapping(os.environ, env) + main(cmd[1:], exit_on_error=False) + except _TestTimeout: # `subprocess` uses `SIGKILL` + returncode, timed_out = -9, True + status = f'error ({returncode})' + except Exception as e: + # Format and output the tracebacks, otherwise we would've + # suppressed them + traceback.print_exception(e) + returncode = 2 if isinstance(e, ArgumentError) else 1 + status = f'error ({returncode})' + else: + status = returncode = 0 + stdout, stderr = get_streams() + + # Turn in-process errors into the corresponding `subprocess` + # errors + if timed_out: + assert timeout is not None # Assure the typechecker + raise subprocess.TimeoutExpired(cmd, timeout, stdout, stderr) + if check and returncode: + raise subprocess.CalledProcessError( + returncode, cmd, stdout, stderr, + ) + return subprocess.CompletedProcess(cmd, returncode, stdout, stderr) + finally: + time = monotonic() - time + captured: str | bytes | None + for name, captured, stream in [ + ('stdout', stdout, sys.stdout), ('stderr', stderr, sys.stderr), + ]: + if captured is None: + continue + if isinstance(captured, bytes): # `text=False` + captured = captured.decode() + print(f'{name}:\n{indent(captured, " ")}', file=stream) + print( + f'-- Emulated process end (time elapsed: {time:.2f} s / ' + f'return status: {status}) --' + ) + + +def _print_env_deltas(env: Mapping[str, str] | None = None) -> None: + if env is None: + return + diff: list[str] = [] + for key in set(os.environ).union(env): + old = os.environ.get(key) + new = env.get(key) + if old is not None is new: + item = f'{old!r} -> (deleted)' + elif old is None is not new: + item = f'{new!r} (added)' + else: + if old == new: + continue + item = f'{old!r} -> {new!r}' + diff.append(f'${{{key}}}: {item}') + if diff: + print('Env:', indent('\n'.join(diff), ' '), sep='\n') def _run_subproc( cmd: Sequence[str] | str, /, *args, - env: Mapping[str, str] | None = None, **kwargs ) -> subprocess.CompletedProcess: """ @@ -1609,22 +1771,7 @@ def stderr(self) -> str | bytes | None: cmd_str = concat_command_line(cmd) print('Command:', cmd_str) - if env is not None: - diff: list[str] = [] - for key in set(os.environ).union(env): - old = os.environ.get(key) - new = env.get(key) - if old is not None is new: - item = f'{old!r} -> (deleted)' - elif old is None is not new: - item = f'{new!r} (added)' - else: - if old == new: - continue - item = f'{old!r} -> {new!r}' - diff.append(f'${{{key}}}: {item}') - if diff: - print('Env:', indent('\n'.join(diff), ' '), sep='\n') + _print_env_deltas(kwargs.get('env')) print('-- Process start --') # Note: somehow `mypy` doesn't agree with simply unpacking the # `*args` into `subprocess.run()`... @@ -1635,9 +1782,7 @@ def stderr(self) -> str | bytes | None: ) time = monotonic() try: - proc = subprocess.run( # type: ignore[call-overload] - cmd, *args, env=env, **kwargs, - ) + proc = subprocess.run(cmd, *args, **kwargs) except Exception as e: status = 'error' if isinstance(e, subproc_errors): @@ -1670,6 +1815,7 @@ def stderr(self) -> str | bytes | None: @_preserve_pth_files() def _run_test_module( run_helper: Callable[..., subprocess.CompletedProcess], + request: pytest.FixtureRequest, test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, runner: str | list[str] = 'kernprof', @@ -1737,7 +1883,7 @@ def _run_test_module( old_pth_files = _preserve_pth_files.get_pth_files() try: proc = run_helper( - runner_args, test_args, test_module, + request, runner_args, test_args, test_module, text=True, capture_output=True, check=(check and not fail), **kwargs ) @@ -2352,7 +2498,7 @@ def test_cache_setup_main_process( @pytest.mark.parametrize(('debug', 'label4'), [(True, 'with-debug'), (False, 'no-debug')]) @pytest.mark.parametrize('n', [100]) -@_preserve_attributes(_GLOBAL_PATCHES) +@_preserve_attributes() def test_cache_setup_child( create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, @@ -2449,7 +2595,7 @@ def list_profiled_funcs() -> list[str]: @pytest.mark.parametrize('ppid_should_match', [True, False, None]) -@_preserve_attributes(_GLOBAL_PATCHES) +@_preserve_attributes() def test_load_pth_hook( create_cache: Callable[..., LineProfilingCache], another_pid: int, @@ -2523,7 +2669,7 @@ def test_load_pth_hook( @_preserve_pth_files() -@_preserve_attributes(_GLOBAL_PATCHES) +@_preserve_attributes() @_timeout def _test_apply_mp_patches_inner( tmp_path_factory: pytest.TempPathFactory, @@ -2835,6 +2981,7 @@ def _get_mp_start_method_fuzzer(label_name: str | None) -> _Params: defaults=(None, None))).sorted() def test_multiproc_script_sanity_check( run_func: Callable[..., subprocess.CompletedProcess], + request: pytest.FixtureRequest, test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, use_local_func: bool, @@ -2850,7 +2997,7 @@ def test_multiproc_script_sanity_check( with vanilla Python. """ run_func( - test_module, tmp_path_factory, + request, test_module, tmp_path_factory, runner=sys.executable, profile=False, fail=fail, use_local_func=use_local_func, @@ -2879,6 +3026,7 @@ def test_multiproc_script_sanity_check( ) def test_running_multiproc_script( run_func: Callable[..., subprocess.CompletedProcess], + request: pytest.FixtureRequest, test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, runner: str | list[str], @@ -2899,7 +3047,7 @@ def test_running_multiproc_script( execution of the code and presence of profiling data thereafter. """ - run_func(test_module, tmp_path_factory, runner, outfile, profile) + run_func(request, test_module, tmp_path_factory, runner, outfile, profile) _fuzz_prof_mp_run_func = _Params.new(('run_func', 'label1'), @@ -2929,6 +3077,7 @@ def test_running_multiproc_script( def _test_profiling_multiproc_script( run_func: Callable[..., subprocess.CompletedProcess], + request: pytest.FixtureRequest, test_module: _ModuleFixture, ext_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, @@ -2970,7 +3119,7 @@ def _test_profiling_multiproc_script( # Also make sure to include the external module in `--prof-mod` runner.append(f'--prof-mod={ext_module.name}') run_func( - test_module, tmp_path_factory, + request, test_module, tmp_path_factory, runner=runner, outfile='out.lprof', profile=True, @@ -2984,6 +3133,7 @@ def _test_profiling_multiproc_script( debug_log=( 'debug.log' if prof_child_procs and _DEBUG else None ), + subproc=False, ) @@ -2995,6 +3145,7 @@ def _test_profiling_multiproc_script( ) def test_profiling_multiproc_script_success( run_func: Callable[..., subprocess.CompletedProcess], + request: pytest.FixtureRequest, test_module: _ModuleFixture, ext_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, @@ -3054,6 +3205,7 @@ def test_profiling_multiproc_script_success( """ _test_profiling_multiproc_script( run_func=run_func, + request=request, test_module=test_module, ext_module=ext_module, tmp_path_factory=tmp_path_factory, @@ -3071,6 +3223,7 @@ def test_profiling_multiproc_script_success( @pytest.mark.parametrize(('nnums', 'nprocs'), [(2000, 3)]) def test_profiling_multiproc_script_failure( run_func: Callable[..., subprocess.CompletedProcess], + request: pytest.FixtureRequest, test_module: _ModuleFixture, ext_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, @@ -3093,6 +3246,7 @@ def test_profiling_multiproc_script_failure( """ _test_profiling_multiproc_script( run_func=run_func, + request=request, test_module=test_module, ext_module=ext_module, tmp_path_factory=tmp_path_factory, From 45b995a4030348ba7734f0371a2efdfd8d305052 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 29 May 2026 04:18:15 +0200 Subject: [PATCH 116/165] Minor global-state fixes kernprof.py::_prepare_child_profiling_cache() Now creating the session-cache directory inside the tempdir already created in `main()` line_profiler/_child_process_profiling/cache.py::LineProfilingCache _warn_possible_lack_of_stats() Now gracefully handling the edge case where this is called (e.g. in `multiprocessing.Process.terminate()`) after the associated `.cache_dir` has been deleted _replace_loaded_instance() Now setting up for the old value of `._loaded_instance` to be restored at `.cleanup()` time tests/test_child_procs.py::test_cache_setup_main_process() Updated order of checks because of the above change in `LineProfilingCache._replace_loaded_instance()` --- kernprof.py | 16 ++++++---------- line_profiler/_child_process_profiling/cache.py | 11 ++++++++--- tests/test_child_procs.py | 4 ++-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/kernprof.py b/kernprof.py index e9612fad..8c8b1455 100755 --- a/kernprof.py +++ b/kernprof.py @@ -241,7 +241,6 @@ def main(): DIAGNOSITICS_VERBOSITY = 2 CLEANUP_PRIORITIES = { # More negative number -> more delayed - 'rm_cache_dir': -1024, 'gather_logs': -1, } @@ -1357,7 +1356,12 @@ def _prepare_child_profiling_cache( # be responsible for managing their lifetimes, while derivative # instances in child processes will merely inherit and use them cache = LineProfilingCache( - cache_dir=tempfile.mkdtemp(), + # We don't need to manage this tempdir separately, + # `options.tmpdir` and its contents will be deleted or + # preserved as is appropriate + cache_dir=tempfile.mkdtemp( + dir=options.tmpdir, prefix='line-profiling-cache-', + ), config=options.config, profiling_targets=options.prof_mod, rewrite_module=script_file, @@ -1367,14 +1371,6 @@ def _prepare_child_profiling_cache( debug=bool(options.debug or options.debug_log), ) clean_up = functools.partial(cache.add_cleanup, _remove, missing_ok=True) - if not diagnostics.KEEP_TEMPDIRS: - # Defer the scrubbing of the cache dir and let the context - # handle it, ideally speaking the cache dir should survive the - # cache object - ctx.add_cleanup_with_priority( - _remove, CLEANUP_PRIORITIES['rm_cache_dir'], cache.cache_dir, - recursive=True, - ) clean_up(cache.filename) # This file is handed to us at the end of diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index b658e8e0..ccb6d2d1 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -645,8 +645,13 @@ def _warn_possible_lack_of_stats( """ if not isinstance(pids, Collection): pids = pids, - with self._empty_stats_pid_registry.open(mode='a') as fobj: - print(*pids, sep='\n', file=fobj) + try: + with self._empty_stats_pid_registry.open(mode='a') as fobj: + print(*pids, sep='\n', file=fobj) + except FileNotFoundError: + # At cleanup time, the tempdir backing the cache instance + # may already have ceased to exist; just let that be + pass def _get_pids_possibly_lacking_stats(self) -> set[int]: """ @@ -692,7 +697,7 @@ def make_tempfile(self, **kwargs) -> Path: def _replace_loaded_instance(self, force: bool = False) -> bool: cls = type(self) if force or self._consistent_with_loaded_instance: - cls._loaded_instance = self + self.patch(cls, '_loaded_instance', self) return True return False diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index e1253876..66c23933 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -2466,11 +2466,11 @@ def test_cache_setup_main_process( new_pth_hook, = _preserve_pth_files.get_pth_files() - original_pths # Check whether the patches are applied compare_patched(operator.is_not, assert_true=patches) + # Check that the instance is set as the `.load()`-ed one + assert cache is cache.load() # Check whether the patches are reversed cache.cleanup() compare_patched() - # Check that the instance is set as the `.load()`-ed one - assert cache is cache.load() # Check the debug-log output patterns: dict[str, bool] = dict.fromkeys( From 5e985fe276f58c62a0adda2d3e0122fc8135bcc4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 8 Jun 2026 07:54:37 +0200 Subject: [PATCH 117/165] Fix unclean cache state line_profiler/_child_process_profiling/cache.py ::LineProfilingCache._consistent_with_loaded_instance Fixed bug where accessing this property causes an instance to be cached if one isn't already `.load()`-ed, making for incomplete `.cleanup()` after which `.load()` returns said created instance instead of a fresh one --- line_profiler/_child_process_profiling/cache.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index ccb6d2d1..8ff40228 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -874,7 +874,15 @@ def _make_debug_entry(self) -> Callable[[str], CacheLoggingEntry]: @cached_property def _consistent_with_loaded_instance(self) -> bool: - return type(self).load()._get_init_args() == self._get_init_args() + cls = type(self) + # Note: calling `.load()` can cause a new instance to be created + # and stored at `._loaded_instance`, if there isn't already one + # such instance; guard against that + already_loaded = cls._loaded_instance + try: + return cls.load()._get_init_args() == self._get_init_args() + finally: + cls._loaded_instance = already_loaded @cached_property def _config_source(self) -> ConfigSource: From ce54d65b8bdeb8af8ea28dbfcad21b01ff9bb679 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 8 Jun 2026 14:47:58 +0200 Subject: [PATCH 118/165] Updated warning handling in tests tests/test_child_procs.py _WarningContext - Added init param `.reissue_warnings` and method `.reissue()` to handle the reissuing of the intercepted warnings - Added methods `.suppress_warnings()` and `.propagate_warnings()` for fine-grained control over which warnings to reissue _check_warnings - Added methods `.suppress_warnings()` and `.propagate_warnings()` (see above) - Updated `.__exit__()` so that captured warnings are conditionally reissued with `_WarningContext.reissue()` _run_as_{script,module,literal_code}() _run_kernprof_main_in_process() When using `subproc=False` (e.g. in `test_profiling_multiproc_script_{success,failure}()`), stray warnings are guarded against with `_check_warnings`, as we're already doing in `test_apply_mp_patches_{success,failure}()` --- tests/test_child_procs.py | 168 ++++++++++++++++++++++++++++++-------- 1 file changed, 133 insertions(+), 35 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 66c23933..c0d6cda9 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1341,37 +1341,44 @@ class _WarningContext: catch_warnings: warnings.catch_warnings = dataclasses.field( default_factory=partial(warnings.catch_warnings, record=True) ) + reissue_warnings: bool = False checks: list[tuple[_WarningMatcher, bool]] = dataclasses.field( default_factory=list, ) + filters: list[_WarningMatcher] = dataclasses.field(default_factory=list) + _reissue_filtered_warnings: bool | None = dataclasses.field( + init=False, default=None, + ) - def forbid_warnings( - self, - message: str | None = None, - category: type[Warning] | None = Warning, - module: str | None = None, - lineno: int | None = None, - ) -> None: - matcher = _WarningMatcher( - message=message, category=category, module=module, lineno=lineno, - ) - self.checks.append((matcher, False)) + def forbid_warnings(self, *args, **kwargs) -> None: + self.checks.append((self._get_matcher(*args, **kwargs), False)) - def expect_warnings( - self, - message: str | None = None, - category: type[Warning] | None = Warning, - module: str | None = None, - lineno: int | None = None, - ) -> None: - matcher = _WarningMatcher( - message=message, category=category, module=module, lineno=lineno, - ) - self.checks.append((matcher, True)) + def expect_warnings(self, *args, **kwargs) -> None: + self.checks.append((self._get_matcher(*args, **kwargs), True)) + + def suppress_warnings(self, *args, **kwargs) -> None: + if self._reissue_filtered_warnings is None: + self._reissue_filtered_warnings = False + elif self._reissue_filtered_warnings: + raise RuntimeError( + 'Either `.suppress_warnings()` or `.propagate_warnings()` can ' + 'be called within the same context, but not both' + ) + self.filters.append(self._get_matcher(*args, **kwargs)) + + def propagate_warnings(self, *args, **kwargs) -> None: + if self._reissue_filtered_warnings is None: + self._reissue_filtered_warnings = True + elif not self._reissue_filtered_warnings: + raise RuntimeError( + 'Either `.suppress_warnings()` or `.propagate_warnings()` can ' + 'be called within the same context, but not both' + ) + self.filters.append(self._get_matcher(*args, **kwargs)) - def check(self, warnings: Sequence[_WarningInfo]) -> None: + def check(self, infos: Sequence[_WarningInfo]) -> None: for matcher, allowed_or_required in self.checks: - matches = [info for info in warnings if matcher.match(info)] + matches = [info for info in infos if matcher.match(info)] if matches and not allowed_or_required: raise ResultMismatch( expected=f'no warnings matching {matcher!r}', @@ -1380,13 +1387,59 @@ def check(self, warnings: Sequence[_WarningInfo]) -> None: if not matches and allowed_or_required: raise ResultMismatch( expected=f'warnings matching {matcher!r}', - actual=f'none out of {len(warnings)} ({warnings!r})', + actual=f'none out of {len(infos)} ({infos!r})', ) + def reissue(self, infos: Sequence[_WarningInfo]) -> None: + def reissue_(info: _WarningInfo) -> None: + warnings.warn_explicit( + message=info.message, + category=info.category, + filename=info.filename, + lineno=info.lineno, + ) + + # If we haven't called `.suppress_warnings()` or + # `.propagate_warnings()`, handle warnings according to the + # default behavior + if not self.filters: + if self.reissue_warnings: + for info in infos: + reissue_(info) + return + # Otherwise we handle the warnings matching any of the filters + # one way, and the remaining the other way: + # - If we used `.suppress_warnings()`, matching warnings are + # suppressed, and the remainder propagated by default + # - If we used `.propagate_warnings()`, matching warnings are + # propagated, and the remainder suppressed by default + # Note that we are not supposed to have called both methods + reissue_by_default = not self._reissue_filtered_warnings + for info in infos: + should_reissue = reissue_by_default + if any(matcher.match(info) for matcher in self.filters): + should_reissue = not should_reissue + if should_reissue: + reissue_(info) + + @staticmethod + def _get_matcher( + message: str | None = None, + category: type[Warning] | None = Warning, + module: str | None = None, + lineno: int | None = None, + ) -> _WarningMatcher: + return _WarningMatcher( + message=message, category=category, module=module, lineno=lineno, + ) + @classmethod - def new(cls, **kwargs) -> Self: + def new(cls, /, *, reissue_warnings: bool = False, **kwargs) -> Self: kwargs['record'] = True - return cls(warnings.catch_warnings(**kwargs)) + return cls( + warnings.catch_warnings(**kwargs), + reissue_warnings=reissue_warnings, + ) class _check_warnings(Sequence[_WarningInfo]): @@ -1426,7 +1479,7 @@ class _check_warnings(Sequence[_WarningInfo]): >>> assert len(cw) == 1 >>> assert str(cw[0].message) == 'foobar' """ - def __init__(self, **kwargs) -> None: + def __init__(self, /, **kwargs) -> None: self._new_context: Callable[[], _WarningContext] = partial( _WarningContext.new, **kwargs, ) @@ -1439,7 +1492,7 @@ def forbid_warnings(self, *args, **kwargs) -> None: """ Equivalent to calling ``filterwarnings('error', *args, **kwargs)``; - at context exit, if ANY matching warning has been issued, an + at context exit, if ANY matching warning has been captured, an error will be raised. """ ctx, _ = self._current_context @@ -1449,12 +1502,42 @@ def expect_warnings(self, *args, **kwargs) -> None: """ Equivalent to calling ``filterwarnings('always', *args, **kwargs)``; - at context exit, if NO matching warnings have been issued, an + at context exit, if NO matching warnings have been captured, an error will be raised. """ ctx, _ = self._current_context ctx.expect_warnings(*args, **kwargs) + def suppress_warnings(self, *args, **kwargs) -> None: + """ + Equivalent to calling + ``filterwarnings('never', *args, **kwargs)``; + at context exit, captured warnings are SUPPRESSED if they match + the filter(s) and REISSUED otherwise. + + Note: + Within each context level, one can call EITHER + :py:meth:`.suppress_warnings` or + :py:meth:`.propagate_warnings` but not both. + """ + ctx, _ = self._current_context + ctx.suppress_warnings(*args, **kwargs) + + def propagate_warnings(self, *args, **kwargs) -> None: + """ + Equivalent to calling + ``filterwarnings('always', *args, **kwargs)``; + at context exit, captured warnings are REISSUED if they match + the filter(s) and SUPPRESSED otherwise. + + Note: + Within each context level, one can call EITHER + :py:meth:`.suppress_warnings` or + :py:meth:`.propagate_warnings` but not both. + """ + ctx, _ = self._current_context + ctx.propagate_warnings(*args, **kwargs) + def __enter__(self) -> Self: ctx = self._new_context() infos: Sequence[_WarningInfo] | None @@ -1470,6 +1553,9 @@ def __exit__(self, *args, **kwargs) -> None: finally: self._last_captured = infos ctx.catch_warnings.__exit__(*args, **kwargs) + # Now that the warning filters are reset we can reissue + # the warnings without interference + ctx.reissue(infos) @overload def __getitem__(self, i: int, /) -> _WarningInfo: @@ -1582,13 +1668,14 @@ def _run_as_script( test_module: _ModuleFixture, *, subproc: bool = True, + check_warnings: bool = True, **kwargs ) -> subprocess.CompletedProcess: cmd = runner_args + [str(test_module.path)] + test_args if subproc: run: Callable[..., subprocess.CompletedProcess] = _run_subproc else: - run = partial(_run_kernprof_main_in_process, request) + run = partial(_run_kernprof_main_in_process, request, check_warnings) test_module.install(children=True, local=not subproc, deps_only=True) return run(cmd, **kwargs) @@ -1600,13 +1687,14 @@ def _run_as_module( test_module: _ModuleFixture, *, subproc: bool = True, + check_warnings: bool = True, **kwargs ) -> subprocess.CompletedProcess: cmd = runner_args + ['-m', test_module.name] + test_args if subproc: run: Callable[..., subprocess.CompletedProcess] = _run_subproc else: - run = partial(_run_kernprof_main_in_process, request) + run = partial(_run_kernprof_main_in_process, request, check_warnings) test_module.install(children=True, local=not subproc) return run(cmd, **kwargs) @@ -1618,13 +1706,14 @@ def _run_as_literal_code( test_module: _ModuleFixture, *, subproc: bool = True, + check_warnings: bool = True, **kwargs ) -> subprocess.CompletedProcess: cmd = runner_args + ['-c', test_module.path.read_text()] + test_args if subproc: run: Callable[..., subprocess.CompletedProcess] = _run_subproc else: - run = partial(_run_kernprof_main_in_process, request) + run = partial(_run_kernprof_main_in_process, request, check_warnings) test_module.install(children=True, local=not subproc, deps_only=True) return run(cmd, **kwargs) @@ -1632,7 +1721,7 @@ def _run_as_literal_code( @_preserve_pth_files() @_preserve_attributes() def _run_kernprof_main_in_process( - request: pytest.FixtureRequest, cmd: Sequence[str], + request: pytest.FixtureRequest, check_warnings: bool, cmd: Sequence[str], *, text: bool = False, capture_output: bool = False, @@ -1682,9 +1771,18 @@ def get_streams( main = kernprof_main try: try: - with Cleanup() as cleanup: + with ExitStack() as stack: + cleanup = stack.enter_context(Cleanup()) if env is not None: cleanup.update_mapping(os.environ, env) + if check_warnings: + # See similar indictions against warnings in + # `_test_apply_mp_patches()` + cw = stack.enter_context(_check_warnings()) + cw.forbid_warnings( + category=UserWarning, module='line_profiler', + ) + cw.forbid_warnings(module='multiprocessing') main(cmd[1:], exit_on_error=False) except _TestTimeout: # `subprocess` uses `SIGKILL` returncode, timed_out = -9, True From 2e05bbf11981d76c68d363f84d41a5aa7b81a614 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 9 Jun 2026 20:57:34 +0200 Subject: [PATCH 119/165] Better handling of error cases tests/test_child_procs.py _run_test_module() - `subproc` is now a named keyword param - if `subproc=False`, try harder to recover and print the debug logs even if `kernprof.main()` hasn't written to `debug_log` test_profiling_multiproc_script_{success,failure}() - Now including `subproc` in the parametrization so that we still have a case testing running `kernprof` in a subprocess --- tests/test_child_procs.py | 45 +++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index c0d6cda9..b1c8a9f6 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1929,6 +1929,7 @@ def _run_test_module( check: bool = True, debug_log: str | None = None, nhits: Mapping[str, int] | None = None, + subproc: bool = True, **kwargs ) -> tuple[subprocess.CompletedProcess, LineStats | None]: """ @@ -1983,6 +1984,7 @@ def _run_test_module( proc = run_helper( request, runner_args, test_args, test_module, text=True, capture_output=True, check=(check and not fail), + subproc=subproc, **kwargs ) # Checks: @@ -2015,6 +2017,26 @@ def _run_test_module( # profiling data will be dropped in the written outfile) for tag, num in (nhits or {}).items(): _check_output(proc.stdout, tag, num) + except subprocess.TimeoutExpired as e: + # If we're not in running in a subproc, while + # `kernprof.main()` shouldn't have gotten around to write + # the debug logs, we MIGHT be able to retrieve them from the + # active `LineProfilingCache` instance... + try: + if not subproc: + cache = LineProfilingCache.load() + if debug_log is None: + alt_debug_log = 'debug.log' + else: + alt_debug_log = debug_log + '.alt' + with open(alt_debug_log, mode='w') as fobj: + for entry in cache._gather_debug_log_entries(): + print(entry.to_text(), file=fobj) + debug_log = alt_debug_log + except Exception: + pass + finally: + raise e finally: if debug_log is not None and os.path.exists(debug_log): with open(debug_log) as fobj: @@ -3170,6 +3192,11 @@ def test_running_multiproc_script( # `run_func()`s + (_fuzz_prof_mp_run_func + _Params.new(('preimports', 'label3'), [(True, 'with-preimports')])) + # Just throw in a case where we actually run `kernprof` in a + # subprocess, otherwise it is more convenient to do so in-process + + _Params.new(('subproc', 'label5'), + [(True, 'subproc'), (False, 'in-proc')], + defaults=(False, 'in-proc')) ).sorted().split_on_params('fail') @@ -3186,6 +3213,7 @@ def _test_profiling_multiproc_script( start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int, nprocs: int, + **kwargs ) -> None: # How many calls do we expect? nhits = dict.fromkeys( @@ -3216,6 +3244,9 @@ def _test_profiling_multiproc_script( if not use_local_func: # Also make sure to include the external module in `--prof-mod` runner.append(f'--prof-mod={ext_module.name}') + kwargs.setdefault('timeout', _TEST_TIMEOUT) + if prof_child_procs and _DEBUG: + kwargs.setdefault('debug_log', 'debug.log') run_func( request, test_module, tmp_path_factory, runner=runner, @@ -3227,11 +3258,7 @@ def _test_profiling_multiproc_script( nhits=nhits, nnums=nnums, nprocs=nprocs, - timeout=_TEST_TIMEOUT, - debug_log=( - 'debug.log' if prof_child_procs and _DEBUG else None - ), - subproc=False, + **kwargs, ) @@ -3253,8 +3280,9 @@ def test_profiling_multiproc_script_success( start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int, nprocs: int, + subproc: bool, # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, label3: str, label4: str, + label1: str, label2: str, label3: str, label4: str, label5: str, ) -> None: """ Check that `kernprof` can PROFILE the test module in various @@ -3314,6 +3342,7 @@ def test_profiling_multiproc_script_success( start_method=start_method, nnums=nnums, nprocs=nprocs, + subproc=subproc, ) @@ -3331,8 +3360,9 @@ def test_profiling_multiproc_script_failure( start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int, nprocs: int, + subproc: bool, # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, label3: str, label4: str, + label1: str, label2: str, label3: str, label4: str, label5: str, ) -> None: """ Check that `kernprof` can PROFILE the test module in various @@ -3355,6 +3385,7 @@ def test_profiling_multiproc_script_failure( start_method=start_method, nnums=nnums, nprocs=nprocs, + subproc=subproc, ) From 6a8ffb828112e99c0ce385e5f9f1fed97bf81df0 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 9 Jun 2026 22:20:19 +0200 Subject: [PATCH 120/165] Fix `sys.argv` pollution in `kernprof` kernprof.py::_prepare_profiler() No longer replacing `sys.argv` because it its protected by the `@_restore.sequence(sys.argv)` decorator on `main()`, and swapping out the object invalidates said protection --- kernprof.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernprof.py b/kernprof.py index 8c8b1455..625d9035 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1261,7 +1261,10 @@ def _prepare_profiler(options, module, exit_on_error): f'Using default output destination {short_string_path(options.outfile)!r}' ) - sys.argv = [options.script] + options.args + # Note: we'll restore the content of `sys.argv` with the decorator + # on `main()`, so it's simpler to just `setitem()` instead of + # replacing the whole list object + sys.argv[:] = [options.script] + options.args if module: # Make sure the current directory is on `sys.path` to emulate # `python -m` From e41fbef38b7058c12e325ded297dd347b8514444 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 9 Jun 2026 22:40:48 +0200 Subject: [PATCH 121/165] Small fix: `~._child_process_profiling` debug logs line_profiler/_child_process_profiling/cache.py ::@LineProfilingCache._method_wrapper Now indicating the wrapper in the emitted debug-log messages so as to reduce confusion for callables wrapped multiple times (e.g. `multiprocessing.pool.worker()`) --- line_profiler/_child_process_profiling/cache.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 8ff40228..ceda3a19 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -792,8 +792,13 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: if debug_ is None: debug_ = cache.debug if debug_: - call_fmt = cache._format_call(name, *args, **kwargs) - write(f'Wrapped call made: {call_fmt}...') + call_fmt = cache._format_call( + vanilla_name, *args, **kwargs, + ) + write( + f'Wrapped call made via `{wrapper_name}`: ' + f'{call_fmt}...', + ) try: result = call() except BaseException as e: @@ -809,11 +814,15 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: outcome = _CALLBACK_REPR_HELPER.repr(result) return result finally: - write(f'Wrapped call {state}: {call_fmt} -> {outcome}') + write( + f'Wrapped call via `{wrapper_name}` {state}: ' + f'{call_fmt} -> {outcome}', + ) else: return call() - name = cls._get_name(vanilla_impl) + vanilla_name = cls._get_name(vanilla_impl) + wrapper_name = cls._get_name(wrapper) return wrapped_impl for field in 'name', 'qualname', 'doc': From dd1462a3ac452e60d89e1654355e55c01d78a6e6 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 10 Jun 2026 17:59:28 +0200 Subject: [PATCH 122/165] Tweaked warning reporting in `test_child_procs.py` tests/test_child_procs.py _check_warnings.__init__() `reissue_warnings` now defaults to true _run_kernprof_main_in_process() Fixed bug where if `_check_warnings` raises a `ResultMismatch` it is erroneously converted into a non-zero return code (instead of just being propagated) test_cache_setup_child() test_apply_mp_patches_{success,failure}(start_method='fork') Updated implementations to explicitly suppress (i.e. not reissue) expected warnings --- tests/test_child_procs.py | 131 ++++++++++++++++++++++---------------- 1 file changed, 76 insertions(+), 55 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index b1c8a9f6..18c62cf2 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1450,7 +1450,7 @@ class _check_warnings(Sequence[_WarningInfo]): Example: >>> import warnings - >>> cw = _check_warnings() + >>> cw = _check_warnings(reissue_warnings=False) >>> with cw: # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... cw.forbid_warnings('foo', UserWarning) @@ -1479,9 +1479,9 @@ class _check_warnings(Sequence[_WarningInfo]): >>> assert len(cw) == 1 >>> assert str(cw[0].message) == 'foobar' """ - def __init__(self, /, **kwargs) -> None: + def __init__(self, /, *, reissue_warnings: bool = True, **kwargs) -> None: self._new_context: Callable[[], _WarningContext] = partial( - _WarningContext.new, **kwargs, + _WarningContext.new, reissue_warnings=reissue_warnings, **kwargs, ) self._contexts: list[ tuple[_WarningContext, Sequence[_WarningInfo]] @@ -1769,59 +1769,63 @@ def get_streams( main = _timeout(kernprof_main, timeout=timeout) else: main = kernprof_main - try: + with ExitStack() as stack: + if check_warnings: + # See similar indictions against warnings in + # `_test_apply_mp_patches()` + cw = stack.enter_context(_check_warnings()) + cw.forbid_warnings( + category=UserWarning, module='line_profiler', + ) + cw.forbid_warnings(module='multiprocessing') + cw.suppress_warnings( + r'.*multi-?threaded.*fork\(\)', + category=DeprecationWarning, + ) try: - with ExitStack() as stack: + try: cleanup = stack.enter_context(Cleanup()) if env is not None: cleanup.update_mapping(os.environ, env) - if check_warnings: - # See similar indictions against warnings in - # `_test_apply_mp_patches()` - cw = stack.enter_context(_check_warnings()) - cw.forbid_warnings( - category=UserWarning, module='line_profiler', - ) - cw.forbid_warnings(module='multiprocessing') main(cmd[1:], exit_on_error=False) - except _TestTimeout: # `subprocess` uses `SIGKILL` - returncode, timed_out = -9, True - status = f'error ({returncode})' - except Exception as e: - # Format and output the tracebacks, otherwise we would've - # suppressed them - traceback.print_exception(e) - returncode = 2 if isinstance(e, ArgumentError) else 1 - status = f'error ({returncode})' - else: - status = returncode = 0 - stdout, stderr = get_streams() - - # Turn in-process errors into the corresponding `subprocess` - # errors - if timed_out: - assert timeout is not None # Assure the typechecker - raise subprocess.TimeoutExpired(cmd, timeout, stdout, stderr) - if check and returncode: - raise subprocess.CalledProcessError( - returncode, cmd, stdout, stderr, + except _TestTimeout: # `subprocess` uses `SIGKILL` + returncode, timed_out = -9, True + status = f'error ({returncode})' + except Exception as e: + # Format and output the tracebacks, otherwise we would've + # suppressed them + traceback.print_exception(e) + returncode = 2 if isinstance(e, ArgumentError) else 1 + status = f'error ({returncode})' + else: + status = returncode = 0 + stdout, stderr = get_streams() + + # Turn in-process errors into the corresponding `subprocess` + # errors + if timed_out: + assert timeout is not None # Assure the typechecker + raise subprocess.TimeoutExpired(cmd, timeout, stdout, stderr) + if check and returncode: + raise subprocess.CalledProcessError( + returncode, cmd, stdout, stderr, + ) + return subprocess.CompletedProcess(cmd, returncode, stdout, stderr) + finally: + time = monotonic() - time + captured: str | bytes | None + for name, captured, stream in [ + ('stdout', stdout, sys.stdout), ('stderr', stderr, sys.stderr), + ]: + if captured is None: + continue + if isinstance(captured, bytes): # `text=False` + captured = captured.decode() + print(f'{name}:\n{indent(captured, " ")}', file=stream) + print( + f'-- Emulated process end (time elapsed: {time:.2f} s / ' + f'return status: {status}) --' ) - return subprocess.CompletedProcess(cmd, returncode, stdout, stderr) - finally: - time = monotonic() - time - captured: str | bytes | None - for name, captured, stream in [ - ('stdout', stdout, sys.stdout), ('stderr', stderr, sys.stderr), - ]: - if captured is None: - continue - if isinstance(captured, bytes): # `text=False` - captured = captured.decode() - print(f'{name}:\n{indent(captured, " ")}', file=stream) - print( - f'-- Emulated process end (time elapsed: {time:.2f} s / ' - f'return status: {status}) --' - ) def _print_env_deltas(env: Mapping[str, str] | None = None) -> None: @@ -2684,11 +2688,15 @@ def list_profiled_funcs() -> list[str]: ]: callback() with _check_warnings() as cw: + handle_warning: list[Callable[..., None]] = [] if has_nonempty_file: - check_warning = cw.forbid_warnings - else: - check_warning = cw.expect_warnings - check_warning(r'.* file\(s\) .* empty', module='line_profiler') + handle_warning.append(cw.forbid_warnings) + else: # Check for the warning but don't reissue it + handle_warning.extend([ + cw.expect_warnings, cw.suppress_warnings, + ]) + for handle in handle_warning: + handle(r'.* file\(s\) .* empty', module='line_profiler') gathered = cache.gather_stats() assert any(gathered.timings.values()) == has_stats, gathered if hasattr(os, 'fork'): @@ -2926,6 +2934,8 @@ def _test_apply_mp_patches( patch_process: bool | None = None, intercept_logs: bool | None = None, trace_pids: bool | None = None, + *, + start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], **kwargs ) -> None: patches = cast(dict[str, bool], _DEFAULT_MP_CONFIG.patches.copy()) @@ -2942,7 +2952,18 @@ def _test_apply_mp_patches( # child processes which didn't perform any work cw.forbid_warnings(category=UserWarning, module='line_profiler') cw.forbid_warnings(module='multiprocessing') - _test_apply_mp_patches_inner(mp_patches=mp_patches, **kwargs) + if start_method == 'fork': + # The `@_timeout` decorator spins up a new thread for + # executing `_test_apply_mp_patches_inner()`; explicitly + # ignore the associated warning when we use + # `start_method='fork'` + cw.suppress_warnings( + r'.*multi-?threaded.*fork\(\)', + category=DeprecationWarning, + ) + _test_apply_mp_patches_inner( + mp_patches=mp_patches, start_method=start_method, **kwargs, + ) @(_Params.new('start_method', From f7a48eda969eb98c97cfe328aa7f0534d4ab9ae2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 12 Jun 2026 21:18:10 +0200 Subject: [PATCH 123/165] Misc. fixes for the session cache line_profiler/_child_process_profiling/cache.py::LineProfilingCache _wrap_os_fork() Updated `os.fork()` wrapper to unregister the `._atexit_hook` of the inherited instance @_method_wrapper Added new parameter `wrapper_name` for more flexible debug-log messages --- line_profiler/_child_process_profiling/cache.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index ceda3a19..d5c70dac 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -626,6 +626,9 @@ def wrapper() -> int: forked._debug_output( 'Superseded cached `.load()`-ed instance in forked process' ) + # Unregister the `._atexit_hook` of the inherited cache + # instance to avoid complications + atexit.unregister(self._atexit_hook) # Note: we can reuse the profiler instance in the fork, but # it needs to go through setup so that the separate # profiling results are dumped into another output file @@ -725,14 +728,18 @@ def _get_filename(cache_dir: os.PathLike[str] | str) -> str: def _method_wrapper( cls, wrapper: Callable[Concatenate[Self, Callable[PS, T], PS], T], + *, debug: bool | None = None, + wrapper_name: str | None = None, ) -> Callable[[Callable[PS, T]], Callable[PS, T]]: ... @overload @classmethod def _method_wrapper( - cls, wrapper: None = None, debug: bool | None = None, + cls, wrapper: None = None, *, + debug: bool | None = None, + wrapper_name: str | None = None, ) -> Callable[ [Callable[Concatenate[Self, Callable[PS, T], PS], T]], Callable[[Callable[PS, T]], Callable[PS, T]] @@ -746,6 +753,7 @@ def _method_wrapper( Callable[Concatenate[Self, Callable[PS, T], PS], T] | None ) = None, debug: bool | None = None, + wrapper_name: str | None = None, ) -> ( Callable[ [Callable[Concatenate[Self, Callable[PS, T], PS], T]], @@ -768,6 +776,9 @@ def _method_wrapper( after the call to the ``wrapper`` callable; if ``debug`` is not set, it will be taken from the session instance. + wrapper_name (str | None) + Optional name to identify the source of the wrapper, + used in debug messages. Returns: inner_wrapper (Callable[[Callable[PS, T]], Callable[PS, T]]) @@ -822,9 +833,10 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: return call() vanilla_name = cls._get_name(vanilla_impl) - wrapper_name = cls._get_name(wrapper) return wrapped_impl + if wrapper_name is None: + wrapper_name = cls._get_name(wrapper) for field in 'name', 'qualname', 'doc': dunder = f'__{field}__' value = getattr(wrapper, dunder, None) From 0198a486d50bd10c3a6b937770863b7319e3ba9b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 12 Jun 2026 21:48:07 +0200 Subject: [PATCH 124/165] Fix conditional skip in `test_child_procs.py` tests/test_child_procs.py::test_apply_mp_patches_{success,failure}() Moved location of `pytest.skip()` call because calling it from within a child thread (such as that created by `@_timeout`) causes issues --- tests/test_child_procs.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 18c62cf2..898a7b9b 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -2881,12 +2881,6 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # Counts from the one final sum over the parallel results nloops_expected += nprocs - if start_method not in ('dummy', *START_METHODS): - pytest.skip( - f'`multiprocessing` start method {start_method!r} ' - 'not available on the platform' - ) - # Note: manually handle the error here instead of using # `pytest.raises()` since we want certain `RuntimeError`s to be # propagated and handled by `@pytest.mark.retry` @@ -2938,6 +2932,12 @@ def _test_apply_mp_patches( start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], **kwargs ) -> None: + if start_method not in ('dummy', *START_METHODS): + pytest.skip( + f'`multiprocessing` start method {start_method!r} ' + 'not available on the platform' + ) + patches = cast(dict[str, bool], _DEFAULT_MP_CONFIG.patches.copy()) for name, applied in { 'pool': patch_pool, 'process': patch_process, @@ -2946,6 +2946,7 @@ def _test_apply_mp_patches( if applied is not None: patches[name] = applied mp_patches = [name for name, applied in patches.items() if applied] + with _check_warnings() as cw: if 'child_pids' in mp_patches: # With PID bookkeeping we should be able to weed out all the From 2507acca98d50c5240401fb73976c2f358560686 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 13 Jun 2026 21:48:34 +0200 Subject: [PATCH 125/165] Various fixes line_profiler/_child_process_profiling/multiprocessing_patches.py ::_setup_in_mp_child() Moved PID check from one of the sub-callbacks to here tests/test_child_procs.py _check_warnings, _WarningContext Added new parameter `format_warnings` for controlling whether to format the recorded warnings that caused the `ResultMismatch` error, because apparently `.reissue()` leaves some warnings out sometimes _run_kernprof_main_in_process(), _test_apply_mp_patches() Updated warning filters to be more specific to avoid bycatches --- .../multiprocessing_patches.py | 4 +- tests/test_child_procs.py | 90 +++++++++++++++---- 2 files changed, 76 insertions(+), 18 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index b30b1bca..4388099c 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -461,6 +461,8 @@ def _setup_in_mp_child(cache: LineProfilingCache) -> None: avoid possible clashes with the profiling-file writing managed by this module. """ + if cache.main_pid == os.getpid(): # Not in a child process + return xc: Exception | None = None for setup in [_add_sigterm_handler_in_child, _unregister_atexit_hook]: try: @@ -479,8 +481,6 @@ def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: key = 'mp_added_sigterm_handler' if not MPConfig.from_cache(cache).catch_sigterm: return - if cache.main_pid == os.getpid(): # Not in a child process - return if cache._additional_data.get(key, False): # Already added (e.g. by another plugin) return diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 898a7b9b..fd5a27c6 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1342,6 +1342,7 @@ class _WarningContext: default_factory=partial(warnings.catch_warnings, record=True) ) reissue_warnings: bool = False + format_warnings: bool = False checks: list[tuple[_WarningMatcher, bool]] = dataclasses.field( default_factory=list, ) @@ -1380,14 +1381,28 @@ def check(self, infos: Sequence[_WarningInfo]) -> None: for matcher, allowed_or_required in self.checks: matches = [info for info in infos if matcher.match(info)] if matches and not allowed_or_required: + if self.format_warnings: + matches_repr = ( + f'{len(matches)}:\n{self._format_warnings(matches)}' + ) + else: + matches_repr = f'{len(matches)} ({matches!r})' raise ResultMismatch( expected=f'no warnings matching {matcher!r}', - actual=f'{len(matches)} ({matches!r})', + actual=matches_repr, ) if not matches and allowed_or_required: + warnings_repr = f'none out of {len(infos)}' + if infos: + if self.format_warnings: + warnings_repr = ( + f'{warnings_repr}:\n{self._format_warnings(infos)}' + ) + else: + warnings_repr = f'{warnings_repr} ({infos!r})' raise ResultMismatch( expected=f'warnings matching {matcher!r}', - actual=f'none out of {len(infos)} ({infos!r})', + actual=warnings_repr, ) def reissue(self, infos: Sequence[_WarningInfo]) -> None: @@ -1422,6 +1437,22 @@ def reissue_(info: _WarningInfo) -> None: if should_reissue: reissue_(info) + @staticmethod + def _format_warnings(infos: Sequence[_WarningInfo]) -> str: + if not infos: + return '' + chunks: list[str] = [] + for info in infos: + text = strip(warnings.formatwarning( + message=info.message, + category=info.category, + filename=info.filename, + lineno=info.lineno, + )) + chunks.append(f'- {info!r}') + chunks.append(indent(text, ' ')) + return '\n'.join(chunks) + @staticmethod def _get_matcher( message: str | None = None, @@ -1434,11 +1465,17 @@ def _get_matcher( ) @classmethod - def new(cls, /, *, reissue_warnings: bool = False, **kwargs) -> Self: + def new( + cls, /, *, + reissue_warnings: bool = False, + format_warnings: bool = False, + **kwargs + ) -> Self: kwargs['record'] = True return cls( warnings.catch_warnings(**kwargs), reissue_warnings=reissue_warnings, + format_warnings=format_warnings, ) @@ -1450,7 +1487,9 @@ class _check_warnings(Sequence[_WarningInfo]): Example: >>> import warnings - >>> cw = _check_warnings(reissue_warnings=False) + >>> cw = _check_warnings( + ... reissue_warnings=False, format_warnings=False, + ... ) >>> with cw: # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... cw.forbid_warnings('foo', UserWarning) @@ -1479,9 +1518,22 @@ class _check_warnings(Sequence[_WarningInfo]): >>> assert len(cw) == 1 >>> assert str(cw[0].message) == 'foobar' """ - def __init__(self, /, *, reissue_warnings: bool = True, **kwargs) -> None: + def __init__( + self, /, *, + reissue_warnings: bool = True, + format_warnings: bool = True, + **kwargs + ) -> None: + # Note: even with `reissue_warnings=True`, it is not guaranteed + # that all the recorded warnings become visible, depending on + # the warning filters in place before entering the context; so + # we also use `format_warnings=True` by default to include the + # warnings in the error message (if any) self._new_context: Callable[[], _WarningContext] = partial( - _WarningContext.new, reissue_warnings=reissue_warnings, **kwargs, + _WarningContext.new, + reissue_warnings=reissue_warnings, + format_warnings=format_warnings, + **kwargs, ) self._contexts: list[ tuple[_WarningContext, Sequence[_WarningInfo]] @@ -1774,14 +1826,17 @@ def get_streams( # See similar indictions against warnings in # `_test_apply_mp_patches()` cw = stack.enter_context(_check_warnings()) - cw.forbid_warnings( - category=UserWarning, module='line_profiler', - ) - cw.forbid_warnings(module='multiprocessing') - cw.suppress_warnings( - r'.*multi-?threaded.*fork\(\)', - category=DeprecationWarning, - ) + cw.forbid_warnings('.*resource_tracker', module='multiprocessing') + if _DEFAULT_MP_CONFIG.patches['child_pids']: + cw.forbid_warnings( + r'.* file\(s\) .* empty', + category=UserWarning, module='line_profiler', + ) + if timeout: + cw.suppress_warnings( + r'.*multi-?threaded.*fork\(\)', + category=DeprecationWarning, + ) try: try: cleanup = stack.enter_context(Cleanup()) @@ -2948,11 +3003,14 @@ def _test_apply_mp_patches( mp_patches = [name for name, applied in patches.items() if applied] with _check_warnings() as cw: + cw.forbid_warnings('.*resource_tracker', module='multiprocessing') if 'child_pids' in mp_patches: # With PID bookkeeping we should be able to weed out all the # child processes which didn't perform any work - cw.forbid_warnings(category=UserWarning, module='line_profiler') - cw.forbid_warnings(module='multiprocessing') + cw.forbid_warnings( + r'.* file\(s\) .* empty', + category=UserWarning, module='line_profiler', + ) if start_method == 'fork': # The `@_timeout` decorator spins up a new thread for # executing `_test_apply_mp_patches_inner()`; explicitly From 36164b892d6cd725aaebf16843474c8db9410fee Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 14 Jun 2026 01:24:42 +0200 Subject: [PATCH 126/165] Streamlined `multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py TaskWrapper, wrap_get_tasks(), wrap_guarded_task_generation() Removed (superseded by `wrap_worker_pool_write_per_task()`) _QueuePIDWrapper Reworked into `_QueuePutWrapper` wrap_worker_pool() Reworked into `wrap_worker_pool_write_on_exit()` wrap_worker_pid() Refactored to use the new `_QueuePutWrapper` wrap_process() Now asserting `BaseProcess.join()` and `.terminate()` to be bound methods --- .../multiprocessing_patches.py | 264 ++++++------------ 1 file changed, 91 insertions(+), 173 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 4388099c..83f8ceb4 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -23,16 +23,14 @@ from collections.abc import Callable, Collection, Mapping, Sequence, Set from functools import partial, wraps from importlib import import_module -from inspect import getattr_static, signature -from multiprocessing.pool import Pool +from inspect import getattr_static from multiprocessing.process import BaseProcess from operator import attrgetter -from queue import SimpleQueue from time import sleep, monotonic from types import MappingProxyType as mappingproxy, MethodType, ModuleType from typing import ( TYPE_CHECKING, - Any, ClassVar, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, + Any, ClassVar, Literal, NamedTuple, Protocol, TypeVar, NoReturn, cast, final, overload, ) from typing_extensions import Concatenate, ParamSpec, Self @@ -59,7 +57,6 @@ _CAN_USE_RESOURCE_TRACKER = True from .. import _diagnostics as diagnostics -from ..cleanup import Cleanup from ..toml_config import ConfigSource from .cache import LineProfilingCache from .runpy_patches import create_runpy_wrapper @@ -88,11 +85,6 @@ # ------------------------------ Helpers ------------------------------- -class _Wrapper(Protocol, Generic[PS, T]): - def __call__(self, func: Callable[PS, T], /) -> Callable[PS, T]: - ... - - class _Queue(Protocol): """ Protocol for methods common to e.g. :py:class:`queue.SimpleQueue` @@ -381,65 +373,32 @@ def _process_has_returned( return result -class TaskWrapper(Generic[PS, T]): - """ - Pickle-able wrapper around the supplied task callable, which writes - to the session's profiling-stats file on exit. - - Note: - Since this produces extra overhead for each invocation of the - callable, it is only used when we can't reliably do - end-of-process cleanup. This mainly happens on Windows, where - we can't catch and handle ``SIGTERM``. - """ - def __init__(self, func: Callable[PS, T]) -> None: - self.func = func - try: - self.__signature__ = signature(func) - except Exception: # nocover - # Can happen with e.g. certain builin/c-based callables - pass - - def __call__(self, *args, **kwargs) -> T: - callback: Callable[[], None] = _no_op - try: - cache = LineProfilingCache.load() - except Exception: - pass - else: - # Note: this doesn't write debugging output... doing so for - # every task would be excessive - callback = partial(_dump_stats_quick, cache) - - try: - return self.func(*args, **kwargs) - finally: - callback() - - -class _QueuePIDWrapper: +class _QueuePutWrapper: # nocover """ - Wrap around a :py:class:`queue.SimpleQueue` (used by - :py:mod:`multiprocessing.dummy`) so that the PID info is attached to - the :py:meth:`.SimpleQueue.put` tuple. - - Notes: - - Used by the ``child_pids`` patch. - - - While the PID info is useless when using - :py:mod:`multiprocessing.dummy`, it is nonetheless necessary - because :py:meth:`multiprocessing.pool.Pool._handle_results` - has also been patched to expect the queue-item-getter function - to return the tuple ``(pid, original_put_value)``. + Wrap around a queue (the ``outqueue`` argument to + :py:func:`multiprocessing.pool.worker`) so that each call to its + ``.put()`` is preceded by calling a ``callback()``; its result is + optionally attached to the tuple pushed back to the parent if + ``push_to_parent`` is true. """ - def __init__(self, queue: _Queue) -> None: + def __init__( + self, + queue: _Queue, + callback: Callable[[], Any], + push_to_parent: bool = False, + ) -> None: self._queue = queue + self._callback = callback + self._push = push_to_parent def __getattr__(self, attr: str) -> Any: return getattr(self._queue, attr) def put(self, obj: Any) -> None: - self._queue.put((os.getpid(), obj)) + data = self._callback() + if self._push: + obj = data, obj + self._queue.put(obj) def get(self) -> Any: return self._queue.get() @@ -449,7 +408,7 @@ def _no_op(*_, **__) -> None: pass -def _setup_in_mp_child(cache: LineProfilingCache) -> None: +def _setup_in_mp_child(cache: LineProfilingCache) -> None: # nocover """ Perform :py:mod:`multiprocessing`-specific setup in a child process curated by the module. Currently it does the following: @@ -784,45 +743,50 @@ def _register_patch(name: str, patch: _Patch | None = None) -> _Patch: # ---------------- `multiprocessing.pool.Pool` patches ----------------- -@LineProfilingCache._method_wrapper -def wrap_get_tasks( - _, # No need to use the cache, but `_method_wrapper` expects it - vanilla_impl: Callable[Concatenate[Callable[PS1, T1], PS2], T2], - func: Callable[PS1, T1], - *args: PS2.args, - **kwargs: PS2.kwargs -) -> T2: +class _PIDQueueGetWrapper: # nocover """ - Wrap around :py:meth:`.Pool._get_tasks` so that the writing of - profiling stats is handled within the callables sent to the child - processes before the parent process assumes control. - - Note: - :py:meth:`.Pool._get_tasks` is a static method. + Wrapper around the ``inqueue`` argument to + :py:func:`multiprocessing.pool.worker` to intercept the sentinel + value (:py:const:`None`) signifying the end of the queue and perform + cleanup. """ - return vanilla_impl(TaskWrapper(func), *args, **kwargs) + def __init__( + self, + queue: _Queue, + cache: LineProfilingCache, + ) -> None: + self._queue = queue + self._cache = cache + def __getattr__(self, attr: str) -> Any: + return getattr(self._queue, attr) -@LineProfilingCache._method_wrapper -def wrap_guarded_task_generation( - _, # No need to use the cache, but `_method_wrapper` expects it - vanilla_impl: Callable[Concatenate[Pool, int, Callable[PS1, T1], PS2], T2], - self: Pool, - result_job: int, - func: Callable[PS1, T1], - *args: PS2.args, - **kwargs: PS2.kwargs -) -> T2: - """ - Wrap around :py:meth:`.Pool._guarded_task_generation` so that the - writing of profiling stats is handled within the callables sent to - the child processes before the parent process assumes control. - """ - return vanilla_impl(self, result_job, TaskWrapper(func), *args, **kwargs) + def put(self, obj: Any) -> None: + self._queue.put(obj) + + def get(self) -> Any: + result = self._queue.get() + cache = self._cache + ntasks: dict[int, int] + ntasks = cache._additional_data.setdefault('mp_queue_ntasks', {}) + queue_id = id(self) + if result is None: + n = ntasks.pop(queue_id, 0) + cache._debug_output( + '`multiprocessing.pool.worker`: ' + f'recieved {n} task(s) in total', + ) + # Got sentinel value, process is about to exit + reason = 'ran out of tasks in `multiprocessing.process.worker()`' + if cache.main_pid != os.getpid(): + _dump_stats_quick(cache, debug=True, reason=reason) + else: + ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 + return result @LineProfilingCache._method_wrapper # nocover -def wrap_worker_pool( +def wrap_worker_pool_write_on_exit( cache: LineProfilingCache, vanilla_impl: Callable[Concatenate[_Queue, PS], None], inqueue: _Queue, @@ -846,60 +810,44 @@ def wrap_worker_pool( # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up _setup_in_mp_child(cache) - # Note: using the `cache` itself as the context manager is prone to - # deadlock - with Cleanup() as cleanup: - if isinstance(inqueue.get, MethodType): - get = partial(_wrap_inqueue_get, cache, inqueue.get.__func__) - cleanup.patch( - inqueue, 'get', MethodType(get, inqueue), - name='._inqueue', - ) - return vanilla_impl(inqueue, *args, **kwargs) + return vanilla_impl(_PIDQueueGetWrapper(inqueue, cache), *args, **kwargs) -def _wrap_inqueue_get( # nocover +@LineProfilingCache._method_wrapper # nocover +def wrap_worker_pool_write_per_task( cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[_Queue, PS], T], - self: _Queue, - /, + vanilla_impl: Callable[Concatenate[_Queue, _Queue, PS], None], + inqueue: _Queue, + outqueue: _Queue, *args: PS.args, **kwargs: PS.kwargs -) -> T: +) -> None: """ - Intercept the sentinel value (:py:const:`None`) signifiying the end - of the queue and perform cleanup. + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes can write profiling output before pushing the result of + each task back to the parent. + + Notes: + - This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. + + - This is only used on Windows where we can't handle ``SIGTERM`` + on child processes, thus necessitating the write to happen + before control flow is passed backed to the parent. """ - result = vanilla_impl(self, *args, **kwargs) - ntasks: dict[int, int] - ntasks = cache._additional_data.setdefault('mp_queue_ntasks', {}) - queue_id = id(self) - if result is None: - n = ntasks.pop(queue_id, 0) - msg = f'`multiprocessing.pool.worker`: recieved {n} task(s) in total' - cache._debug_output(msg) - # Got sentinel value, process is about to exit - reason = 'ran out of tasks in `multiprocessing.process.worker()`' - if cache.main_pid != os.getpid(): - _dump_stats_quick(cache, debug=True, reason=reason) - else: - ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 - return result + outqueue = _QueuePutWrapper(outqueue, partial(_dump_stats_quick, cache)) + return vanilla_impl(inqueue, outqueue, *args, **kwargs) -_patch_pool = _register_patch('pool', Patch('pool')).add_method if _CAN_CATCH_SIGTERM: - # Only write profiling output once per process if it can be helped - _patch_pool('', 'worker', wrap_worker_pool) + wrap_worker_pool: Callable[[Callable[..., None]], Callable[..., None]] + wrap_worker_pool = wrap_worker_pool_write_on_exit else: - # Don't have a choice on platform like Windows, the only reliable - # way to ensure that the child survives until profiling output is - # written is to write it for every task before control is returned - # to the parent - _patch_pool('Pool', '_get_tasks', wrap_get_tasks, 'static') - _patch_pool( - 'Pool', '_guarded_task_generation', wrap_guarded_task_generation, - ) + wrap_worker_pool = wrap_worker_pool_write_per_task +_register_patch('pool', Patch('pool')).add_method( + '', 'worker', wrap_worker_pool, +) # ----------- `multiprocessing.process.BaseProcess` patches ------------ @@ -1027,22 +975,8 @@ def wrap_worker_pid( reliably measure coverage thereon; see also :py:func:`wrap_bootstrap`. """ - # Note: using the `cache` itself as the context manager is prone to - # deadlock - with Cleanup() as cleanup: - if isinstance(outqueue, SimpleQueue): - # `multiprocessing.dummy` instantiates C-based queue - # objects, which doesn't permit assigning to the instance - # (because it doesn't have an instance dict)... - # so just wrap the queue with a helper class - outqueue = _QueuePIDWrapper(outqueue) - elif isinstance(outqueue.put, MethodType): - put = partial(_wrap_outqueue_put, outqueue.put.__func__) - cleanup.patch( - outqueue, 'put', MethodType(put, outqueue), - name='._outqueue', - ) - return vanilla_impl(inqueue, outqueue, *args, **kwargs) + outqueue = _QueuePutWrapper(outqueue, os.getpid, push_to_parent=True) + return vanilla_impl(inqueue, outqueue, *args, **kwargs) @LineProfilingCache._method_wrapper @@ -1068,9 +1002,9 @@ def wrap_process( patch = partial(cache.patch, cleanup=False, name=name) for method, action in ('join', 'joining'), ('terminate', 'terminating'): bound = getattr(proc, method) - if isinstance(bound, MethodType): - finalize = _wrap_process_finalize(cache, bound.__func__, action) - patch(proc, method, MethodType(finalize, proc)) + assert isinstance(bound, MethodType) + finalize = _wrap_process_finalize(cache, bound.__func__, action) + patch(proc, method, MethodType(finalize, proc)) return proc @@ -1125,22 +1059,6 @@ def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: return finalize -def _wrap_outqueue_put( - vanilla_impl: Callable[ - Concatenate[_Queue, tuple[Any, ...], PS], None - ], - self: _Queue, - obj: tuple[Any, ...], - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Smuggle in the PID of the child process so that the parent can keep - track of which child completed what task. - """ - vanilla_impl(self, (os.getpid(), obj), *args, **kwargs) - - def _wrap_outqueue_quick_get( cache: LineProfilingCache, vanilla_impl: Callable[PS, tuple[int, tuple[Any, ...]] | None], @@ -1202,7 +1120,7 @@ def tee_log( **kwargs: PS.kwargs ) -> None: """ -Wrap around logging functions like + Wrap around logging functions like :py:func:`multiprocessing.util.debug` so that we can tee log messages from the package to our own logs. """ From b3f05e337fffba6323058d8f3743229963ef36a2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 15 Jun 2026 01:44:30 +0200 Subject: [PATCH 127/165] WIP: prioritization of `multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py _PatchRegistry New mapping class to allow for prioritization of the registered plugins _PATCHES Now a `_PatchRegistry` _register_patch() - Now `_PATCHES.register()` - Now taking the optional argument `priority` wrap_worker_pool_write_per_task() Now also calling `_setup_in_mp_child()` apply() Now making use of `_PatchRegistry.select()` for selecting the patches to apply --- .../multiprocessing_patches.py | 175 +++++++++++++----- 1 file changed, 129 insertions(+), 46 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 83f8ceb4..da35d240 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -20,7 +20,9 @@ import os import sys import warnings -from collections.abc import Callable, Collection, Mapping, Sequence, Set +from collections.abc import ( + Callable, Collection, Generator, Mapping, Sequence, Set, +) from functools import partial, wraps from importlib import import_module from inspect import getattr_static @@ -66,21 +68,15 @@ T = TypeVar('T') -T1 = TypeVar('T1') -T2 = TypeVar('T2') P = TypeVar('P', bound=BaseProcess) Pt = TypeVar('Pt', bound='_Patch') PS = ParamSpec('PS') -PS1 = ParamSpec('PS1') -PS2 = ParamSpec('PS2') _OnTimeout = Literal['ignore', 'warn', 'error'] PublicPatch = Literal['pool', 'process', 'logging', 'child_pids'] _CAN_CATCH_SIGTERM = sys.platform != 'win32' _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' _LOGGERS = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] -_PATCHES: dict[str, '_Patch'] = {} - # ------------------------------ Helpers ------------------------------- @@ -438,6 +434,8 @@ def _setup_in_mp_child(cache: LineProfilingCache) -> None: # nocover def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: key = 'mp_added_sigterm_handler' + if not _CAN_CATCH_SIGTERM: + return if not MPConfig.from_cache(cache).catch_sigterm: return if cache._additional_data.get(key, False): @@ -710,35 +708,128 @@ def summary(self) -> mappingproxy[str, frozenset[str]]: }) -@overload -def _register_patch(name: str, patch: Pt) -> Pt: - ... +class _PatchRegistry(Mapping[str, _Patch]): + """ + Mapping subclass for managing patches. + """ + def __init__(self) -> None: + self._patches: dict[str, tuple[int, _Patch]] = {} + + def __repr__(self) -> str: + if self._patches: + patches = ', '.join( + f'{name!r} (priority {priority})' if priority else repr(name) + for name, (priority, _) in self._patches.items() + ) + patches = f'({len(self)} patch(es)): {patches}' + else: + patches = '(0 patches)' + return f'<{type(self).__name__} @ {id(self):#x} {patches}>' + def __getitem__(self, key: str) -> _Patch: + return self._patches[key][1] -@overload -def _register_patch(name: str, patch: None = None) -> _Patch: - ... + def __iter__(self) -> Generator[str, None, None]: + for name, _ in self._iter_patches(): + yield name + def __len__(self) -> int: + return len(self._patches) -def _register_patch(name: str, patch: _Patch | None = None) -> _Patch: - """ - Register the ``patch`` under ``name`` and return it as-is. If - ``patch`` isn't provided, look for the existing patch registered - under the name. + def __contains__(self, key: Any) -> bool: + return key in self._patches - Note: - Patches named with leading double underscores are applied no - matter the user input (e.g. via ``apply(..., patches=...)`` or - the config file). - """ - if patch is not None: - if _PATCHES.setdefault(name, patch) is not patch: - raise ValueError( - f'name = {name!r}, patch = {patch!r}: ' - 'name already in use by {_PATCHES[name]}' - ) - return _PATCHES[name] + @overload + def register(self, name: str, patch: Pt, *, priority: int = 0) -> Pt: + ... + + @overload + def register( + self, name: str, patch: None = None, *, priority: int = 0, + ) -> _Patch: + ... + + def register( + self, name: str, patch: _Patch | None = None, *, priority: int = 0, + ) -> _Patch: + """ + Register the ``patch`` under ``name`` with the specified + ``priority`` and return it as-is. If ``patch`` isn't provided, + look for the existing patch registered under the name. + + Note: + Patches named with leading double underscores are applied no + matter the user input (e.g. via ``apply(..., patches=...)`` + or the config file). + """ + if patch is not None: + old_pri, stored = self._patches.setdefault(name, (priority, patch)) + if stored is not patch: + raise ValueError( + f'name = {name!r}, patch = {patch!r}: ' + f'name already in use by {stored!r}' + ) + if old_pri != priority: # Update priority + self._patches[name] = priority, patch + return self[name] + + def select(self, patches: Collection[str]) -> Self: + """ + Returns: + New instance with the selected patches + + Note: + Patches whose names are prefixed with double underscores are + considered mandatory and are always selected. + """ + new = type(self)() + new._patches.update( + (name, patch_info) for name, patch_info in self._patches.items() + if name.startswith('__') or name in patches + ) + return new + def _iter_patches( + self, reverse: bool = False, + ) -> Generator[tuple[str, _Patch], None, None]: + """ + Iterate over the available patches. + + Note: + Since patches typically consists of function wrappers, and + outer wrappers are both called first and responsible for + calling inner wrappers, patches with a HIGHER priority are + yielded LATER (or earlier if ``reverse=True``. + """ + for _, patches in sorted(self._prioritized.items(), reverse=reverse): + yield from patches.items() + + @property + def summary(self) -> dict[str, frozenset[str]]: + """ + Mapping from the names of the entities affected by the patches + to sets of attributes patched thereon. + """ + summaries = [patch.summary for patch in self.values()] + return { + target: frozenset().union(*(s.get(target, ()) for s in summaries)) + for target in frozenset().union(*summaries) + } + + @property + def _prioritized(self) -> dict[int, dict[str, _Patch]]: + # This could've been a static attribute maintained by the + # `.register()` method, but we don't call this a bunch so it + # isn't like we incur a lot of overhead by calculating it + # on-the-fly, and it's less error-prone this way. + result: dict[int, dict[str, _Patch]] = {} + for name, (priority, patch) in self._patches.items(): + result.setdefault(priority, {})[name] = patch + return result + + +_PATCHES = _PatchRegistry() +_register_patch = _PATCHES.register # ---------------- `multiprocessing.pool.Pool` patches ----------------- @@ -807,8 +898,6 @@ def wrap_worker_pool_write_on_exit( ``SIGTERM`` on child processes and ensure that they aren't prematurely terminated. """ - # Set a signal handler for SIGTERM to help child processes with - # consistently cleaning up _setup_in_mp_child(cache) return vanilla_impl(_PIDQueueGetWrapper(inqueue, cache), *args, **kwargs) @@ -836,6 +925,7 @@ def wrap_worker_pool_write_per_task( on child processes, thus necessitating the write to happen before control flow is passed backed to the parent. """ + _setup_in_mp_child(cache) outqueue = _QueuePutWrapper(outqueue, partial(_dump_stats_quick, cache)) return vanilla_impl(inqueue, outqueue, *args, **kwargs) @@ -905,8 +995,6 @@ def wrap_bootstrap( there. Hence :py:func:`wrap_terminate` remains necessary for mitigating unclean exits. """ - # Set a signal handler for SIGTERM to help child processes with - # consistently cleaning up _setup_in_mp_child(cache) try: return vanilla_impl(self, *args, **kwargs) @@ -1378,17 +1466,12 @@ def apply( patches_: set[str] = {p for p, use in patches_dict.items() if use} else: patches_ = {p.lower() for p in patches} - for name, patch in _PATCHES.items(): - if name in patches_: - should_apply = True - elif name.startswith('__'): - should_apply = (name != '__reboot_forkserver' or reboot_forkserver) - else: - should_apply = False - if should_apply: - msg = f'applying `multiprocessing` patch {name!r}' - cache._debug_output(msg.capitalize() + '...') - patch.apply(cache) - cache._debug_output('Done with ' + msg) + for name, patch in _PATCHES.select(patches_).items(): + if name == '__reboot_forkserver' and not reboot_forkserver: + continue + msg = f'applying `multiprocessing` patch {name!r}' + cache._debug_output(msg.capitalize() + '...') + patch.apply(cache) + cache._debug_output('Done with ' + msg) # Mark `multiprocessing` as having been patched cache.patch(multiprocessing, _PATCHED_MARKER, True) From 1dd05ba0c6bfcb6f3602e93f7543bb26fc6bbe94 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 15 Jun 2026 03:03:17 +0200 Subject: [PATCH 128/165] WIP: make `multiprocessing_patches` a subpackage --- .../__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename line_profiler/_child_process_profiling/{multiprocessing_patches.py => multiprocessing_patches/__init__.py} (100%) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py similarity index 100% rename from line_profiler/_child_process_profiling/multiprocessing_patches.py rename to line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py From d1b69bc6e91799531091ddd8563c6140a7bf356d Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 16 Jun 2026 00:05:44 +0200 Subject: [PATCH 129/165] Migrate `multiprocessing_patches/__init__.py` line_profiler/_child_process_profiling/multiprocessing_patches/ Split the contents of `__init__.py` into separate submodules: poller.py::Poller Migrated from `__init__.py::_Poller` _queue.py::PutWrapper Migrated from `__init__.py::_QueuePutWrapper` config.py::MPConfig Migrated from `__init__.py::MPConfig` _infrastructure.py Patch - Migrated from `__init__.py::_Patch` - Added attribute `.priority` to the interface SingleModulePatch Migrated from `__init__.py::Patch` Registry - Migrated from `__init__.py::_PatchRegistry` - Added class method `.from_entry_point()` for constructing an instance from patches loaded from an entry point _mandatory_patches.py::{RebootForkserver,ResourceTracker,RunPy}Patch - Migrated from the eponymous classes in `__init__.py` - Now declared as `line_profiler._multiproc_patches` entry points _profiling_patches.py POOL_PATCH, PROCESS_PATCH `SingleModulePatch` objects that are now exposed as `line_profiler._multiproc_patches` entry points wrap_worker[_{write_on_exit,write_per_task}]() Migrated from `__init__.py::wrap_worker_pool*()` wrap_bootstrap(), wrap_terminate() Migrated from the eponymous functions in `__init__.py` _optional_patches.py CHILD_PIDS_PATCH, LOGGING_PATCH `SingleModulePatch` objects that are now exposed as `line_profiler._multiproc_patches` entry points wrap_handle_results(), wrap_process() Migrated from the eponymous functions in `__init__.py` wrap_worker() Migrated from `__init__.py::wrap_worker_pid()` __init__.py::__all__ Now including the following: - `MPConfig` (= `config.py::MPConfig`) - `Poller` (= `poller.py::Poller`) - `Registry` (= `_infrastructure.py::Registry`) - `Timeout` (= `poller.py::Poller.Timeout`) tests/test_child_procs.py Updated imports from `line_profiler._child_process_profiling.multiprocessing_patches` setup.py Added the aforementioned `line_profiler._multiproc_patches` entry points --- .../multiprocessing_patches/__init__.py | 1356 +---------------- .../_infrastructure.py | 497 ++++++ .../_mandatory_patches.py | 168 ++ .../_optional_patches.py | 242 +++ .../_profiling_patches.py | 308 ++++ .../multiprocessing_patches/_queue.py | 50 + .../multiprocessing_patches/mp_config.py | 78 + .../multiprocessing_patches/poller.py | 198 +++ setup.py | 32 + tests/test_child_procs.py | 4 +- 10 files changed, 1587 insertions(+), 1346 deletions(-) create mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py create mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py create mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches/_optional_patches.py create mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py create mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py create mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py create mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches/poller.py diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py index da35d240..675db30c 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py @@ -14,1359 +14,24 @@ """ from __future__ import annotations -import atexit -import dataclasses import multiprocessing -import os -import sys -import warnings -from collections.abc import ( - Callable, Collection, Generator, Mapping, Sequence, Set, -) -from functools import partial, wraps -from importlib import import_module -from inspect import getattr_static -from multiprocessing.process import BaseProcess -from operator import attrgetter -from time import sleep, monotonic -from types import MappingProxyType as mappingproxy, MethodType, ModuleType -from typing import ( - TYPE_CHECKING, - Any, ClassVar, Literal, NamedTuple, Protocol, TypeVar, NoReturn, - cast, final, overload, -) -from typing_extensions import Concatenate, ParamSpec, Self +from collections.abc import Collection +from typing import Literal, get_args -try: - from multiprocessing import spawn -except ImportError: - _CAN_USE_SPAWN = False -else: - _CAN_USE_SPAWN = True -try: - from multiprocessing import forkserver -except ImportError: - _CAN_USE_FORKSERVER = False -else: - _CAN_USE_FORKSERVER = ( - 'forkserver' in multiprocessing.get_all_start_methods() - ) -try: - from multiprocessing import resource_tracker -except ImportError: - _CAN_USE_RESOURCE_TRACKER = False -else: - _CAN_USE_RESOURCE_TRACKER = True +from ..cache import LineProfilingCache +from ._infrastructure import Registry +from .mp_config import MPConfig +from .poller import Poller -from .. import _diagnostics as diagnostics -from ..toml_config import ConfigSource -from .cache import LineProfilingCache -from .runpy_patches import create_runpy_wrapper +__all__ = ('MPConfig', 'Poller', 'Registry', 'Timeout', 'apply') -__all__ = ('apply',) - - -T = TypeVar('T') -P = TypeVar('P', bound=BaseProcess) -Pt = TypeVar('Pt', bound='_Patch') -PS = ParamSpec('PS') -_OnTimeout = Literal['ignore', 'warn', 'error'] PublicPatch = Literal['pool', 'process', 'logging', 'child_pids'] -_CAN_CATCH_SIGTERM = sys.platform != 'win32' _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' -_LOGGERS = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] - -# ------------------------------ Helpers ------------------------------- - - -class _Queue(Protocol): - """ - Protocol for methods common to e.g. :py:class:`queue.SimpleQueue` - and :py:class:`multiprocessing.queues.SimpleQueue`. - """ - def put(self, obj: Any) -> None: - ... - - def get(self) -> Any: - ... - - -class _Poller: - """ - Poll a callable until it returns true-y. - - Example: - >>> import warnings - >>> from contextlib import ExitStack - >>> from functools import partial - >>> from itertools import count - >>> from typing import Iterator, Literal - - >>> def count_until( - ... limit: int, mode: Literal['until', 'while'] = 'until', - ... ) -> bool: - ... def counter_is_big_enough( - ... counter: Iterator[int], limit: int, - ... ) -> bool: - ... return next(counter) >= limit - ... - ... def counter_is_small_enough( - ... counter: Iterator[int], limit: int, - ... ) -> bool: - ... return next(counter) < limit - ... - ... # The branches are ultimately equal in results, but we - ... # want to explicitly test both `.poll_until()` and - ... # `.poll_while()` - ... if mode == 'until': - ... get_poller = partial( - ... _Poller.poll_until, counter_is_big_enough, - ... ) - ... else: - ... get_poller = partial( - ... _Poller.poll_while, counter_is_small_enough, - ... ) - ... return get_poller(count(), limit) - - >>> with count_until(10).with_cooldown(.01).with_timeout(1): - ... # Note: we shouldn't really need that much time, but - ... # something in CI seems to be slowing down the polling - ... # loop... - ... print('We counted up to 10') - We counted up to 10 - - >>> with ( - ... count_until(100) - ... .with_cooldown(.01) - ... .with_timeout(.5) # `[on_]timeout` separately supplied - ... .with_timeout(on_timeout='ignore') - ... ): - ... print("We probably didn't count up to 100 but whatever") - We probably didn't count up to 100 but whatever - - >>> with ( # doctest: +NORMALIZE_WHITESPACE - ... count_until(30).with_cooldown(.01).with_timeout(.25) - ... ): - ... print('We counted up to 30') - Traceback (most recent call last): - ... - line_profiler..._Poller.Timeout: ... - timed out (... s >= 0.25 s) waiting for - callback ...counter_is_big_enough... to return true - - >>> with ExitStack() as stack: # doctest: +NORMALIZE_WHITESPACE - ... enter = stack.enter_context - ... enter(warnings.catch_warnings()) - ... warnings.simplefilter('error', _Poller.TimeoutWarning) - ... enter( - ... count_until(30, 'while') - ... .with_cooldown(.01) - ... .with_timeout(.25, 'warn') - ... ) - ... print('We counted up to 30 again') - Traceback (most recent call last): - ... - line_profiler..._Poller.TimeoutWarning: ... - timed out (... s >= 0.25 s) waiting for - callback ...counter_is_small_enough... to return true - """ - def __init__( - self, - func: Callable[[], Any], - cooldown: float = 0, - timeout: float = 0, - on_timeout: _OnTimeout = 'error', - ) -> None: - self._func: Callable[[], Any] = func - self._cooldown = max(0, cooldown) - self._timeout = max(0, timeout) - self._on_timeout = on_timeout - - def sleep(self): - cd = self._cooldown - if cd > 0: - sleep(cd) - - def with_cooldown(self, cooldown: float) -> Self: - return type(self)( - self._func, cooldown, self._timeout, self._on_timeout, - ) - - def with_timeout( - self, - timeout: float | None = None, - on_timeout: _OnTimeout | None = None, - ) -> Self: - if timeout is None: - timeout = self._timeout - if on_timeout is None: - on_timeout = self._on_timeout - return type(self)(self._func, self._cooldown, timeout, on_timeout) - - @classmethod - def poll_until( - cls, func: Callable[PS, Any], /, *args: PS.args, **kwargs: PS.kwargs - ) -> Self: - if args or kwargs: - func = partial(func, *args, **kwargs) - return cls(func) - - @classmethod - def poll_while( - cls, func: Callable[PS, Any], /, *args: PS.args, **kwargs: PS.kwargs - ) -> Self: - def negated( - func: Callable[PS, Any], *a: PS.args, **k: PS.kwargs - ) -> bool: - return not func(*a, **k) - - return cls(partial(negated, func, *args, **kwargs)) - - def __enter__(self) -> Self: - def error(msg: str) -> NoReturn: - raise type(self).Timeout(msg) - - def warn(msg: str) -> None: - # Write log before issuing the warning because that may be - # promoted to an exception - diagnostics.log.warning(msg) - warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) - - timeout = self._timeout - callback = self._func - - handle_timeout: Callable[[str], Any] = { - 'error': error, 'warn': warn, 'ignore': _no_op, - }[self._on_timeout] - fmt = '.3g' - timeout_msg_header = f'{type(self).__name__} at {id(self):#x}' - - start = monotonic() - while not callback(): - elapsed = monotonic() - start - if timeout and elapsed >= timeout: - handle_timeout( - f'{timeout_msg_header}: ' - f'timed out ({elapsed:{fmt}} s >= {timeout:{fmt}} s) ' - f'waiting for callback {callback!r} to return true' - ) - break - self.sleep() - return self - - def __exit__(self, *_, **__) -> None: - pass - - class Timeout(RuntimeError): - """ - Raised when a :py:class:`_Poller` is timed out when polling. - """ - pass - - class TimeoutWarning(Timeout, UserWarning): - """ - Issued when a :py:class:`_Poller` is timed out when polling. - """ - pass - - -@final -class _PollerArgs(NamedTuple): - cooldown: float - timeout: float - on_timeout: str | None - - @classmethod - def new(cls, cooldown: Any, timeout: Any, on_timeout: Any) -> Self: - try: - cd = max(float(cooldown), 0) - except (TypeError, ValueError): - cd = 0 - try: - to = max(float(timeout), 0) - except (TypeError, ValueError): - to = 0 - try: - ot: str | None = on_timeout.lower() - except Exception: # Fallback (use `_Poller`'s default) - ot = None - return cls(cd, to, ot) - - -@final -@dataclasses.dataclass -class MPConfig: - """ - Consolidate the config options into a structured object. - """ - catch_sigterm: bool - patches: dict[PublicPatch, bool] - polling: _PollerArgs - - def _get_terminate_poller( - self, cache: LineProfilingCache, process: BaseProcess, - ) -> _Poller: - cd, timeout, on_timeout = self.polling - if on_timeout not in ('ignore', 'warn', 'error'): - on_timeout = self.get_defaults().polling.on_timeout - # `_process_has_returned()` takes a `timeout` which it passes to - # `popen.wait()`; said timeout is essentially a limit as to how - # often the function is called, hence our cooldown - poller = _Poller.poll_until( - self._process_has_returned, process, cache, cd, - ) - return poller.with_timeout(timeout, cast(_OnTimeout, on_timeout)) - - @classmethod - def from_config(cls, config: ConfigSource) -> Self: - loaded = ( - config - .get_subconfig('child_processes', 'multiprocessing') - .conf_dict - ) - polling = _PollerArgs.new(**loaded['polling']) - return cls( - catch_sigterm=loaded['catch_sigterm'], - patches=dict(loaded['patches']), - polling=polling, - ) - - @classmethod - def from_cache(cls, cache: LineProfilingCache) -> Self: - key = 'mp_config' - try: - return cache._additional_data[key] - except KeyError: - config = cls.from_config(cache._config_source) - return cache._additional_data.setdefault(key, config) - - @classmethod - def get_defaults(cls) -> Self: - namespace = globals() - name = '_DEFAULT_CONFIG' - try: - return namespace[name] - except KeyError: - defaults = cls.from_config(ConfigSource.from_default(copy=False)) - return namespace.setdefault(name, defaults) - - @staticmethod - def _process_has_returned( - proc: BaseProcess, cache: LineProfilingCache, timeout: float, - ) -> bool: - popen = getattr(proc, '_popen', None) - if popen is None: - msg, result = 'No associated process', True - else: - result = popen.wait(timeout) is not None - if result: - msg = f'Process {popen.pid} has returned' - else: - msg = f'Waiting for process {popen.pid} to return...' - cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') - return result - - -class _QueuePutWrapper: # nocover - """ - Wrap around a queue (the ``outqueue`` argument to - :py:func:`multiprocessing.pool.worker`) so that each call to its - ``.put()`` is preceded by calling a ``callback()``; its result is - optionally attached to the tuple pushed back to the parent if - ``push_to_parent`` is true. - """ - def __init__( - self, - queue: _Queue, - callback: Callable[[], Any], - push_to_parent: bool = False, - ) -> None: - self._queue = queue - self._callback = callback - self._push = push_to_parent - - def __getattr__(self, attr: str) -> Any: - return getattr(self._queue, attr) - - def put(self, obj: Any) -> None: - data = self._callback() - if self._push: - obj = data, obj - self._queue.put(obj) - - def get(self) -> Any: - return self._queue.get() - - -def _no_op(*_, **__) -> None: - pass - - -def _setup_in_mp_child(cache: LineProfilingCache) -> None: # nocover - """ - Perform :py:mod:`multiprocessing`-specific setup in a child process - curated by the module. Currently it does the following: - - - Set up ``cache`` to handle ``SIGTERM`` on POSIX if not already - set. - - - Unregister the :py:mod:`atexit` hook associated with ``cache`` to - avoid possible clashes with the profiling-file writing managed by - this module. - """ - if cache.main_pid == os.getpid(): # Not in a child process - return - xc: Exception | None = None - for setup in [_add_sigterm_handler_in_child, _unregister_atexit_hook]: - try: - setup(cache) - except Exception as e: - xc = e - if xc is not None: - xc_str = type(xc).__name__ - if str(xc): - xc_str = f'{xc_str}: {xc}' - cache._debug_output(f'Setup failed in process {os.getpid()}: {xc_str}') - raise xc - - -def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: - key = 'mp_added_sigterm_handler' - if not _CAN_CATCH_SIGTERM: - return - if not MPConfig.from_cache(cache).catch_sigterm: - return - if cache._additional_data.get(key, False): - # Already added (e.g. by another plugin) - return - cache._add_signal_handler() - cache._additional_data[key] = True - - -def _unregister_atexit_hook(cache: LineProfilingCache) -> None: - atexit.unregister(cache._atexit_hook) - - -def _dump_stats_quick( - cache: LineProfilingCache, - *, - reason: str | None = None, - debug: bool = False, -) -> None: - """ - Note: - We don't really care about cleanup in the child process, so just - dump the stats and bail to reduce the chance of end-of-process - shenanigans causing a deadlock... - but do use ``._stats_dumper.cleanup()`` instead of - ``.__call__()`` so that we get debugging output (if ``debug`` is - true) - """ - stats_dumper = cache._stats_dumper - if stats_dumper is None: - return - if debug: - stats_dumper.cleanup(force=True, reason=reason) - else: - stats_dumper() - - -# ---------------------- Patching infrastructure ----------------------- - - -class _Patch(Protocol): - """ - Interface for patches. - """ - def apply( - self, - cache: LineProfilingCache, - *, - cleanup: bool = True, - **kwargs - ) -> Any: - """ - Apply the patch. - - Args: - cache (LineProfilingCache): - Session cache - cleanup (bool): - Whether ``cache.cleanup()`` should reverse the patch - **kwargs - Individual implementations should pick the ones they - need and ignore the rest. - """ - ... - - @property - def summary(self) -> Mapping[str, Set[str]]: - """ - A mapping from dotted-path names of objects to the set of - attributes patched thereon. - """ - ... - - -@dataclasses.dataclass -class Patch: - """ - Patch to apply to a component in :py:mod:`multiprocessing`. - - Attributes: - submodule (str): - Name of the :py:mod:`multiprocessing` submodule. - targets (dict[str,\ -dict[str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]]]]): - Dictionary mapping (dot-chained) names in said submodule to - a dictionary of patches; said patches dictionary should have - the format of - ``dict[simple_attribute, wrapper | [wrapper1, ...]]``. See - Example for details. - - Example: - Consider - ``Patch('foo', {'bar.baz': {'foobar': foofoo},\ -'': {'spam': [ham, eggs]}})``. - This instance would perform the following patches on the module - ``multiprocessing.foo``: - - - Replace ``multiprocessing.foo.bar.baz.foobar`` with - ``foofoo(multiprocessing.foo.bar.baz.foobar)`` - - - Replace ``multiprocessing.foo.spam`` with - ``eggs(ham(multiprocessing.foo.spam))``; - note that the two wrappers are applied in order to the - original attribute. - """ - submodule: str - targets: dict[ - str, dict[str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]]] - ] = dataclasses.field(default_factory=dict) - package: ClassVar[str] = 'multiprocessing' - - def add_target( - self, - target: str, - patches: Mapping[ - str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]] - ], - ) -> Self: - """ - Convenience method for gradually constructing the patch with a - fluent interface. - - Args: - target (str): - Dotted path to the object in :py:attr:`.submodule` - patches (Mapping[str, Callable[[Any], Any] \ -| Sequence[Callable[[Any], Any]]]): - Mapping from patched attrbute names to the wrappers to - apply thereto; sequences of wrappers are applied in - order - - Returns: - This instance - """ - self.targets.setdefault(target, {}).update(patches) - return self - - def add_method( - self, - target: str, - method: str, - wrapper: Callable[[Any], Any], - methodtype: ( - type[classmethod] | type[staticmethod] - | Literal['class', 'static'] | None - ) = None, - ) -> Self: - """ - Convenience method for gradually constructing the patch with a - fluent interface. - - Args: - target (str): - Dotted path to the object in :py:attr:`.submodule` - method (str): - Name of the (class, static, or instance) method to patch - wrapper (Callable[[Any], Any]): - Wrapping callable which takes the method-implementaion - callable and returns a wrapper thereof - methodtype (type[classmethod] | type[staticmethod] | \ -Literal['class', 'static'] | None): - Optional type of the method if not an instance method; - the strings ``'class'`` and ``'static'`` are respective - shorthands for :py:class:`classmethod` and - :py:class:`staticmethod` - - Returns: - This instance - """ - wrappers: Callable[[Any], Any] | list[Callable[[Any], Any]] - if methodtype is None: - wrappers = wrapper - else: - if methodtype == 'class': - methodtype = classmethod - elif methodtype == 'static': - methodtype = staticmethod - wrappers = [attrgetter('__func__'), wrapper, methodtype] - return self.add_target(target, {method: wrappers}) - - def apply( - self, - cache: LineProfilingCache, - *, - cleanup: bool = True, - static: bool = True, - **_ - ) -> list[str]: - """ - Apply the patch. - - Args: - cache (LineProfilingCache): - Session cache - cleanup (bool): - Whether ``cache.cleanup()`` should reverse the patch - static (bool): - Whether to use :py:func:`inspect.getattr_static` to - retrieve to the attributes to be patched on the patch - targets - - Returns: - replacements (list[str]): - Names of entities replaced - """ - submod_name = f'{self.package}.{self.submodule}' - get_attribute = getattr_static if static else getattr - result: list[str] = [] - try: - mod = self.load_module() - except ImportError: # nocover - return [] - - for target in sorted(self.targets, key=len, reverse=True): - if TYPE_CHECKING: - # See `ty` issue #2572 - assert isinstance(target, str) - if target: - try: - obj: Any = attrgetter(target)(mod) - except AttributeError: # nocover - continue - name = f'{submod_name}.{target}' - else: - obj, name = mod, submod_name - replace = partial(cache.patch, obj, cleanup=cleanup, name=name) - for method, method_wrappers in self.targets[target].items(): - if callable(method_wrappers): - method_wrappers = cast( - Sequence[Callable[[Any], Any]], (method_wrappers,), - ) - try: - impl = get_attribute(obj, method) - except AttributeError: - continue - for wrapper in method_wrappers: - impl = wrapper(impl) - replace(method, impl) - result.append(f'{name}.{method}') - return result - - def load_module(self) -> ModuleType: - """ - Returns: - Module object :py:attr:`.module` points to - """ - return import_module(self.module) - - @staticmethod - def _join(s: str, *strs: str, sep: str = '.') -> str: - return sep.join(string for string in (s, *strs) if string) - - @property - def module(self) -> str: - """ - Module where the patches are applied - """ - return self._join(self.package, self.submodule) - - @property - def summary(self) -> mappingproxy[str, frozenset[str]]: - """ - Summary of the dotted paths to the patched objects and their - patched attributes - """ - add_prefix = partial(self._join, self.module) - return mappingproxy({ - add_prefix(target): frozenset(patches) - for target, patches in self.targets.items() - }) - - -class _PatchRegistry(Mapping[str, _Patch]): - """ - Mapping subclass for managing patches. - """ - def __init__(self) -> None: - self._patches: dict[str, tuple[int, _Patch]] = {} - - def __repr__(self) -> str: - if self._patches: - patches = ', '.join( - f'{name!r} (priority {priority})' if priority else repr(name) - for name, (priority, _) in self._patches.items() - ) - patches = f'({len(self)} patch(es)): {patches}' - else: - patches = '(0 patches)' - return f'<{type(self).__name__} @ {id(self):#x} {patches}>' - - def __getitem__(self, key: str) -> _Patch: - return self._patches[key][1] - - def __iter__(self) -> Generator[str, None, None]: - for name, _ in self._iter_patches(): - yield name - - def __len__(self) -> int: - return len(self._patches) - - def __contains__(self, key: Any) -> bool: - return key in self._patches - - @overload - def register(self, name: str, patch: Pt, *, priority: int = 0) -> Pt: - ... - - @overload - def register( - self, name: str, patch: None = None, *, priority: int = 0, - ) -> _Patch: - ... - - def register( - self, name: str, patch: _Patch | None = None, *, priority: int = 0, - ) -> _Patch: - """ - Register the ``patch`` under ``name`` with the specified - ``priority`` and return it as-is. If ``patch`` isn't provided, - look for the existing patch registered under the name. - - Note: - Patches named with leading double underscores are applied no - matter the user input (e.g. via ``apply(..., patches=...)`` - or the config file). - """ - if patch is not None: - old_pri, stored = self._patches.setdefault(name, (priority, patch)) - if stored is not patch: - raise ValueError( - f'name = {name!r}, patch = {patch!r}: ' - f'name already in use by {stored!r}' - ) - if old_pri != priority: # Update priority - self._patches[name] = priority, patch - return self[name] - - def select(self, patches: Collection[str]) -> Self: - """ - Returns: - New instance with the selected patches - - Note: - Patches whose names are prefixed with double underscores are - considered mandatory and are always selected. - """ - new = type(self)() - new._patches.update( - (name, patch_info) for name, patch_info in self._patches.items() - if name.startswith('__') or name in patches - ) - return new - - def _iter_patches( - self, reverse: bool = False, - ) -> Generator[tuple[str, _Patch], None, None]: - """ - Iterate over the available patches. - - Note: - Since patches typically consists of function wrappers, and - outer wrappers are both called first and responsible for - calling inner wrappers, patches with a HIGHER priority are - yielded LATER (or earlier if ``reverse=True``. - """ - for _, patches in sorted(self._prioritized.items(), reverse=reverse): - yield from patches.items() - - @property - def summary(self) -> dict[str, frozenset[str]]: - """ - Mapping from the names of the entities affected by the patches - to sets of attributes patched thereon. - """ - summaries = [patch.summary for patch in self.values()] - return { - target: frozenset().union(*(s.get(target, ()) for s in summaries)) - for target in frozenset().union(*summaries) - } - - @property - def _prioritized(self) -> dict[int, dict[str, _Patch]]: - # This could've been a static attribute maintained by the - # `.register()` method, but we don't call this a bunch so it - # isn't like we incur a lot of overhead by calculating it - # on-the-fly, and it's less error-prone this way. - result: dict[int, dict[str, _Patch]] = {} - for name, (priority, patch) in self._patches.items(): - result.setdefault(priority, {})[name] = patch - return result - - -_PATCHES = _PatchRegistry() -_register_patch = _PATCHES.register - -# ---------------- `multiprocessing.pool.Pool` patches ----------------- - - -class _PIDQueueGetWrapper: # nocover - """ - Wrapper around the ``inqueue`` argument to - :py:func:`multiprocessing.pool.worker` to intercept the sentinel - value (:py:const:`None`) signifying the end of the queue and perform - cleanup. - """ - def __init__( - self, - queue: _Queue, - cache: LineProfilingCache, - ) -> None: - self._queue = queue - self._cache = cache - - def __getattr__(self, attr: str) -> Any: - return getattr(self._queue, attr) - - def put(self, obj: Any) -> None: - self._queue.put(obj) - - def get(self) -> Any: - result = self._queue.get() - cache = self._cache - ntasks: dict[int, int] - ntasks = cache._additional_data.setdefault('mp_queue_ntasks', {}) - queue_id = id(self) - if result is None: - n = ntasks.pop(queue_id, 0) - cache._debug_output( - '`multiprocessing.pool.worker`: ' - f'recieved {n} task(s) in total', - ) - # Got sentinel value, process is about to exit - reason = 'ran out of tasks in `multiprocessing.process.worker()`' - if cache.main_pid != os.getpid(): - _dump_stats_quick(cache, debug=True, reason=reason) - else: - ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 - return result - - -@LineProfilingCache._method_wrapper # nocover -def wrap_worker_pool_write_on_exit( - cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[_Queue, PS], None], - inqueue: _Queue, - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around :py:func:`multiprocessing.pool.worker` so that child - processes can write profiling output as soon as the pool runs out of - tasks. - - Notes: - - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. - - - This only works reliably for POSIX because we can handle - ``SIGTERM`` on child processes and ensure that they aren't - prematurely terminated. - """ - _setup_in_mp_child(cache) - return vanilla_impl(_PIDQueueGetWrapper(inqueue, cache), *args, **kwargs) - - -@LineProfilingCache._method_wrapper # nocover -def wrap_worker_pool_write_per_task( - cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[_Queue, _Queue, PS], None], - inqueue: _Queue, - outqueue: _Queue, - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around :py:func:`multiprocessing.pool.worker` so that child - processes can write profiling output before pushing the result of - each task back to the parent. - - Notes: - - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. - - - This is only used on Windows where we can't handle ``SIGTERM`` - on child processes, thus necessitating the write to happen - before control flow is passed backed to the parent. - """ - _setup_in_mp_child(cache) - outqueue = _QueuePutWrapper(outqueue, partial(_dump_stats_quick, cache)) - return vanilla_impl(inqueue, outqueue, *args, **kwargs) - - -if _CAN_CATCH_SIGTERM: - wrap_worker_pool: Callable[[Callable[..., None]], Callable[..., None]] - wrap_worker_pool = wrap_worker_pool_write_on_exit -else: - wrap_worker_pool = wrap_worker_pool_write_per_task -_register_patch('pool', Patch('pool')).add_method( - '', 'worker', wrap_worker_pool, -) - -# ----------- `multiprocessing.process.BaseProcess` patches ------------ - - -@LineProfilingCache._method_wrapper -def wrap_terminate( - cache: LineProfilingCache, - vanilla_impl: Callable[[BaseProcess], None], - self: BaseProcess, -) -> None: - """ - Wrap around :py:meth:`.BaseProcess.terminate` to make sure that we - don't actually kill the child (OS-level) process before it has the - chance to properly clean up. - - Note: - We're technically polling in a loop, but it isn't actually - *that* bad: typically ``.terminate()`` is only called when we're - on the bad path (e.g. the parallel workload errored out), and - after the performance-critical part of the code (said workload). - """ - try: - config = MPConfig.from_cache(cache) - with config._get_terminate_poller(cache, self): - pass - except _Poller.Timeout as e: # Also handles `~.TimeoutWarning` - cache._debug_output(f'{type(e).__qualname__}: {e}') - raise - finally: # Always call `Process.terminate()` to avoid orphans - vanilla_impl(self) - - -@LineProfilingCache._method_wrapper # nocover -def wrap_bootstrap( - cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], - self: BaseProcess, - /, - *args: PS.args, **kwargs: PS.kwargs -) -> T: - """ - Wrap around :py:meth:`.BaseProcess._bootstrap` so that profiling - stats are written at the end. - - Notes: - - - This is only invoked in child processes, and - :py:mod:`coverage` seems to be having trouble with them in the - current setup, probably due to issues with .pth file - precendence causing :py:mod:`line_profiler` to be loaded - before it. Hence the ``# nocover``. - - - ``SIGTERM`` handling is not consistent on Windows, so we made - :py:meth:`.LineProfilingCache._add_signal_handler` a no-op - there. Hence :py:func:`wrap_terminate` remains necessary for - mitigating unclean exits. - """ - _setup_in_mp_child(cache) - try: - return vanilla_impl(self, *args, **kwargs) - finally: - reason = 'exiting `multiprocessing.process.BaseProcess._bootstrap`' - _dump_stats_quick(cache, debug=True, reason=reason) - - -_patch_process = partial( - _register_patch('process', Patch('process')).add_method, 'BaseProcess', -) -_patch_process('_bootstrap', wrap_bootstrap) -# We only need to patch `Process.terminate()` if we can't do SIGTERM -# handling, i.e. on Windows -if not _CAN_CATCH_SIGTERM: - _patch_process('terminate', wrap_terminate) - -# ---------------------- PID bookkeeping patches ----------------------- - - -@LineProfilingCache._method_wrapper -def wrap_handle_results( - cache: LineProfilingCache, - vanilla_impl: Callable[ - Concatenate[_Queue, Callable[[], tuple[Any, ...] | None], PS], - None - ], - outqueue: _Queue, - # Since we patched `outqueue.put()` in the child process, the result - # tuple pushed to the parent has an extra item (the child PID) - get: Callable[[], tuple[int, tuple[Any, ...]] | None], - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around :py:meth:`multiprocessing.pool.Pool._handle_results` so - that it handles the extra info (PID of child process handling the - task) included by :py:func:`.wrap_worker_pid`. - - Note: - :py:meth:`.Pool._handle_results` is a static method. - """ - # Somehow this doesn't type-check with either `mypy` or `ty` when - # we use a `TypeVar` instead of `Any` with the tuple items... - # (see `ty` issue #3467) - wrapped_get = partial(_wrap_outqueue_quick_get, cache, get) - vanilla_impl(outqueue, wrapped_get, *args, **kwargs) - - -@LineProfilingCache._method_wrapper # nocover -def wrap_worker_pid( - _, # We don't need the cache instance, but `@_method_wrapper` does - vanilla_impl: Callable[Concatenate[_Queue, _Queue, PS], None], - inqueue: _Queue, - outqueue: _Queue, - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around :py:func:`multiprocessing.pool.worker` so that child - processes report their PIDs as they pass the task results back to - the parent. - - Note: - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. - """ - outqueue = _QueuePutWrapper(outqueue, os.getpid, push_to_parent=True) - return vanilla_impl(inqueue, outqueue, *args, **kwargs) - - -@LineProfilingCache._method_wrapper -def wrap_process( - cache: LineProfilingCache, - vanilla_impl: Callable[PS, P], - *args: PS.args, - **kwargs: PS.kwargs -) -> P: - """ - Wrap around :py:func:`multiprocessing.pool.Pool.Process` so that the - processes created can report on usage when - :py:meth:`.BaseProcess.join`-ed or - :py:meth:`.BaseProcess.terminate`-ed. - - Note: - :py:meth:`.Pool.Process` is a static method. - """ - proc = vanilla_impl(*args, **kwargs) - # Note: since we don't clean up here, there's no need to instantiate - # another `Cleanup` helper - name = f'<{type(proc).__name__} @ {hex(id(proc))}>' - patch = partial(cache.patch, cleanup=False, name=name) - for method, action in ('join', 'joining'), ('terminate', 'terminating'): - bound = getattr(proc, method) - assert isinstance(bound, MethodType) - finalize = _wrap_process_finalize(cache, bound.__func__, action) - patch(proc, method, MethodType(finalize, proc)) - return proc - - -def _wrap_process_finalize( - cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[P, PS], None], - action: str, -) -> Callable[Concatenate[P, PS], None]: - """ - Check if the process has run any tasks; - if not, report to the cache. - - Note: - Since the process object is pickled, this method has to directly - return a function object instead of merely being - :py:func:`partial`-ed and wrapped in a - :py:class:`types.MethodType`. - """ - @wraps(vanilla_impl) - def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: - log = cache._debug_output - call = cache._format_call(vanilla_impl, self, *args, **kwargs) - try: - log(f'Wrapped call made: {call}') - pid: int | None = getattr(self, 'pid', None) - checked_procs = _get_checked_processes(cache) - identifier = id(self), pid - if not (pid is None or identifier in checked_procs): - ntasks = _get_ntasks(cache).pop(pid, 0) - if not ntasks: - cache._warn_possible_lack_of_stats(pid) - log(f'{action} process {pid} which ran {ntasks} task(s)...') - checked_procs.add(cast(tuple[int, int], identifier)) - except BaseException as e: - log( - f'Error in bookkeeping ({cache._format_exception(e)}), ' - 'invoking base implementation nonetheless...' - ) - raise e - finally: - try: - vanilla_impl(self, *args, **kwargs) - except BaseException as e: - state = f'failed ({cache._format_exception(e)})' - raise e - else: - state = 'succeeded' - finally: - log(f'Wrapped call {call} {state}') - - action = action.capitalize() - return finalize - - -def _wrap_outqueue_quick_get( - cache: LineProfilingCache, - vanilla_impl: Callable[PS, tuple[int, tuple[Any, ...]] | None], - *args: PS.args, - **kwargs: PS.kwargs -) -> tuple[Any, ...] | None: - """ - Take and process the PID of the child process completing the task. - """ - result = vanilla_impl(*args, **kwargs) - if result is None: - return None - pid, orig_result = result - ntasks = _get_ntasks(cache) - ntasks[pid] = ntasks.get(pid, 0) + 1 - return orig_result - - -def _get_ntasks(cache: LineProfilingCache) -> dict[int, int]: - key = 'mp_proc_ntasks' - return cache._additional_data.setdefault(key, cast(dict[int, int], {})) - - -def _get_checked_processes( - cache: LineProfilingCache, -) -> set[tuple[int, int]]: - key = 'mp_proc_checked_workload' - return cache._additional_data.setdefault( - key, cast(set[tuple[int, int]], set()), - ) - - -_patch_pid = _register_patch('child_pids', Patch('pool')).add_method -_patch_pid('', 'worker', wrap_worker_pid) -_patch_pid('Pool', '_handle_results', wrap_handle_results, 'static') -_patch_pid('Pool', 'Process', wrap_process, 'static') - -# --------------- `multiprocessing.util` logging patches --------------- - - -def _cache_hook( - vanilla_impl: Callable[PS, T], - get_logging_message: Callable[PS, str], - /, - *args: PS.args, - **kwargs: PS.kwargs -) -> T: - msg = get_logging_message(*args, **kwargs) - LineProfilingCache.load()._debug_output(msg) - return vanilla_impl(*args, **kwargs) - - -def tee_log( - marker: str, - vanilla_impl: Callable[Concatenate[str, PS], None], - /, - msg: str, - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around logging functions like - :py:func:`multiprocessing.util.debug` so that we can tee log - messages from the package to our own logs. - """ - def get_msg(msg: str, *_, **__) -> str: - return f'`multiprocessing` logging ({marker}): {msg}' - - _cache_hook( - vanilla_impl, get_msg, # type: ignore[arg-type] - msg, *args, **kwargs, - ) - - -_register_patch('logging', Patch('util')).add_target( - # The logging functions exists directly in the module namespace so - # no further attribute access is needed - '', {func: partial(partial, tee_log, func) for func in _LOGGERS}, -) - -# --------------------------- Misc. patches ---------------------------- - - -class RebootForkserverPatch: - """ - Reboot the process backing the global - :py:class:`multiprocessing.forkserver.ForkServer` instance: - - - When the patch is applied, so as to ensure that child processes - forked therefrom actually receives the active patches; and - - - When the session cache is cleaned up, so that child processes - forked therefrom is no longer polluted by the patches. - - Note: - This uses - :py:method:`multiprocessing.forkserver.ForkServer._stop()` which - is private API, but it's the same hack used in Python's own test - suite -- see the comment to said method. - """ - summary: ClassVar[mappingproxy[str, frozenset[str]]] = mappingproxy({}) - - @classmethod - def apply(cls, cache: LineProfilingCache, **_) -> None: - if not _CAN_USE_FORKSERVER: - return - cls.reboot() - cache.add_cleanup(cls.reboot) - - @staticmethod - def reboot() -> None: - # Appease the type-checker since `._stop()` is not public API - stop = getattr(forkserver._forkserver, '_stop', None) - assert callable(stop) - stop() - - -class ResourceTrackerPatch: - """ - Patch :py:mod:`multiprocessing.resource_tracker` so that - :py:func:`multiprocessing.resource_tracker.ensure_running` and the - eponymous method of - :py:class:`multiprocessing.resource_tracker.ResourceTracker` report - the resource-tracker server PIDs to the session cache. - - Note: - The ``ResourceTracker`` server process is spawned when the first - :py:mod:`multiprocessing` child process is created via the - ``spawn`` or ``forkserver`` start methods. While this server - process does not meaningfully contribute to the profiling result - either way, since it can be created with profiling set up, its - longevity means that :py:meth:`.LineProfilingCache.gather_stats` - often catches empty .lprof files which it has occupied but not - written to. - - To reduce noise while keeping the empty-file warning for other - output files, we report the PIDs used by the server to the - session cache so that they can be ignored if necessary. - """ - if _CAN_USE_RESOURCE_TRACKER: - summary: ClassVar[mappingproxy[str, frozenset[str]]] = mappingproxy({ - 'multiprocessing.resource_tracker': - frozenset({'ensure_running'}), - 'multiprocessing.resource_tracker.ResourceTracker': - frozenset({'ensure_running'}), - }) - else: - summary = mappingproxy({}) - - @staticmethod - @LineProfilingCache._method_wrapper - def wrap_ensure_running( - cache: LineProfilingCache, - vanilla_impl: Callable[['resource_tracker.ResourceTracker'], None], - self: 'resource_tracker.ResourceTracker', - ) -> None: - """ - Wrap around :py:meth:`multiprocessing.resource_tracker\ -.ResourceTracker.ensure_running` - so that the session cache can keep track of the PIDs used by the - resource-tracer server. - """ - maybe_pids: set[int | None] = {getattr(self, '_pid', None)} - try: - vanilla_impl(self) - finally: - maybe_pids.add(getattr(self, '_pid', None)) - pids = cast(set[int], maybe_pids - {None}) - if pids: - cache._warn_possible_lack_of_stats(pids) - - @classmethod - def apply( - cls, cache: LineProfilingCache, *, cleanup: bool = True, **_, - ) -> list[str]: - if _CAN_USE_RESOURCE_TRACKER: - patch = partial(cache.patch, cleanup=cleanup) - # Patch the method on the class - method = resource_tracker.ResourceTracker.ensure_running - method = cls.wrap_ensure_running(method) - patch(resource_tracker.ResourceTracker, 'ensure_running', method) - # Patch the preexisting bound method on the module - instance = resource_tracker._resource_tracker - bound_method = MethodType(method, instance) - patch(resource_tracker, 'ensure_running', bound_method) - return list(cls.summary) - - -class RunpyPatch: - """ - Patch the copy of :py:mod:`runpy` in the - :py:mod:`multiprocessing.spawn` namespace so that subprocesses can - perform rewrite-based profiling as with - :py:func:`line_profiler.autoprofile.autoprofile.run`. - - See also: - :py:mod:`line_profiler._child_process_profiling.runpy_patches` - """ - summary: ClassVar[mappingproxy[str, frozenset[str]]] - if _CAN_USE_SPAWN and hasattr(spawn, 'runpy'): - summary = mappingproxy({'multiprocessing.spawn': frozenset({'runpy'})}) - else: - summary = mappingproxy({}) - - @classmethod - def apply( - cls, cache: LineProfilingCache, *, cleanup: bool = True, **_, - ) -> list[str]: - if cls.summary: - patch = partial(cache.patch, cleanup=cleanup) - patch(spawn, 'runpy', create_runpy_wrapper(cache)) - return list(cls.summary) - - -# See `ty` issue #3429 for why we need the casts -_register_patch('__reboot_forkserver', cast(_Patch, RebootForkserverPatch)) -_register_patch('__resource_tracker', cast(_Patch, ResourceTrackerPatch)) -_register_patch('__spawn_runpy', cast(_Patch, RunpyPatch)) +_PATCHES = Registry.from_entry_point() -# -------------------------- Applying patches -------------------------- +Timeout = Poller.Timeout def apply( @@ -1466,6 +131,9 @@ def apply( patches_: set[str] = {p for p, use in patches_dict.items() if use} else: patches_ = {p.lower() for p in patches} + # Sanity check on `_PATCHES` + for patch in get_args(PublicPatch): + assert patch in _PATCHES for name, patch in _PATCHES.select(patches_).items(): if name == '__reboot_forkserver' and not reboot_forkserver: continue diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py new file mode 100644 index 00000000..3e3e6413 --- /dev/null +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py @@ -0,0 +1,497 @@ +from __future__ import annotations + +import dataclasses +import warnings +from collections.abc import ( + Callable, Collection, Generator, Mapping, Sequence, Set, +) +from functools import partial +from importlib import import_module +from importlib.metadata import entry_points +from inspect import getattr_static +from operator import attrgetter +from types import MappingProxyType as mappingproxy, ModuleType +from typing import ( + TYPE_CHECKING, + Any, ClassVar, Literal, Protocol, TypeVar, + cast, final, overload, +) +from typing_extensions import Self + +from ... import _diagnostics as diagnostics +from ..cache import LineProfilingCache + + +__all__ = ('Patch', 'SingleModulePatch', 'Registry') + +P = TypeVar('P', bound='Patch') + + +class Patch(Protocol): + """ + Interface for patches. + """ + def apply( + self, + cache: LineProfilingCache, + *, + cleanup: bool = True, + **kwargs + ) -> Any: + """ + Apply the patch. + + Args: + cache (LineProfilingCache): + Session cache + cleanup (bool): + Whether ``cache.cleanup()`` should reverse the patch + **kwargs + Individual implementations should pick the ones they + need and ignore the rest. + + Note: + The patch is responsible for registering the requisite + cleanup callbacks with ``cache`` so that it is reversed when + ``cache.cleanup()`` is called. + """ + ... + + @property + def summary(self) -> Mapping[str, Set[str]]: + """ + A mapping from dotted-path names of objects to the set of + attributes patched thereon. + """ + ... + + @property + def priority(self) -> float | None: + """ + Real number representing how the patch is to be prioritized. A + patch with a HIGHER priority should be applied LATER, that way + it gets to wrap wrappers created by patches applied earlier. + """ + ... + + +@dataclasses.dataclass +class SingleModulePatch: + """ + Patch to apply to a module component in :py:mod:`multiprocessing`. + + Attributes: + submodule (str): + Name of the :py:mod:`multiprocessing` submodule. + targets (dict[str,\ +dict[str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]]]]): + Dictionary mapping (dot-chained) names in said submodule to + a dictionary of patches; said patches dictionary should have + the format of + ``dict[simple_attribute, wrapper | [wrapper1, ...]]``. See + Example for details. + priority (float | None): + Optional numerical value for how to prioritize the patch + (see :py:class:`.Patch`) + + Example: + Consider + ``SingleModulePatch('foo', {'bar.baz': {'foobar': foofoo},\ +'': {'spam': [ham, eggs]}})``. + This instance would perform the following patches on the module + ``multiprocessing.foo``: + + - Replace ``multiprocessing.foo.bar.baz.foobar`` with + ``foofoo(multiprocessing.foo.bar.baz.foobar)`` + + - Replace ``multiprocessing.foo.spam`` with + ``eggs(ham(multiprocessing.foo.spam))``; + note that the two wrappers are applied in order to the + original attribute. + """ + submodule: str + targets: dict[ + str, dict[str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]]] + ] = dataclasses.field(default_factory=dict) + priority: float | None = None + + package: ClassVar[str] = 'multiprocessing' + + def add_target( + self, + target: str, + patches: Mapping[ + str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]] + ], + ) -> Self: + """ + Convenience method for gradually constructing the patch with a + fluent interface. + + Args: + target (str): + Dotted path to the object in :py:attr:`.submodule` + patches (Mapping[str, Callable[[Any], Any] \ +| Sequence[Callable[[Any], Any]]]): + Mapping from patched attrbute names to the wrappers to + apply thereto; sequences of wrappers are applied in + order + + Returns: + This instance + """ + self.targets.setdefault(target, {}).update(patches) + return self + + def add_method( + self, + target: str, + method: str, + wrapper: Callable[[Any], Any], + methodtype: ( + type[classmethod] | type[staticmethod] + | Literal['class', 'static'] | None + ) = None, + ) -> Self: + """ + Convenience method for gradually constructing the patch with a + fluent interface. + + Args: + target (str): + Dotted path to the object in :py:attr:`.submodule` + method (str): + Name of the (class, static, or instance) method to patch + wrapper (Callable[[Any], Any]): + Wrapping callable which takes the method-implementaion + callable and returns a wrapper thereof + methodtype (type[classmethod] | type[staticmethod] | \ +Literal['class', 'static'] | None): + Optional type of the method if not an instance method; + the strings ``'class'`` and ``'static'`` are respective + shorthands for :py:class:`classmethod` and + :py:class:`staticmethod` + + Returns: + This instance + """ + wrappers: Callable[[Any], Any] | list[Callable[[Any], Any]] + if methodtype is None: + wrappers = wrapper + else: + if methodtype == 'class': + methodtype = classmethod + elif methodtype == 'static': + methodtype = staticmethod + wrappers = [attrgetter('__func__'), wrapper, methodtype] + return self.add_target(target, {method: wrappers}) + + def apply( + self, + cache: LineProfilingCache, + *, + cleanup: bool = True, + static: bool = True, + **_ + ) -> list[str]: + """ + Apply the patch. + + Args: + cache (LineProfilingCache): + Session cache + cleanup (bool): + Whether ``cache.cleanup()`` should reverse the patch + static (bool): + Whether to use :py:func:`inspect.getattr_static` to + retrieve to the attributes to be patched on the patch + targets + + Returns: + replacements (list[str]): + Names of entities replaced + """ + submod_name = f'{self.package}.{self.submodule}' + get_attribute = getattr_static if static else getattr + result: list[str] = [] + try: + mod = self.load_module() + except ImportError: # nocover + return [] + + for target in sorted(self.targets, key=len, reverse=True): + if TYPE_CHECKING: + # See `ty` issue #2572 + assert isinstance(target, str) + if target: + try: + obj: Any = attrgetter(target)(mod) + except AttributeError: # nocover + continue + name = f'{submod_name}.{target}' + else: + obj, name = mod, submod_name + replace = partial(cache.patch, obj, cleanup=cleanup, name=name) + for method, method_wrappers in self.targets[target].items(): + if callable(method_wrappers): + method_wrappers = cast( + Sequence[Callable[[Any], Any]], (method_wrappers,), + ) + try: + impl = get_attribute(obj, method) + except AttributeError: + continue + for wrapper in method_wrappers: + impl = wrapper(impl) + replace(method, impl) + result.append(f'{name}.{method}') + return result + + def load_module(self) -> ModuleType: + """ + Returns: + Module object :py:attr:`.module` points to + """ + return import_module(self.module) + + @staticmethod + def _join(s: str, *strs: str, sep: str = '.') -> str: + return sep.join(string for string in (s, *strs) if string) + + @property + def module(self) -> str: + """ + Module where the patches are applied + """ + return self._join(self.package, self.submodule) + + @property + def summary(self) -> mappingproxy[str, frozenset[str]]: + """ + Summary of the dotted paths to the patched objects and their + patched attributes + """ + add_prefix = partial(self._join, self.module) + return mappingproxy({ + add_prefix(target): frozenset(patches) + for target, patches in self.targets.items() + }) + + +@final +class Registry(Mapping[str, Patch]): + """ + Mapping subclass for managing patches. + """ + _loaded_instances: ClassVar[dict[str, Registry]] = {} + DEFAULT_ENTRY_POINT: ClassVar[str] = 'line_profiler._multiproc_patches' + + def __init__(self) -> None: + self._patches: dict[str, tuple[float, Patch]] = {} + + def __repr__(self) -> str: + if self._patches: + patches = ', '.join( + f'{name!r} (priority {priority})' if priority else repr(name) + for name, (priority, _) in self._patches.items() + ) + patches = f'({len(self)} patch(es)): {patches}' + else: + patches = '(0 patches)' + return f'<{type(self).__name__} @ {id(self):#x} {patches}>' + + def __getitem__(self, key: str) -> Patch: + return self._patches[key][1] + + def __iter__(self) -> Generator[str, None, None]: + for name, _ in self._iter_patches(): + yield name + + def __len__(self) -> int: + return len(self._patches) + + def __contains__(self, key: Any) -> bool: + return key in self._patches + + @overload + def register( + self, name: str, patch: P, *, priority: float | None = None, + ) -> P: + ... + + @overload + def register( + self, name: str, patch: None = None, *, priority: float | None = None, + ) -> Patch: + ... + + def register( + self, name: str, patch: Patch | None = None, *, + priority: float | None = None, + ) -> Patch: + """ + Register/look up a patch. + + Args: + name (str): + Name of the patch; patches named with leading double + underscores are considered MANDATORY, and are applied no + matter the user input (e.g. via + ``apply(..., patches=...)`` or the config file). + patch (Patch | None): + Patch object to register; if not provided, look for the + existing patch registered under the name. + priority (float | None): + Optional priority to assign to the patch; default is to + look at :py:attr:`Patch.priority` and to resolve + :py:const:`None` to 0. + + Returns: + Patch object (``patch`` if provided, looked up otherwise) + """ + if patch is not None: + if priority is None: + priority = patch.priority + if not priority: + priority = 0 + old_pri, stored = self._patches.setdefault(name, (priority, patch)) + if stored is not patch: + raise ValueError( + f'name = {name!r}, patch = {patch!r}: ' + f'name already in use by {stored!r}' + ) + if old_pri != priority: # Update priority + self._patches[name] = priority, patch + elif priority is not None: + # Reassign priority of an existing patch + self._patches[name] = priority, self[name] + return self[name] + + def select(self, patches: Collection[str]) -> Registry: + """ + Returns: + New instance with the selected patches + + Note: + Patches whose names are prefixed with double underscores are + considered mandatory and are always selected. + """ + new = Registry() + new._patches.update( + (name, patch_info) for name, patch_info in self._patches.items() + if name.startswith('__') or name in patches + ) + return new + + def _iter_patches( + self, reverse: bool = False, + ) -> Generator[tuple[str, Patch], None, None]: + """ + Iterate over the available patches. + + Note: + Since patches typically consists of function wrappers, and + outer wrappers are both called first and responsible for + calling inner wrappers, patches with a HIGHER priority are + yielded LATER (or earlier if ``reverse=True``. + """ + for _, patches in sorted(self._prioritized.items(), reverse=reverse): + yield from patches.items() + + @property + def summary(self) -> dict[str, frozenset[str]]: + """ + Mapping from the names of the entities affected by the patches + to sets of attributes patched thereon. + """ + summaries = [patch.summary for patch in self.values()] + return { + target: frozenset().union(*(s.get(target, ()) for s in summaries)) + for target in frozenset().union(*summaries) + } + + @property + def _prioritized(self) -> dict[float, dict[str, Patch]]: + """ + Note: + This could've been a static attribute maintained by the + :py:meth:`.register` method, but we don't call this a bunch + so it isn't like we incur a lot of overhead by calculating + it on-the-fly; and it's less error-prone this way. + """ + result: dict[float, dict[str, Patch]] = {} + for name, (priority, patch) in self._patches.items(): + result.setdefault(priority, {})[name] = patch + return result + + @classmethod + def from_entry_point(cls, entry_point: str | None = None) -> Registry: + """ + Args: + entry_point (str | None): + Entry point to load patches from; + Returns: + Instance representing the patches loaded from the provided + ``entry_point``; default is :py:attr:`.DEFAULT_ENTRY_POINT` + + Note: + This method does NOT create a copy. + + Example: + >>> reg = Registry.from_entry_point() + >>> assert reg.from_entry_point() is reg + + Check for the default plugins that should be installed and + their contents: + + >>> assert 'pool' in reg + >>> assert 'process' in reg + >>> assert 'logging' in reg + >>> assert 'child_pids' in reg + + >>> assert ( + ... 'multiprocessing.process.BaseProcess' in reg.summary + ... ) + >>> assert ( + ... 'worker' + ... in reg.summary.get('multiprocessing.pool', set()) + ... ) + """ + def check(patch: P) -> P: + error: str | None = None + if not hasattr(patch, 'priority'): + error = 'expected a `.priority: float | None` field' + elif not isinstance(getattr(patch, 'summary', None), Mapping): + error = 'expected a `.summary: Mapping[str, Set[str]]` field' + elif not callable(getattr(patch, 'apply', None)): + error = ( + 'expected an `.apply(cache: LineProfilingCache, ...)` ' + 'method' + ) + if error: + raise TypeError(f'patch `{patch!r}`: {error}') + return patch + + if entry_point is None: + entry_point = cls.DEFAULT_ENTRY_POINT + try: + return cls._loaded_instances[entry_point] + except KeyError: + pass + instance = Registry() + for ep_obj in entry_points(group=entry_point): + try: + patch = check(cast(Patch, ep_obj.load())) + except Exception as e: + error = type(e).__name__ + if str(error): + error = f'{error}: {e}' + msg = ( + f'failed to load patch {ep_obj.name!r} ' + f'from entry point {ep_obj!r}: {error}' + ) + diagnostics.log.warning(msg) + warnings.warn(msg) + else: + instance.register(ep_obj.name, patch) + return cls._loaded_instances.setdefault(entry_point, instance) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py new file mode 100644 index 00000000..ff508bb1 --- /dev/null +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import multiprocessing +from collections.abc import Callable +from functools import partial +from types import MappingProxyType as mappingproxy, MethodType +from typing import ClassVar, cast + +try: + from multiprocessing import spawn +except ImportError: + _CAN_USE_SPAWN = False +else: + _CAN_USE_SPAWN = True +try: + from multiprocessing import forkserver +except ImportError: + _CAN_USE_FORKSERVER = False +else: + _CAN_USE_FORKSERVER = ( + 'forkserver' in multiprocessing.get_all_start_methods() + ) +try: + from multiprocessing import resource_tracker +except ImportError: + _CAN_USE_RESOURCE_TRACKER = False +else: + _CAN_USE_RESOURCE_TRACKER = True + +from ..cache import LineProfilingCache +from ..runpy_patches import create_runpy_wrapper + + +__all__ = ('RebootForkserverPatch', 'ResourceTrackerPatch', 'RunpyPatch') + + +class RebootForkserverPatch: + """ + Reboot the process backing the global + :py:class:`multiprocessing.forkserver.ForkServer` instance: + + - When the patch is applied, so as to ensure that child processes + forked therefrom actually receives the active patches; and + + - When the session cache is cleaned up, so that child processes + forked therefrom is no longer polluted by the patches. + + Note: + This uses + :py:method:`multiprocessing.forkserver.ForkServer._stop()` which + is private API, but it's the same hack used in Python's own test + suite -- see the comment to said method. + """ + summary: ClassVar[mappingproxy[str, frozenset[str]]] = mappingproxy({}) + priority: ClassVar[float | None] = None + + @classmethod + def apply(cls, cache: LineProfilingCache, **_) -> None: + if not _CAN_USE_FORKSERVER: + return + cls.reboot() + cache.add_cleanup(cls.reboot) + + @staticmethod + def reboot() -> None: + # Appease the type-checker since `._stop()` is not public API + stop = getattr(forkserver._forkserver, '_stop', None) + assert callable(stop) + stop() + + +class ResourceTrackerPatch: + """ + Patch :py:mod:`multiprocessing.resource_tracker` so that + :py:func:`multiprocessing.resource_tracker.ensure_running` and the + eponymous method of + :py:class:`multiprocessing.resource_tracker.ResourceTracker` report + the resource-tracker server PIDs to the session cache. + + Note: + The ``ResourceTracker`` server process is spawned when the first + :py:mod:`multiprocessing` child process is created via the + ``spawn`` or ``forkserver`` start methods. While this server + process does not meaningfully contribute to the profiling result + either way, since it can be created with profiling set up, its + longevity means that :py:meth:`.LineProfilingCache.gather_stats` + often catches empty .lprof files which it has occupied but not + written to. + + To reduce noise while keeping the empty-file warning for other + output files, we report the PIDs used by the server to the + session cache so that they can be ignored if necessary. + """ + if _CAN_USE_RESOURCE_TRACKER: + summary: ClassVar[mappingproxy[str, frozenset[str]]] = mappingproxy({ + 'multiprocessing.resource_tracker': + frozenset({'ensure_running'}), + 'multiprocessing.resource_tracker.ResourceTracker': + frozenset({'ensure_running'}), + }) + else: + summary = mappingproxy({}) + priority: ClassVar[float | None] = None + + @staticmethod + @LineProfilingCache._method_wrapper + def wrap_ensure_running( + cache: LineProfilingCache, + vanilla_impl: Callable[['resource_tracker.ResourceTracker'], None], + self: 'resource_tracker.ResourceTracker', + ) -> None: + """ + Wrap around :py:meth:`multiprocessing.resource_tracker\ +.ResourceTracker.ensure_running` + so that the session cache can keep track of the PIDs used by the + resource-tracer server. + """ + maybe_pids: set[int | None] = {getattr(self, '_pid', None)} + try: + vanilla_impl(self) + finally: + maybe_pids.add(getattr(self, '_pid', None)) + pids = cast(set[int], maybe_pids - {None}) + if pids: + cache._warn_possible_lack_of_stats(pids) + + @classmethod + def apply( + cls, cache: LineProfilingCache, *, cleanup: bool = True, **_, + ) -> list[str]: + if _CAN_USE_RESOURCE_TRACKER: + patch = partial(cache.patch, cleanup=cleanup) + # Patch the method on the class + method = resource_tracker.ResourceTracker.ensure_running + method = cls.wrap_ensure_running(method) + patch(resource_tracker.ResourceTracker, 'ensure_running', method) + # Patch the preexisting bound method on the module + instance = resource_tracker._resource_tracker + bound_method = MethodType(method, instance) + patch(resource_tracker, 'ensure_running', bound_method) + return list(cls.summary) + + +class RunpyPatch: + """ + Patch the copy of :py:mod:`runpy` in the + :py:mod:`multiprocessing.spawn` namespace so that subprocesses can + perform rewrite-based profiling as with + :py:func:`line_profiler.autoprofile.autoprofile.run`. + + See also: + :py:mod:`line_profiler._child_process_profiling.runpy_patches` + """ + summary: ClassVar[mappingproxy[str, frozenset[str]]] + if _CAN_USE_SPAWN and hasattr(spawn, 'runpy'): + summary = mappingproxy({'multiprocessing.spawn': frozenset({'runpy'})}) + else: + summary = mappingproxy({}) + priority: ClassVar[float | None] = None + + @classmethod + def apply( + cls, cache: LineProfilingCache, *, cleanup: bool = True, **_, + ) -> list[str]: + if cls.summary: + patch = partial(cache.patch, cleanup=cleanup) + patch(spawn, 'runpy', create_runpy_wrapper(cache)) + return list(cls.summary) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_optional_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_optional_patches.py new file mode 100644 index 00000000..1ad7e08a --- /dev/null +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_optional_patches.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import os +from collections.abc import Callable +from functools import partial, wraps +from multiprocessing.process import BaseProcess +from types import MethodType +from typing import Any, TypeVar, cast +from typing_extensions import Concatenate, ParamSpec + +from ..cache import LineProfilingCache +from ._infrastructure import SingleModulePatch +from ._queue import Queue, PutWrapper + + +__all__ = ( + 'CHILD_PIDS_PATCH', 'LOGGING_PATCH', + 'tee_log', 'wrap_handle_results', 'wrap_process', 'wrap_worker', +) + +T = TypeVar('T') +P = TypeVar('P', bound=BaseProcess) +PS = ParamSpec('PS') + +_LOGGERS = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] + +# ---------------------- PID bookkeeping patches ----------------------- + + +@LineProfilingCache._method_wrapper +def wrap_handle_results( + cache: LineProfilingCache, + vanilla_impl: Callable[ + Concatenate[Queue, Callable[[], tuple[Any, ...] | None], PS], + None + ], + outqueue: Queue, + # Since we patched `outqueue.put()` in the child process, the result + # tuple pushed to the parent has an extra item (the child PID) + get: Callable[[], tuple[int, tuple[Any, ...]] | None], + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:meth:`multiprocessing.pool.Pool._handle_results` so + that it handles the extra info (PID of child process handling the + task) included by :py:func:`.wrap_worker`. + + Note: + :py:meth:`.Pool._handle_results` is a static method. + """ + # Somehow this doesn't type-check with either `mypy` or `ty` when + # we use a `TypeVar` instead of `Any` with the tuple items... + # (see `ty` issue #3467) + wrapped_get = partial(_wrap_outqueue_quick_get, cache, get) + vanilla_impl(outqueue, wrapped_get, *args, **kwargs) + + +@LineProfilingCache._method_wrapper # nocover +def wrap_worker( + _, # We don't need the cache instance, but `@_method_wrapper` does + vanilla_impl: Callable[Concatenate[Queue, Queue, PS], None], + inqueue: Queue, + outqueue: Queue, + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes report their PIDs as they pass the task results back to + the parent. + + Note: + This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. + """ + outqueue = PutWrapper(outqueue, os.getpid, push_to_parent=True) + return vanilla_impl(inqueue, outqueue, *args, **kwargs) + + +@LineProfilingCache._method_wrapper +def wrap_process( + cache: LineProfilingCache, + vanilla_impl: Callable[PS, P], + *args: PS.args, + **kwargs: PS.kwargs +) -> P: + """ + Wrap around :py:func:`multiprocessing.pool.Pool.Process` so that the + processes created can report on usage when + :py:meth:`.BaseProcess.join`-ed or + :py:meth:`.BaseProcess.terminate`-ed. + + Note: + :py:meth:`.Pool.Process` is a static method. + """ + proc = vanilla_impl(*args, **kwargs) + # Note: since we don't clean up here, there's no need to instantiate + # another `Cleanup` helper + name = f'<{type(proc).__name__} @ {hex(id(proc))}>' + patch = partial(cache.patch, cleanup=False, name=name) + for method, action in ('join', 'joining'), ('terminate', 'terminating'): + bound = getattr(proc, method) + assert isinstance(bound, MethodType) + finalize = _wrap_process_finalize(cache, bound.__func__, action) + patch(proc, method, MethodType(finalize, proc)) + return proc + + +def _wrap_process_finalize( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[P, PS], None], + action: str, +) -> Callable[Concatenate[P, PS], None]: + """ + Check if the process has run any tasks; + if not, report to the cache. + + Note: + Since the process object is pickled, this method has to directly + return a function object instead of merely being + :py:func:`partial`-ed and wrapped in a + :py:class:`types.MethodType`. + """ + @wraps(vanilla_impl) + def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: + log = cache._debug_output + call = cache._format_call(vanilla_impl, self, *args, **kwargs) + try: + log(f'Wrapped call made: {call}') + pid: int | None = getattr(self, 'pid', None) + checked_procs = _get_checked_processes(cache) + identifier = id(self), pid + if not (pid is None or identifier in checked_procs): + ntasks = _get_ntasks(cache).pop(pid, 0) + if not ntasks: + cache._warn_possible_lack_of_stats(pid) + log(f'{action} process {pid} which ran {ntasks} task(s)...') + checked_procs.add(cast(tuple[int, int], identifier)) + except BaseException as e: + log( + f'Error in bookkeeping ({cache._format_exception(e)}), ' + 'invoking base implementation nonetheless...' + ) + raise e + finally: + try: + vanilla_impl(self, *args, **kwargs) + except BaseException as e: + state = f'failed ({cache._format_exception(e)})' + raise e + else: + state = 'succeeded' + finally: + log(f'Wrapped call {call} {state}') + + action = action.capitalize() + return finalize + + +def _wrap_outqueue_quick_get( + cache: LineProfilingCache, + vanilla_impl: Callable[PS, tuple[int, tuple[Any, ...]] | None], + *args: PS.args, + **kwargs: PS.kwargs +) -> tuple[Any, ...] | None: + """ + Take and process the PID of the child process completing the task. + """ + result = vanilla_impl(*args, **kwargs) + if result is None: + return None + pid, orig_result = result + ntasks = _get_ntasks(cache) + ntasks[pid] = ntasks.get(pid, 0) + 1 + return orig_result + + +def _get_ntasks(cache: LineProfilingCache) -> dict[int, int]: + key = 'mp_proc_ntasks' + return cache._additional_data.setdefault(key, cast(dict[int, int], {})) + + +def _get_checked_processes( + cache: LineProfilingCache, +) -> set[tuple[int, int]]: + key = 'mp_proc_checked_workload' + return cache._additional_data.setdefault( + key, cast(set[tuple[int, int]], set()), + ) + + +CHILD_PIDS_PATCH = ( + SingleModulePatch('pool') + .add_method('', 'worker', wrap_worker) + .add_method('Pool', '_handle_results', wrap_handle_results, 'static') + .add_method('Pool', 'Process', wrap_process, 'static') +) + +# --------------- `multiprocessing.util` logging patches --------------- + + +def _cache_hook( + vanilla_impl: Callable[PS, T], + get_logging_message: Callable[PS, str], + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> T: + msg = get_logging_message(*args, **kwargs) + LineProfilingCache.load()._debug_output(msg) + return vanilla_impl(*args, **kwargs) + + +def tee_log( + marker: str, + vanilla_impl: Callable[Concatenate[str, PS], None], + /, + msg: str, + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around logging functions like + :py:func:`multiprocessing.util.debug` so that we can tee log + messages from the package to our own logs. + """ + def get_msg(msg: str, *_, **__) -> str: + return f'`multiprocessing` logging ({marker}): {msg}' + + _cache_hook( + vanilla_impl, get_msg, # type: ignore[arg-type] + msg, *args, **kwargs, + ) + + +LOGGING_PATCH = SingleModulePatch('util').add_target( + # The logging functions exists directly in the module namespace so + # no further attribute access is needed + '', {func: partial(partial, tee_log, func) for func in _LOGGERS}, +) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py new file mode 100644 index 00000000..2486b01e --- /dev/null +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import atexit +import os +import sys +from collections.abc import Callable +from functools import partial +from multiprocessing.process import BaseProcess +from typing import Any, TypeVar, cast +from typing_extensions import Concatenate, ParamSpec + +from ..cache import LineProfilingCache +from ._infrastructure import SingleModulePatch +from ._queue import Queue, PutWrapper +from .mp_config import MPConfig +from .poller import Poller, OnTimeout + + +__all__ = ( + 'POOL_PATCH', 'PROCESS_PATCH', + 'wrap_bootstrap', 'wrap_terminate', 'wrap_worker', +) + +T = TypeVar('T') +PS = ParamSpec('PS') + +_CAN_CATCH_SIGTERM = sys.platform != 'win32' + +# ------------------------------ Helpers ------------------------------- + + +def setup_mp_child(cache: LineProfilingCache) -> None: # nocover + """ + Perform :py:mod:`multiprocessing`-specific setup in a child process + curated by the module. Currently it does the following: + + - Set up ``cache`` to handle ``SIGTERM`` on POSIX if not already + set. + + - Unregister the :py:mod:`atexit` hook associated with ``cache`` to + avoid possible clashes with the profiling-file writing managed by + this module. + """ + if cache.main_pid == os.getpid(): # Not in a child process + return + xc: Exception | None = None + for callback in [_add_sigterm_handler_in_child, _unregister_atexit_hook]: + try: + callback(cache) + except Exception as e: + xc = e + if xc is not None: + xc_str = type(xc).__name__ + if str(xc): + xc_str = f'{xc_str}: {xc}' + cache._debug_output(f'Setup failed in process {os.getpid()}: {xc_str}') + raise xc + + +def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: + key = 'mp_added_sigterm_handler' + if not _CAN_CATCH_SIGTERM: + return + if not MPConfig.from_cache(cache).catch_sigterm: + return + if cache._additional_data.get(key, False): + # Already added (e.g. by another plugin) + return + cache._add_signal_handler() + cache._additional_data[key] = True + + +def _unregister_atexit_hook(cache: LineProfilingCache) -> None: + atexit.unregister(cache._atexit_hook) + + +def dump_stats_quick( + cache: LineProfilingCache, + *, + reason: str | None = None, + debug: bool = False, +) -> None: + """ + Note: + We don't really care about cleanup in the child process, so just + dump the stats and bail to reduce the chance of end-of-process + shenanigans causing a deadlock... + but do use ``._stats_dumper.cleanup()`` instead of + ``.__call__()`` so that we get debugging output (if ``debug`` is + true) + """ + stats_dumper = cache._stats_dumper + if stats_dumper is None: + return + if debug: + stats_dumper.cleanup(force=True, reason=reason) + else: + stats_dumper() + + +# ---------------- `multiprocessing.pool.Pool` patches ----------------- + + +class _PIDQueueGetWrapper: # nocover + """ + Wrapper around the ``inqueue`` argument to + :py:func:`multiprocessing.pool.worker` to intercept the sentinel + value (:py:const:`None`) signifying the end of the queue and perform + cleanup. + """ + def __init__( + self, + queue: Queue, + cache: LineProfilingCache, + ) -> None: + self._queue = queue + self._cache = cache + + def __getattr__(self, attr: str) -> Any: + return getattr(self._queue, attr) + + def put(self, obj: Any) -> None: + self._queue.put(obj) + + def get(self) -> Any: + result = self._queue.get() + cache = self._cache + ntasks: dict[int, int] + ntasks = cache._additional_data.setdefault('mp_queue_ntasks', {}) + queue_id = id(self) + if result is None: + n = ntasks.pop(queue_id, 0) + cache._debug_output( + '`multiprocessing.pool.worker`: ' + f'recieved {n} task(s) in total', + ) + # Got sentinel value, process is about to exit + reason = 'ran out of tasks in `multiprocessing.process.worker()`' + if cache.main_pid != os.getpid(): + dump_stats_quick(cache, debug=True, reason=reason) + else: + ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 + return result + + +@LineProfilingCache._method_wrapper # nocover +def wrap_worker_write_on_exit( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[Queue, PS], None], + inqueue: Queue, + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes can write profiling output as soon as the pool runs out of + tasks. + + Notes: + - This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. + + - This only works reliably for POSIX because we can handle + ``SIGTERM`` on child processes and ensure that they aren't + prematurely terminated. + """ + setup_mp_child(cache) + return vanilla_impl(_PIDQueueGetWrapper(inqueue, cache), *args, **kwargs) + + +@LineProfilingCache._method_wrapper # nocover +def wrap_worker_write_per_task( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[Queue, Queue, PS], None], + inqueue: Queue, + outqueue: Queue, + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes can write profiling output before pushing the result of + each task back to the parent. + + Notes: + - This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. + + - This is only used on Windows where we can't handle ``SIGTERM`` + on child processes, thus necessitating the write to happen + before control flow is passed backed to the parent. + """ + setup_mp_child(cache) + outqueue = PutWrapper(outqueue, partial(dump_stats_quick, cache)) + return vanilla_impl(inqueue, outqueue, *args, **kwargs) + + +if _CAN_CATCH_SIGTERM: + wrap_worker: Callable[[Callable[..., None]], Callable[..., None]] + wrap_worker = wrap_worker_write_on_exit +else: + wrap_worker = wrap_worker_write_per_task +POOL_PATCH = SingleModulePatch('pool').add_method( + '', 'worker', wrap_worker, +) + +# ----------- `multiprocessing.process.BaseProcess` patches ------------ + + +@LineProfilingCache._method_wrapper +def wrap_terminate( + cache: LineProfilingCache, + vanilla_impl: Callable[[BaseProcess], None], + self: BaseProcess, +) -> None: + """ + Wrap around :py:meth:`.BaseProcess.terminate` to make sure that we + don't actually kill the child (OS-level) process before it has the + chance to properly clean up. + + Note: + We're technically polling in a loop, but it isn't actually + *that* bad: typically ``.terminate()`` is only called when we're + on the bad path (e.g. the parallel workload errored out), and + after the performance-critical part of the code (said workload). + """ + try: + with _get_terminate_poller(cache, self): + pass + except Poller.Timeout as e: # Also handles `~.TimeoutWarning` + cache._debug_output(f'{type(e).__qualname__}: {e}') + raise + finally: # Always call `Process.terminate()` to avoid orphans + vanilla_impl(self) + + +@LineProfilingCache._method_wrapper # nocover +def wrap_bootstrap( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], + self: BaseProcess, + /, + *args: PS.args, **kwargs: PS.kwargs +) -> T: + """ + Wrap around :py:meth:`.BaseProcess._bootstrap` so that profiling + stats are written at the end. + + Notes: + + - This is only invoked in child processes, and + :py:mod:`coverage` seems to be having trouble with them in the + current setup, probably due to issues with .pth file + precendence causing :py:mod:`line_profiler` to be loaded + before it. Hence the ``# nocover``. + + - ``SIGTERM`` handling is not consistent on Windows, so we made + :py:meth:`.LineProfilingCache._add_signal_handler` a no-op + there. Hence :py:func:`wrap_terminate` remains necessary for + mitigating unclean exits. + """ + setup_mp_child(cache) + try: + return vanilla_impl(self, *args, **kwargs) + finally: + reason = 'exiting `multiprocessing.process.BaseProcess._bootstrap`' + dump_stats_quick(cache, debug=True, reason=reason) + + +def _get_terminate_poller( + cache: LineProfilingCache, process: BaseProcess, +) -> Poller: + config = MPConfig.from_cache(cache) + cd, timeout, on_timeout = config.polling + if on_timeout not in ('ignore', 'warn', 'error'): + on_timeout = config.get_defaults().polling.on_timeout + # `_process_has_returned()` takes a `timeout` which it passes to + # `popen.wait()`; said timeout is essentially a limit as to how + # often the function is called, hence our cooldown + poller = Poller.poll_until(_process_has_returned, process, cache, cd) + return poller.with_timeout(timeout, cast(OnTimeout, on_timeout)) + + +def _process_has_returned( + proc: BaseProcess, cache: LineProfilingCache, timeout: float, +) -> bool: + popen = getattr(proc, '_popen', None) + if popen is None: + msg, result = 'No associated process', True + else: + result = popen.wait(timeout) is not None + if result: + msg = f'Process {popen.pid} has returned' + else: + msg = f'Waiting for process {popen.pid} to return...' + cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') + return result + + +PROCESS_PATCH = SingleModulePatch('process').add_method( + 'BaseProcess', '_bootstrap', wrap_bootstrap, +) +# We only need to patch `Process.terminate()` if we can't do SIGTERM +# handling, i.e. on Windows +if not _CAN_CATCH_SIGTERM: + PROCESS_PATCH.add_method('BaseProcess', 'terminate', wrap_terminate) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py new file mode 100644 index 00000000..f2eac0b8 --- /dev/null +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, Protocol + + +__all__ = ('Queue', 'PutWrapper') + + +class Queue(Protocol): + """ + Protocol for methods common to e.g. :py:class:`queue.SimpleQueue` + and :py:class:`multiprocessing.queues.SimpleQueue`. + """ + def put(self, obj: Any) -> None: + ... + + def get(self) -> Any: + ... + + +class PutWrapper: + """ + Wrap around a queue (the ``outqueue`` argument to + :py:func:`multiprocessing.pool.worker`) so that each call to its + ``.put()`` is preceded by calling a ``callback()``; its result is + optionally attached to the tuple pushed back to the parent if + ``push_to_parent`` is true. + """ + def __init__( + self, + queue: Queue, + callback: Callable[[], Any], + push_to_parent: bool = False, + ) -> None: + self._queue = queue + self._callback = callback + self._push = push_to_parent + + def __getattr__(self, attr: str) -> Any: + return getattr(self._queue, attr) + + def put(self, obj: Any) -> None: + data = self._callback() + if self._push: + obj = data, obj + self._queue.put(obj) + + def get(self) -> Any: + return self._queue.get() diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py b/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py new file mode 100644 index 00000000..a3f3a3fa --- /dev/null +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import dataclasses +from typing import Any, NamedTuple, final +from typing_extensions import Self + +from ...toml_config import ConfigSource +from ..cache import LineProfilingCache + + +__all__ = ('MPConfig',) + + +@final +class _PollerArgs(NamedTuple): + cooldown: float + timeout: float + on_timeout: str | None + + @classmethod + def new(cls, cooldown: Any, timeout: Any, on_timeout: Any) -> Self: + try: + cd = max(float(cooldown), 0) + except (TypeError, ValueError): + cd = 0 + try: + to = max(float(timeout), 0) + except (TypeError, ValueError): + to = 0 + try: + ot: str | None = on_timeout.lower() + except Exception: # Fallback (use `_Poller`'s default) + ot = None + return cls(cd, to, ot) + + +@final +@dataclasses.dataclass +class MPConfig: + """ + Consolidate the config options into a structured object. + """ + catch_sigterm: bool + patches: dict[str, bool] + polling: _PollerArgs + + @classmethod + def from_config(cls, config: ConfigSource) -> Self: + loaded = ( + config + .get_subconfig('child_processes', 'multiprocessing') + .conf_dict + ) + polling = _PollerArgs.new(**loaded['polling']) + return cls( + catch_sigterm=loaded['catch_sigterm'], + patches=dict(loaded['patches']), + polling=polling, + ) + + @classmethod + def from_cache(cls, cache: LineProfilingCache) -> Self: + key = 'mp_config' + try: + return cache._additional_data[key] + except KeyError: + config = cls.from_config(cache._config_source) + return cache._additional_data.setdefault(key, config) + + @classmethod + def get_defaults(cls) -> Self: + namespace = globals() + name = '_DEFAULT_CONFIG' + try: + return namespace[name] + except KeyError: + defaults = cls.from_config(ConfigSource.from_default(copy=False)) + return namespace.setdefault(name, defaults) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/poller.py b/line_profiler/_child_process_profiling/multiprocessing_patches/poller.py new file mode 100644 index 00000000..cb8b60ff --- /dev/null +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/poller.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import warnings +from collections.abc import Callable +from functools import partial +from time import sleep, monotonic +from typing import Any, Literal, NoReturn +from typing_extensions import ParamSpec, Self + +from ... import _diagnostics as diagnostics + + +__all__ = ('Poller', 'OnTimeout') + +PS = ParamSpec('PS') +OnTimeout = Literal['ignore', 'warn', 'error'] + + +class Poller: + """ + Poll a callable until it returns true-y. + + Example: + >>> import warnings + >>> from contextlib import ExitStack + >>> from functools import partial + >>> from itertools import count + >>> from typing import Iterator, Literal + + >>> def count_until( + ... limit: int, mode: Literal['until', 'while'] = 'until', + ... ) -> bool: + ... def counter_is_big_enough( + ... counter: Iterator[int], limit: int, + ... ) -> bool: + ... return next(counter) >= limit + ... + ... def counter_is_small_enough( + ... counter: Iterator[int], limit: int, + ... ) -> bool: + ... return next(counter) < limit + ... + ... # The branches are ultimately equal in results, but we + ... # want to explicitly test both `.poll_until()` and + ... # `.poll_while()` + ... if mode == 'until': + ... get_poller = partial( + ... Poller.poll_until, counter_is_big_enough, + ... ) + ... else: + ... get_poller = partial( + ... Poller.poll_while, counter_is_small_enough, + ... ) + ... return get_poller(count(), limit) + + >>> with count_until(10).with_cooldown(.01).with_timeout(1): + ... # Note: we shouldn't really need that much time, but + ... # something in CI seems to be slowing down the polling + ... # loop... + ... print('We counted up to 10') + We counted up to 10 + + >>> with ( + ... count_until(100) + ... .with_cooldown(.01) + ... .with_timeout(.5) # `[on_]timeout` separately supplied + ... .with_timeout(on_timeout='ignore') + ... ): + ... print("We probably didn't count up to 100 but whatever") + We probably didn't count up to 100 but whatever + + >>> with ( # doctest: +NORMALIZE_WHITESPACE + ... count_until(30).with_cooldown(.01).with_timeout(.25) + ... ): + ... print('We counted up to 30') + Traceback (most recent call last): + ... + line_profiler...Poller.Timeout: ... + timed out (... s >= 0.25 s) waiting for + callback ...counter_is_big_enough... to return true + + >>> with ExitStack() as stack: # doctest: +NORMALIZE_WHITESPACE + ... enter = stack.enter_context + ... enter(warnings.catch_warnings()) + ... warnings.simplefilter('error', Poller.TimeoutWarning) + ... enter( + ... count_until(30, 'while') + ... .with_cooldown(.01) + ... .with_timeout(.25, 'warn') + ... ) + ... print('We counted up to 30 again') + Traceback (most recent call last): + ... + line_profiler...Poller.TimeoutWarning: ... + timed out (... s >= 0.25 s) waiting for + callback ...counter_is_small_enough... to return true + """ + def __init__( + self, + func: Callable[[], Any], + cooldown: float = 0, + timeout: float = 0, + on_timeout: OnTimeout = 'error', + ) -> None: + self._func: Callable[[], Any] = func + self._cooldown = max(0, cooldown) + self._timeout = max(0, timeout) + self._on_timeout = on_timeout + + def sleep(self): + cd = self._cooldown + if cd > 0: + sleep(cd) + + def with_cooldown(self, cooldown: float) -> Self: + return type(self)( + self._func, cooldown, self._timeout, self._on_timeout, + ) + + def with_timeout( + self, + timeout: float | None = None, + on_timeout: OnTimeout | None = None, + ) -> Self: + if timeout is None: + timeout = self._timeout + if on_timeout is None: + on_timeout = self._on_timeout + return type(self)(self._func, self._cooldown, timeout, on_timeout) + + @classmethod + def poll_until( + cls, func: Callable[PS, Any], /, *args: PS.args, **kwargs: PS.kwargs + ) -> Self: + if args or kwargs: + func = partial(func, *args, **kwargs) + return cls(func) + + @classmethod + def poll_while( + cls, func: Callable[PS, Any], /, *args: PS.args, **kwargs: PS.kwargs + ) -> Self: + def negated( + func: Callable[PS, Any], *a: PS.args, **k: PS.kwargs + ) -> bool: + return not func(*a, **k) + + return cls(partial(negated, func, *args, **kwargs)) + + def __enter__(self) -> Self: + def error(msg: str) -> NoReturn: + raise type(self).Timeout(msg) + + def warn(msg: str) -> None: + # Write log before issuing the warning because that may be + # promoted to an exception + diagnostics.log.warning(msg) + warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) + + def no_op(_) -> None: + pass + + timeout = self._timeout + callback = self._func + + handle_timeout: Callable[[str], Any] = { + 'error': error, 'warn': warn, 'ignore': no_op, + }[self._on_timeout] + fmt = '.3g' + timeout_msg_header = f'{type(self).__name__} at {id(self):#x}' + + start = monotonic() + while not callback(): + elapsed = monotonic() - start + if timeout and elapsed >= timeout: + handle_timeout( + f'{timeout_msg_header}: ' + f'timed out ({elapsed:{fmt}} s >= {timeout:{fmt}} s) ' + f'waiting for callback {callback!r} to return true' + ) + break + self.sleep() + return self + + def __exit__(self, *_, **__) -> None: + pass + + class Timeout(RuntimeError): + """ + Raised when a :py:class:`Poller` is timed out when polling. + """ + pass + + class TimeoutWarning(Timeout, UserWarning): + """ + Issued when a :py:class:`Poller` is timed out when polling. + """ + pass diff --git a/setup.py b/setup.py index a0810cc6..542f1ff4 100755 --- a/setup.py +++ b/setup.py @@ -303,6 +303,38 @@ def run_cythonize(force=False): 'console_scripts': [ 'kernprof=kernprof:main', ], + 'line_profiler._multiproc_patches': [ + '__reboot_forkserver' + '=line_profiler._child_process_profiling' + '.multiprocessing_patches._mandatory_patches' + ':RebootForkserverPatch', + '__resource_tracker' + '=line_profiler._child_process_profiling' + '.multiprocessing_patches._mandatory_patches' + ':ResourceTrackerPatch', + '__spawn_runpy' + '=line_profiler._child_process_profiling' + '.multiprocessing_patches._mandatory_patches' + ':RunpyPatch', + + 'child_pids' + '=line_profiler._child_process_profiling' + '.multiprocessing_patches._optional_patches' + ':CHILD_PIDS_PATCH', + 'logging' + '=line_profiler._child_process_profiling' + '.multiprocessing_patches._optional_patches' + ':LOGGING_PATCH', + + 'pool' + '=line_profiler._child_process_profiling' + '.multiprocessing_patches._profiling_patches' + ':POOL_PATCH', + 'process' + '=line_profiler._child_process_profiling' + '.multiprocessing_patches._profiling_patches' + ':PROCESS_PATCH', + ], } setupkw['name'] = NAME setupkw['version'] = VERSION diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index fd5a27c6..d3489859 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -53,7 +53,7 @@ create_runpy_wrapper, ) from line_profiler._child_process_profiling.multiprocessing_patches import ( - _Poller, MPConfig, _PATCHED_MARKER, _PATCHES as MP_PATCHES, + MPConfig, Timeout as MPTimeout, _PATCHED_MARKER, _PATCHES as MP_PATCHES, ) from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler.cleanup import Cleanup @@ -3097,7 +3097,7 @@ def test_apply_mp_patches_success( @pytest.mark.retry( retries=2, condition='_WINDOWS and not patch_pool', - exceptions=(ResultMismatch, _Poller.Timeout), + exceptions=(ResultMismatch, MPTimeout), ) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) From bb5fcfeb12e7a3a825ba36756a6601e8756344c1 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 16 Jun 2026 16:33:29 +0200 Subject: [PATCH 130/165] Refactored child-process setup line_profiler/_child_process_profiling/multiprocessing_patches/ _mandatory_patches.py::wrap_{terminate,start,bootstrap}() _mandatory_patches.py::PROCESS_TERMINATION_PATCH New patch for `multiprocessing.process.BaseProcess` with increased priority, which ensures that SIGTERM isn't sent to child processes until they are finished with setting up; exposed as entry point `__process_termination` _mandatory_patches.py::setup_mp_child() Refactored and migrated from `_profiling_patches.py::setup_mp_child()` _profiling_patches.py::wrap_{worker_*,bootstrap}() Removed basic setup (migrated to `PROCESS_TERMINATION_PATCH`) setup.py Added entry point `__process_termination` to `line_profiler._multiproc_patches` --- .../_mandatory_patches.py | 227 +++++++++++++++++- .../_profiling_patches.py | 49 ---- setup.py | 4 + 3 files changed, 229 insertions(+), 51 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py index ff508bb1..2b0cf26b 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -1,10 +1,16 @@ from __future__ import annotations +import atexit +import os import multiprocessing +import sys from collections.abc import Callable from functools import partial +from multiprocessing.process import BaseProcess +from pathlib import Path from types import MappingProxyType as mappingproxy, MethodType -from typing import ClassVar, cast +from typing import Any, ClassVar, TypeVar, cast +from typing_extensions import Concatenate, ParamSpec try: from multiprocessing import spawn @@ -29,9 +35,226 @@ from ..cache import LineProfilingCache from ..runpy_patches import create_runpy_wrapper +from ._infrastructure import SingleModulePatch +from .mp_config import MPConfig +from .poller import Poller, OnTimeout -__all__ = ('RebootForkserverPatch', 'ResourceTrackerPatch', 'RunpyPatch') +__all__ = ( + 'PROCESS_TERMINATION_PATCH', + 'RebootForkserverPatch', 'ResourceTrackerPatch', 'RunpyPatch', + 'wrap_terminate', 'wrap_start', 'wrap_bootstrap', +) + +_LOCK_FILE_LOC = '__line_profiler_multiprocessing_process_lock_file__' + +T = TypeVar('T') +PS = ParamSpec('PS') + +# ------------------------------ Helpers ------------------------------- + + +def setup_mp_child( # nocover + cache: LineProfilingCache, proc: BaseProcess, +) -> None: + """ + Perform :py:mod:`multiprocessing`-specific setup in a child process + curated by the module. Currently it does the following: + + - Set up ``cache`` to handle ``SIGTERM`` on POSIX if not already + set. + + - Unregister the :py:mod:`atexit` hook associated with ``cache`` to + avoid possible clashes with the profiling-file writing managed by + this module. + + - Remove the per-child-process lock file which prevents the parent + from :py:meth:`.BaseProcess.terminate`-ing it before it can be + properly set up and causing a hang. + """ + if cache.main_pid == os.getpid(): # Not in a child process + return + xc: Exception | None = None + msg = 'Performing setup for `multiprocessing` child processes...' + cache._debug_output(msg) + setup: Callable[[LineProfilingCache, BaseProcess], Any] + for setup in [ + _add_sigterm_handler_in_child, + _unregister_atexit_hook, + _remove_lock_file, + ]: + try: + setup(cache, proc) + except Exception as e: + xc = e + if xc is None: + msg = 'Setup for `multiprocessing` child process succeeded' + cache._debug_output(msg) + else: + xc_str = type(xc).__name__ + if str(xc): + xc_str = f'{xc_str}: {xc}' + cache._debug_output(f'Setup failed: {xc_str}') + raise xc + + +def _add_sigterm_handler_in_child( # nocover + cache: LineProfilingCache, _, +) -> None: + key = 'mp_added_sigterm_handler' + if sys.platform == 'win32': # Can't handle `SIGTERM` on Windows + return + if not MPConfig.from_cache(cache).catch_sigterm: + return + if cache._additional_data.get(key, False): + # Already added (e.g. by another plugin) + return + cache._add_signal_handler() + cache._additional_data[key] = True + + +def _unregister_atexit_hook( # nocover + cache: LineProfilingCache, _, +) -> None: + atexit.unregister(cache._atexit_hook) + + +def _remove_lock_file( # nocover + cache: LineProfilingCache, proc: BaseProcess, +) -> None: + lock_file = _get_lock_file(proc) + if lock_file is None: + return + lock_file.unlink(missing_ok=True) + cache._debug_output(f'Removed lock file {lock_file.name!r}') + + +def _get_lock_file(proc: BaseProcess) -> Path | None: + return getattr(proc, _LOCK_FILE_LOC, None) + + +def _set_lock_file(proc: BaseProcess, lock_file: Path) -> None: + setattr(proc, _LOCK_FILE_LOC, lock_file) + + +# ----------- `multiprocessing.process.BaseProcess` patches ------------ + + +@LineProfilingCache._method_wrapper +def wrap_terminate( + cache: LineProfilingCache, + vanilla_impl: Callable[[BaseProcess], None], + self: BaseProcess, +) -> None: + """ + Wrap around :py:meth:`.BaseProcess.terminate` to make sure that we + don't attempt to kill the child (OS-level) process before it has + been set up, by polling for when the child process has completed + setup and deleted its lock file. + + See also: + :py:func:`line_profiler._child_process_profiling.\ +multiprocessing_patches._profiling_patches.wrap_terminate` + """ + lock_file = _get_lock_file(self) + if lock_file is None: + cache._debug_output(f'no lock file associated with {self!r}') + vanilla_impl(self) + return + try: + with _get_terminate_poller(cache, self, lock_file): + pass + except Poller.Timeout as e: # Also handles `~.TimeoutWarning` + cache._debug_output(f'{type(e).__qualname__}: {e}') + raise + finally: # Always call `Process.terminate()` to avoid orphans + vanilla_impl(self) + + +@LineProfilingCache._method_wrapper +def wrap_start( + cache: LineProfilingCache, + vanilla_impl: Callable[[BaseProcess], None], + self: BaseProcess, +) -> None: + """ + Wrap around :py:meth:`.BaseProcess.start` to make sure that we + don't attempt to kill the child (OS-level) process before it has + been set up, by setting up a lock file which the child process + should delete upon completing setup. + """ + prefix = f'process-termination-lock-{os.getpid()}-{id(self):#x}-' + # This assigns the tempfile to the instance dict, which should be + # pickled along with the rest of the instance and sent to the child + # process + _set_lock_file(self, cache.make_tempfile(prefix=prefix, suffix='.lock')) + vanilla_impl(self) + + +@LineProfilingCache._method_wrapper # nocover +def wrap_bootstrap( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], + self: BaseProcess, + /, + *args: PS.args, **kwargs: PS.kwargs +) -> T: + """ + Wrap around :py:meth:`.BaseProcess._bootstrap` to perform setups + and signal to the parent process thereafter. + + Notes: + This is only invoked in child processes, and + :py:mod:`coverage` seems to be having trouble with them in the + current setup, probably due to issues with .pth file + precendence causing :py:mod:`line_profiler` to be loaded + before it. Hence the ``# nocover``. + """ + setup_mp_child(cache, self) + return vanilla_impl(self, *args, **kwargs) + + +def _get_terminate_poller( + cache: LineProfilingCache, process: BaseProcess, lock_file: Path, +) -> Poller: + config = MPConfig.from_cache(cache) + cd, timeout, on_timeout = config.polling + if on_timeout not in ('ignore', 'warn', 'error'): + on_timeout = config.get_defaults().polling.on_timeout + return ( + Poller.poll_until(_lock_file_removed, cache, process, lock_file) + .with_cooldown(cd) + .with_timeout(timeout, cast(OnTimeout, on_timeout)) + ) + + +def _lock_file_removed( + cache: LineProfilingCache, proc: BaseProcess, path: Path, +) -> bool: + exists = path.exists() + if exists: + msg = ( + f'Waiting for process {proc.ident} to set up and ' + f'delete the lock file {path.name!r}...' + ) + else: + msg = f'Process {proc.ident} has been set up' + cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') + return not exists + + +PROCESS_TERMINATION_PATCH = SingleModulePatch( + 'process', priority=1, +).add_target( + 'BaseProcess', + { + 'terminate': wrap_terminate, + 'start': wrap_start, + '_bootstrap': wrap_bootstrap, + }, +) + +# --------------------------- Misc. patches ---------------------------- class RebootForkserverPatch: diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py index 2486b01e..d5b0b56f 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py @@ -1,6 +1,5 @@ from __future__ import annotations -import atexit import os import sys from collections.abc import Callable @@ -29,51 +28,6 @@ # ------------------------------ Helpers ------------------------------- -def setup_mp_child(cache: LineProfilingCache) -> None: # nocover - """ - Perform :py:mod:`multiprocessing`-specific setup in a child process - curated by the module. Currently it does the following: - - - Set up ``cache`` to handle ``SIGTERM`` on POSIX if not already - set. - - - Unregister the :py:mod:`atexit` hook associated with ``cache`` to - avoid possible clashes with the profiling-file writing managed by - this module. - """ - if cache.main_pid == os.getpid(): # Not in a child process - return - xc: Exception | None = None - for callback in [_add_sigterm_handler_in_child, _unregister_atexit_hook]: - try: - callback(cache) - except Exception as e: - xc = e - if xc is not None: - xc_str = type(xc).__name__ - if str(xc): - xc_str = f'{xc_str}: {xc}' - cache._debug_output(f'Setup failed in process {os.getpid()}: {xc_str}') - raise xc - - -def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: - key = 'mp_added_sigterm_handler' - if not _CAN_CATCH_SIGTERM: - return - if not MPConfig.from_cache(cache).catch_sigterm: - return - if cache._additional_data.get(key, False): - # Already added (e.g. by another plugin) - return - cache._add_signal_handler() - cache._additional_data[key] = True - - -def _unregister_atexit_hook(cache: LineProfilingCache) -> None: - atexit.unregister(cache._atexit_hook) - - def dump_stats_quick( cache: LineProfilingCache, *, @@ -165,7 +119,6 @@ def wrap_worker_write_on_exit( ``SIGTERM`` on child processes and ensure that they aren't prematurely terminated. """ - setup_mp_child(cache) return vanilla_impl(_PIDQueueGetWrapper(inqueue, cache), *args, **kwargs) @@ -192,7 +145,6 @@ def wrap_worker_write_per_task( on child processes, thus necessitating the write to happen before control flow is passed backed to the parent. """ - setup_mp_child(cache) outqueue = PutWrapper(outqueue, partial(dump_stats_quick, cache)) return vanilla_impl(inqueue, outqueue, *args, **kwargs) @@ -261,7 +213,6 @@ def wrap_bootstrap( there. Hence :py:func:`wrap_terminate` remains necessary for mitigating unclean exits. """ - setup_mp_child(cache) try: return vanilla_impl(self, *args, **kwargs) finally: diff --git a/setup.py b/setup.py index 542f1ff4..c75f8125 100755 --- a/setup.py +++ b/setup.py @@ -304,6 +304,10 @@ def run_cythonize(force=False): 'kernprof=kernprof:main', ], 'line_profiler._multiproc_patches': [ + '__process_termination' + '=line_profiler._child_process_profiling' + '.multiprocessing_patches._mandatory_patches' + ':PROCESS_TERMINATION_PATCH', '__reboot_forkserver' '=line_profiler._child_process_profiling' '.multiprocessing_patches._mandatory_patches' From 52b43e56f4f6cddb94b16e69e14751fe872f703f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 25 Jun 2026 21:52:03 +0200 Subject: [PATCH 131/165] Minor fixes line_profiler/_child_process_profiling/cache.py ::LineProfilingCache._wrap_os_fork() `os.fork()` wrapper now clearing the cleanup-callback stacks on the pre-fork instance in the forked process to avoid clashes with the new, post-fork instance line_profiler/rc/line_profiler.toml Updated comments --- line_profiler/_child_process_profiling/cache.py | 7 +++++-- line_profiler/rc/line_profiler.toml | 16 ++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index d5c70dac..089abd55 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -626,9 +626,12 @@ def wrapper() -> int: forked._debug_output( 'Superseded cached `.load()`-ed instance in forked process' ) - # Unregister the `._atexit_hook` of the inherited cache - # instance to avoid complications + # To avoid complications with the inherited cache instance + # `self`: + # - Unregister its `._atexit_hook` + # - Discard its cleanup callbacks atexit.unregister(self._atexit_hook) + self._contexts.clear() # Note: we can reuse the profiler instance in the fork, but # it needs to go through setup so that the separate # profiling results are dumped into another output file diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index bdd8549b..829d32a2 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -234,14 +234,9 @@ catch_sigterm = true [tool.line_profiler.child_processes.multiprocessing.patches] # - `multiprocessing.patches.pool` (bool): -# Whether to patch `multiprocessing.pool`: -# - POSIX: -# Patch `multiprocessing.pool.worker()` so that each child process -# write profiling output as it runs out of tasks -# - Windows: -# Patch `multiprocessing.pool.Pool` so that each task writes -# profiling output before pushing the result back to the parent -# process +# Whether to patch `multiprocessing.pool.worker()` so that each child +# process write profiling output when it runs out of tasks (POSIX) or +# as each task is completed (Windows) pool = true # - `multiprocessing.patches.process` (bool): # Whether to patch `multiprocessing.process.BaseProcess`, so that each @@ -271,8 +266,9 @@ child_pids = true cooldown = 0.03125 # 1/32-nd of a second # - `multiprocessing.polling.timeout` (float): # Time (seconds) before the main process disregards the alive-ness of -# child processes, and unblocks calls to `.terminate()` a (most -# probably errored-out) child anyway (set to <= 0 to disable timeouts) +# child processes/existence of lock files, and unblocks calls to +# `.terminate()` a (most probably errored-out) child anyway (set to +# <= 0 to disable timeouts) timeout = 0.25 # - `multiprocessing.polling.on_timeout` # (Literal['error', 'warn', 'ignore']): From e7e605fb37b5d6e71a2c7a1ac7465243baf7a2ad Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 25 Jun 2026 23:03:16 +0200 Subject: [PATCH 132/165] Removed signal handling We no longer catch and handle SIGTERM because that apparently causes deadlocks in `multiprocessing` child processes (see CPython GitHub issues 73945 & 82408, coverage issue 1310)... have to figure out another way to ensure the writing of profiling data. line_profiler/_child_process_profiling/ cache.py::LineProfilingCache Removed private attributes and methods related to signal handling multiprocessing_patches/_mandatory_patches.py::setup_mp_child() No longer adding `SIGTERM` handling multiprocessing_patches/_profiling_patches.py dump_stats_quick() Updated call signature wrap_worker_write_*(), wrap_bootstrap() Removed reference to `SIGTERM` handling multiprocessing_patches/mp_config.py::MPConfig Removed attribute `.catch_sigterm` line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes.multiprocessing] Removed config item `catch_sigterm` TODO: Update `_profiling_patches` to mitigate `BaseProcess.terminate()` --- .../_child_process_profiling/cache.py | 80 ++----------------- .../_mandatory_patches.py | 25 +----- .../_profiling_patches.py | 66 +++++---------- .../multiprocessing_patches/mp_config.py | 7 +- line_profiler/rc/line_profiler.toml | 7 -- 5 files changed, 29 insertions(+), 156 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 089abd55..aeaedcdc 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -7,8 +7,6 @@ import atexit import dataclasses import os -import signal -import sys import sysconfig try: import _pickle as pickle @@ -22,8 +20,7 @@ from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent -from threading import current_thread, main_thread -from types import FrameType, ModuleType +from types import ModuleType from typing import Any, ClassVar, Literal, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self @@ -43,7 +40,6 @@ PS = ParamSpec('PS') # Note: `typing.AnyStr` deprecated since 3.13 AnyStr = TypeVar('AnyStr', str, bytes) -_SignalHandler = Callable[[int, FrameType | None], Any] _THIS_SUBPACKAGE, *_ = (lambda: None).__module__.rpartition('.') INHERITED_CACHE_ENV_VARNAME_PREFIX = ( @@ -110,9 +106,6 @@ class LineProfilingCache(Cleanup): debug: bool = diagnostics.DEBUG profiler: LineProfiler | None = _private_field(default=None) - _sighandlers: dict[int, _SignalHandler | int | None] = ( - _private_field(default_factory=dict) - ) _stats_dumper: _DumpStatsHelper | None = _private_field(default=None) # These are unstructured fields; other components can decide on what # to put in them. They are also pickled by `.dump()`, and are thus @@ -490,7 +483,7 @@ def wrap_ctx_debug( # - Set the profiler up to write thereto when the process # terminates (with high priority) # (Also keep a separate reference to the callback for e.g. - # dumping stats ASAP when a signal is caught) + # dumping stats ASAP at process exit) prof_outfile = self.make_tempfile( prefix=_PROFILING_OUTPUT_PREFIX_PATTERN.format( main_pid=self.main_pid, @@ -503,8 +496,9 @@ def wrap_ctx_debug( self._stats_dumper = dumper = _DumpStatsHelper(prof, prof_outfile) self.patch( # If we call `dumper.cleanup()` instead of `dumper` (e.g. - # in `._handle_signal()`), the subsequent debug-log messages - # are attributed to and handled by this cache instance + # in some `multiprocessing` patches), the subsequent + # debug-log messages are attributed to and handled by this + # cache instance dumper, '_debug_output', self._debug_output, cleanup=False, name='._stats_dumper', ) @@ -531,70 +525,6 @@ def _setup_common( self, **(mp_apply_kwargs or {}), ) - def _handle_signal(self, signum: int, *_) -> None: # nocover - """ - See also: - :py:meth:`coverage.control.Converage._on_sigterm` - """ - name = self._get_signal_name(signum) - # Shouldn't happen, but all kinds of weird things happen at the - # interpreter's EoL... - state = 'not initiated?!' - try: - # Just use the `._stats_dumper` to dump the stats ASAP - # without running this cache's `.cleanup()` to avoid - # deadlocks - if self._stats_dumper is None: - state = 'unavailable' - else: - reason = f'caught `{name}` ({signum})' - self._stats_dumper.cleanup(force=True, reason=reason) - except BaseException as e: - state = f'failed ({self._format_exception(e)})' - raise e - else: - state = 'succeeded' - finally: - handler = self._sighandlers.pop(signum, None) - msg = f'Stat-dumping {state}, passing `{name}` onto {handler!r}...' - self._debug_output(msg) - if handler is None: - msg = 'original handler set from outside of Python' - raise RuntimeError(msg) - else: - signal.signal(signum, handler) - signal.raise_signal(signum) - - def _add_signal_handler( - self, signum: int = signal.SIGTERM, - ) -> None: # nocover - """ - Side effects: - If on the main thread and not on Windows: - - - :py:func:`signal.signal` called to set - :py:meth:`~._handle_signal` as the ``SIGTERM`` handler - - - :py:meth:`~.cleanup` callback registered undoing that - - Note: - ``SIGTERM`` handling is known to be faulty on Windows; see - previous discussions at (examples `1`_, `2`_). - - .. _1: https://github.com/coveragepy/coveragepy/blob/main/\ -coverage/control.py - .. _2: https://stackoverflow.com/questions/35772001/ - """ - if current_thread() != main_thread() or sys.platform == 'win32': - return - name = self._get_signal_name(signum) - self._debug_output(f'Adding `{name}` handler...') - self._sighandlers[signum] = signal.signal(signum, self._handle_signal) - - @staticmethod - def _get_signal_name(signum: int) -> str: - return signal.Signals(signum).name - def _wrap_os_fork(self) -> None: """ Create a wrapper around :py:func:`os.fork` which handles diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py index 2b0cf26b..4ce1ec57 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -3,7 +3,6 @@ import atexit import os import multiprocessing -import sys from collections.abc import Callable from functools import partial from multiprocessing.process import BaseProcess @@ -61,9 +60,6 @@ def setup_mp_child( # nocover Perform :py:mod:`multiprocessing`-specific setup in a child process curated by the module. Currently it does the following: - - Set up ``cache`` to handle ``SIGTERM`` on POSIX if not already - set. - - Unregister the :py:mod:`atexit` hook associated with ``cache`` to avoid possible clashes with the profiling-file writing managed by this module. @@ -78,11 +74,7 @@ def setup_mp_child( # nocover msg = 'Performing setup for `multiprocessing` child processes...' cache._debug_output(msg) setup: Callable[[LineProfilingCache, BaseProcess], Any] - for setup in [ - _add_sigterm_handler_in_child, - _unregister_atexit_hook, - _remove_lock_file, - ]: + for setup in [_unregister_atexit_hook, _remove_lock_file]: try: setup(cache, proc) except Exception as e: @@ -98,21 +90,6 @@ def setup_mp_child( # nocover raise xc -def _add_sigterm_handler_in_child( # nocover - cache: LineProfilingCache, _, -) -> None: - key = 'mp_added_sigterm_handler' - if sys.platform == 'win32': # Can't handle `SIGTERM` on Windows - return - if not MPConfig.from_cache(cache).catch_sigterm: - return - if cache._additional_data.get(key, False): - # Already added (e.g. by another plugin) - return - cache._add_signal_handler() - cache._additional_data[key] = True - - def _unregister_atexit_hook( # nocover cache: LineProfilingCache, _, ) -> None: diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py index d5b0b56f..0951b21e 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py @@ -1,7 +1,6 @@ from __future__ import annotations import os -import sys from collections.abc import Callable from functools import partial from multiprocessing.process import BaseProcess @@ -23,16 +22,11 @@ T = TypeVar('T') PS = ParamSpec('PS') -_CAN_CATCH_SIGTERM = sys.platform != 'win32' - # ------------------------------ Helpers ------------------------------- def dump_stats_quick( - cache: LineProfilingCache, - *, - reason: str | None = None, - debug: bool = False, + cache: LineProfilingCache, *, reason: str | None = None, ) -> None: """ Note: @@ -46,7 +40,7 @@ def dump_stats_quick( stats_dumper = cache._stats_dumper if stats_dumper is None: return - if debug: + if cache.debug: stats_dumper.cleanup(force=True, reason=reason) else: stats_dumper() @@ -91,7 +85,7 @@ def get(self) -> Any: # Got sentinel value, process is about to exit reason = 'ran out of tasks in `multiprocessing.process.worker()`' if cache.main_pid != os.getpid(): - dump_stats_quick(cache, debug=True, reason=reason) + dump_stats_quick(cache, reason=reason) else: ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 return result @@ -110,14 +104,10 @@ def wrap_worker_write_on_exit( processes can write profiling output as soon as the pool runs out of tasks. - Notes: - - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. - - - This only works reliably for POSIX because we can handle - ``SIGTERM`` on child processes and ensure that they aren't - prematurely terminated. + Note: + This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. """ return vanilla_impl(_PIDQueueGetWrapper(inqueue, cache), *args, **kwargs) @@ -136,20 +126,17 @@ def wrap_worker_write_per_task( processes can write profiling output before pushing the result of each task back to the parent. - Notes: - - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. - - - This is only used on Windows where we can't handle ``SIGTERM`` - on child processes, thus necessitating the write to happen - before control flow is passed backed to the parent. + Note: + This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. """ - outqueue = PutWrapper(outqueue, partial(dump_stats_quick, cache)) + dump = partial(dump_stats_quick, cache, reason='processed task') + outqueue = PutWrapper(outqueue, dump) return vanilla_impl(inqueue, outqueue, *args, **kwargs) -if _CAN_CATCH_SIGTERM: +if True: # FIXME wrap_worker: Callable[[Callable[..., None]], Callable[..., None]] wrap_worker = wrap_worker_write_on_exit else: @@ -200,24 +187,18 @@ def wrap_bootstrap( Wrap around :py:meth:`.BaseProcess._bootstrap` so that profiling stats are written at the end. - Notes: - - - This is only invoked in child processes, and - :py:mod:`coverage` seems to be having trouble with them in the - current setup, probably due to issues with .pth file - precendence causing :py:mod:`line_profiler` to be loaded - before it. Hence the ``# nocover``. - - - ``SIGTERM`` handling is not consistent on Windows, so we made - :py:meth:`.LineProfilingCache._add_signal_handler` a no-op - there. Hence :py:func:`wrap_terminate` remains necessary for - mitigating unclean exits. + Note: + This is only invoked in child processes, and :py:mod:`coverage` + seems to be having trouble with them in the current setup, + probably due to issues with .pth file precendence causing + :py:mod:`line_profiler` to be loaded before it. Hence the + ``# nocover``. """ try: return vanilla_impl(self, *args, **kwargs) finally: reason = 'exiting `multiprocessing.process.BaseProcess._bootstrap`' - dump_stats_quick(cache, debug=True, reason=reason) + dump_stats_quick(cache, reason=reason) def _get_terminate_poller( @@ -253,7 +234,4 @@ def _process_has_returned( PROCESS_PATCH = SingleModulePatch('process').add_method( 'BaseProcess', '_bootstrap', wrap_bootstrap, ) -# We only need to patch `Process.terminate()` if we can't do SIGTERM -# handling, i.e. on Windows -if not _CAN_CATCH_SIGTERM: - PROCESS_PATCH.add_method('BaseProcess', 'terminate', wrap_terminate) +PROCESS_PATCH.add_method('BaseProcess', 'terminate', wrap_terminate) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py b/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py index a3f3a3fa..cd2a19ce 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py @@ -40,7 +40,6 @@ class MPConfig: """ Consolidate the config options into a structured object. """ - catch_sigterm: bool patches: dict[str, bool] polling: _PollerArgs @@ -52,11 +51,7 @@ def from_config(cls, config: ConfigSource) -> Self: .conf_dict ) polling = _PollerArgs.new(**loaded['polling']) - return cls( - catch_sigterm=loaded['catch_sigterm'], - patches=dict(loaded['patches']), - polling=polling, - ) + return cls(patches=dict(loaded['patches']), polling=polling) @classmethod def from_cache(cls, cache: LineProfilingCache) -> Self: diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 829d32a2..b6dd2ab1 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -222,13 +222,6 @@ percent = 8 prefix = '_line_profiler-profiling-hook-' suffix = '' -[tool.line_profiler.child_processes.multiprocessing] - -# - `multiprocessing.catch_sigterm` (bool): -# Whether to set a hook for the child-process session caches to -# `.cleanup()` on SIGTERM -catch_sigterm = true - # List of individual patches to apply to `multiprocessing` [tool.line_profiler.child_processes.multiprocessing.patches] From 7b5952fd21c6d3200216387d81bc22feda6364e0 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 26 Jun 2026 03:19:35 +0200 Subject: [PATCH 133/165] Always track pool-worker PIDs line_profiler/_child_process_profiling/multiprocessing_patches (The `child_pids` optional patch has been reworked into the `__pool_worker_pid` mandatory patch.) __init__.py::apply() Updated docstring _infrastructure.py::Registry.from_entry_point() Updated doctest _mandatory_patches.py POOL_WORKER_PID_PATCH Migrated from `_optional_patches.py::CHILD_PIDS_PATCH` wrap_handle_results(), wrap_process(), wrap_worker() Migrated from eponymous functions in `_optional_patches.py` _profiling_patches.py wrap_worker() Now always using the previous `wrap_worker_write_per_task()` implementation, because we're no longer catching SIGTERM even on POSIX wrap_terminate() Now only blocking the call to `BaseProcess.terminate()` if `self` is a pool worker which has run at least 1 task; this avoids having idle workers in a "dirty" state interfering with pool termination line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes.multiprocessing.patches] Removed item `child_pids` setup.py Renamed entry point: `line_profiler._multiproc_patches.child_pids` -> `.__pool_worker_pid` tests/test_child_procs.py _run_kernprof_main_in_process() _test_apply_mp_patches() test_apply_mp_patches_success() Removed references to the `child_pids` patch --- .../multiprocessing_patches/__init__.py | 17 +- .../_infrastructure.py | 1 - .../_mandatory_patches.py | 181 ++++++++++++++++- .../_optional_patches.py | 188 +----------------- .../_profiling_patches.py | 138 ++++++------- line_profiler/rc/line_profiler.toml | 9 +- setup.py | 8 +- tests/test_child_procs.py | 41 ++-- 8 files changed, 258 insertions(+), 325 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py index 675db30c..3fb62409 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py @@ -26,7 +26,7 @@ __all__ = ('MPConfig', 'Poller', 'Registry', 'Timeout', 'apply') -PublicPatch = Literal['pool', 'process', 'logging', 'child_pids'] +PublicPatch = Literal['pool', 'process', 'logging'] _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' _PATCHES = Registry.from_entry_point() @@ -52,8 +52,7 @@ def apply( so as to ensure that profiling happens on processes forked therefrom (see Note). patches \ -(Collection[Literal['pool', 'process', 'logging', 'child_pids'] \ -| None]): +(Collection[Literal['pool', 'process', 'logging'] | None]): Patches to apply to :py:mod:`multiprocessing`; see the following section for a description of each; the default is taken from the TOML config file. @@ -77,18 +76,6 @@ def apply( Patch :py:mod:`multiprocessing.util`'s logging methods (e.g. ``debug()`` and ``info()``) so that their messages are teed to the cache's debug log. - ``'child_pids'``: - Patch the following components of - :py:mod:`multiprocess.pool` so that the parent process keeps - track of the workload executed by each child process, - reducing stray warnings about the lack of profiling stats - reported thereby: - - - :py:func:`multiprocessing.pool.worker` - - - :py:meth:`multiprocessing.pool.Pool._handle_results` - - - :py:meth:`multiprocessing.pool.Pool.Process` Side effects: - The aforementioned patches applied diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py index 3e3e6413..b2973b05 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py @@ -447,7 +447,6 @@ def from_entry_point(cls, entry_point: str | None = None) -> Registry: >>> assert 'pool' in reg >>> assert 'process' in reg >>> assert 'logging' in reg - >>> assert 'child_pids' in reg >>> assert ( ... 'multiprocessing.process.BaseProcess' in reg.summary diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py index 4ce1ec57..731d12da 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -4,7 +4,7 @@ import os import multiprocessing from collections.abc import Callable -from functools import partial +from functools import partial, wraps from multiprocessing.process import BaseProcess from pathlib import Path from types import MappingProxyType as mappingproxy, MethodType @@ -35,19 +35,22 @@ from ..cache import LineProfilingCache from ..runpy_patches import create_runpy_wrapper from ._infrastructure import SingleModulePatch +from ._queue import Queue, PutWrapper from .mp_config import MPConfig from .poller import Poller, OnTimeout __all__ = ( - 'PROCESS_TERMINATION_PATCH', + 'POOL_WORKER_PID_PATCH', 'PROCESS_TERMINATION_PATCH', 'RebootForkserverPatch', 'ResourceTrackerPatch', 'RunpyPatch', 'wrap_terminate', 'wrap_start', 'wrap_bootstrap', + 'wrap_handle_results', 'wrap_process', 'wrap_worker', ) _LOCK_FILE_LOC = '__line_profiler_multiprocessing_process_lock_file__' T = TypeVar('T') +P = TypeVar('P', bound=BaseProcess) PS = ParamSpec('PS') # ------------------------------ Helpers ------------------------------- @@ -231,6 +234,180 @@ def _lock_file_removed( }, ) +# ---------------------- PID bookkeeping patches ----------------------- + + +@LineProfilingCache._method_wrapper +def wrap_handle_results( + cache: LineProfilingCache, + vanilla_impl: Callable[ + Concatenate[Queue, Callable[[], tuple[Any, ...] | None], PS], + None + ], + outqueue: Queue, + # Since we patched `outqueue.put()` in the child process, the result + # tuple pushed to the parent has an extra item (the child PID) + get: Callable[[], tuple[int, tuple[Any, ...]] | None], + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:meth:`multiprocessing.pool.Pool._handle_results` so + that it handles the extra info (PID of child process handling the + task) included by :py:func:`.wrap_worker`. + + Note: + :py:meth:`.Pool._handle_results` is a static method. + """ + # Somehow this doesn't type-check with either `mypy` or `ty` when + # we use a `TypeVar` instead of `Any` with the tuple items... + # (see `ty` issue #3467) + wrapped_get = partial(_wrap_outqueue_quick_get, cache, get) + vanilla_impl(outqueue, wrapped_get, *args, **kwargs) + + +@LineProfilingCache._method_wrapper # nocover +def wrap_worker( + _, # We don't need the cache instance, but `@_method_wrapper` does + vanilla_impl: Callable[Concatenate[Queue, Queue, PS], None], + inqueue: Queue, + outqueue: Queue, + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes report their PIDs as they pass the task results back to + the parent. + + Note: + This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. + """ + outqueue = PutWrapper(outqueue, os.getpid, push_to_parent=True) + return vanilla_impl(inqueue, outqueue, *args, **kwargs) + + +@LineProfilingCache._method_wrapper +def wrap_process( + cache: LineProfilingCache, + vanilla_impl: Callable[PS, P], + *args: PS.args, + **kwargs: PS.kwargs +) -> P: + """ + Wrap around :py:func:`multiprocessing.pool.Pool.Process` so that the + processes created can report on usage when + :py:meth:`.BaseProcess.join`-ed or + :py:meth:`.BaseProcess.terminate`-ed. + + Note: + :py:meth:`.Pool.Process` is a static method. + """ + proc = vanilla_impl(*args, **kwargs) + # Note: since we don't clean up here, there's no need to instantiate + # another `Cleanup` helper + name = f'<{type(proc).__name__} @ {hex(id(proc))}>' + patch = partial(cache.patch, cleanup=False, name=name) + for method, action in ('join', 'joining'), ('terminate', 'terminating'): + bound = getattr(proc, method) + assert isinstance(bound, MethodType) + finalize = _wrap_process_finalize(cache, bound.__func__, action) + patch(proc, method, MethodType(finalize, proc)) + return proc + + +def _wrap_process_finalize( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[P, PS], None], + action: str, +) -> Callable[Concatenate[P, PS], None]: + """ + Check if the process has run any tasks; + if not, report to the cache. + + Note: + Since the process object is pickled, this method has to directly + return a function object instead of merely being + :py:func:`partial`-ed and wrapped in a + :py:class:`types.MethodType`. + """ + @wraps(vanilla_impl) + def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: + log = cache._debug_output + call = cache._format_call(vanilla_impl, self, *args, **kwargs) + try: + log(f'Wrapped call made: {call}') + pid: int | None = getattr(self, 'pid', None) + ntasks_finalized = _get_ntasks_finalized(cache) + identifier = id(self), pid + if not (pid is None or identifier in ntasks_finalized): + ntasks = _get_ntasks(cache).pop(pid, 0) + if not ntasks: + cache._warn_possible_lack_of_stats(pid) + log(f'{action} process {pid} which ran {ntasks} task(s)...') + ntasks_finalized[cast(tuple[int, int], identifier)] = ntasks + except BaseException as e: + log( + f'Error in bookkeeping ({cache._format_exception(e)}), ' + 'invoking base implementation nonetheless...' + ) + raise e + finally: + try: + vanilla_impl(self, *args, **kwargs) + except BaseException as e: + state = f'failed ({cache._format_exception(e)})' + raise e + else: + state = 'succeeded' + finally: + log(f'Wrapped call {call} {state}') + + action = action.capitalize() + return finalize + + +def _wrap_outqueue_quick_get( + cache: LineProfilingCache, + vanilla_impl: Callable[PS, tuple[int, tuple[Any, ...]] | None], + *args: PS.args, + **kwargs: PS.kwargs +) -> tuple[Any, ...] | None: + """ + Take and process the PID of the child process completing the task. + """ + result = vanilla_impl(*args, **kwargs) + if result is None: + return None + pid, orig_result = result + ntasks = _get_ntasks(cache) + ntasks[pid] = ntasks.get(pid, 0) + 1 + return orig_result + + +def _get_ntasks(cache: LineProfilingCache) -> dict[int, int]: + key = 'mp_proc_ntasks' + return cache._additional_data.setdefault(key, cast(dict[int, int], {})) + + +def _get_ntasks_finalized( + cache: LineProfilingCache, +) -> dict[tuple[int, int], int]: + key = 'mp_proc_ntasks_finalized' + return cache._additional_data.setdefault( + key, cast(dict[tuple[int, int], int], {}) + ) + + +POOL_WORKER_PID_PATCH = ( + SingleModulePatch('pool') + .add_method('', 'worker', wrap_worker) + .add_method('Pool', '_handle_results', wrap_handle_results, 'static') + .add_method('Pool', 'Process', wrap_process, 'static') +) + # --------------------------- Misc. patches ---------------------------- diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_optional_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_optional_patches.py index 1ad7e08a..0fe0f328 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_optional_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_optional_patches.py @@ -1,203 +1,21 @@ from __future__ import annotations -import os from collections.abc import Callable -from functools import partial, wraps -from multiprocessing.process import BaseProcess -from types import MethodType -from typing import Any, TypeVar, cast +from functools import partial +from typing import TypeVar from typing_extensions import Concatenate, ParamSpec from ..cache import LineProfilingCache from ._infrastructure import SingleModulePatch -from ._queue import Queue, PutWrapper -__all__ = ( - 'CHILD_PIDS_PATCH', 'LOGGING_PATCH', - 'tee_log', 'wrap_handle_results', 'wrap_process', 'wrap_worker', -) +__all__ = ('LOGGING_PATCH', 'tee_log') T = TypeVar('T') -P = TypeVar('P', bound=BaseProcess) PS = ParamSpec('PS') _LOGGERS = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] -# ---------------------- PID bookkeeping patches ----------------------- - - -@LineProfilingCache._method_wrapper -def wrap_handle_results( - cache: LineProfilingCache, - vanilla_impl: Callable[ - Concatenate[Queue, Callable[[], tuple[Any, ...] | None], PS], - None - ], - outqueue: Queue, - # Since we patched `outqueue.put()` in the child process, the result - # tuple pushed to the parent has an extra item (the child PID) - get: Callable[[], tuple[int, tuple[Any, ...]] | None], - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around :py:meth:`multiprocessing.pool.Pool._handle_results` so - that it handles the extra info (PID of child process handling the - task) included by :py:func:`.wrap_worker`. - - Note: - :py:meth:`.Pool._handle_results` is a static method. - """ - # Somehow this doesn't type-check with either `mypy` or `ty` when - # we use a `TypeVar` instead of `Any` with the tuple items... - # (see `ty` issue #3467) - wrapped_get = partial(_wrap_outqueue_quick_get, cache, get) - vanilla_impl(outqueue, wrapped_get, *args, **kwargs) - - -@LineProfilingCache._method_wrapper # nocover -def wrap_worker( - _, # We don't need the cache instance, but `@_method_wrapper` does - vanilla_impl: Callable[Concatenate[Queue, Queue, PS], None], - inqueue: Queue, - outqueue: Queue, - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around :py:func:`multiprocessing.pool.worker` so that child - processes report their PIDs as they pass the task results back to - the parent. - - Note: - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. - """ - outqueue = PutWrapper(outqueue, os.getpid, push_to_parent=True) - return vanilla_impl(inqueue, outqueue, *args, **kwargs) - - -@LineProfilingCache._method_wrapper -def wrap_process( - cache: LineProfilingCache, - vanilla_impl: Callable[PS, P], - *args: PS.args, - **kwargs: PS.kwargs -) -> P: - """ - Wrap around :py:func:`multiprocessing.pool.Pool.Process` so that the - processes created can report on usage when - :py:meth:`.BaseProcess.join`-ed or - :py:meth:`.BaseProcess.terminate`-ed. - - Note: - :py:meth:`.Pool.Process` is a static method. - """ - proc = vanilla_impl(*args, **kwargs) - # Note: since we don't clean up here, there's no need to instantiate - # another `Cleanup` helper - name = f'<{type(proc).__name__} @ {hex(id(proc))}>' - patch = partial(cache.patch, cleanup=False, name=name) - for method, action in ('join', 'joining'), ('terminate', 'terminating'): - bound = getattr(proc, method) - assert isinstance(bound, MethodType) - finalize = _wrap_process_finalize(cache, bound.__func__, action) - patch(proc, method, MethodType(finalize, proc)) - return proc - - -def _wrap_process_finalize( - cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[P, PS], None], - action: str, -) -> Callable[Concatenate[P, PS], None]: - """ - Check if the process has run any tasks; - if not, report to the cache. - - Note: - Since the process object is pickled, this method has to directly - return a function object instead of merely being - :py:func:`partial`-ed and wrapped in a - :py:class:`types.MethodType`. - """ - @wraps(vanilla_impl) - def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: - log = cache._debug_output - call = cache._format_call(vanilla_impl, self, *args, **kwargs) - try: - log(f'Wrapped call made: {call}') - pid: int | None = getattr(self, 'pid', None) - checked_procs = _get_checked_processes(cache) - identifier = id(self), pid - if not (pid is None or identifier in checked_procs): - ntasks = _get_ntasks(cache).pop(pid, 0) - if not ntasks: - cache._warn_possible_lack_of_stats(pid) - log(f'{action} process {pid} which ran {ntasks} task(s)...') - checked_procs.add(cast(tuple[int, int], identifier)) - except BaseException as e: - log( - f'Error in bookkeeping ({cache._format_exception(e)}), ' - 'invoking base implementation nonetheless...' - ) - raise e - finally: - try: - vanilla_impl(self, *args, **kwargs) - except BaseException as e: - state = f'failed ({cache._format_exception(e)})' - raise e - else: - state = 'succeeded' - finally: - log(f'Wrapped call {call} {state}') - - action = action.capitalize() - return finalize - - -def _wrap_outqueue_quick_get( - cache: LineProfilingCache, - vanilla_impl: Callable[PS, tuple[int, tuple[Any, ...]] | None], - *args: PS.args, - **kwargs: PS.kwargs -) -> tuple[Any, ...] | None: - """ - Take and process the PID of the child process completing the task. - """ - result = vanilla_impl(*args, **kwargs) - if result is None: - return None - pid, orig_result = result - ntasks = _get_ntasks(cache) - ntasks[pid] = ntasks.get(pid, 0) + 1 - return orig_result - - -def _get_ntasks(cache: LineProfilingCache) -> dict[int, int]: - key = 'mp_proc_ntasks' - return cache._additional_data.setdefault(key, cast(dict[int, int], {})) - - -def _get_checked_processes( - cache: LineProfilingCache, -) -> set[tuple[int, int]]: - key = 'mp_proc_checked_workload' - return cache._additional_data.setdefault( - key, cast(set[tuple[int, int]], set()), - ) - - -CHILD_PIDS_PATCH = ( - SingleModulePatch('pool') - .add_method('', 'worker', wrap_worker) - .add_method('Pool', '_handle_results', wrap_handle_results, 'static') - .add_method('Pool', 'Process', wrap_process, 'static') -) - # --------------- `multiprocessing.util` logging patches --------------- diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py index 0951b21e..a93daa0a 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py @@ -1,7 +1,7 @@ from __future__ import annotations -import os from collections.abc import Callable +from contextlib import AbstractContextManager, nullcontext from functools import partial from multiprocessing.process import BaseProcess from typing import Any, TypeVar, cast @@ -9,6 +9,7 @@ from ..cache import LineProfilingCache from ._infrastructure import SingleModulePatch +from ._mandatory_patches import _get_ntasks_finalized from ._queue import Queue, PutWrapper from .mp_config import MPConfig from .poller import Poller, OnTimeout @@ -46,74 +47,19 @@ def dump_stats_quick( stats_dumper() -# ---------------- `multiprocessing.pool.Pool` patches ----------------- - - -class _PIDQueueGetWrapper: # nocover - """ - Wrapper around the ``inqueue`` argument to - :py:func:`multiprocessing.pool.worker` to intercept the sentinel - value (:py:const:`None`) signifying the end of the queue and perform - cleanup. - """ - def __init__( - self, - queue: Queue, - cache: LineProfilingCache, - ) -> None: - self._queue = queue - self._cache = cache - - def __getattr__(self, attr: str) -> Any: - return getattr(self._queue, attr) - - def put(self, obj: Any) -> None: - self._queue.put(obj) - - def get(self) -> Any: - result = self._queue.get() - cache = self._cache - ntasks: dict[int, int] - ntasks = cache._additional_data.setdefault('mp_queue_ntasks', {}) - queue_id = id(self) - if result is None: - n = ntasks.pop(queue_id, 0) - cache._debug_output( - '`multiprocessing.pool.worker`: ' - f'recieved {n} task(s) in total', - ) - # Got sentinel value, process is about to exit - reason = 'ran out of tasks in `multiprocessing.process.worker()`' - if cache.main_pid != os.getpid(): - dump_stats_quick(cache, reason=reason) - else: - ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 - return result +def _get_worker_ntasks(cache: LineProfilingCache, worker: BaseProcess) -> int: + ntasks = _get_ntasks_finalized(cache) + pid: int | None = getattr(worker, 'pid', None) + if pid is None: + return 0 + return ntasks.get((id(worker), pid), 0) -@LineProfilingCache._method_wrapper # nocover -def wrap_worker_write_on_exit( - cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[Queue, PS], None], - inqueue: Queue, - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around :py:func:`multiprocessing.pool.worker` so that child - processes can write profiling output as soon as the pool runs out of - tasks. - - Note: - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. - """ - return vanilla_impl(_PIDQueueGetWrapper(inqueue, cache), *args, **kwargs) +# ---------------- `multiprocessing.pool.Pool` patches ----------------- @LineProfilingCache._method_wrapper # nocover -def wrap_worker_write_per_task( +def wrap_worker( cache: LineProfilingCache, vanilla_impl: Callable[Concatenate[Queue, Queue, PS], None], inqueue: Queue, @@ -126,21 +72,38 @@ def wrap_worker_write_per_task( processes can write profiling output before pushing the result of each task back to the parent. - Note: - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. + Notes: + + - This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. + + - In an ideal world, we would have just written profiling output + once as :py:func:`multiprocessing.pool.worker` returns. But: + + - Worker sometimes end up in "dirty" states and deadlock, and + thus has to be terminated. + + - However, terminating a Python process bypasses the + interpreter control flow, meaning that :py:mod:`atexit` + hooks and ``try``-``finally`` blocks aren't executed. + + - On POSIX, this can be mitigated by setting signal handlers, + but signal handling is infamously unreliable on + :py:mod:`multiprocessing` child processes (examples: + `1`_, `2`_, `3`_), causing hangs that are hard to remedy. + + So this is about as good as we can do. + + .. _1: https://github.com/python/cpython/issues/73945 + .. _2: https://github.com/python/cpython/issues/82408 + .. _3: https://github.com/coveragepy/coveragepy/issues/1310 """ dump = partial(dump_stats_quick, cache, reason='processed task') outqueue = PutWrapper(outqueue, dump) return vanilla_impl(inqueue, outqueue, *args, **kwargs) -if True: # FIXME - wrap_worker: Callable[[Callable[..., None]], Callable[..., None]] - wrap_worker = wrap_worker_write_on_exit -else: - wrap_worker = wrap_worker_write_per_task POOL_PATCH = SingleModulePatch('pool').add_method( '', 'worker', wrap_worker, ) @@ -159,14 +122,26 @@ def wrap_terminate( don't actually kill the child (OS-level) process before it has the chance to properly clean up. - Note: - We're technically polling in a loop, but it isn't actually - *that* bad: typically ``.terminate()`` is only called when we're - on the bad path (e.g. the parallel workload errored out), and - after the performance-critical part of the code (said workload). + Notes: + + - We're technically polling in a loop, but it isn't actually + *that* bad: typically ``.terminate()`` is only called when + we're on the bad path (e.g. the parallel workload errored + out), and after the performance-critical part of the code + (said workload). + + - For currently unclear reasons, worker processes created by a + pool seem to sometimes deadlock, and thus we can't and don't + wait for their clean exit. This doesn't affect the profiling + results since they haven't run any task anyways, """ + block: AbstractContextManager[Any] + if _get_worker_ntasks(cache, self): + block = _get_terminate_poller(cache, self) + else: # Don't block if we don't have to + block = nullcontext() try: - with _get_terminate_poller(cache, self): + with block: pass except Poller.Timeout as e: # Also handles `~.TimeoutWarning` cache._debug_output(f'{type(e).__qualname__}: {e}') @@ -231,7 +206,6 @@ def _process_has_returned( return result -PROCESS_PATCH = SingleModulePatch('process').add_method( - 'BaseProcess', '_bootstrap', wrap_bootstrap, -) +PROCESS_PATCH = SingleModulePatch('process') +PROCESS_PATCH.add_method('BaseProcess', '_bootstrap', wrap_bootstrap) PROCESS_PATCH.add_method('BaseProcess', 'terminate', wrap_terminate) diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index b6dd2ab1..f7776531 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -228,8 +228,7 @@ suffix = '' # - `multiprocessing.patches.pool` (bool): # Whether to patch `multiprocessing.pool.worker()` so that each child -# process write profiling output when it runs out of tasks (POSIX) or -# as each task is completed (Windows) +# process write profiling output as each task is completed pool = true # - `multiprocessing.patches.process` (bool): # Whether to patch `multiprocessing.process.BaseProcess`, so that each @@ -242,12 +241,6 @@ process = true # that the internal logs of `multiprocessing` are teed to the session # cache's debug logs logging = false -# - `multiprocessing.patches.child_pids` (bool): -# Whether to patch `multiprocessing.pool.worker()` and -# `multiprocessing.pool.Pool` so that the the parent process keeps -# track of the workload sent to each child, suppressing stray warnings -# about empty output files where appropriate -child_pids = true # Polling controls, used for the `pool` patch diff --git a/setup.py b/setup.py index c75f8125..05e3fdc2 100755 --- a/setup.py +++ b/setup.py @@ -320,11 +320,11 @@ def run_cythonize(force=False): '=line_profiler._child_process_profiling' '.multiprocessing_patches._mandatory_patches' ':RunpyPatch', - - 'child_pids' + '__pool_worker_pid' '=line_profiler._child_process_profiling' - '.multiprocessing_patches._optional_patches' - ':CHILD_PIDS_PATCH', + '.multiprocessing_patches._mandatory_patches' + ':POOL_WORKER_PID_PATCH', + 'logging' '=line_profiler._child_process_profiling' '.multiprocessing_patches._optional_patches' diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index d3489859..a0dc622b 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -53,7 +53,7 @@ create_runpy_wrapper, ) from line_profiler._child_process_profiling.multiprocessing_patches import ( - MPConfig, Timeout as MPTimeout, _PATCHED_MARKER, _PATCHES as MP_PATCHES, + MPConfig, _PATCHED_MARKER as MP_PATCHED_MARKER, _PATCHES as MP_PATCHES, ) from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler.cleanup import Cleanup @@ -1827,11 +1827,10 @@ def get_streams( # `_test_apply_mp_patches()` cw = stack.enter_context(_check_warnings()) cw.forbid_warnings('.*resource_tracker', module='multiprocessing') - if _DEFAULT_MP_CONFIG.patches['child_pids']: - cw.forbid_warnings( - r'.* file\(s\) .* empty', - category=UserWarning, module='line_profiler', - ) + cw.forbid_warnings( + r'.* file\(s\) .* empty', + category=UserWarning, module='line_profiler', + ) if timeout: cw.suppress_warnings( r'.*multi-?threaded.*fork\(\)', @@ -2394,7 +2393,7 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: _GLOBAL_MINIMAL_PATCHES = { - 'multiprocessing': frozenset({_PATCHED_MARKER}), + 'multiprocessing': frozenset({MP_PATCHED_MARKER}), } # Get patches that are dynamically resolved: while these patches are # always applied, some of the patch targets are @@ -2936,9 +2935,6 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # Counts from the one final sum over the parallel results nloops_expected += nprocs - # Note: manually handle the error here instead of using - # `pytest.raises()` since we want certain `RuntimeError`s to be - # propagated and handled by `@pytest.mark.retry` fail_msg = 'forced failure' try: result = called_func(n) @@ -2982,7 +2978,6 @@ def _test_apply_mp_patches( patch_pool: bool | None = None, patch_process: bool | None = None, intercept_logs: bool | None = None, - trace_pids: bool | None = None, *, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], **kwargs @@ -2996,7 +2991,7 @@ def _test_apply_mp_patches( patches = cast(dict[str, bool], _DEFAULT_MP_CONFIG.patches.copy()) for name, applied in { 'pool': patch_pool, 'process': patch_process, - 'logging': intercept_logs, 'child_pids': trace_pids, + 'logging': intercept_logs, }.items(): if applied is not None: patches[name] = applied @@ -3004,13 +2999,10 @@ def _test_apply_mp_patches( with _check_warnings() as cw: cw.forbid_warnings('.*resource_tracker', module='multiprocessing') - if 'child_pids' in mp_patches: - # With PID bookkeeping we should be able to weed out all the - # child processes which didn't perform any work - cw.forbid_warnings( - r'.* file\(s\) .* empty', - category=UserWarning, module='line_profiler', - ) + cw.forbid_warnings( + r'.* file\(s\) .* empty', + category=UserWarning, module='line_profiler', + ) if start_method == 'fork': # The `@_timeout` decorator spins up a new thread for # executing `_test_apply_mp_patches_inner()`; explicitly @@ -3032,12 +3024,8 @@ def _test_apply_mp_patches( # parametrizations don't matter + _Params.new(('intercept_logs', 'label1'), [(True, 'with-logging'), (False, 'no-logging')], - defaults=(None, 'default-logging')) - # Same deal with `trace_pids = child_pids` - + _Params.new(('trace_pids', 'label2'), - [(True, 'with-child_pids'), (False, 'no-child-pids')], - defaults=(None, 'default-child-pids'))).sorted() -@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label3'), + defaults=(None, 'default-logging'))).sorted() +@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label2'), [(True, True, 'pool-and-process'), (True, False, 'pool-only'), (False, True, 'process-only')]) @@ -3051,12 +3039,10 @@ def test_apply_mp_patches_success( patch_pool: bool, patch_process: bool, intercept_logs: bool | None, - trace_pids: bool | None, n: int, nprocs: int, label1: str, label2: str, - label3: str, ) -> None: """ Test that :py:func:`line_profiler._child_process_profiling\ @@ -3070,7 +3056,6 @@ def test_apply_mp_patches_success( patch_pool, patch_process, intercept_logs, - trace_pids, tmp_path_factory=tmp_path_factory, create_cache=create_cache, ext_module_object=ext_module_object, From 96d1f525b5eba65b4f119c7e04cc9a63de0325a4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 26 Jun 2026 03:10:02 +0200 Subject: [PATCH 134/165] No longer using `@pytest.mark.retry` tests/test_child_procs.py::test_apply_mp_patches_failure() Removed `@pytest.mark.retry` marker tests/conftest.py, tests/test_child_procs.py Removed because `@pytest.mark.retry` is no longer used anywhere --- tests/conftest.py | 790 ---------------------------- tests/test_child_procs.py | 17 - tests/test_retry_tests.py | 1039 ------------------------------------- 3 files changed, 1846 deletions(-) delete mode 100644 tests/conftest.py delete mode 100644 tests/test_retry_tests.py diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 205ecf8d..00000000 --- a/tests/conftest.py +++ /dev/null @@ -1,790 +0,0 @@ -""" -A simple :py:deco:`pytest.mark.retry` decorator. -Function-scoped fixtures are re-fetched between retries. - -Note: - - This file is designed to also function as a standalone - ``conftest.py`` file. - - - Adapted from `pytest-mark-retry`_. - -.. _pytest-mark-retry: https://gitlab.com/TTsangSC/pytest-mark-retry -""" -from __future__ import annotations - -import ast -import dataclasses -import os -import sys -import warnings -from collections.abc import ( - Callable, Collection, Generator, Hashable, Iterable, Mapping, -) -from functools import cached_property, lru_cache, partial -from importlib.util import find_spec -from inspect import Signature, signature -from operator import contains -from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, ClassVar, Literal, Protocol, TypedDict, TypeVar, cast, overload, -) -from typing_extensions import Self - -import pytest -from _pytest.compat import NOTSET -from _pytest.fixtures import SubRequest -from _pytest.nodes import Node -from _pytest.scope import Scope -from _pytest.unittest import TestCaseFunction -try: - from pytest import TerminalReporter # type: ignore -except ImportError: # pytest < ~8.4 - from _pytest.terminal import TerminalReporter # type: ignore - - -__all__ = ( - 'RetryMarker', - 'RetryMarkerWarning', - 'RetryConditionFailure', - 'pytest_addhooks', - 'pytest_configure', - 'pytest_terminal_summary', -) - -_Status = Literal['passed', 'failed', 'skipped'] -_Require = Literal['any', 'all'] -F = TypeVar('F', bound=TestCaseFunction) -FCls = TypeVar('FCls', bound='type[TestCaseFunction]') -T = TypeVar('T') - -_FUNCTION_SCOPE = Scope.Function - - -class _PyfuncCallImpl(Protocol): - def __call__(self, *, pyfuncitem: pytest.Function) -> Any: - ... - - -class _RetryError(RuntimeError): - """ - Base class for errors associated with retrying tests. - """ - - -class RetryMarkerWarning(_RetryError, UserWarning): - """ - Warning issued when the :deco:`pytest.mark.retry` markers on a test - fail to resolve to a valid :py:class:`RetryMarker` instance. - """ - @classmethod - def warn_from_error( - cls, xc: Exception, *args, **kwargs - ) -> None: - msg = 'disregarding invalid `@pytest.mark.retry` marker' - msg = f'{msg}: ({_format_exception(xc)})' - if 'PYTEST_CURRENT_TEST' in os.environ: - msg = f'{os.environ["PYTEST_CURRENT_TEST"]}: {msg}' - if sys.version_info < (3, 12): # Compatibility - kwargs.pop('skip_file_prefixes', None) - return warnings.warn(msg, cls, *args, **kwargs) - - -class RetryConditionFailure(_RetryError): - """ - Error raised when an attempt to retry a test failed because we - can't :py:func:`eval` the condition. - """ - def __init__( - self, - previous_error: Exception, - condition_error: Exception, - # Should't be here if the condition isn't a string, but whatever - condition: str | bool | None = None - ) -> None: - self.previous_error = previous_error - self.condition_error = condition_error - self.condition = condition - super().__init__(self._format_message()) - - condition_error.__cause__ = previous_error - self.__cause__ = condition_error - - def _format_message(self) -> str: - prev = _format_exception(self.previous_error) - if len(prev.split()) > 1: - prev = f'({prev})' - condition = _format_exception(self.condition_error) - if self.condition: - condition = f'(condition: {self.condition!r} -> {condition})' - return '{} -> {}'.format(prev, condition) - - -@dataclasses.dataclass -class _RetryEntry: - func: pytest.Function - retries: int - status: _Status - - def _get_name(self, with_params: bool) -> str: - path, prefix = self._name_prefixes - if with_params: - name = self.func.name - else: - name = self.func.originalname - if prefix: - name = f'{prefix}.{name}' - if path: - name = f'{path}::{name}' - return name - - @classmethod - def add_entry(cls, func: pytest.Function, *args, **kwargs) -> Self: - assert func.config is not None - entry = cls(func, *args, **kwargs) - entry.get_entries(func.config).append(entry) - return entry - - @staticmethod - def get_entries(config: pytest.Config) -> list[_RetryEntry]: - return config.stash.setdefault(_RETRY_ENTRIES_KEY, []) - - @property - def full_name(self) -> str: - return self._get_name(True) - - @property - def full_original_name(self) -> str: - return self._get_name(False) - - @cached_property - def _name_prefixes(self) -> tuple[str, str]: - chunks: list[str] = [] - node: Node | None = self.func.parent - path = '' - seen: set[int] = set() - while True: - if id(node) in seen: - break - else: - seen.add(id(node)) - if isinstance(node, (pytest.Module, pytest.Package)): - if node.path: - npath = node.path - try: - npath = npath.relative_to(Path.cwd()) - except ValueError: # Not a subpath - pass - path = str(npath) - else: - path = repr(node) - break - name: str | None = getattr(node, 'name', None) - if not name: - break - chunks.append(name) - return path, '.'.join(reversed(chunks)) - - -_RETRY_ENTRIES_KEY = pytest.StashKey[list[_RetryEntry]]() - - -def _retry_marker_sig_helper( - retries=1, *, - exceptions=None, reset_fixtures=None, condition=None, require=None, -): - """ - Dummy callable helping with :py:meth:`RetryMarker.from_test_func`, - so that we can handle marker stacking in a sane way withou having to - special-case the defaults of the class constructor. - """ - pass - - -if sys.version_info >= (3, 10): - _keyword = partial(dataclasses.field, kw_only=True) -else: - _keyword = dataclasses.field - - -class _RetryMarkerArgs(TypedDict, total=False): - retries: int - exceptions: type[Exception] | tuple[type[Exception], ...] - reset_fixtures: bool | Collection[str] - condition: str | bool | None - require: _Require - - -@dataclasses.dataclass -class RetryMarker: - """ - Object representing the :deco:`pytest.mark.retry` marks on a test - function, managing test retries. - - Attributes: - retries (int): - Number of retries to attempt; should be positive. - exceptions (type[Exception] | tuple[type[Exception], ...]): - "Allowed" exception type(s) which result in retries; - mismatching exceptions are propagated normally, resuling in - a failure - reset_fixtures (bool | Collection[str]): - Whether to reset the function-scoped fixtures when retrying; - if a collection of names, only reset matching fixtures - condition (str | bool | None): - Only attempt retries if this is true (or :py:const:`None`); - if a string, it is :py:func:`eval`-ed to the condition - before each retry using the globals of the test function, - and the fixtures and parametrizations as the locals - require (Literal['any', 'all']): - Whether 'any' or 'all' attempts to run a test function - should pass for the test to pass - - See also: - :py:meth:`.from_arguments` for examples - """ - retries: int = 1 - exceptions: type[Exception] | tuple[type[Exception], ...] = ( - _keyword(default=Exception) - ) - reset_fixtures: bool | Collection[str] = _keyword(default=True) - condition: str | bool | None = _keyword(default=None) - require: _Require = _keyword(default='any') - - name: ClassVar[str] = 'retry' - _sig: ClassVar[Signature] = signature(_retry_marker_sig_helper) - - def __post_init__(self) -> None: - # Normalize `.retries` - try: - self.retries = max(0, int(self.retries)) - except Exception as e: - msg = f'.retries = {self.retries!r}' - msg = f'{msg}: not a valid number {_format_exception(e)}' - raise TypeError(msg).with_traceback(e.__traceback__) - # Check `.exceptions` - if isinstance(self.exceptions, tuple): - xc: tuple[type[Exception], ...] = self.exceptions - else: - xc = self.exceptions, - if not all( - isinstance(X, type) and issubclass(X, Exception) for X in xc - ): - raise TypeError( - f'.exceptions = {self.exceptions!r}: ' - 'expected an exception type or a tuple thereof' - ) - # Check `.condition` - if isinstance(self.condition, str): - try: - ast.parse(self.condition, mode='eval') - except Exception as e: - msg = f'.condition = {self.condition!r}' - msg = f'{msg}: not a valid expression ({_format_exception(e)})' - raise ValueError(msg).with_traceback(e.__traceback__) - # Check `.require` - if self.require not in ('all', 'any'): - msg = f'.require = {self.require!r}: expected \'any\' or \'all\'' - raise TypeError(msg) - - def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: - """ - Manage the call(s) to a function. - - Args: - impl (Callable): - Implementation of - ``pytest_pyfunc_call(pyfuncitem: pytest.Function) \ --> Any`` - func (pytest.Function): - Test function item - - Returns: - Value of the first successful call to - ``pytest_pyfunc_call(pyfuncitem=func)`` - """ - check_fixture_name: Callable[[str], bool] - reset_fixtures: Callable[[pytest.Function], None] - if self.reset_fixtures: - if isinstance(self.reset_fixtures, Collection): - check_fixture_name = partial(contains, self.reset_fixtures) - else: - check_fixture_name = lambda _: True # noqa: E731 - reset_fixtures = partial( - self._reset_between_retries, - reset_fixtures=True, - should_reset=check_fixture_name, - ) - else: - reset_fixtures = self._reset_between_retries - - result: Any = None - xc: Exception | None = None - for i in range(1 + self.retries): - if i: - reset_fixtures(func) - cond, error = self._check_condition(self.condition, func) - if error: # Bail - # XXX: would be nice if we can directly force an - # internal error, but that doesn't seem to be - # possible from within `pytest_pyfunc_call()`; - # directly calling `pytest_internalerror()` - # results in botched teardown and weird - # tracebacks, and leaves the test session in a - # bad state... - assert xc is not None - raise RetryConditionFailure(xc, cond, self.condition) - if not cond: - i -= 1 - break - try: - result = impl(pyfuncitem=func) - except self.exceptions as e: - # `ty` doesn't agree that `e` is an exception (#3432)... - xc = cast(Exception, e) - if self.require == 'all': - break - except Exception as e: # Uncaught exc. -> break to raise - xc = e - break - else: # Correct execution -> break to return - xc = None - if self.require == 'any': - break - if i: - if xc is None: - status: _Status = 'passed' - elif isinstance(xc, pytest.skip.Exception): - status = 'skipped' - else: - status = 'failed' - _RetryEntry.add_entry(func, i, status) - if xc is None: - return result - else: - raise xc - - @staticmethod - def _check_condition( - condition: str | bool | None, func: pytest.Function, - ) -> tuple[Any, Literal[False]] | tuple[Exception, Literal[True]]: - if condition in (True, None): # Always retry - return (True, False) - if condition in (False,): # Never retry - return (False, False) - - if TYPE_CHECKING: # Help narrowing - assert isinstance(condition, str) - global_ns: dict[str, Any] | None = None - try: - global_ns = func.obj.__globals__ - except AttributeError: # Not a `types.FunctionType` - pass - local_ns = func.funcargs - try: - return (eval(condition, global_ns, local_ns), False) - except Exception as e: - return (e, True) - - @staticmethod - def _reset_between_retries( - func: pytest.Function, - reset_fixtures: bool = False, - should_reset: Callable[[str], bool] = lambda _: False, - ) -> None: - """ - Note: - This makes HEAVY use of :py:mod`_pytest` internals. - """ - def cleanup_fixture(fdef: pytest.FixtureDef[Any]) -> None: - if not ( - fdef.scope == 'function' - and should_reset(fdef.argname) - and getattr(fdef, 'cached_result', None) is not None - ): - return - fdef.cached_result = None - finalize(fdef) - - def finalize(fdef: pytest.FixtureDef[Any]) -> None: - assert fdef.scope == 'function' - - # Plagiarized code from - # `FixtureRequest._get_active_fixture_def()` - try: - callspec = func.callspec - except AttributeError: - callspec = None - if callspec is not None and fdef.argname in callspec.params: - value = callspec.params[fdef.argname] - index = callspec.indices[fdef.argname] - else: - value, index = NOTSET, 0 - - with warnings.catch_warnings(): - warnings.simplefilter( - 'ignore', pytest.PytestDeprecationWarning, - ) - fdef.finish(SubRequest( - request=func._request, - scope=_FUNCTION_SCOPE, - param=value, - param_index=index, - fixturedef=fdef, - )) - - def unique( - items: Iterable[T], key: Callable[[T], Hashable] = id, - ) -> Generator[T, None, None]: - seen: set[Hashable] = set() - for item in items: - hashed = key(item) - if hashed in seen: - continue - seen.add(hashed) - yield item - - def iter_all_fixture_defs( - func: pytest.Function, - ) -> Generator[pytest.FixtureDef[Any], None, None]: - fdef_mapping: Mapping[Any, Iterable[pytest.FixtureDef[Any]]] - # Somehow `mypy` doesn't trust the below but `ty` does... - for fdef_mapping in [ # type:ignore[assignment] - func._fixtureinfo.name2fixturedefs, - func._request._arg2fixturedefs, - func.session._fixturemanager._arg2fixturedefs, - ]: - for fixture_defs in fdef_mapping.values(): - yield from fixture_defs - - if reset_fixtures: - # Beside clearing `.funcargs`, `._initrequest()` also resets - # the `TopRequest` instance that `func` has (`._request`) - func._initrequest() - for fixture_def in unique(iter_all_fixture_defs(func)): - cleanup_fixture(fixture_def) - else: - # Fixture values will naturally refill, possibly from caches - func.funcargs.clear() - func.setup() - - @classmethod - def pytest_pyfunc_call(cls, pyfuncitem: pytest.Function) -> Any: - """ - Run the :py:class:`pytest.Function` object with the requisite - number of retries if necessary. - """ - pm = pyfuncitem.config.pluginmanager - pytest_pyfunc_call: _PyfuncCallImpl = pm.subset_hook_caller( - 'pytest_pyfunc_call', [cls], - ) - try: - helper = cls.from_test_func(pyfuncitem) - except Exception as e: - # Level 1 is the `.warn_from_error()` frame, 2 is here, 3 is - # where the error actually happened - warn = RetryMarkerWarning.warn_from_error - skip_ = {_find_module_path('_pytest'), _find_module_path('pluggy')} - skip = cast(tuple[str, ...], tuple(skip_ - {None})) - warn(e, stacklevel=3, skip_file_prefixes=skip) - else: - if helper.is_active: - return helper.manage_call(pytest_pyfunc_call, pyfuncitem) - return pytest_pyfunc_call(pyfuncitem=pyfuncitem) - - @classmethod - def from_test_func(cls, func: pytest.Function, /) -> Self: - """ - Returns: - Instance combining the stack of :deco:`pytest.mark.retry` - decorators on the :py:class:`pytest.Function`. - """ - marks = (m for m in func.iter_markers() if m.name == cls.name) - return cls.from_arguments(cls._get_marker_args(mark) for mark in marks) - - @classmethod - @overload - def from_arguments(cls, args: Iterable[_RetryMarkerArgs] = (), /) -> Self: - ... - - @classmethod - @overload - def from_arguments(cls, *args: _RetryMarkerArgs) -> Self: - ... - - @classmethod - def from_arguments(cls, *args) -> Self: - """ - Invocations: - (, ...) -> RetryMarker - ([, ...]) -> RetryMarker - - Examples: - >>> empty = RetryMarker.from_arguments() - >>> assert not empty.retries - >>> assert empty == RetryMarker.from_arguments([]) - - >>> default = RetryMarker.from_arguments({}) - >>> assert default.retries == 1 - >>> assert default.exceptions in ((Exception,), Exception) - >>> assert default.reset_fixtures == True - >>> assert default.condition is None - >>> assert default.require == 'any' - >>> assert default.is_active - >>> assert default == RetryMarker.from_arguments([{}]) - - Some arguments result in inactive instances (i.e. no - retries): - - >>> bad_xc = RetryMarker.from_arguments({'exceptions': ()}) - >>> assert not bad_xc.exceptions - >>> assert not bad_xc.is_active - - >>> bad_retries = RetryMarker.from_arguments( - ... {}, {}, {'retries': -5}, - ... ) - >>> assert not bad_retries.retries - >>> assert not bad_retries.is_active - - >>> bad_cond = RetryMarker.from_arguments( - ... {'condition': False}, - ... ) - >>> assert bad_cond.condition == False - >>> assert not bad_cond.is_active - - Congruent values are unioned: - - >>> stacked_xcs = RetryMarker.from_arguments([ - ... {'exceptions': ()}, - ... {'exceptions': ValueError}, - ... {'retries': 3, 'exceptions': (TypeError, OSError)}, - ... ]) - >>> assert stacked_xcs.retries == 5 - >>> assert set(stacked_xcs.exceptions) == { - ... ValueError, TypeError, OSError, - ... } - - >>> stacked_resets_1 = RetryMarker.from_arguments( - ... {'reset_fixtures': ['foo', 'bar']}, - ... {'reset_fixtures': ['baz']}, - ... ) - >>> sorted(stacked_resets_1.reset_fixtures) - ['bar', 'baz', 'foo'] - - Incongruent values override one another: - - >>> stacked_resets_2 = RetryMarker.from_arguments( - ... {'reset_fixtures': ['foo', 'bar']}, - ... {'reset_fixtures': False}, - ... ) - >>> assert stacked_resets_2.reset_fixtures == False - - >>> stacked_conditions = RetryMarker.from_arguments( - ... {'condition': 'foo==bar'}, - ... {'condition': False}, - ... ) - >>> assert stacked_conditions.condition == False - >>> assert not stacked_conditions.is_active - - >>> stacked_requires = RetryMarker.from_arguments( - ... {'require': 'any'}, - ... {'require': 'all'}, - ... ) - >>> assert stacked_requires.require == 'all' - """ - retries: int = 0 - xc: set[type[Exception]] | None = None - reset_fixtures: bool | set[str] = True - condition: bool | str | None = None - require: _Require | None = None - - if args: - if isinstance(args[0], Mapping): - iter_args = args - else: - assert len(args) == 1 - iter_args = args[0] - else: - iter_args = args - - # `ty` needs some help here... hence the `cast()` - for bound_args in cast(Iterable[_RetryMarkerArgs], iter_args): - retries += bound_args.get('retries', 1) - if 'exceptions' in bound_args: - xc_new = bound_args['exceptions'] - if xc is None: - xc = set() - if isinstance(xc_new, type): - xc.add(xc_new) - else: - xc.update(xc_new) - if 'reset_fixtures' in bound_args: - rf_new = bound_args['reset_fixtures'] - if isinstance(rf_new, Collection): - if isinstance(reset_fixtures, Collection): - reset_fixtures.update(rf_new) - else: - reset_fixtures = set(rf_new) - else: - reset_fixtures = bool(rf_new) - if 'condition' in bound_args: - condition = bound_args['condition'] - if 'require' in bound_args: - require = bound_args['require'] - - kwargs: _RetryMarkerArgs = { - 'retries': retries, 'reset_fixtures': reset_fixtures, - } - if xc is not None: - kwargs['exceptions'] = tuple(xc) - if condition is not None: - kwargs['condition'] = condition - if require is not None: - kwargs['require'] = require - return cls(**kwargs) - - @classmethod - def _get_marker_args(cls, mark: pytest.Mark) -> _RetryMarkerArgs: - args = cls._sig.bind(*mark.args, **mark.kwargs).arguments - return cast(_RetryMarkerArgs, args) - - @property - def is_active(self) -> bool: - """ - Whether the instance should possibly attempt retries in any - condition - """ - if not self.retries: - return False - if self.exceptions == (): - return False - if self.condition in (False,): - return False - return True - - -def _pluralize(noun: str, count: int, plural: str | None = None) -> str: - if plural is None: - plural = noun + 's' - return f'{count} {noun if count == 1 else plural}' - - -def _format_exception(xc: Exception) -> str: - msg = type(xc).__name__ - if str(xc): - return f'{msg}: {xc}' - return msg - - -@lru_cache() -def _find_module_path(module: str) -> str | None: - spec = find_spec(module) - if spec is None or spec.origin is None: - return None - file = Path(spec.origin) - if not file.exists(): - return None - if file.name == '__init__.py': # Package - file = file.parent - return str(file) - - -def pytest_addhooks(pluginmanager: pytest.PytestPluginManager) -> None: - """ - Register :py:class:`RetryMarker` as a plugin so that its - :py:meth:`pytest_pyfunc_call` method can safely call other - implementations without recursing to itself. - """ - pluginmanager.register(RetryMarker) - - -def pytest_configure(config: pytest.Config) -> None: - """ - Register the :py:deco:`pytest.mark.retry` marker. - """ - help_text = ' '.join(""" - retry(retries=1, *, exceptions=Exception, \ -reset_fixtures=True, condition=None, require='any'): - - mark the test for retrying upon failure. - - Args (all optional): - retries (int): - Max number of retries for the (sub-)test; - exceptions (type[Exception] | tuple[type[Exception], ...]): - Error types which trigger a retry when caught; - reset_fixtures (bool | Collection[str]): - Names of function-scoped fixtures to reset between retries, - `True` reset all such fixtures, - `False` none thereof; - condition (bool | str | None): - Optional condition for retry: - if a boolean, only retry if true; - if a string, only retry if it `eval()`s true - (w/globals of the test module and locals from the fixtures - and parametrizations); - require (Literal['any', 'all']): - If 'any', stop retrying and record a test pass if ANY - attempt passes; - if 'all', only record a test pass if ALL attempts pass - """.split()) - config.addinivalue_line('markers', help_text) - - -def pytest_terminal_summary( - terminalreporter: TerminalReporter, config: pytest.Config, -) -> None: - """ - Write a summary section about rerun tests. - """ - def get_summary(status: _Status, entries: list[_RetryEntry]) -> str: - return f'{_pluralize("test", len(entries))} {status} with retries' - - def group_subtests( - entries: list[_RetryEntry] - ) -> dict[str, list[_RetryEntry]]: - result: dict[str, list[_RetryEntry]] = {} - for entry in entries: - result.setdefault(entry.full_original_name, []).append(entry) - return result - - formatting = {'yellow': True} - write_line: Callable[[str], None] = partial( - terminalreporter.write_line, **formatting # type: ignore - ) - write_header = partial( - terminalreporter.write_sep, '=', 'retries summary', **formatting - ) - write_newline = partial(write_line, '') - try: - verbosity: int = config.get_verbosity() # type: ignore - except AttributeError: # pytest < 8.0 - verbosity = int(config.option.verbose) - retry_entries: dict[_Status, list[_RetryEntry]] = {} - for entry in _RetryEntry.get_entries(config): - retry_entries.setdefault(entry.status, []).append(entry) - if not retry_entries: - return - if verbosity > 0: - write_newline() - write_header() - write_newline() - for status, entries in retry_entries.items(): - write_line(get_summary(status, entries) + ':') - for entry in entries: - write_line( - f' {entry.full_name}: ' - f'retried {_pluralize("time", entry.retries)}' - ) - write_newline() - else: - write_header() - for status, entries in retry_entries.items(): - tests: list[str] = [] - for name, children in group_subtests(entries).items(): - if len(children) == 1: - tests.append(children[0].full_name) - else: - msg = f'{name} ({_pluralize("subtest", len(children))})' - tests.append(msg) - write_line(f'{get_summary(status, entries)}: {", ".join(tests)}') diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index a0dc622b..92b3b5e2 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -3067,23 +3067,6 @@ def test_apply_mp_patches_success( ) -# XXX: on POSIX child processes can hang around for long enough for -# profiling-stats collection to occur somewhat robustly, thanks to -# signal handling. But unfortunately on Windows: -# - When `patch_pool` is true, we wrap the task callables so that they -# always write profiling stats before returning/erroring out. This -# incurs extra overhead, but effectively prevents the reliquishing of -# control back to the parent process before the stats are ready. -# - However, when `patch_pool` is false, we can only try to block/delay -# child-process termination. A timeout is used to prevent indefinite -# waits for them to finish, and there's always the off chance that the -# end-of-process cleanup still haven't finished at the end. -# Hence the conditional need for retries... -@pytest.mark.retry( - retries=2, - condition='_WINDOWS and not patch_pool', - exceptions=(ResultMismatch, MPTimeout), -) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) @pytest.mark.parametrize(('patch_pool', 'patch_process', 'label'), diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py deleted file mode 100644 index a3251671..00000000 --- a/tests/test_retry_tests.py +++ /dev/null @@ -1,1039 +0,0 @@ -""" -Tests to make sure that our :py:deco:`pytest.mark.retry` decorator -works. - -Notes: - This test module is written to work both: - - - When :py:mod:`pytest_mark_retry` (`link`_) is installed from - source along with this file and the rest of the test suite, or - - - In a test directory containing (among other things): - - - This file as a standalone test module, and - - - A ``conftest.py`` containing the content of single-file module - ``pytest_mark_retry.py``. - -.. _link: https://gitlab.com/TTsangSC/pytest-mark-retry -""" -from __future__ import annotations - -import re -import pprint -import textwrap -from collections.abc import Collection, Iterable, Sequence -from dataclasses import dataclass -from functools import cached_property, partial -from importlib.util import find_spec -from operator import attrgetter -from pathlib import Path -from shutil import rmtree -from typing import Any, Literal, cast -from typing_extensions import Self - -import pytest - - -pytest_plugins = ('pytester',) - -_Status = Literal['passed', 'failed', 'skipped'] -_RunPytestMethod = Literal[ - 'runpytest', 'runpytest_inprocess', 'runpytest_subprocess', -] - -PROJECT_MODULE = 'pytest_mark_retry' - -TEST_COUNTERS = """ -from __future__ import annotations -from itertools import count -from typing import Literal - -import pytest - - -@pytest.fixture -def func_scoped_counter() -> count: - return count() - - -@pytest.fixture(scope='module') -def module_scoped_counter() -> count: - return count() - - -@pytest.mark.parametrize( - ('scope', 'n'), - [('func', 0), # This passes - ('func', 2), # This passes with 2 retries - ('func', 6), # This fails with 3 retries - ('module', 4), # This fails with 3 retries (counter now at 3) - ('module', 5)] # This passes with 1 retry (counter now at 5) -) -@pytest.mark.retry(3, reset_fixtures=False) -def test_dynamic_fixtures_persisted( - request: pytest.FixtureRequest, scope: Literal['func', 'module'], n: int, -) -> None: - ''' - Test counter fixtures that are requested dynamically via the - ``request`` fixture; function-scoped fixtures persist between - test retries. - ''' - counter = request.getfixturevalue(scope + '_scoped_counter') - assert next(counter) >= n - - -@pytest.mark.parametrize( - ('scope', 'n'), - [('func', 3), # This passes with 3 retries - ('func', 4), # This fails with 3 retries - ('module', 4), # This passes (counter now at 6) - ('module', 9)] # This passes with 2 retries (counter now at 9) -) -@pytest.mark.retry(3, reset_fixtures=False) -def test_static_fixtures_persisted( - func_scoped_counter: Iterable[int], - module_scoped_counter: Iterable[int], - scope: Literal['func', 'module'], - n: int, -) -> None: - ''' - Test counter fixtures that are requested by name; function-scoped - fixtures persist between test retries. - ''' - if scope == 'func': - counter = func_scoped_counter - else: - counter = module_scoped_counter - assert next(counter) >= n - - -@pytest.mark.parametrize( - ('scope', 'n'), - [('func', 0), # This passes - ('func', 1), # This fails with 1 retry - ('module', 11)] # This passes with 1 retry (counter now at 11) -) -@pytest.mark.retry # Counters reset between retries -def test_dynamic_fixtures_reset( - request: pytest.FixtureRequest, scope: Literal['func', 'module'], n: int, -) -> None: - ''' - Test counter fixtures that are requested dynamically via the - ``request`` fixture; function-scoped fixtures are reset between - test retries. - ''' - counter = request.getfixturevalue(scope + '_scoped_counter') - assert next(counter) >= n - - -@pytest.mark.parametrize( - ('scope', 'n'), - [('func', 0), # This passes - ('func', 1), # This fails with 2 retries - ('module', 14)] # This passes with 2 retries (counter now at 14) -) -@pytest.mark.retry(2) # Ditto above -def test_static_fixtures_reset( - func_scoped_counter: Iterable[int], - module_scoped_counter: Iterable[int], - scope: Literal['func', 'module'], - n: int, -) -> None: - ''' - Test counter fixtures that are requested by name; function-scoped - fixtures are reset between test retries. - ''' - if scope == 'func': - counter = func_scoped_counter - else: - counter = module_scoped_counter - assert next(counter) >= n -""" -TEST_TEARDOWN = """ -from __future__ import annotations - -import os -import tempfile -from collections.abc import Callable, Generator -from functools import partial -from pathlib import Path - -import pytest - - -@pytest.fixture(scope='module') -def my_temp_dir(pytestconfig: pytest.Config) -> Generator[Path, None, None]: - path: Path | None = getattr(pytestconfig.option, 'my_temp_dir', None) - if path is None: - with tempfile.TemporaryDirectory() as tmpdir: - yield Path(tmpdir) - else: - yield path - -@pytest.fixture(scope='module') -def my_log(pytestconfig: pytest.Config) -> Path | None: - path: Path | None = getattr(pytestconfig.option, 'my_log', None) - return path - - -def _tempfile(*args, **kwargs) -> Path: - handle, path = tempfile.mkstemp(*args, **kwargs) - try: - return Path(path) - finally: - os.close(handle) - - -@pytest.fixture -def maketemp( - my_temp_dir: Path, my_log: Path | None, -) -> Generator[Callable[..., Path], None, None]: - paths: list[Path] = [] - - def _maketemp(*args, **kwargs) -> Path: - path = _tempfile(*args, **kwargs) - paths.append(path) - log(f'created tempfile {path}') - return path - - log = partial(_log, _maketemp, my_log) - try: - yield _maketemp - finally: - for path in paths: - path.unlink(missing_ok=True) - log(f'removed tempfile {path}') - - -def _log(maketemp: Any, my_log: Path | None, msg: str) -> None: - chunks: list[str] = [ - os.environ['PYTEST_CURRENT_TEST'], - f'maketemp() @ {id(maketemp):#x}', - msg, - ] - msg = ': '.join(chunks) - print(msg) - if my_log is None: - return - with my_log.open(mode='a') as fobj: - print(msg, file=fobj) - - -@pytest.mark.retry(reset_fixtures=True) -def test_with_fixture_reset( - my_temp_dir: Path, maketemp: Callable[..., Path], -) -> None: - path = maketemp(dir=my_temp_dir) - assert False - - -@pytest.mark.retry(2, reset_fixtures=False) -def test_no_fixture_reset( - my_temp_dir: Path, maketemp: Callable[..., Path], -) -> None: - path = maketemp(dir=my_temp_dir) - assert False -""" -TEST_EXCEPTIONS = """ -from __future__ import annotations - -from collections.abc import Iterable -from typing import Any - -import pytest - - -@pytest.fixture -def items() -> Iterable[Any]: - return iter(['1', None, '', '-1']) - - -@pytest.mark.retry(3, reset_fixtures=('foo',)) # Not resetting `items` -def test_all_xc_types(items: Iterable[Any]) -> None: - ''' - This should pass after 3 retries because the last item fulfills the - criterion. - ''' - assert int(next(items)) < 0 - - -@pytest.mark.retry(3, exceptions=AssertionError, reset_fixtures=()) -def test_one_xc_type(items: Iterable[Any]) -> None: - ''' - This should fail after 1 retry because the second item triggers a - :py:class:`TypeError`. - ''' - assert int(next(items)) < 0 - - -@pytest.mark.retry(reset_fixtures=False) -@pytest.mark.retry(exceptions=TypeError) -@pytest.mark.retry(exceptions=AssertionError) -def test_two_xc_types(items: Iterable[Any]) -> None: - ''' - This should fail after 2 retries because the third item triggers a - :py:class:`ValueError`. - - Note: - The three decorators stack to give 3 retries and to accept both - :py:class:`AssertionError` and :py:class:`TypeError`. - ''' - assert int(next(items)) < 0 - - -@pytest.mark.retry( - 3, - exceptions=(AssertionError, TypeError, ValueError), - reset_fixtures=False, -) -def test_three_xc_types(items: Iterable[Any]) -> None: - ''' - This should pass after 3 retries because the last item fulfills the - criterion, and the preceding errors are all included in the - ``exceptions`` argument to the wrapper. - ''' - assert int(next(items)) < 0 -""" -TEST_CONDITIONS = """ -from __future__ import annotations - -from sys import version_info - -import pytest - - -@pytest.mark.retry(2, condition=(11 % 2)) -def test_concrete_positive_condition() -> None: - ''' - This should fail after 2 retries because its condition is true. - ''' - raise RuntimeError - - -@pytest.mark.retry(condition=('a' in 'foo')) -def test_concrete_negative_condition() -> None: - ''' - This should fail without retries because its condition is false. - ''' - raise RuntimeError - - -@pytest.mark.retry(condition='version_info.major >= 3') -def test_dynamic_positive_condition_test_module_globals() -> None: - ''' - This should fail after 1 retry because the condition evaluates to - true on the test module's ``globals()``. - ''' - raise RuntimeError - - -@pytest.mark.retry(condition='version_info.major < 3') -def test_dynamic_negative_condition_test_module_globals() -> None: - ''' - This should fail without retries because the condition evaluates to - false on the test module's ``globals()``. - ''' - raise RuntimeError - - -@pytest.mark.retry(condition='foo == 1') -def test_bad_dynamic_condition() -> None: - ''' - This should fail without retries because the condition cannot be - evaluated (``NameError: name 'foo' is not defined``). - ''' - raise RuntimeError('bar') - - -@pytest.mark.retry(condition='n % 2') -@pytest.mark.parametrize('n', [0, 1, 2]) -def test_dynamic_condition_test_params(n: int) -> None: - ''' - Subtests ``[0]`` and ``[2]`` (resp. subtest ``[1]``) should fail - without retries (resp. with 1 retry) because the condition evaluates - to false (resp. true) on the test's parametrization. - ''' - raise RuntimeError -""" -TEST_BAD_MARKERS = """ -from __future__ import annotations - -import pytest - - -@pytest.mark.retry(1, 2) # `exceptions` cannot be 2 -def test_passing_bad_exceptions() -> None: - ''' - This test passes with a warning because its retry marker has an - invalid :py:attr:`RetryMarker.exceptions`. - ''' - pass - - -@pytest.mark.retry(foo=1) # No argument named `foo` -def test_passing_stray_arg() -> None: - ''' - This test also passes with a warning because its retry marker has am - stray argument ``foo`` - ''' - pass - - -@pytest.mark.retry(condition='') # Syntax error -def test_failing_bad_condition() -> None: - ''' - This test fails with a warning and without retries, because its - retry marker got a bad :py:attr:`RetryMarker.condition`. - ''' - assert False -""" -TEST_REQUIRE = """ -from __future__ import annotations - -import itertools - -import pytest - - -@pytest.fixture(scope='module') -def counter() -> itertools.count: - return itertools.count() - - -@pytest.fixture -def index(counter: itertools.count) -> int: - return next(counter) - - -@pytest.mark.retry(3) -def test_passing_retry_require_any(index: int) -> None: - ''' - This passes with two retries and leave ``index`` at 2. - ''' - assert index >= 2 - - -@pytest.mark.retry(3, require='any') -def test_failing_retry_require_any(index: int) -> None: - ''' - This fails with three retries and leave ``index`` at 6. - ''' - assert index < 3 - - -@pytest.mark.retry(3, require='all') -def test_failing_retry_require_all(index: int) -> None: - ''' - This fails with zero retries and leave ``index`` at 7. - ''' - # Fails right out the gate, no need to continue retrying - assert index > 7 - - -@pytest.mark.retry(3, require='all') -def test_passing_retry_require_all(index: int) -> None: - ''' - This passes with three retries and leave ``index`` at 11. - ''' - # All attempts pass, but we are instructed to exhaust the retries - assert index > 0 -""" - - -@dataclass -class _TestOutcome: - name: str = '' - status: _Status = 'passed' - retries: int = 0 - - def subtest( - self, - *params: str, - status: _Status | None = None, - retries: int | None = None, - ) -> Self: - if status is None: - status = self.status - if retries is None: - retries = self.retries - name = f'{self.name}[{"-".join(params)}]' - return type(self)(name, status, retries) - - -@dataclass -class _TestModule: - """ - Helper object for running a test module. - """ - name: str - content: str - expected_outcomes: dict[str, list[_TestOutcome]] - pytester: pytest.Pytester - conftest: str | None = None - - def __post_init__(self) -> None: - self.content = self._strip(self.content) - if self.conftest: - self.conftest = self._strip(self.conftest) - - def run( - self, - *args: str, - check_results: bool = False, - check_summary: Literal['verbose', 'concise'] | None = None, - check_warnings: int | None = None, - runner: _RunPytestMethod = 'runpytest', - additional_stdout_lines: Collection[str] = (), - additional_stderr_lines: Collection[str] = (), - ) -> pytest.RunResult: - """ - Args: - *args (str): - Passed to :py:meth:`pytester.Pytester.runpytest` - check_results (bool): - If true, check that the test outcomes are as expected - using :py:meth:`pytester.Pytester.assert_outcomes` - check_summary (bool): - If true, check that the 'retries summary' report section - is written with the expected content indicating test - results and number of retries - check_warnings (int | None): - If an integer and if ``check_results`` is true, also - check that the number of captured warnings match - runner (Literal['runpytest', 'runpytest_inprocess', \ -'runpytest_subprocess']): - The :py:class:`pytest.Pytester` method used to run the - test module - additional_stdout_lines, additional_stderr_lines \ -(Collection[str]): - Additional regex patterns (other than the - automatically-generated ones) to match against the - output streams - - Returns: - :py:class:`pytest.RunResult` object returned by the - :py:class:`pytest.Pytester` method - """ - tempfiles: list[Path] = [] - tempdirs: list[Path] = [] - try: - conftests: list[str] = [] - if not self.marker_plugin_globally_installed: - # If we don't do this the project will be loaded twice - # as a plugin, leading to a clash - conftests.append(self.marker_plugin_path.read_text()) - if self.conftest: - conftests.append(self.conftest) - # Create separate conftest.py in nested subdirs to avoid - # hook-func implementations stepping oer one another - path = self.pytester.path - for i, conftest in enumerate(conftests): - if i: - path /= 'nested' - path.mkdir() - tempdirs.append(path) - conftest_file = path / 'conftest.py' - conftest_file.write_text(conftest) - tempfiles.append(conftest_file) - module = path / f'{self.name}.py' - module.write_text(self.content) - tempfiles.append(module) - result = getattr(self.pytester, runner)(*args, str(module)) - if check_results: - self.check_results(result, check_warnings) - if check_summary is not None: - if check_summary == 'verbose': - checker = self.check_verbose_summary - else: - checker = self.check_concise_summary - checker( - result, - stdout=additional_stdout_lines, - stderr=additional_stderr_lines, - ) - return result - finally: - for path in tempfiles: - try: - path.unlink(missing_ok=True) - except OSError: - pass - else: - print('Removed temppath', path) - for path in reversed(tempdirs): - try: - rmtree(path) - except OSError: - pass - else: - print('Removed tempdir', path) - - def check_results( - self, result: pytest.RunResult, warnings: int | None = None, - ) -> None: - counts: dict[_Status, int] = {} - for outcomes in self.expected_outcomes.values(): - for outcome in outcomes: - counts[outcome.status] = counts.get(outcome.status, 0) + 1 - result.assert_outcomes( - warnings=warnings, **cast(dict[str, int], counts), - ) - - def check_verbose_summary( - self, - result: pytest.RunResult, - stdout: Collection[str] = (), - stderr: Collection[str] = (), - ) -> None: - lines: list[str] = [] - counts: dict[_Status, int] = {} - for outcomes in self.expected_outcomes.values(): - for outcome in outcomes: - lines.append( - f'.*::{re.escape(outcome.name)} +{outcome.status.upper()}', - ) - if not outcome.retries: - continue - counts[outcome.status] = counts.get(outcome.status, 0) + 1 - lines.append(r'.*{}.*retried {} time{}'.format( - re.escape(outcome.name), - outcome.retries, - '' if outcome.retries == 1 else 's', - )) - lines.extend( - self._format_header(status, n) for status, n in counts.items() - ) - self._check_lines(result, [*lines, *stdout], stderr) - - def check_concise_summary( - self, - result: pytest.RunResult, - stdout: Collection[str] = (), - stderr: Collection[str] = (), - ) -> None: - lines: list[str] = [] - counts: dict[_Status, int] = {} - test_names: dict[_Status, dict[str, set[str]]] = {} - consolidated_names: dict[_Status, set[str]] = {} - for parent_test, outcomes in self.expected_outcomes.items(): - for outcome in outcomes: - if outcome.status == 'failed': - lines.append( - f'{outcome.status.upper()} +' - f'.*::{re.escape(outcome.name)}', - ) - if not outcome.retries: - continue - counts[outcome.status] = counts.get(outcome.status, 0) + 1 - ( - test_names - .setdefault(outcome.status, {}) - .setdefault(parent_test, set()) - .add(outcome.name) - ) - - for status, tests in test_names.items(): - for parent_test, subtests in tests.items(): - names = consolidated_names.setdefault(status, set()) - n = len(subtests) - if n == 1: - names.add(*subtests) - else: - names.add('{} ({} subtest{})'.format( - parent_test, n, '' if n == 1 else 's', - )) - - self._check_lines(result, [*lines, *stdout], stderr) - - for status, n in counts.items(): - header = self._format_header(status, n) - names = consolidated_names[status] - print(f'Expecting line in the output: "{header}: <...>"...') - print(f'Expecting these names in said line: {names!r}...') - line = self._find_line(header + ':', str(result.stdout)) - for test_name in names: - assert test_name in line - - @property - def marker_plugin_path(self) -> Path: - return self._source[0] - - @property - def marker_plugin_globally_installed(self) -> bool: - return self._source[1] - - @cached_property - def _source(self) -> tuple[Path, bool]: - sources = { - f'module `{PROJECT_MODULE}`': (self._get_proj_module_path, True), - repr('conftest.py'): (self._get_proj_conftest, False), - } - for src, (get_path, retry_globally_installed) in sources.items(): - try: - path = get_path() - assert 'class RetryMarker' in path.read_text() - print(f'Loaded project source from {src}: {str(path)!r}') - return path, retry_globally_installed - except Exception: - pass - raise RuntimeError( - f'Failed to load the project source from any of: {sources!r}', - ) - - @staticmethod - def _check_lines( - result: pytest.RunResult, - stdout: Collection[str], - stderr: Collection[str], - ) -> None: - for stream, lines in { - 'stdout': list(stdout), 'stderr': list(stderr), - }.items(): - if not lines: - continue - print(f'Expecting these lines in the {stream}: {lines!r}...') - getattr(result, stream).re_match_lines_random(lines) - - @staticmethod - def _find_line(pattern: str, text: str) -> str: - pattern = f'^.*{pattern}.*' - maybe_match = re.search(pattern, text, re.MULTILINE) - if not maybe_match: - raise ValueError(f'Cannot find {pattern!r} in {text!r}') - return maybe_match.group() - - @staticmethod - def _format_header(status: _Status, n: int) -> str: - return '{} test{} {} with retries'.format( - n, '' if n == 1 else 's', status, - ) - - @staticmethod - def _get_proj_conftest() -> Path: # If installed as the conftest - return Path(__file__).parent / 'conftest.py' - - @staticmethod - def _get_proj_module_path() -> Path: # If installed as a module - spec = find_spec(PROJECT_MODULE) - assert spec and spec.origin - return Path(spec.origin) - - @staticmethod - def _strip(text: str) -> str: - return textwrap.dedent(text).strip('\n') - - -def _identical_items_are_adjacent(items: Iterable[Any]) -> bool: - """ - Example: - >>> _identical_items_are_adjacent([]) - True - >>> _identical_items_are_adjacent([1]) - True - >>> _identical_items_are_adjacent([1, 10]) - True - >>> _identical_items_are_adjacent([1, 10, 1]) - False - >>> _identical_items_are_adjacent('AAcCb') - True - >>> _identical_items_are_adjacent('AcCAb') - False - """ - past: set[Any] = set() - sentinel = object() - last: Any = sentinel - for item in items: - if last is not sentinel and last != item: - past.add(last) - if item in past: - return False - last = item - return True - - -def _outcomes_to_outcome_dict( - outcomes: Iterable[_TestOutcome], -) -> dict[str, list[_TestOutcome]]: - """ - Example: - >>> o0 = _TestOutcome('foo', 'passed', 0) - >>> o1 = _TestOutcome('bar[1-2-3]', 'failed', 1) - >>> o2 = _TestOutcome('bar[4-5-6]', 'passed', 2) - >>> outcomes = {'foo': [o0], 'bar': [o1, o2]} - >>> assert _outcomes_to_outcome_dict([o1, o0, o2]) == outcomes - """ - result: dict[str, list[_TestOutcome]] = {} - for outcome in outcomes: - name = outcome.name - if name.endswith(']') and '[' in name: # Subtest - base_name, *_ = name.partition('[') - else: - base_name = name - result.setdefault(base_name, []).append(outcome) - return result - - -@pytest.fixture -def counters_module(pytester: pytest.Pytester) -> _TestModule: - dynamic_p = _TestOutcome('test_dynamic_fixtures_persisted').subtest - static_p = _TestOutcome('test_static_fixtures_persisted').subtest - dynamic_r = _TestOutcome('test_dynamic_fixtures_reset').subtest - static_r = _TestOutcome('test_static_fixtures_reset').subtest - outcomes = _outcomes_to_outcome_dict([ - dynamic_p('func-0'), - dynamic_p('func-2', retries=2), - dynamic_p('func-6', status='failed', retries=3), - dynamic_p('module-4', status='failed', retries=3), - dynamic_p('module-5', retries=1), - static_p('func-3', retries=3), - static_p('func-4', status='failed', retries=3), - static_p('module-4'), - static_p('module-9', retries=2), - dynamic_r('func-0'), - dynamic_r('func-1', status='failed', retries=1), - dynamic_r('module-11', retries=1), - static_r('func-0'), - static_r('func-1', status='failed', retries=2), - static_r('module-14', retries=2), - ]) - return _TestModule('test_counters', TEST_COUNTERS, outcomes, pytester) - - -@pytest.fixture -def teardown_module(pytester: pytest.Pytester) -> _TestModule: - outcomes = _outcomes_to_outcome_dict([ - _TestOutcome('test_no_fixture_reset', 'failed', 2), - _TestOutcome('test_with_fixture_reset', 'failed', 1), - ]) - cf = """ - from __future__ import annotations - - from pathlib import Path - - import pytest - - - def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption( - '--my-temp-dir', - type=Path, - help=f'persisted tempdir location for {__file__!r}', - ) - parser.addoption( - '--my-log', - type=Path, - help=f'log file location for tempfile creation/deletion', - ) - """ - return _TestModule('test_teardown', TEST_TEARDOWN, outcomes, pytester, cf) - - -@pytest.fixture -def exceptions_module(pytester: pytest.Pytester) -> _TestModule: - outcomes = _outcomes_to_outcome_dict([ - _TestOutcome('test_all_xc_types', retries=3), - _TestOutcome('test_one_xc_type', 'failed', 1), - _TestOutcome('test_two_xc_types', 'failed', 2), - _TestOutcome('test_three_xc_types', retries=3), - ]) - return _TestModule('test_exceptions', TEST_EXCEPTIONS, outcomes, pytester) - - -@pytest.fixture -def conditions_module(pytester: pytest.Pytester) -> _TestModule: - test = partial(_TestOutcome, status='failed') - param_test_name = 'test_dynamic_condition_test_params' - param_test = partial(test(param_test_name).subtest, status='failed') - outcomes = _outcomes_to_outcome_dict([ - test('test_concrete_positive_condition', retries=2), - test('test_concrete_negative_condition'), - test('test_dynamic_positive_condition_test_module_globals', retries=1), - test('test_dynamic_negative_condition_test_module_globals'), - test('test_bad_dynamic_condition'), - param_test('0'), - param_test('1', retries=1), - param_test('2'), - ]) - return _TestModule('test_conditions', TEST_CONDITIONS, outcomes, pytester) - - -@pytest.fixture -def bad_markers_module(pytester: pytest.Pytester) -> _TestModule: - outcomes = _outcomes_to_outcome_dict([ - _TestOutcome('test_passing_bad_exceptions'), - _TestOutcome('test_passing_stray_arg'), - _TestOutcome('test_failing_bad_condition', 'failed'), - ]) - return _TestModule('test_bad', TEST_BAD_MARKERS, outcomes, pytester) - - -@pytest.fixture -def require_module(pytester: pytest.Pytester) -> _TestModule: - outcomes = _outcomes_to_outcome_dict([ - _TestOutcome('test_passing_retry_require_any', retries=2), - _TestOutcome('test_failing_retry_require_any', 'failed', 3), - _TestOutcome('test_failing_retry_require_all', 'failed'), - _TestOutcome('test_passing_retry_require_all', retries=3), - ]) - return _TestModule('test_require', TEST_REQUIRE, outcomes, pytester) - - -@pytest.mark.parametrize('verbose', [True, False]) -def test_fixture_scoping(counters_module: _TestModule, verbose: bool) -> None: - """ - Test that the decorator correctly handles scoped fixtures. - """ - run = partial(counters_module.run, check_results=True, check_warnings=0) - if verbose: - run('--verbose', check_summary='verbose') - else: - run(check_summary='concise') - - -def test_fixture_teardown( - tmp_path_factory: pytest.TempPathFactory, teardown_module: _TestModule, -) -> None: - """ - Test that the decorator correctly handles teardown for additional - fixture copies incurred by retries; in particular, superseded - function-scoped fixtures should be torn down before their - replacements are set up. - """ - Stage = Literal['setup', 'call', 'teardown'] - - @dataclass - class LogEntry: - test: str - stage: Stage - fixture_id: int - msg: str - - @classmethod - def parse_line(cls, line: str) -> Self: - test, ident, *remainder = line.split(': ') - msg = ': '.join(remainder) - test_match = re.fullmatch( - r'(.+) +\((setup|call|teardown)\)', test, - ) - assert test_match - test, stage = test_match.group(1, 2) - assert stage in ('setup', 'call', 'teardown') - ident_match = re.fullmatch( - r'maketemp\(\) @ 0x([0-9a-f]+)', ident, - ) - assert ident_match - fixture_id = int(ident_match.group(1), base=16) - return cls(test, cast(Stage, stage), fixture_id, msg) - - tempdir = tmp_path_factory.mktemp('my_temp') - log = tempdir / 'tempfiles.log' - teardown_module.run( - '--verbose', f'--my-temp-dir={tempdir}', f'--my-log={log}', - check_results=True, check_summary='verbose', check_warnings=0, - ) - - # Check that all the tempfiles ahve been wiped - files = {path.name for path in tempdir.iterdir()} - assert not (files - {log.name}) - - # Check that tempfiles are deleted as soon as the fixture value - # that created them went obsolete, before the next rerun; - # we can verify that by checking that the ids of the `makefile()` - # fixtures appear in contiguous blocks - - # Note: there seems to be a weird corner case where neighboring - # tests may reuse the same fixture id (see `line_profiler` failing - # job 73520441960 in pipeline 25091142386); probably has to do with - # object lifetime. - # So instead of just checking the `fixture_id`, also consult - # `test`; it suffices to see that WITHIN THE SAME TEST we don't have - # fixture values stepping over one another - with log.open() as fobj: - entries = [LogEntry.parse_line(line.rstrip('\n')) for line in fobj] - pprint.pprint(entries) - for fields in ('test', 'stage'), ('test', 'fixture_id'): - getter = attrgetter(*fields) - values = [getter(entry) for entry in entries] - assert _identical_items_are_adjacent(values), ( - f'Inconsistency in {fields} order: {values!r}' - ) - - -def test_exception_restrictions(exceptions_module: _TestModule) -> None: - """ - Test that the decorator correctly handles failures owing to - different exception classes. - """ - exceptions_module.run( - '--verbose', - check_results=True, check_summary='verbose', check_warnings=0, - ) - - -def test_retry_conditions(conditions_module: _TestModule) -> None: - """ - Test that the decorator correctly handles retry conditions. - """ - # `test_bad_dynamic_condition()` should have failed with a - # `RetryConditionFailure`, listing the error encountered in the last - # trial and the error encountered when `eval()`-ing the condition - # (Note: grepping for the error details in the short test summary is - # fragile since it may be elided or even entirely omitted; so we - # just use a separate pattern to grep it from the tracebacks) - lines = [ - r'.*RetryConditionFailure: \(RuntimeError: bar\) ' - r"-> \(condition: 'foo == 1' -> NameError: .*'foo'.*\)", - ] - conditions_module.run( - '--verbose', - check_results=True, check_summary='verbose', check_warnings=0, - additional_stdout_lines=lines, - ) - - -def test_bad_markers(bad_markers_module: _TestModule) -> None: - """ - Test that the decorator gracefully handles incorrect constructions. - """ - stdout = bad_markers_module.run( - '--verbose', - check_results=True, check_summary='verbose', check_warnings=3, - ).stdout - # Check the warnings emitted - # (Since we want to match across multiple lines we can't use - # `additional_stdout_lines`) - errors: str | Sequence[str] - pattern = ( - '{0}\n' - r'.*RetryMarkerWarning: .*{0}.*: disregarding .* marker: \(.*{1}.*\)' - ) - for test, errors in { - 'test_passing_bad_exceptions': [ - r'TypeError: \.exceptions = .*2.*: expected .*exception', - r'TypeError: .*not iterable', - r'TypeError: too many positional arguments', - r'TypeError: .*takes 1 positional argument but' - ], - 'test_passing_stray_arg': - r'TypeError: .*unexpected keyword argument \'foo\'', - 'test_failing_bad_condition': - r'ValueError: \.condition = \'\': not a valid expression ' - r'\(SyntaxError.*\)', - }.items(): - if isinstance(errors, str): - errors = [errors] - messages = [pattern.format(re.escape(test), error) for error in errors] - if not any(re.search(msg, str(stdout)) for msg in messages): - msg = f'none of the patterns {messages!r} matched {stdout!r}' - raise AssertionError(msg) - - -def test_requirement(require_module: _TestModule) -> None: - """ - Test that the decorator correctly handles requirements that all - trials should pass (via ``require='all'``). - """ - require_module.run( - '--verbose', check_results=True, check_summary='verbose', - ) From 7f35b117e9d4a521db0bd9f4ba94388a1d8a7ae4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 26 Jun 2026 04:35:42 +0200 Subject: [PATCH 135/165] Updated `ty` config --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 3abd4137..daee1b2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,3 +126,6 @@ docstring-code-format = false unused-ignore-comment = "ignore" unused-type-ignore-comment = "ignore" unresolved-import = "ignore" + +[tool.ty.terminal] +error-on-warning = false # No longer the default since 0.0.52 From 5ca8d3e1fb9b741a4fca8187c8391e84f54a99ed Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 27 Jun 2026 21:43:14 +0200 Subject: [PATCH 136/165] Debug info on assertions line_profiler/_child_process_profiling/_cache_logging.py Updated assertions to emit more helpful error messages when they fail --- .../_cache_logging.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/line_profiler/_child_process_profiling/_cache_logging.py b/line_profiler/_child_process_profiling/_cache_logging.py index 992f68d0..8190059a 100644 --- a/line_profiler/_child_process_profiling/_cache_logging.py +++ b/line_profiler/_child_process_profiling/_cache_logging.py @@ -74,7 +74,9 @@ def parse_timestamp(ts: str) -> datetime: Turn a formatted string timestamp back to a :py:class:`datetime.datetime` object. """ - assert TIMESTAMP_MICROSECOND_SEP in ts + assert TIMESTAMP_MICROSECOND_SEP in ts, ( + f'{ts=!r}, {TIMESTAMP_MICROSECOND_SEP=!r}' + ) base, _, fractional = ts.rpartition(TIMESTAMP_MICROSECOND_SEP) # The microsecond field %f must be 6 digits long if len(fractional) < 6: @@ -181,7 +183,7 @@ def fmt_to_regex( if field is None: break # Suffix -> we're done if field: # Named fields - assert field.isidentifier() + assert field.isidentifier(), f'{field=!r}' if field in seen_fields: chunks.append(f'(?P={field})') else: @@ -263,7 +265,7 @@ def from_file(cls, file: os.PathLike[str] | str | TextIO) -> list[Self]: except TypeError: # File object # If we're here, `file` is a file object if TYPE_CHECKING: - assert isinstance(file, TextIO) + assert isinstance(file, TextIO), f'{file=!r}' content = file.read() else: content = path.read_text() @@ -293,27 +295,28 @@ def gen_message_blocks(text: str) -> Generator[ # Handle all the entries up till the 2nd-to-last one for this_match, next_match in pairwise(timestamps): ts = parse_timestamp(this_match.group('timestamp')) - # The -1 is for stripping the trailing newline - text_block = text[this_match.start():next_match.start() - 1] - yield (ts, this_match, text_block) + text_block = text[this_match.start():next_match.start()] + yield (ts, this_match, text_block.rstrip('\n')) # Handle the last entry last_match = timestamps[-1] yield ( parse_timestamp(last_match.group('timestamp')), last_match, - text[last_match.start():], + text[last_match.start():].rstrip('\n'), ) def get_entries(text: str) -> Generator[Self, None, None]: for timestamp, ts_match, text_block in gen_message_blocks(text): # Strip the block indent ts_text = ts_match.group(0) - assert text_block.startswith(ts_text) + assert text_block.startswith(ts_text), ( + f'{text_block=!r}, {ts_text=!r}' + ) ts_width = len(ts_text) text_block = dedent(' ' * ts_width + text_block[ts_width:]) # Strip the header and parse the relevant info from it header_match = header_regex.match(text_block) - assert header_match + assert header_match, f'{header_regex=!r}, {text_block=!r}' current_pid = int(header_match.group('current_pid')) main_pid_ = header_match.group('main_pid') if main_pid_ == HEADER_MAIN_INDICATOR: From 499d8e8df124e14181d44f47f9b52161ce270cd4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 27 Jun 2026 22:18:44 +0200 Subject: [PATCH 137/165] Fix for worker-process PID tracking line_profiler/_child_process_profiling/multiprocessing_patches /_mandatory_patches.py wrap_process() Removed (superseded by `wrap_terminate_pool()`) wrap_terminate_pool() New wrapper for `multiprocessing.pool.Pool._terminate_pool()` for handling end-of-process bookkeeping, so that workers that exited before they can be `.terminate()`-ed or `.join()`-ed are also accounted for POOL_WORKER_PID_PATCH.targets Now patching `multiprocessing.pool.Pool._terminate_pool()` instead of `.Process` --- .../_mandatory_patches.py | 138 ++++++++---------- 1 file changed, 57 insertions(+), 81 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py index 731d12da..8830bc7e 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -4,7 +4,8 @@ import os import multiprocessing from collections.abc import Callable -from functools import partial, wraps +from functools import partial +from multiprocessing.pool import Pool from multiprocessing.process import BaseProcess from pathlib import Path from types import MappingProxyType as mappingproxy, MethodType @@ -44,7 +45,7 @@ 'POOL_WORKER_PID_PATCH', 'PROCESS_TERMINATION_PATCH', 'RebootForkserverPatch', 'ResourceTrackerPatch', 'RunpyPatch', 'wrap_terminate', 'wrap_start', 'wrap_bootstrap', - 'wrap_handle_results', 'wrap_process', 'wrap_worker', + 'wrap_handle_results', 'wrap_terminate_pool', 'wrap_worker', ) _LOCK_FILE_LOC = '__line_profiler_multiprocessing_process_lock_file__' @@ -252,9 +253,9 @@ def wrap_handle_results( **kwargs: PS.kwargs ) -> None: """ - Wrap around :py:meth:`multiprocessing.pool.Pool._handle_results` so - that it handles the extra info (PID of child process handling the - task) included by :py:func:`.wrap_worker`. + Wrap around :py:meth:`.Pool._handle_results` so that it handles the + extra info (PID of child process handling the task) included by + :py:func:`.wrap_worker`. Note: :py:meth:`.Pool._handle_results` is a static method. @@ -266,6 +267,39 @@ def wrap_handle_results( vanilla_impl(outqueue, wrapped_get, *args, **kwargs) +@LineProfilingCache._method_wrapper +def wrap_terminate_pool( + cache: LineProfilingCache, + vanilla_impl: Callable[ + Concatenate[type[Pool], Queue, Queue, Queue, list[P], PS], None + ], + cls: type[Pool], + taskqueue: Queue, + inqueue: Queue, + outqueue: Queue, + pool: list[P], + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:meth:`.Pool._terminate_pool` so that we recover task + info from the worker processes. If a worker was idle and hasn't + processed any task, it is reported to the cache. + + Note: + :py:meth:`.Pool._terminate_pool` is a class method. + """ + try: + vanilla_impl(cls, taskqueue, inqueue, outqueue, pool, *args, **kwargs) + finally: + # Guard against dummy ppol; see similar code in + # `multiprocessing.pool` + if pool and hasattr(pool[0], 'terminate'): + for worker in pool: + assert not worker.is_alive() + _get_worker_ntasks(worker, cache) + + @LineProfilingCache._method_wrapper # nocover def wrap_worker( _, # We don't need the cache instance, but `@_method_wrapper` does @@ -289,84 +323,26 @@ def wrap_worker( return vanilla_impl(inqueue, outqueue, *args, **kwargs) -@LineProfilingCache._method_wrapper -def wrap_process( - cache: LineProfilingCache, - vanilla_impl: Callable[PS, P], - *args: PS.args, - **kwargs: PS.kwargs -) -> P: +def _get_worker_ntasks(worker: BaseProcess, cache: LineProfilingCache) -> int: """ - Wrap around :py:func:`multiprocessing.pool.Pool.Process` so that the - processes created can report on usage when - :py:meth:`.BaseProcess.join`-ed or - :py:meth:`.BaseProcess.terminate`-ed. + Check if the process has run any tasks; if not, report to the cache. - Note: - :py:meth:`.Pool.Process` is a static method. - """ - proc = vanilla_impl(*args, **kwargs) - # Note: since we don't clean up here, there's no need to instantiate - # another `Cleanup` helper - name = f'<{type(proc).__name__} @ {hex(id(proc))}>' - patch = partial(cache.patch, cleanup=False, name=name) - for method, action in ('join', 'joining'), ('terminate', 'terminating'): - bound = getattr(proc, method) - assert isinstance(bound, MethodType) - finalize = _wrap_process_finalize(cache, bound.__func__, action) - patch(proc, method, MethodType(finalize, proc)) - return proc - - -def _wrap_process_finalize( - cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[P, PS], None], - action: str, -) -> Callable[Concatenate[P, PS], None]: + Returns: + Number of tasks run by ``worker`` """ - Check if the process has run any tasks; - if not, report to the cache. - - Note: - Since the process object is pickled, this method has to directly - return a function object instead of merely being - :py:func:`partial`-ed and wrapped in a - :py:class:`types.MethodType`. - """ - @wraps(vanilla_impl) - def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: - log = cache._debug_output - call = cache._format_call(vanilla_impl, self, *args, **kwargs) - try: - log(f'Wrapped call made: {call}') - pid: int | None = getattr(self, 'pid', None) - ntasks_finalized = _get_ntasks_finalized(cache) - identifier = id(self), pid - if not (pid is None or identifier in ntasks_finalized): - ntasks = _get_ntasks(cache).pop(pid, 0) - if not ntasks: - cache._warn_possible_lack_of_stats(pid) - log(f'{action} process {pid} which ran {ntasks} task(s)...') - ntasks_finalized[cast(tuple[int, int], identifier)] = ntasks - except BaseException as e: - log( - f'Error in bookkeeping ({cache._format_exception(e)}), ' - 'invoking base implementation nonetheless...' - ) - raise e - finally: - try: - vanilla_impl(self, *args, **kwargs) - except BaseException as e: - state = f'failed ({cache._format_exception(e)})' - raise e - else: - state = 'succeeded' - finally: - log(f'Wrapped call {call} {state}') - - action = action.capitalize() - return finalize + pid: int | None = getattr(worker, 'pid', None) + ntasks_finalized = _get_ntasks_finalized(cache) + if pid is None: # Dummy process + return 0 + key = id(worker), pid + try: + return ntasks_finalized[key] + except KeyError: + pass + ntasks = _get_ntasks(cache).pop(pid, 0) + if not ntasks: + cache._warn_possible_lack_of_stats(pid) + return ntasks_finalized.setdefault(key, ntasks) def _wrap_outqueue_quick_get( @@ -405,7 +381,7 @@ def _get_ntasks_finalized( SingleModulePatch('pool') .add_method('', 'worker', wrap_worker) .add_method('Pool', '_handle_results', wrap_handle_results, 'static') - .add_method('Pool', 'Process', wrap_process, 'static') + .add_method('Pool', '_terminate_pool', wrap_terminate_pool, 'class') ) # --------------------------- Misc. patches ---------------------------- From 3d27ede540a35abe0ea639893b47a48e3f3f6f52 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 27 Jun 2026 23:41:57 +0200 Subject: [PATCH 138/165] Stop intefering with worker-process termination line_profiler/_child_process_profiling/multiprocessing_patches /_profiling_patches.py wrap_terminate() Removed because relying on timeouts is inherently unreliable PROCESS_PATCH.targets No longer patching `multiprocessing.process.BaseProcess.terminate()` --- .../_profiling_patches.py | 79 ++++--------------- 1 file changed, 17 insertions(+), 62 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py index a93daa0a..e97c8e2a 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py @@ -1,23 +1,20 @@ from __future__ import annotations from collections.abc import Callable -from contextlib import AbstractContextManager, nullcontext from functools import partial from multiprocessing.process import BaseProcess -from typing import Any, TypeVar, cast +from typing import TypeVar, cast from typing_extensions import Concatenate, ParamSpec from ..cache import LineProfilingCache from ._infrastructure import SingleModulePatch -from ._mandatory_patches import _get_ntasks_finalized from ._queue import Queue, PutWrapper from .mp_config import MPConfig from .poller import Poller, OnTimeout __all__ = ( - 'POOL_PATCH', 'PROCESS_PATCH', - 'wrap_bootstrap', 'wrap_terminate', 'wrap_worker', + 'POOL_PATCH', 'PROCESS_PATCH', 'wrap_bootstrap', 'wrap_worker', ) T = TypeVar('T') @@ -47,14 +44,6 @@ def dump_stats_quick( stats_dumper() -def _get_worker_ntasks(cache: LineProfilingCache, worker: BaseProcess) -> int: - ntasks = _get_ntasks_finalized(cache) - pid: int | None = getattr(worker, 'pid', None) - if pid is None: - return 0 - return ntasks.get((id(worker), pid), 0) - - # ---------------- `multiprocessing.pool.Pool` patches ----------------- @@ -104,52 +93,12 @@ def wrap_worker( return vanilla_impl(inqueue, outqueue, *args, **kwargs) -POOL_PATCH = SingleModulePatch('pool').add_method( - '', 'worker', wrap_worker, -) +POOL_PATCH = SingleModulePatch('pool') +POOL_PATCH.add_method('', 'worker', wrap_worker) # ----------- `multiprocessing.process.BaseProcess` patches ------------ -@LineProfilingCache._method_wrapper -def wrap_terminate( - cache: LineProfilingCache, - vanilla_impl: Callable[[BaseProcess], None], - self: BaseProcess, -) -> None: - """ - Wrap around :py:meth:`.BaseProcess.terminate` to make sure that we - don't actually kill the child (OS-level) process before it has the - chance to properly clean up. - - Notes: - - - We're technically polling in a loop, but it isn't actually - *that* bad: typically ``.terminate()`` is only called when - we're on the bad path (e.g. the parallel workload errored - out), and after the performance-critical part of the code - (said workload). - - - For currently unclear reasons, worker processes created by a - pool seem to sometimes deadlock, and thus we can't and don't - wait for their clean exit. This doesn't affect the profiling - results since they haven't run any task anyways, - """ - block: AbstractContextManager[Any] - if _get_worker_ntasks(cache, self): - block = _get_terminate_poller(cache, self) - else: # Don't block if we don't have to - block = nullcontext() - try: - with block: - pass - except Poller.Timeout as e: # Also handles `~.TimeoutWarning` - cache._debug_output(f'{type(e).__qualname__}: {e}') - raise - finally: # Always call `Process.terminate()` to avoid orphans - vanilla_impl(self) - - @LineProfilingCache._method_wrapper # nocover def wrap_bootstrap( cache: LineProfilingCache, @@ -162,12 +111,19 @@ def wrap_bootstrap( Wrap around :py:meth:`.BaseProcess._bootstrap` so that profiling stats are written at the end. - Note: - This is only invoked in child processes, and :py:mod:`coverage` - seems to be having trouble with them in the current setup, - probably due to issues with .pth file precendence causing - :py:mod:`line_profiler` to be loaded before it. Hence the - ``# nocover``. + Notes: + + - This is only invoked in child processes, and + :py:mod:`coverage` seems to be having trouble with them in the + current setup, probably due to issues with .pth file + precendence causing :py:mod:`line_profiler` to be loaded + before it. Hence the ``# nocover``. + + - Since process termination bypasses the Python interpreter (see + notes in :py:func:`wrap_worker`), if a child process is + terminated prematurely (e.g. via + :py:meth:`.BaseProcess.terminate`), profiling data may be + missing. """ try: return vanilla_impl(self, *args, **kwargs) @@ -208,4 +164,3 @@ def _process_has_returned( PROCESS_PATCH = SingleModulePatch('process') PROCESS_PATCH.add_method('BaseProcess', '_bootstrap', wrap_bootstrap) -PROCESS_PATCH.add_method('BaseProcess', 'terminate', wrap_terminate) From 72f4fe9c76f2a943efe02bd008339f9037b31366 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 28 Jun 2026 00:23:35 +0200 Subject: [PATCH 139/165] WIP: break down the test_child_procs tests --- tests/{test_child_procs.py => test_child_procs/tests.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_child_procs.py => test_child_procs/tests.py} (100%) diff --git a/tests/test_child_procs.py b/tests/test_child_procs/tests.py similarity index 100% rename from tests/test_child_procs.py rename to tests/test_child_procs/tests.py From 1ba227cb781ff93135badc995180423813b04ac7 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 29 Jun 2026 03:38:29 +0200 Subject: [PATCH 140/165] Break `test_child_procs` down tests/test_child_procs/ Separated `tests.py` into these files: - `_test_child_procs_utils.py`: Util constants, functions and classes used by the tests and the fixtures - `multiproc_examples/*.py`: Example scripts that are profiled - `conftest.py`: Fixtures and fixture functions - `test_child_procs.py`: Actual tests TODO: Fixes; new tests for non-`Pool`-based parallelism --- tests/test_child_procs/__init__.py | 0 .../{tests.py => _test_child_procs_utils.py} | 3209 +++++------------ tests/test_child_procs/conftest.py | 332 ++ .../multiproc_examples/external_module.py | 10 + .../multiproc_examples/pool_test_module.py | 80 + tests/test_child_procs/test_child_procs.py | 1205 +++++++ 6 files changed, 2512 insertions(+), 2324 deletions(-) create mode 100644 tests/test_child_procs/__init__.py rename tests/test_child_procs/{tests.py => _test_child_procs_utils.py} (50%) create mode 100644 tests/test_child_procs/conftest.py create mode 100644 tests/test_child_procs/multiproc_examples/external_module.py create mode 100644 tests/test_child_procs/multiproc_examples/pool_test_module.py create mode 100644 tests/test_child_procs/test_child_procs.py diff --git a/tests/test_child_procs/__init__.py b/tests/test_child_procs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_child_procs/tests.py b/tests/test_child_procs/_test_child_procs_utils.py similarity index 50% rename from tests/test_child_procs/tests.py rename to tests/test_child_procs/_test_child_procs_utils.py index 92b3b5e2..cdeeb4ed 100644 --- a/tests/test_child_procs/tests.py +++ b/tests/test_child_procs/_test_child_procs_utils.py @@ -19,20 +19,18 @@ from abc import ABC, abstractmethod from argparse import ArgumentError from collections.abc import ( - Callable, - Collection, Generator, Iterable, Iterator, Mapping, Sequence, Set, + Callable, Collection, Generator, Iterable, Iterator, Mapping, + Sequence, Set, ) from contextlib import ExitStack from functools import lru_cache, partial, wraps -from io import BytesIO, StringIO +from io import BytesIO from importlib import import_module from multiprocessing.pool import ( # type: ignore ExceptionWithTraceback as ExceptionHelper, ) from numbers import Real from pathlib import Path -from runpy import run_path -from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic from types import MappingProxyType, ModuleType, TracebackType @@ -40,7 +38,7 @@ TYPE_CHECKING, Any, Generic, IO, Literal, Protocol, TypeVar, cast, final, overload, ) -from typing_extensions import Self, ParamSpec +from typing_extensions import Concatenate, Self, ParamSpec from uuid import uuid4 import pytest @@ -49,485 +47,54 @@ from _line_profiler_hooks import load_pth_hook from kernprof import main as kernprof_main from line_profiler._child_process_profiling.cache import LineProfilingCache -from line_profiler._child_process_profiling.runpy_patches import ( - create_runpy_wrapper, -) from line_profiler._child_process_profiling.multiprocessing_patches import ( - MPConfig, _PATCHED_MARKER as MP_PATCHED_MARKER, _PATCHES as MP_PATCHES, + _PATCHED_MARKER as MP_PATCHED_MARKER, _PATCHES as _MP_PATCHES, ) from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler.cleanup import Cleanup -from line_profiler.curated_profiling import ( - CuratedProfilerContext, ClassifiedPreimportTargets, -) from line_profiler.line_profiler import LineProfiler, LineStats from line_profiler.toml_config import ConfigSource +__all__ = ( + 'DEBUG', 'DEFAULT_TIMEOUT', 'NOT_SUPPLIED', 'START_METHODS', + 'PATCH_SUMMARIES', + 'StartMethod', 'ModuleFixture', 'Params', + 'ResultMismatch', 'TestTimeout', + 'preserve_object_attrs', 'preserve_targets', + 'cleanup_extra_pth_files', 'add_timeout', + 'CheckWarnings', + 'mp_patch_is_internal', 'get_mp_patches_toml_text', + 'summarize_mp_patches', 'filter_mp_patch_summary', + 'concat_command_line', 'run_subproc', + 'run_module', 'run_script', 'run_literal_code', 'check_tagged_line_nhits', + 'strip', 'search_cache_logs', +) + + T = TypeVar('T') T1 = TypeVar('T1') T2 = TypeVar('T2') TCtx_ = TypeVar('TCtx_') PS = ParamSpec('PS') C = TypeVar('C', bound=Callable[..., Any]) +StartMethod = Literal['spawn', 'fork', 'forkserver', 'dummy'] -NUM_NUMBERS = 100 -NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) -_TEST_TIMEOUT = 5 # Seconds -_DEBUG = True -_WINDOWS = sys.platform == 'win32' - - -def strip(s: str) -> str: - return dedent(s).strip('\n') - - -EXTERNAL_MODULE_BODY = strip(""" -from __future__ import annotations - - -def my_external_sum(x: list[int], fail: bool = False) -> int: - result: int = 0 # GREP_MARKER[EXT-INVOCATION] - for item in x: - result += item # GREP_MARKER[EXT-LOOP] - if fail: - raise RuntimeError('forced failure') - return result -""") - -TEST_MODULE_TEMPLATE = strip(""" -from __future__ import annotations - -from argparse import ArgumentParser -from collections.abc import Callable -from multiprocessing import dummy, get_context, Pool -from typing import Literal - -from {EXT_MODULE} import my_external_sum - - -def my_local_sum(x: list[int], fail: bool = False) -> int: - result: int = 0 # GREP_MARKER[LOCAL-INVOCATION] - # The reversing is to prevent bytecode aliasing with - # `my_external_sum()` (see issue #424, PR #425) - for item in reversed(x): - result += item # GREP_MARKER[LOCAL-LOOP] - if fail: - raise RuntimeError('forced failure') - return result - - -def sum_in_child_procs( - length: int, n: int, my_sum: Callable[[list[int]], int], - start_method: Literal[ - 'fork', 'forkserver', 'spawn', 'dummy' - ] | None = None, - fail: bool = False, -) -> int: - my_list: list[int] = list(range(1, length + 1)) - sublists: list[list[int]] = [] - subsums: list[int] - sublength = length // n - if sublength * n < length: - sublength += 1 - while my_list: - sublist, my_list = my_list[:sublength], my_list[sublength:] - sublists.append(sublist) - if start_method == 'dummy': - pool = dummy.Pool(n) - elif start_method: - pool = get_context(start_method).Pool(n) - else: - pool = Pool(n) - with pool: - subsums = pool.starmap(my_sum, [(sl, fail) for sl in sublists]) - pool.close() - pool.join() - return my_sum(subsums, fail) - - -def main(args: list[str] | None = None) -> None: - parser = ArgumentParser() - parser.add_argument('-l', '--length', type=int, default={NUM_NUMBERS}) - parser.add_argument('-n', type=int, default={NUM_PROCS}) - parser.add_argument( - '-s', '--start-method', - choices=['fork', 'forkserver', 'spawn'], default=None, - ) - parser.add_argument('-f', '--force-failure', action='store_true') - parser.add_argument( - '--local', - action='store_const', - dest='my_sum', - default=my_external_sum, - const=my_local_sum, - ) - options = parser.parse_args(args) - print(sum_in_child_procs( - options.length, options.n, options.my_sum, - start_method=options.start_method, - fail=options.force_failure, - )) - - -if __name__ == '__main__': - main() -""") - - -# ============================== Fixtures ============================== - - -@dataclasses.dataclass -class _ModuleFixture: - """ - Convenience wrapper around a Python source file which represents an - importable module. - """ - path: Path - monkeypatch: pytest.MonkeyPatch - dependencies: Collection[_ModuleFixture] = () - - def install( - self, *, - local: bool = False, children: bool = False, deps_only: bool = False, - ) -> None: - """ - Set the module at :py:attr:`~.path` up to be importable. - - Args: - local (bool): - Make it importable for the CURRENT process (via - :py:data:`sys.path`). - children (bool): - Make it importable for CHILD processes (via - ``os.environ['PYTHONPATH']``). - deps_only (bool): - If true, only does the equivalent setup for - dependencies. - """ - for dep in self.dependencies: - dep.install(local=local, children=children) - if deps_only: - return - path = str(self.path.parent) - if local: - self.monkeypatch.syspath_prepend(path) - if children: - self.monkeypatch.setenv('PYTHONPATH', path, prepend=os.pathsep) - - def _import_module_helper(self) -> Generator[ModuleType, None, None]: - def iter_module_names( - module: _ModuleFixture, - ) -> Generator[str, None, None]: - yield module.name - for dep in module.dependencies: - yield from iter_module_names(dep) - - self.install(local=True, children=True) - try: - yield import_module(self.name) - finally: - for name in set(iter_module_names(self)): - sys.modules.pop(name, None) - - @staticmethod - def propose_name(prefix: str) -> Generator[str, None, None]: - """ - Propose a valid module name that isn't already occupied. - """ - while True: - name = '_'.join([prefix] + str(uuid4()).split('-')) - if name not in sys.modules: - assert name.isidentifier() - yield name - - @property - def name(self) -> str: - return self.path.stem - - -# Only write the files once per test session - - -@pytest.fixture(scope='session') -def _ext_module() -> Generator[Path, None, None]: - name = next(_ModuleFixture.propose_name('my_ext_module')) - with TemporaryDirectory() as mydir_str: - my_dir = Path(mydir_str) - my_dir.mkdir(exist_ok=True) - my_module = my_dir / f'{name}.py' - my_module.write_text(EXTERNAL_MODULE_BODY) - yield my_module - - -@pytest.fixture(scope='session') -def _test_module(_ext_module: Path) -> Generator[Path, None, None]: - name = next(_ModuleFixture.propose_name('my_test_module')) - body = TEST_MODULE_TEMPLATE.format( - EXT_MODULE=_ext_module.stem, - NUM_NUMBERS=NUM_NUMBERS, - NUM_PROCS=NUM_PROCS, - ) - with TemporaryDirectory() as mydir_str: - my_dir = Path(mydir_str) - my_dir.mkdir(exist_ok=True) - my_module = my_dir / f'{name}.py' - my_module.write_text(body) - yield my_module - - -@pytest.fixture -def ext_module( - _ext_module: Path, monkeypatch: pytest.MonkeyPatch, -) -> Generator[_ModuleFixture, None, None]: - """ - Yields: - :py:class:`_ModuleFixture` helper object containing the code at - :py:data:`EXTERNAL_MODULE_BODY` - """ - yield _ModuleFixture(_ext_module, monkeypatch) - - -@pytest.fixture -def test_module( - _test_module: Path, - ext_module: _ModuleFixture, - monkeypatch: pytest.MonkeyPatch, -) -> Generator[_ModuleFixture, None, None]: - """ - Yields: - :py:class:`_ModuleFixture` helper object containing the code at - :py:data:`TEST_MODULE_TEMPLATE` - """ - yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) - - -@pytest.fixture -def test_module_clone( - tmp_path_factory: pytest.TempPathFactory, - monkeypatch: pytest.MonkeyPatch, - _test_module: Path, - ext_module: _ModuleFixture, -) -> Generator[_ModuleFixture, None, None]: - """ - Yields: - :py:class:`_ModuleFixture` helper object containing the same - code as :py:data:`test_module` - """ - tmpdir = tmp_path_factory.mktemp('my_path') - name = next(_ModuleFixture.propose_name('my_cloned_module')) - path = tmpdir / f'{name}.py' - path.write_text(_test_module.read_text()) - yield _ModuleFixture(path, monkeypatch, [ext_module]) - - -@pytest.fixture -def ext_module_object( - ext_module: _ModuleFixture, -) -> Generator[ModuleType, None, None]: - """ - Yields: - :py:class:`ModuleType` object containing the code at - :py:data:`EXTERNAL_MODULE_BODY`, and is torn down at the end of - the test - """ - yield from ext_module._import_module_helper() - - -@pytest.fixture -def test_module_object( - test_module: _ModuleFixture, ext_module_object: ModuleType, -) -> Generator[ModuleType, None, None]: - """ - Yields: - :py:class:`ModuleType` object containing the code at - :py:data:`TEST_MODULE_TEMPLATE`, and is torn down at the end of - the test - """ - yield from test_module._import_module_helper() - - -@pytest.fixture -def create_cache( - tmp_path_factory: pytest.TempPathFactory, - request: pytest.FixtureRequest, -) -> Generator[Callable[..., LineProfilingCache], None, None]: - """ - Wrapper around the :py:class:`LineProfilingCache` instantiator - which: - - - Automatically creates a tempdir and provides it as the - :py:attr:`LineProfilingCache.cache_dir`, - - - Extends the argument ``preimports_module`` to allow for taking - boolean values: - - - ``True``: a temporary preimports module is automatically written - based on ``profiling_targets`` and supplied to the base - constructor. - - - ``False``: equivalent to ``None``. - - - Unless the argument ``_use_curated_profiler: bool = True`` is set - to :py:const:`False`, automatically creates an instance of - :py:class:`LineProfiler` that is curated by a - :py:class:`CuratedProfilerContext` and provides it as the - :py:attr:`LineProfilingCache.profiler`, and - - - At teardown: - - - Removes tempdirs and tempfiles generated. - - - Restores the value of the class' internal reference to the - :py:meth:`LineProfilingCache.load`-ed instance. - - - Calls the `.cleanup()` method of each instance created. - - - Prints these diagnostics for each instance: - - - The stats on the ``.profiler`` associated with each instance - (if any) - - - The stats gathered by - :py:meth:`LineProfilingCache.gather_stats()` - - - The debug logs (if ``.debug`` is true) - """ - def instantiate( - *, - profiling_targets: Collection[str] = (), - preimports_module: os.PathLike[str] | str | bool | None = None, - _use_curated_profiler: bool = True, - **kwargs - ) -> LineProfilingCache: - tmpdir = tmp_path_factory.mktemp('my_cache_dir') - pim: os.PathLike[str] | str | None - if preimports_module in (True, False): - if preimports_module: - targets = ( - ClassifiedPreimportTargets.from_targets(profiling_targets) - ) - if targets: - pim = tmpdir / 'preimports.py' - with pim.open(mode='w') as fobj: - targets.write_preimport_module(fobj) - else: - pim = None - else: - pim = None - else: - # The type checker needs some convincing... - assert not isinstance(preimports_module, bool) - pim = preimports_module - cache = LineProfilingCache( - tmpdir, - profiling_targets=profiling_targets, - preimports_module=pim, - **kwargs, - ) - if _use_curated_profiler: - cache.profiler = request.getfixturevalue('curated_profiler') - instances.append(cache) - return cache - - def print_result( - cache: LineProfilingCache, topic: str, result: str, *notes: str, - ) -> None: - header = '{} ({}):'.format( - topic, '; '.join([f'cache instance {id(cache):#x}', *notes]), - ) - print(header, indent(result, ' '), sep='\n') - - def print_profiler_stats(cache: LineProfilingCache) -> None: - if cache.profiler is None: - result = '' - notes = [] - else: - with StringIO() as sio: - cache.profiler.print_stats(sio) - result = sio.getvalue() - notes = [f'profiler instance {id(cache.profiler):#x}'] - print_result(cache, 'Native profiler stats', result, *notes) - - def print_gathered_stats(cache: LineProfilingCache) -> None: - with StringIO() as sio: - cache.gather_stats().print(sio) - result = sio.getvalue() - print_result(cache, 'Gathered profiler stats', result) - - def print_debug_logs(cache: LineProfilingCache) -> None: - if cache.debug: - result = '\n'.join( - entry.to_text() for entry in cache._gather_debug_log_entries() - ) - else: - result = '' - print_result(cache, 'Gathered debug logs', result) - - instances: list[LineProfilingCache] = [] - handlers: list[Callable[[LineProfilingCache], None]] - handlers = [print_profiler_stats, print_gathered_stats, print_debug_logs] - try: - with _preserve_obj_attributes( - LineProfilingCache, ['_loaded_instance'], - ): - yield instantiate - finally: - for cache in instances: - callbacks: list[Callable[[], Any]] = [cache.cleanup] - callbacks.extend(partial(func, cache) for func in handlers) - for callback in callbacks: - try: - callback() - except Exception: - pass - - -@pytest.fixture -def curated_profiler() -> Generator[LineProfiler, None, None]: - """ - Yields: - Fresh instance of :py:class:`LineProfiler` that is managed by a - :py:class:`CuratedProfilerContext` - """ - prof = LineProfiler() - with CuratedProfilerContext(prof, insert_builtin=True): - yield prof - - -@pytest.fixture -def another_pid() -> int: - """ - Get a PID which is distinct from the current one. - """ - curr_pid = os.getpid() - pid = (curr_pid - 42) % (2 * 16) - assert pid != curr_pid - return pid - - -@pytest.fixture(autouse=True) -def _trim_mismatch_traceback(pytestconfig: pytest.Config) -> None: - """ - Truncate the traceback of raised :py:class`ResultMismatch` for more - useful error attribution. - """ - try: - pytestconfig.pluginmanager.register(ResultMismatch) - except ValueError: # Already registered - pass +DEFAULT_TIMEOUT = 5 # Seconds +DEBUG = True - -# ========================== Helper functions ========================== +# ======================== Misc. helper classes ======================== class _NotSupplied(enum.Enum): NOT_SUPPLIED = enum.auto() +NOT_SUPPLIED = _NotSupplied.NOT_SUPPLIED + + class _GetAttr(Protocol): """ Function signature for functions that behave like @@ -550,10 +117,10 @@ class ResultMismatch(ValueError): def __init__( self, expected: Any, - actual: Any | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + actual: Any | _NotSupplied = NOT_SUPPLIED, _trunc_tb: int = 0, ) -> None: - if actual == _NotSupplied.NOT_SUPPLIED: + if actual == NOT_SUPPLIED: msg = f'expected: {expected}' else: msg = f'expected {expected}, got {actual}' @@ -620,105 +187,297 @@ def rich_message(self) -> str: return msg -class _TestTimeout(RuntimeError): +class TestTimeout(RuntimeError): """ - Error raised by the :py:func:`_timeout` decorator. + Error raised by the :py:func:`add_timeout` decorator. """ pass -@final -@dataclasses.dataclass -class _Params: - """ - Convenience wrapper around :py:func:`pytest.mark.parametrize`. - """ - params: tuple[str, ...] - values: list[tuple[Any, ...]] - defaults: tuple[Any, ...] +class _CallableContextManager(ABC, Generic[TCtx_]): + debug: bool - def __post_init__(self) -> None: - n = len(self.params) - assert all(p.isidentifier() for p in self.params) # Validity - assert len(set(self.params)) == n # Uniqueness - assert len(self.defaults) == n # Consistency - self.values = list(self._unique(self.values)) - assert all(len(v) == n for v in self.values) + @abstractmethod + def __enter__(self) -> TCtx_: + ... - def __mul__(self, other: Self) -> Self: - """ - Form a Cartesian product between the two instances with disjoint - :py:attr:`~.params`, like stacking the - :py:func:`pytest.mark.parametrize `decorators. + @abstractmethod + def __exit__(self, *a, **k) -> Any: + ... - Example: - >>> p1 = _Params.new(('a', 'b'), [(0, 0), (1, 2), (3, 4)], - ... defaults=(1, 2)) - >>> p2 = _Params.new('c', [0, 5, 6]) - >>> p1 * p2 # doctest: +NORMALIZE_WHITESPACE - _Params(params=('a', 'b', 'c'), - values=[(0, 0, 0), (0, 0, 5), (0, 0, 6), - (1, 2, 0), (1, 2, 5), (1, 2, 6), - (3, 4, 0), (3, 4, 5), (3, 4, 6)], - defaults=(1, 2, 0)) + def __call__(self, func: Callable[PS, T]) -> Callable[PS, T]: """ - assert not set(self.params) & set(other.params) - return type(self)( - self.params + other.params, - [sv + ov for sv in self.values for ov in other.values], - self.defaults + other.defaults, - ) - - def __add__(self, other: Self) -> Self: + Wrap ``func()`` so that its calls always happen in the context + of the instance. """ - Concatenate two instances: + @wraps(func) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + with self: + return func(*args, **kwargs) - - For parameters appearing in both, their lists of values are - concatenated. + return wrapper - - For parameters appearing in either instance, the missing - values are taken from the other instance's - :py:attr:`~.defaults`. + def _debug(self, msg: str, **kwargs) -> None: + if not self.debug: + return + header = f'{os.environ["PYTEST_CURRENT_TEST"]}: {type(self).__name__}' + print(f'{header}: {msg}', **kwargs) - Note: - In the case of clashes, the :py:attr:`~.defaults` and the - order of the :py:attr:`~.params` of ``self`` (the left - operand) take precedence. - Example: - >>> p1 = _Params.new(('a', 'b', 'c'), - ... [(0, 0, 0), # defaults - ... (1, 2, 3), (4, 5, 6)]) - >>> p2 = _Params.new(('c', 'd'), [(7, 8), (9, 10)], - ... defaults=(-1, -1)) - >>> p1 + p2 # doctest: +NORMALIZE_WHITESPACE - _Params(params=('a', 'b', 'c', 'd'), - values=[(0, 0, 0, -1), - (1, 2, 3, -1), - (4, 5, 6, -1), - (0, 0, 7, 8), - (0, 0, 9, 10)], - defaults=(0, 0, 0, -1)) - """ - self_defaults = dict(zip(self.params, self.defaults)) - other_defaults = dict(zip(other.params, other.defaults)) - new_params = tuple(self._unique(self.params + other.params)) +# ========================== Misc. functions =========================== - defaults = {**other_defaults, **self_defaults} - new_defaults_tuple = tuple(defaults[p] for p in new_params) - new_values: list[tuple[Any, ...]] = [] - for old_values, old_params in [ - (self.values, self.params), (other.values, other.params), - ]: - indices: list[ - tuple[Literal[True], int] | tuple[Literal[False], str] - ] = [ - (True, old_params.index(p)) if p in old_params else (False, p) - for p in new_params - ] - new_values.extend( - tuple( +def strip(s: str) -> str: + return dedent(s).strip('\n') + + +def import_target(target: str) -> Any: + try: + return import_module(target) + except ImportError: # Not a module + assert '.' in target + module, _, attr = target.rpartition('.') + return getattr(import_module(module), attr) + + +def search_cache_logs( + cache: LineProfilingCache, + expecting_logs: bool, + patterns: Mapping[str, bool] | Collection[str], + match_individual_messages: bool = False, + flags: int = 0, +) -> None: + entries = cache._gather_debug_log_entries() + ResultMismatch.compare( + expecting_logs, bool(entries), + expected='logs' if expecting_logs else 'no logs', + actual=repr(entries) if entries else 'nothing', + ) + if not expecting_logs: + return + text_chunks: list[str] = [entry.to_text() for entry in entries] + if not match_individual_messages: + text_chunks = ['\n'.join(text_chunks)] + if isinstance(patterns, Mapping): + to_match: dict[str, bool] = { + str(pat): bool(should_match) + for pat, should_match in patterns.items() + } + else: + to_match = dict.fromkeys(patterns, True) + for pat, should_match in to_match.items(): + pattern = re.compile(pat, flags) + if any(pattern.search(chunk) for chunk in text_chunks) == should_match: + continue + raise ResultMismatch( + f'pattern {pattern!r} to {"" if should_match else "not "}match ' + f'{cache!r}\'s logs: {text_chunks!r}' + ) + + +# ================ `pytest` stuff: fixtures and markers ================ + + +@dataclasses.dataclass +class ModuleFixture: + """ + Convenience wrapper around a Python source file which represents an + importable module. + + Attributes: + path (Path): + File path to the module content. + monkeypatch (pytest.MonkeyPatch): + Monkey-patch object. + dependencies (Collection[ModuleFixture[Any]]): + Other :py:class:`ModuleFixture` objects this instance + depends on. + build_command_line \ +(Callable[Concatenate[bool, StartMethod | None, ...], Sequence[str]] \ +| None): + If the module is supposed to be callable (via + ``python -m ...``), a callable which takes the leading + positional arguments + ``fail: bool, start_method: Literal[...] | None`` followed by + arbitrary arguments, and constructs the command-line + arguments to pass to the module; + note that: + + - ``fail=True`` should result in command-line arguments + causing the executed module to fail. + + - ``start_method`` should select the + :py:mod:`multiprocessing` start method, where ``dummy`` + should cause the analogous APIs at + :py:mod:`multiprocessing.dummy` to be used instead; + if it is :py:const:`None`, the OS default should be used. + get_output (Callable[..., Any] | None): + If the module is supposed to be callable (via + ``python -m ...``), a callable which takes arbitrary + arguments (consistent with those to + ``build_command_line()``) and returns the expected output of + the executed module. + """ + path: Path + monkeypatch: pytest.MonkeyPatch + dependencies: Collection[ModuleFixture] = () + build_command_line: ( + Callable[Concatenate[bool, StartMethod | None, ...], Sequence[str]] + | None + ) = None + get_output: Callable[..., Any] | None = None + + def install( + self, *, + local: bool = False, children: bool = False, deps_only: bool = False, + ) -> None: + """ + Set the module at :py:attr:`~.path` up to be importable. + + Args: + local (bool): + Make it importable for the CURRENT process (via + :py:data:`sys.path`). + children (bool): + Make it importable for CHILD processes (via + ``os.environ['PYTHONPATH']``). + deps_only (bool): + If true, only does the equivalent setup for + dependencies. + """ + for dep in self.dependencies: + dep.install(local=local, children=children) + if deps_only: + return + path = str(self.path.parent) + if local: + self.monkeypatch.syspath_prepend(path) + if children: + self.monkeypatch.setenv('PYTHONPATH', path, prepend=os.pathsep) + + def _import_module_helper(self) -> Generator[ModuleType, None, None]: + def iter_module_names( + module: ModuleFixture, + ) -> Generator[str, None, None]: + yield module.name + for dep in module.dependencies: + yield from iter_module_names(dep) + + self.install(local=True, children=True) + try: + yield import_module(self.name) + finally: + for name in set(iter_module_names(self)): + sys.modules.pop(name, None) + + @staticmethod + def propose_name(prefix: str) -> Generator[str, None, None]: + """ + Propose a valid module name that isn't already occupied. + """ + while True: + name = '_'.join([prefix] + str(uuid4()).split('-')) + if name not in sys.modules: + assert name.isidentifier() + yield name + + @property + def name(self) -> str: + return self.path.stem + + +@final +@dataclasses.dataclass +class Params: + """ + Convenience wrapper around :py:func:`pytest.mark.parametrize`. + """ + params: tuple[str, ...] + values: list[tuple[Any, ...]] + defaults: tuple[Any, ...] + + def __post_init__(self) -> None: + n = len(self.params) + assert all(p.isidentifier() for p in self.params) # Validity + assert len(set(self.params)) == n # Uniqueness + assert len(self.defaults) == n # Consistency + self.values = list(self._unique(self.values)) + assert all(len(v) == n for v in self.values) + + def __mul__(self, other: Self) -> Self: + """ + Form a Cartesian product between the two instances with disjoint + :py:attr:`~.params`, like stacking the + :py:func:`pytest.mark.parametrize `decorators. + + Example: + >>> p1 = Params.new(('a', 'b'), [(0, 0), (1, 2), (3, 4)], + ... defaults=(1, 2)) + >>> p2 = Params.new('c', [0, 5, 6]) + >>> p1 * p2 # doctest: +NORMALIZE_WHITESPACE + Params(params=('a', 'b', 'c'), + values=[(0, 0, 0), (0, 0, 5), (0, 0, 6), + (1, 2, 0), (1, 2, 5), (1, 2, 6), + (3, 4, 0), (3, 4, 5), (3, 4, 6)], + defaults=(1, 2, 0)) + """ + assert not set(self.params) & set(other.params) + return type(self)( + self.params + other.params, + [sv + ov for sv in self.values for ov in other.values], + self.defaults + other.defaults, + ) + + def __add__(self, other: Self) -> Self: + """ + Concatenate two instances: + + - For parameters appearing in both, their lists of values are + concatenated. + + - For parameters appearing in either instance, the missing + values are taken from the other instance's + :py:attr:`~.defaults`. + + Note: + In the case of clashes, the :py:attr:`~.defaults` and the + order of the :py:attr:`~.params` of ``self`` (the left + operand) take precedence. + + Example: + >>> p1 = Params.new(('a', 'b', 'c'), + ... [(0, 0, 0), # defaults + ... (1, 2, 3), (4, 5, 6)]) + >>> p2 = Params.new(('c', 'd'), [(7, 8), (9, 10)], + ... defaults=(-1, -1)) + >>> p1 + p2 # doctest: +NORMALIZE_WHITESPACE + Params(params=('a', 'b', 'c', 'd'), + values=[(0, 0, 0, -1), + (1, 2, 3, -1), + (4, 5, 6, -1), + (0, 0, 7, 8), + (0, 0, 9, 10)], + defaults=(0, 0, 0, -1)) + """ + self_defaults = dict(zip(self.params, self.defaults)) + other_defaults = dict(zip(other.params, other.defaults)) + new_params = tuple(self._unique(self.params + other.params)) + + defaults = {**other_defaults, **self_defaults} + new_defaults_tuple = tuple(defaults[p] for p in new_params) + + new_values: list[tuple[Any, ...]] = [] + for old_values, old_params in [ + (self.values, self.params), (other.values, other.params), + ]: + indices: list[ + tuple[Literal[True], int] | tuple[Literal[False], str] + ] = [ + (True, old_params.index(p)) if p in old_params else (False, p) + for p in new_params + ] + new_values.extend( + tuple( ( value[cast(int, index)] if available else @@ -780,9 +539,9 @@ def drop_params(self, params: Collection[str] | str) -> Self: that don't match :py:attr:`.params` are ignored. Example: - >>> p = _Params.new(('a', 'b'), [(1, 2), (3, 4)]) + >>> p = Params.new(('a', 'b'), [(1, 2), (3, 4)]) >>> p.drop_params('a') - _Params(params=('b',), values=[(2,), (4,)], defaults=(2,)) + Params(params=('b',), values=[(2,), (4,)], defaults=(2,)) >>> assert p.drop_params(['c', 'd']) == p """ def drop(t: tuple[T, ...]) -> tuple[T, ...]: @@ -818,38 +577,38 @@ def split_on_params( in an error. Example: - >>> p = _Params.new(('a', 'b', 'c'), - ... [(1, 2, True), - ... (1, 2, False), - ... (3, 4, True)]) + >>> p = Params.new(('a', 'b', 'c'), + ... [(1, 2, True), + ... (1, 2, False), + ... (3, 4, True)]) >>> p.split_on_params('a') # doctest: +NORMALIZE_WHITESPACE - {1: _Params(params=('b', 'c'), - values=[(2, True), (2, False)], - defaults=(2, True)), - 3: _Params(params=('b', 'c'), - values=[(4, True)], - defaults=(2, True))} + {1: Params(params=('b', 'c'), + values=[(2, True), (2, False)], + defaults=(2, True)), + 3: Params(params=('b', 'c'), + values=[(4, True)], + defaults=(2, True))} >>> p.split_on_params( # doctest: +NORMALIZE_WHITESPACE ... ('a', 'b'), ... ) - {(1, 2): _Params(params=('c',), - values=[(True,), (False,)], - defaults=(True,)), - (3, 4): _Params(params=('c',), - values=[(True,)], - defaults=(True,))} + {(1, 2): Params(params=('c',), + values=[(True,), (False,)], + defaults=(True,)), + (3, 4): Params(params=('c',), + values=[(True,)], + defaults=(True,))} >>> p.split_on_params( # doctest: +NORMALIZE_WHITESPACE ... 'a', drop_split_params=False, ... ) - {1: _Params(params=('a', 'b', 'c'), - values=[(1, 2, True), (1, 2, False)], - defaults=(1, 2, True)), - 3: _Params(params=('a', 'b', 'c'), - values=[(3, 4, True)], - defaults=(1, 2, True))} + {1: Params(params=('a', 'b', 'c'), + values=[(1, 2, True), (1, 2, False)], + defaults=(1, 2, True)), + 3: Params(params=('a', 'b', 'c'), + values=[(3, 4, True)], + defaults=(1, 2, True))} >>> p.split_on_params( # doctest: +NORMALIZE_WHITESPACE ... ('c', 'd'), @@ -916,7 +675,7 @@ def new( cls, params: Sequence[str] | str, values: Sequence[Sequence[Any]], - defaults: Sequence[Any] | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + defaults: Sequence[Any] | _NotSupplied = NOT_SUPPLIED, ) -> Self: ... @@ -926,7 +685,7 @@ def new( cls, params: str, values: Sequence[Any], - defaults: Any | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + defaults: Any | _NotSupplied = NOT_SUPPLIED, ) -> Self: ... @@ -935,9 +694,7 @@ def new( cls, params: Sequence[str] | str, values: Sequence[Sequence[Any]] | Sequence[Any], - defaults: ( - Sequence[Any] | Any | _NotSupplied - ) = _NotSupplied.NOT_SUPPLIED, + defaults: Sequence[Any] | Any | _NotSupplied = NOT_SUPPLIED, ) -> Self: """ Instantiator more akin to :py:func:`pytest.mark.parametrize`: @@ -958,7 +715,7 @@ def new( else: param_list = tuple(params) unpacked = False - if defaults == _NotSupplied.NOT_SUPPLIED: + if defaults == NOT_SUPPLIED: defaults, *_ = values if unpacked: default_values: tuple[Any, ...] = defaults, @@ -969,40 +726,16 @@ def new( return cls(param_list, value_tuple_list, default_values) -class _CallableContextManager(ABC, Generic[TCtx_]): - debug: bool - - @abstractmethod - def __enter__(self) -> TCtx_: - ... - - @abstractmethod - def __exit__(self, *a, **k) -> Any: - ... - - def __call__(self, func: Callable[PS, T]) -> Callable[PS, T]: - """ - Wrap ``func()`` so that its calls always happen in the context - of the instance. - """ - @wraps(func) - def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: - with self: - return func(*args, **kwargs) - - return wrapper - - def _debug(self, msg: str, **kwargs) -> None: - if not self.debug: - return - header = f'{os.environ["PYTEST_CURRENT_TEST"]}: {type(self).__name__}' - print(f'{header}: {msg}', **kwargs) +# ============================= Failsafes ============================== -class _preserve_obj_attributes(_CallableContextManager[dict[str, Any]]): +class preserve_object_attrs(_CallableContextManager[dict[str, Any]]): + """ + Protect attributes on a single supplied object. + """ def __init__( self, obj: Any, attrs: Collection[str], *, - static: bool = True, debug: bool = _DEBUG, + static: bool = True, debug: bool = DEBUG, ) -> None: self.obj = obj self.attrs = set(attrs) @@ -1041,8 +774,8 @@ def reset(attr: str, value: Any) -> None: result: dict[str, Any] = {} for attr in self.attrs: - old = get_attribute(self.obj, attr, _NotSupplied.NOT_SUPPLIED) - if old is _NotSupplied.NOT_SUPPLIED: + old = get_attribute(self.obj, attr, NOT_SUPPLIED) + if old is NOT_SUPPLIED: callback = partial(delete, attr) else: callback = partial(reset, attr, old) @@ -1058,8 +791,11 @@ def __exit__(self, *_, **__) -> None: pass -class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): +class preserve_targets(_CallableContextManager[dict[str, dict[str, Any]]]): """ + Protect attributes on multiple target objects, which are resolved at + context entry. + Example: >>> from functools import wraps >>> from line_profiler.curated_profiling import ( @@ -1085,12 +821,10 @@ class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): ... '.CuratedProfilerContext': {'foo'}, ... 'line_profiler.line_profiler': {'main'}, ... } - >>> with _preserve_attributes(preserved, debug=False) as old: + >>> with preserve_targets(preserved, debug=False) as old: ... assert old == { ... 'line_profiler.curated_profiling' - ... '.CuratedProfilerContext': { - ... 'foo': _NotSupplied.NOT_SUPPLIED, - ... }, + ... '.CuratedProfilerContext': {'foo': NOT_SUPPLIED}, ... 'line_profiler.line_profiler': {'main': old_main}, ... } ... CuratedProfilerContext.foo = foo @@ -1106,7 +840,7 @@ class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): """ def __init__( self, targets: Mapping[str, Collection[str]] | None = None, *, - static: bool = True, debug: bool = _DEBUG, + static: bool = True, debug: bool = DEBUG, ) -> None: self.targets = targets self._stacks: list[ExitStack] = [] @@ -1118,8 +852,8 @@ def __enter__(self) -> dict[str, dict[str, Any]]: self._stacks.append(stack) result: dict[str, Any] = {} for target, attrs in self.targets.items(): - result[target] = stack.enter_context(_preserve_obj_attributes( - _import_target(target), attrs, + result[target] = stack.enter_context(preserve_object_attrs( + import_target(target), attrs, debug=self.debug, static=self.static, )) return result @@ -1132,13 +866,13 @@ def fetch_current_values( targets: Mapping[str, Collection[str]], static: bool = True, ) -> dict[str, dict[str, Any]]: result: dict[str, dict[str, Any]] = {} - na = _NotSupplied.NOT_SUPPLIED + na = NOT_SUPPLIED if static: get: _GetAttr = inspect.getattr_static else: get = getattr for target, attrs in targets.items(): - obj = _import_target(target) + obj = import_target(target) result[target] = {attr: get(obj, attr, na) for attr in attrs} return result @@ -1206,7 +940,7 @@ def get_from_boolean(*_, **__) -> bool: def targets(self) -> dict[str, set[str]]: if self._targets is None: # Defer resolution to context entrance - self.targets = _GLOBAL_PATCHES + self.targets = PATCH_SUMMARIES['maximal'] assert self._targets is not None return self._targets @@ -1220,8 +954,8 @@ def targets(self, targets: Mapping[str, Collection[str]] | None) -> None: } -class _preserve_pth_files(_CallableContextManager[frozenset[str]]): - def __init__(self, debug: bool = _DEBUG) -> None: +class cleanup_extra_pth_files(_CallableContextManager[frozenset[str]]): + def __init__(self, debug: bool = DEBUG) -> None: self.debug = debug def __enter__(self) -> frozenset[str]: @@ -1246,30 +980,181 @@ def _get_path() -> Path: return Path(sysconfig.get_path('purelib')) -class _WarningInfo(Protocol): - @property - def message(self) -> str | Warning: - ... - - @property - def category(self) -> type[Warning]: - ... - - @property - def filename(self) -> str: - ... - - @property - def lineno(self) -> int: - ... - - @property - def line(self) -> str | None: - ... +def _cleanup_profiling_in_current_thread() -> None: + """ + Disable all active profiler instances on the current thread, so that + we don't trip over ourselves if a new thread down the road happens + to reuse the thread ID. + Note: + The thread-local profiler state is supposed to be handled by + :py:mod:`line_profiler._threading_patches` and + :py:class:`.CuratedProfilerContext`, but since a test function + decorated with :py:deco:`.add_timeout` will be isolated in a new + thread BEFORE the fixture setting up such management + (:py:func:`curated_profiler`) is invoked, we don't get that + managing and will have to deal with profilers ourselves. + """ + class Manager(Protocol): + @property + def active_instances(self) -> set[LineProfiler]: + ... -@dataclasses.dataclass -class _WarningMatcher: + lp_managers = cast( + dict[int, Manager], getattr(LineProfiler, '_managers', {}), + ) + thread_id = threading.get_ident() + if thread_id not in lp_managers: + return + instances = lp_managers[thread_id].active_instances + for prof in set(instances): + count = cast(int, getattr(prof, 'enable_count', 0)) + for _ in range(count): + prof.disable_by_count() + if count: + print('Disabled', prof, 'in thread', hex(thread_id)) + # Removed after the last `.disable_by_count()` + assert prof not in instances + + +@overload +def add_timeout( + func: Callable[PS, T], *, timeout: float = DEFAULT_TIMEOUT, +) -> Callable[PS, T]: + ... + + +@overload +def add_timeout( + func: None = None, *, timeout: float = DEFAULT_TIMEOUT, +) -> Callable[[Callable[PS, T]], Callable[PS, T]]: + ... + + +def add_timeout( + func: Callable[PS, T] | None = None, *, + timeout: float = DEFAULT_TIMEOUT, +) -> Callable[PS, T] | Callable[[Callable[PS, T]], Callable[PS, T]]: + """ + Decorate the test function so that it is run in another thread and + can be timed out. + + Example: + >>> from time import sleep + + >>> @add_timeout(timeout=.5) + ... def my_func( + ... n: int, delay: float = 1, error: bool = False, + ... ) -> list[int]: + ... sleep(delay) + ... if error: + ... raise RuntimeError('my error message') + ... return list(range(n)) + + Normal execution: + + >>> my_func(3, 0) + [3] + [0, 1, 2, 3] + + Erroring out: + + >>> my_func(3, 0, error=True) + Traceback (most recent call last): + ... + RuntimeError: my error message + + Timing out: + + >>> my_func(4, delay=5) # doctest: +NORMALIZE_WHITESPACE + Traceback (most recent call last): + ... + test_child_procs.TestTimeout: + my_func(4, delay=5): timed out after 0.5 s + """ + if func is None: + return cast( + Callable[[Callable[PS, T]], Callable[PS, T]], + partial(add_timeout, timeout=timeout), + ) + + @wraps(func) + def worker( + fobj: IO[bytes], /, *args: PS.args, **kwargs: PS.kwargs + ) -> None: + try: + try: + result = True, func(*args, **kwargs) + except Exception as e: + result = False, ExceptionHelper(e, e.__traceback__) + # Do this instead of directly using `pickle.dump(..., fobj)` + # so that pickling errors and file-handle-related errors are + # handled separately + serialized = pickle.dumps(result, protocol=pickle.HIGHEST_PROTOCOL) + try: + fobj.write(serialized) + fobj.flush() + except Exception: + # Since this is run in a daemon thread, by the time this + # write happens the main thread could've already timed + # out and destroyed `fobj`... in that case just + # gracefully exit + pass + finally: + # See the docstring for that function to see why this is + # needed. + _cleanup_profiling_in_current_thread() + + @wraps(func) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + with BytesIO() as bio: + thread = new_thread(args=(bio, *args), kwargs=kwargs) + thread.start() + thread.join(timeout) + if not thread.is_alive(): + successful, result = pickle.loads(bio.getvalue()) + if successful: + return result + assert isinstance(result, Exception) + raise result + args_repr = [repr(a) for a in args] + args_repr.extend(f'{k}={v!r}' for k, v in kwargs.items()) + name = getattr(func, '__name__', repr(func)) + call_repr = f'{name}({", ".join(args_repr)})' + msg = f'{call_repr}: timed out after {timeout:.2g} s' + raise TestTimeout(msg) + + new_thread = partial(threading.Thread, target=worker, daemon=True) + + return wrapper + + +# =================== Warning handling/verification ==================== + + +class _WarningInfo(Protocol): + @property + def message(self) -> str | Warning: + ... + + @property + def category(self) -> type[Warning]: + ... + + @property + def filename(self) -> str: + ... + + @property + def lineno(self) -> int: + ... + + @property + def line(self) -> str | None: + ... + + +@dataclasses.dataclass +class _WarningMatcher: message: str | None = None category: type[Warning] | None = None module: str | None = None @@ -1479,7 +1364,7 @@ def new( ) -class _check_warnings(Sequence[_WarningInfo]): +class CheckWarnings(Sequence[_WarningInfo]): """ Helper context for deferring the checking of warnings to until context exit. @@ -1487,7 +1372,7 @@ class _check_warnings(Sequence[_WarningInfo]): Example: >>> import warnings - >>> cw = _check_warnings( + >>> cw = CheckWarnings( ... reissue_warnings=False, format_warnings=False, ... ) @@ -1656,106 +1541,302 @@ def _current_warnings(self) -> Sequence[_WarningInfo]: return self._last_captured -def _import_target(target: str) -> Any: - try: - return import_module(target) - except ImportError: # Not a module - assert '.' in target - module, _, attr = target.rpartition('.') - return getattr(import_module(module), attr) +# ================= `multiprocessing` patch resolution ================= +_PatchSummary = Mapping[str, Set[str]] -def _search_cache_logs( - cache: LineProfilingCache, - expecting_logs: bool, - patterns: Mapping[str, bool] | Collection[str], - match_individual_messages: bool = False, - flags: int = 0, -) -> None: - entries = cache._gather_debug_log_entries() - ResultMismatch.compare( - expecting_logs, bool(entries), - expected='logs' if expecting_logs else 'no logs', - actual=repr(entries) if entries else 'nothing', - ) - if not expecting_logs: - return - text_chunks: list[str] = [entry.to_text() for entry in entries] - if not match_individual_messages: - text_chunks = ['\n'.join(text_chunks)] - if isinstance(patterns, Mapping): - to_match: dict[str, bool] = { - str(pat): bool(should_match) - for pat, should_match in patterns.items() - } - else: - to_match = dict.fromkeys(patterns, True) - for pat, should_match in to_match.items(): - pattern = re.compile(pat, flags) - if any(pattern.search(chunk) for chunk in text_chunks) == should_match: - continue - raise ResultMismatch( - f'pattern {pattern!r} to {"" if should_match else "not "}match ' - f'{cache!r}\'s logs: {text_chunks!r}' - ) +mp_patch_is_internal: Callable[[str], bool] +mp_patch_is_internal = operator.methodcaller('startswith', '__') -# `shlex.join()` doesn't work properly on Windows, so use -# `subprocess.list2cmdline()` instead; -# though an "intentionally" undocumented API (cpython issue #10308), -# it's been around since 2.4, seems stable enough, and does exactly what -# is needed -if _WINDOWS: - concat_command_line: Callable[ - [Sequence[str]], str - ] = subprocess.list2cmdline -else: - concat_command_line = shlex.join +def get_patched_attributes( + applied_mp_patches: Collection[str] | None = None, +) -> MappingProxyType[str, frozenset[str]]: + if applied_mp_patches is None: + applied_mp_patches = { + patch for patch, applied in ( + ConfigSource.from_default() + .get_subconfig('child_processes', 'multiprocessing', 'patches') + .conf_dict.items() + ) if applied + } + return _get_patched_attributes(frozenset(applied_mp_patches)) -def _run_as_script( - request: pytest.FixtureRequest, - runner_args: list[str], - test_args: list[str], - test_module: _ModuleFixture, - *, - subproc: bool = True, - check_warnings: bool = True, - **kwargs -) -> subprocess.CompletedProcess: - cmd = runner_args + [str(test_module.path)] + test_args - if subproc: - run: Callable[..., subprocess.CompletedProcess] = _run_subproc - else: - run = partial(_run_kernprof_main_in_process, request, check_warnings) - test_module.install(children=True, local=not subproc, deps_only=True) - return run(cmd, **kwargs) +@lru_cache() +def _get_patched_attributes( + applied_mp_patches: frozenset[str], +) -> MappingProxyType[str, frozenset[str]]: + # Get the contents of the individual patches + patches = PATCH_SUMMARIES['minimal'].copy() + iter_summaries = ( + _MP_PATCHES[patch].summary + for patch in applied_mp_patches if patch in _MP_PATCHES + ) + patches = _get_patch_summary_union(patches, *iter_summaries) + return MappingProxyType({ + target: frozenset(attrs) + for target, attrs in filter_mp_patch_summary(patches).items() + }) -def _run_as_module( - request: pytest.FixtureRequest, - runner_args: list[str], - test_args: list[str], - test_module: _ModuleFixture, - *, - subproc: bool = True, - check_warnings: bool = True, - **kwargs -) -> subprocess.CompletedProcess: - cmd = runner_args + ['-m', test_module.name] + test_args - if subproc: - run: Callable[..., subprocess.CompletedProcess] = _run_subproc - else: - run = partial(_run_kernprof_main_in_process, request, check_warnings) - test_module.install(children=True, local=not subproc) - return run(cmd, **kwargs) +def get_mp_patches_toml_text(mp_patches: Collection[str]) -> str: + """ + Return the TOML section configuring whether to apply individual + :py:mod:`multiprocessing` patches. + """ + mp_patches_as_dict = { + name: name in mp_patches for name in _MP_PATCHES + if not mp_patch_is_internal(name) + } + return ( + '[tool.line_profiler.child_processes.multiprocessing.patches]\n' + + '\n'.join( + f'{patch} = {str(applied).lower()}' + for patch, applied in mp_patches_as_dict.items() + ) + ) -def _run_as_literal_code( - request: pytest.FixtureRequest, - runner_args: list[str], +def _get_patch_summary_union( + *summaries: _PatchSummary, +) -> dict[str, frozenset[str]]: + result: dict[str, frozenset[str]] = {} + for summary in summaries: + for target, attrs in summary.items(): + result[target] = result.get(target, frozenset()) | frozenset(attrs) + return result + + +def summarize_mp_patches( + summaries: Collection[tuple[bool, _PatchSummary]] +) -> dict[str, dict[str, bool]]: + """ + Take individual patch summaries and whether each of them is applied, + and return a dictionary representing whether each of the patch + targets is to be patched. + + Example: + >>> summarize_mp_patches([(False, {'foo': {'bar'}})]) + {'foo': {'bar': False}} + >>> summarize_mp_patches([ # doctest: +NORMALIZE_WHITESPACE + ... (False, {'foo': {'bar', 'baz'}}), + ... (True, {'foo': {'baz', 'foobar'}, 'spam': {'ham'}}), + ... (False, {'foo': {'baz'}, 'spam': {'eggs'}}) + ... ]) + {'foo': {'bar': False, 'baz': True, 'foobar': True}, + 'spam': {'eggs': False, 'ham': True}} + """ + def get_all_mentioned(s: Iterable[_PatchSummary]) -> dict[str, set[str]]: + all_items: dict[str, set[str]] = {} + for summary in s: + for target, attrs in summary.items(): + all_items.setdefault(target, set()).update(attrs) + return all_items + + all_items = get_all_mentioned(s for _, s in summaries) + all_patched = get_all_mentioned(s for applied, s in summaries if applied) + result: dict[str, dict[str, bool]] = { + target: + {attr: attr in all_patched.get(target, set()) for attr in attrs} + for target, attrs in all_items.items() + } + # Normalize the order for convenience + return { + target: dict(sorted(attrs.items())) + for target, attrs in sorted(result.items()) + } + + +def filter_mp_patch_summary(summary: _PatchSummary) -> dict[str, set[str]]: + """ + Filter the content of a + :py:attr:`line_profiler._child_process_profiling.multiprocessing\ +._infrastructure.Patch.summary` + so that only patch targets that actually exists remains. + """ + result: dict[str, set[str]] = {} + for target, attrs in summary.items(): + try: + obj = import_target(target) + except ImportError: + continue + present_attrs = {a for a in attrs if hasattr(obj, a)} + # Drop if none of the attributes is present + if present_attrs: + result[target] = present_attrs + return result + + +PATCH_SUMMARIES: dict[ + Literal['minimal', 'maximal', 'default', 'pth_hook'], + dict[str, frozenset[str]] +] = { + 'minimal': {'multiprocessing': frozenset({MP_PATCHED_MARKER})}, +} +# Get patches that are dynamically resolved: while these patches are +# always applied, some of the patch targets are +# platform-/Pyhon-version-specific and may not always exist +_dynamically_resolved_patch_summaries: Iterable[_PatchSummary] = ( + patch.summary for name, patch in _MP_PATCHES.items() + # Basic `multiprocessing` patches are always applied + if mp_patch_is_internal(name) +) +_dynamically_resolved_patch_summaries = itertools.chain( + _dynamically_resolved_patch_summaries, + # some platforms e.g. Windows don't have `fork()` + [{'os': frozenset({'fork'})}], +) +_dynamically_resolved_patch_summaries = cast( # See `ty` issue #3428 + Iterable[_PatchSummary], + map(filter_mp_patch_summary, _dynamically_resolved_patch_summaries), +) +PATCH_SUMMARIES['minimal'] = _get_patch_summary_union( + PATCH_SUMMARIES['minimal'], *_dynamically_resolved_patch_summaries, +) + +# This is only patched if we called +# `_line_profiler_hooks.load_pth_hook()` +PATCH_SUMMARIES['pth_hook'] = { + f'{load_pth_hook.__module__}.{load_pth_hook.__qualname__}': + frozenset({'called'}), +} +# Upper limit of what we could've patched +PATCH_SUMMARIES['maximal'] = _get_patch_summary_union( + PATCH_SUMMARIES['minimal'], + PATCH_SUMMARIES['pth_hook'], + get_patched_attributes([ + name for name in _MP_PATCHES if not mp_patch_is_internal(name) + ]), +) +# Actual patches using the default config +PATCH_SUMMARIES['default'] = _get_patch_summary_union( + PATCH_SUMMARIES['minimal'], get_patched_attributes(), +) + +# ========================= Command execution ========================== + +# `shlex.join()` doesn't work properly on Windows, so use +# `subprocess.list2cmdline()` instead; +# though an "intentionally" undocumented API (cpython issue #10308), +# it's been around since 2.4, seems stable enough, and does exactly what +# is needed +concat_command_line: Callable[[Sequence[str]], str] +if sys.platform == 'win32': + concat_command_line = subprocess.list2cmdline +else: + concat_command_line = shlex.join + + +def run_subproc( + cmd: Sequence[str] | str, /, *args, **kwargs +) -> subprocess.CompletedProcess: + """ + Wrapper around :py:func:`subprocess.run` which writes debugging + output. + """ + class HasStreams(Protocol): + @property + def stdout(self) -> str | bytes | None: + ... + + @property + def stderr(self) -> str | bytes | None: + ... + + if isinstance(cmd, str): + cmd_str = cmd + else: + cmd_str = concat_command_line(cmd) + + print('Command:', cmd_str) + _print_env_deltas(kwargs.get('env')) + print('-- Process start --') + # Note: somehow `mypy` doesn't agree with simply unpacking the + # `*args` into `subprocess.run()`... + status: int | str = '???' + result: HasStreams | None = None + subproc_errors = ( + subprocess.CalledProcessError, subprocess.TimeoutExpired, + ) + time = monotonic() + try: + proc = subprocess.run(cmd, *args, **kwargs) + except Exception as e: + status = 'error' + if isinstance(e, subproc_errors): + result = e + if hasattr(e, 'returncode'): # `CalledProcessError` + status = f'{status} ({e.returncode})' + raise + else: + result, status = proc, proc.returncode + return proc + finally: + time = monotonic() - time + if result is not None: + captured: str | bytes | None + for name, captured, stream in [ + ('stdout', result.stdout, sys.stdout), + ('stderr', result.stderr, sys.stderr), + ]: + if captured is None: + continue + if isinstance(captured, bytes): # `text=False` + captured = captured.decode() + print(f'{name}:\n{indent(captured, " ")}', file=stream) + print( + f'-- Process end (time elapsed: {time:.2f} s / ' + f'return status: {status}) --' + ) + + +# ======================== `kernprof` execution ======================== + + +def _run_as_script( + request: pytest.FixtureRequest, + runner_args: list[str], + test_args: list[str], + test_module: ModuleFixture, + *, + subproc: bool = True, + check_warnings: bool = True, + **kwargs +) -> subprocess.CompletedProcess: + cmd = runner_args + [str(test_module.path)] + test_args + if subproc: + run: Callable[..., subprocess.CompletedProcess] = run_subproc + else: + run = partial(_run_kernprof_main_in_process, request, check_warnings) + test_module.install(children=True, local=not subproc, deps_only=True) + return run(cmd, **kwargs) + + +def _run_as_module( + request: pytest.FixtureRequest, + runner_args: list[str], + test_args: list[str], + test_module: ModuleFixture, + *, + subproc: bool = True, + check_warnings: bool = True, + **kwargs +) -> subprocess.CompletedProcess: + cmd = runner_args + ['-m', test_module.name] + test_args + if subproc: + run: Callable[..., subprocess.CompletedProcess] = run_subproc + else: + run = partial(_run_kernprof_main_in_process, request, check_warnings) + test_module.install(children=True, local=not subproc) + return run(cmd, **kwargs) + + +def _run_as_literal_code( + request: pytest.FixtureRequest, + runner_args: list[str], test_args: list[str], - test_module: _ModuleFixture, + test_module: ModuleFixture, *, subproc: bool = True, check_warnings: bool = True, @@ -1763,15 +1844,15 @@ def _run_as_literal_code( ) -> subprocess.CompletedProcess: cmd = runner_args + ['-c', test_module.path.read_text()] + test_args if subproc: - run: Callable[..., subprocess.CompletedProcess] = _run_subproc + run: Callable[..., subprocess.CompletedProcess] = run_subproc else: run = partial(_run_kernprof_main_in_process, request, check_warnings) test_module.install(children=True, local=not subproc, deps_only=True) return run(cmd, **kwargs) -@_preserve_pth_files() -@_preserve_attributes() +@cleanup_extra_pth_files() +@preserve_targets() def _run_kernprof_main_in_process( request: pytest.FixtureRequest, check_warnings: bool, cmd: Sequence[str], *, @@ -1818,14 +1899,14 @@ def get_streams( timed_out = False time = monotonic() if timeout: - main = _timeout(kernprof_main, timeout=timeout) + main = add_timeout(kernprof_main, timeout=timeout) else: main = kernprof_main with ExitStack() as stack: if check_warnings: # See similar indictions against warnings in # `_test_apply_mp_patches()` - cw = stack.enter_context(_check_warnings()) + cw = stack.enter_context(CheckWarnings()) cw.forbid_warnings('.*resource_tracker', module='multiprocessing') cw.forbid_warnings( r'.* file\(s\) .* empty', @@ -1842,7 +1923,7 @@ def get_streams( if env is not None: cleanup.update_mapping(os.environ, env) main(cmd[1:], exit_on_error=False) - except _TestTimeout: # `subprocess` uses `SIGKILL` + except TestTimeout: # `subprocess` uses `SIGKILL` returncode, timed_out = -9, True status = f'error ({returncode})' except Exception as e: @@ -1902,145 +1983,73 @@ def _print_env_deltas(env: Mapping[str, str] | None = None) -> None: print('Env:', indent('\n'.join(diff), ' '), sep='\n') -def _run_subproc( - cmd: Sequence[str] | str, - /, - *args, +@cleanup_extra_pth_files() +def _run_test_module( + run_helper: Callable[..., subprocess.CompletedProcess], + request: pytest.FixtureRequest, + test_module: ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + runner: str | list[str] = 'kernprof', + outfile: str | None = None, + profile: bool = True, + *, + profiled_code_is_tempfile: bool = False, + fail: bool = False, + start_method: StartMethod | None = None, + test_module_args: Sequence[Any] | None = None, + test_module_kwargs: Mapping[str, Any] | None = None, + check: bool = True, + debug_log: str | None = None, + nhits: Mapping[str, int] | None = None, + subproc: bool = True, **kwargs -) -> subprocess.CompletedProcess: +) -> tuple[subprocess.CompletedProcess, LineStats | None]: """ - Wrapper around :py:func:`subprocess.run` which writes debugging - output. + Returns: + process_running_the_test_module (subprocess.CompletedProcess): + Process object + profliing_stats (LineStats | None): + Line-profiling stats (where available) """ - class HasStreams(Protocol): - @property - def stdout(self) -> str | bytes | None: - ... + if isinstance(runner, str): + runner_cli_args: list[str] = [runner] + else: + runner_cli_args = list(runner) - @property - def stderr(self) -> str | bytes | None: - ... + if not profile: + nhits = None - if isinstance(cmd, str): - cmd_str = cmd - else: - cmd_str = concat_command_line(cmd) + if profile and not profiled_code_is_tempfile: + runner_cli_args.extend(['--prof-mod', str(test_module.path)]) + if nhits is not None: + # We need `kernprof` to write the profliing results immediately + # to preserve data from tempfiles (see note below) + runner_cli_args.append('--view') - print('Command:', cmd_str) - _print_env_deltas(kwargs.get('env')) - print('-- Process start --') - # Note: somehow `mypy` doesn't agree with simply unpacking the - # `*args` into `subprocess.run()`... - status: int | str = '???' - result: HasStreams | None = None - subproc_errors = ( - subprocess.CalledProcessError, subprocess.TimeoutExpired, + if start_method and start_method not in ('dummy', *START_METHODS): + pytest.skip( + f'`multiprocessing` start method {start_method!r} ' + 'not available on the platform' + ) + if test_module_args is None: + test_module_args = () + if test_module_kwargs is None: + test_module_kwargs = {} + assert test_module.build_command_line + assert test_module.get_output + test_cli_args = test_module.build_command_line( + fail, start_method, *test_module_args, **test_module_kwargs, ) - time = monotonic() - try: - proc = subprocess.run(cmd, *args, **kwargs) - except Exception as e: - status = 'error' - if isinstance(e, subproc_errors): - result = e - if hasattr(e, 'returncode'): # `CalledProcessError` - status = f'{status} ({e.returncode})' - raise - else: - result, status = proc, proc.returncode - return proc - finally: - time = monotonic() - time - if result is not None: - captured: str | bytes | None - for name, captured, stream in [ - ('stdout', result.stdout, sys.stdout), - ('stderr', result.stderr, sys.stderr), - ]: - if captured is None: - continue - if isinstance(captured, bytes): # `text=False` - captured = captured.decode() - print(f'{name}:\n{indent(captured, " ")}', file=stream) - print( - f'-- Process end (time elapsed: {time:.2f} s / ' - f'return status: {status}) --' - ) - - -@_preserve_pth_files() -def _run_test_module( - run_helper: Callable[..., subprocess.CompletedProcess], - request: pytest.FixtureRequest, - test_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - runner: str | list[str] = 'kernprof', - outfile: str | None = None, - profile: bool = True, - *, - profiled_code_is_tempfile: bool = False, - use_local_func: bool = False, - fail: bool = False, - start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, - nnums: int | None = None, - nprocs: int | None = None, - check: bool = True, - debug_log: str | None = None, - nhits: Mapping[str, int] | None = None, - subproc: bool = True, - **kwargs -) -> tuple[subprocess.CompletedProcess, LineStats | None]: - """ - Returns: - process_running_the_test_module (subprocess.CompletedProcess): - Process object - profliing_stats (LineStats | None): - Line-profiling stats (where available) - """ - if isinstance(runner, str): - runner_args: list[str] = [runner] - else: - runner_args = list(runner) - - if not profile: - nhits = None - - if profile and not profiled_code_is_tempfile: - runner_args.extend(['--prof-mod', str(test_module.path)]) - if nhits is not None: - # We need `kernprof` to write the profliing results immediately - # to preserve data from tempfiles (see note below) - runner_args.append('--view') - - test_args: list[str] = [] - if use_local_func: - test_args.append('--local') - if fail: - test_args.append('--force-failure') - if start_method: - if start_method in START_METHODS: - test_args.extend(['-s', start_method]) - else: - pytest.skip( - f'`multiprocessing` start method {start_method!r} ' - 'not available on the platform' - ) - if nnums is None: - nnums = NUM_NUMBERS - else: - test_args.extend(['-l', str(nnums)]) - if nprocs is not None: - test_args.extend(['-n', str(nprocs)]) with ub.ChDir(tmp_path_factory.mktemp('mytemp')): if outfile is not None: - runner_args.extend(['--outfile', outfile]) + runner_cli_args.extend(['--outfile', outfile]) if debug_log: - runner_args.extend(['--debug-log', debug_log]) - old_pth_files = _preserve_pth_files.get_pth_files() + runner_cli_args.extend(['--debug-log', debug_log]) + old_pth_files = cleanup_extra_pth_files.get_pth_files() try: proc = run_helper( - request, runner_args, test_args, test_module, + request, runner_cli_args, test_cli_args, test_module, text=True, capture_output=True, check=(check and not fail), subproc=subproc, **kwargs @@ -2052,12 +2061,14 @@ def _run_test_module( assert proc.returncode else: # - The result is correctly calculated - expected = nnums * (nnums + 1) // 2 + expected = test_module.get_output( + *test_module_args, **test_module_kwargs, + ) output_lines = proc.stdout.splitlines() ResultMismatch.compare(str(expected), output_lines[0]) # - Temporary `.pth` file(s) created by # `LineProfilingCache.write_pth_hook()` has been cleaned up - assert _preserve_pth_files.get_pth_files() == old_pth_files + assert cleanup_extra_pth_files.get_pth_files() == old_pth_files # - Profiling results are written to the specified file prof_result: LineStats | None = None if outfile is None: @@ -2074,7 +2085,7 @@ def _run_test_module( # because if the profiled code is in a tempfile the # profiling data will be dropped in the written outfile) for tag, num in (nhits or {}).items(): - _check_output(proc.stdout, tag, num) + check_tagged_line_nhits(proc.stdout, tag, num) except subprocess.TimeoutExpired as e: # If we're not in running in a subproc, while # `kernprof.main()` shouldn't have gotten around to write @@ -2104,7 +2115,11 @@ def _run_test_module( return proc, prof_result -def _check_output(output: str, tag: str, nhits: int) -> None: +def check_tagged_line_nhits(output: str, tag: str, nhits: int) -> None: + """ + Check the output of :py:meth:`LineStats.print` for the number of + hits on the line tagged with the comment ``# GREP_MARKER[<...>]``. + """ # The line should be preixed with 5 numbers: # lineno, nhits, time, time-per-hit, % time actual_nhits = 0 @@ -2126,1457 +2141,3 @@ def _check_output(output: str, tag: str, nhits: int) -> None: run_literal_code = partial( _run_test_module, _run_as_literal_code, profiled_code_is_tempfile=True, ) - - -def _cleanup_profiling_in_current_thread() -> None: - """ - Disable all active profiler instances on the current thread, so that - we don't trip over ourselves if a new thread down the road happens - to reuse the thread ID. - - Note: - The thread-local profiler state is supposed to be handled by - :py:mod:`line_profiler._threading_patches` and - :py:class:`.CuratedProfilerContext`, but since a test function - decorated with :py:deco:`._timeout` will be isolated in a new - thread BEFORE the fixture setting up such management - (:py:func:`curated_profiler`) is invoked, we don't get that - managing and will have to deal with profilers ourselves. - """ - class Manager(Protocol): - @property - def active_instances(self) -> set[LineProfiler]: - ... - - lp_managers = cast( - dict[int, Manager], getattr(LineProfiler, '_managers', {}), - ) - thread_id = threading.get_ident() - if thread_id not in lp_managers: - return - instances = lp_managers[thread_id].active_instances - for prof in set(instances): - count = cast(int, getattr(prof, 'enable_count', 0)) - for _ in range(count): - prof.disable_by_count() - if count: - print('Disabled', prof, 'in thread', hex(thread_id)) - # Removed after the last `.disable_by_count()` - assert prof not in instances - - -@overload -def _timeout( - func: Callable[PS, T], *, timeout: float = _TEST_TIMEOUT, -) -> Callable[PS, T]: - ... - - -@overload -def _timeout( - func: None = None, *, timeout: float = _TEST_TIMEOUT, -) -> Callable[[Callable[PS, T]], Callable[PS, T]]: - ... - - -def _timeout( - func: Callable[PS, T] | None = None, *, - timeout: float = _TEST_TIMEOUT, -) -> Callable[PS, T] | Callable[[Callable[PS, T]], Callable[PS, T]]: - """ - Decorate the test function so that it is run in another thread and - can be timed out. - - Example: - >>> from time import sleep - - >>> @_timeout(timeout=.5) - ... def my_func( - ... n: int, delay: float = 1, error: bool = False, - ... ) -> list[int]: - ... sleep(delay) - ... if error: - ... raise RuntimeError('my error message') - ... return list(range(n)) - - Normal execution: - - >>> my_func(3, 0) + [3] - [0, 1, 2, 3] - - Erroring out: - - >>> my_func(3, 0, error=True) - Traceback (most recent call last): - ... - RuntimeError: my error message - - Timing out: - - >>> my_func(4, delay=5) # doctest: +NORMALIZE_WHITESPACE - Traceback (most recent call last): - ... - test_child_procs._TestTimeout: - my_func(4, delay=5): timed out after 0.5 s - """ - if func is None: - return cast( - Callable[[Callable[PS, T]], Callable[PS, T]], - partial(_timeout, timeout=timeout), - ) - - @wraps(func) - def worker( - fobj: IO[bytes], /, *args: PS.args, **kwargs: PS.kwargs - ) -> None: - try: - try: - result = True, func(*args, **kwargs) - except Exception as e: - result = False, ExceptionHelper(e, e.__traceback__) - # Do this instead of directly using `pickle.dump(..., fobj)` - # so that pickling errors and file-handle-related errors are - # handled separately - serialized = pickle.dumps(result, protocol=pickle.HIGHEST_PROTOCOL) - try: - fobj.write(serialized) - fobj.flush() - except Exception: - # Since this is run in a daemon thread, by the time this - # write happens the main thread could've already timed - # out and destroyed `fobj`... in that case just - # gracefully exit - pass - finally: - # See the docstring for that function to see why this is - # needed. - _cleanup_profiling_in_current_thread() - - @wraps(func) - def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: - with BytesIO() as bio: - thread = new_thread(args=(bio, *args), kwargs=kwargs) - thread.start() - thread.join(timeout) - if not thread.is_alive(): - successful, result = pickle.loads(bio.getvalue()) - if successful: - return result - assert isinstance(result, Exception) - raise result - args_repr = [repr(a) for a in args] - args_repr.extend(f'{k}={v!r}' for k, v in kwargs.items()) - name = getattr(func, '__name__', repr(func)) - call_repr = f'{name}({", ".join(args_repr)})' - msg = f'{call_repr}: timed out after {timeout:.2g} s' - raise _TestTimeout(msg) - - new_thread = partial(threading.Thread, target=worker, daemon=True) - - return wrapper - - -# ============================= Unit tests ============================= - -# XXX: Tests in this section concern implementation details, and the -# tested APIs and behaviors MUST NOT be relied upon by end-users. - -_PatchSummary = Mapping[str, Set[str]] - -_mp_patch_is_internal: Callable[[str], bool] -_mp_patch_is_internal = operator.methodcaller('startswith', '__') - - -def get_patched_attributes( - applied_mp_patches: Collection[str] | None = None, -) -> MappingProxyType[str, frozenset[str]]: - if applied_mp_patches is None: - applied_mp_patches = { - patch for patch, applied in ( - ConfigSource.from_config() - .get_subconfig('child_processes', 'multiprocessing', 'patches') - .conf_dict.items() - ) if applied - } - return _get_patched_attributes(frozenset(applied_mp_patches)) - - -@lru_cache() -def _get_patched_attributes( - applied_mp_patches: frozenset[str], -) -> MappingProxyType[str, frozenset[str]]: - # Get the contents of the individual patches - patches = _GLOBAL_MINIMAL_PATCHES.copy() - iter_summaries = ( - MP_PATCHES[patch].summary - for patch in applied_mp_patches if patch in MP_PATCHES - ) - patches = _get_patch_summary_union(patches, *iter_summaries) - return MappingProxyType({ - target: frozenset(attrs) - for target, attrs in _filter_patches(patches).items() - }) - - -def _get_toml_patches_section(mp_patches: Collection[str]) -> str: - mp_patches_as_dict = { - name: name in mp_patches for name in MP_PATCHES - if not _mp_patch_is_internal(name) - } - return ( - '[tool.line_profiler.child_processes.multiprocessing.patches]\n' - + '\n'.join( - f'{patch} = {str(applied).lower()}' - for patch, applied in mp_patches_as_dict.items() - ) - ) - - -def _get_patch_summary_union( - *summaries: _PatchSummary, -) -> dict[str, frozenset[str]]: - result: dict[str, frozenset[str]] = {} - for summary in summaries: - for target, attrs in summary.items(): - result[target] = result.get(target, frozenset()) | frozenset(attrs) - return result - - -def _summarize_patches( - summaries: Collection[tuple[bool, _PatchSummary]] -) -> dict[str, dict[str, bool]]: - """ - Example: - >>> _summarize_patches([(False, {'foo': {'bar'}})]) - {'foo': {'bar': False}} - >>> _summarize_patches([ # doctest: +NORMALIZE_WHITESPACE - ... (False, {'foo': {'bar', 'baz'}}), - ... (True, {'foo': {'baz', 'foobar'}, 'spam': {'ham'}}), - ... (False, {'foo': {'baz'}, 'spam': {'eggs'}}) - ... ]) - {'foo': {'bar': False, 'baz': True, 'foobar': True}, - 'spam': {'eggs': False, 'ham': True}} - """ - def get_all_mentioned(s: Iterable[_PatchSummary]) -> dict[str, set[str]]: - all_items: dict[str, set[str]] = {} - for summary in s: - for target, attrs in summary.items(): - all_items.setdefault(target, set()).update(attrs) - return all_items - - all_items = get_all_mentioned(s for _, s in summaries) - all_patched = get_all_mentioned(s for applied, s in summaries if applied) - result: dict[str, dict[str, bool]] = { - target: - {attr: attr in all_patched.get(target, set()) for attr in attrs} - for target, attrs in all_items.items() - } - # Normalize the order for convenience - return { - target: dict(sorted(attrs.items())) - for target, attrs in sorted(result.items()) - } - - -def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: - result: dict[str, set[str]] = {} - for target, attrs in summary.items(): - try: - obj = _import_target(target) - except ImportError: - continue - present_attrs = {a for a in attrs if hasattr(obj, a)} - # Drop if none of the attributes is present - if present_attrs: - result[target] = present_attrs - return result - - -_GLOBAL_MINIMAL_PATCHES = { - 'multiprocessing': frozenset({MP_PATCHED_MARKER}), -} -# Get patches that are dynamically resolved: while these patches are -# always applied, some of the patch targets are -# platform-/Pyhon-version-specific and may not always exist -_dynamically_resolved_patch_summaries: Iterable[_PatchSummary] = ( - patch.summary for name, patch in MP_PATCHES.items() - # Basic `multiprocessing` patches are always applied - if _mp_patch_is_internal(name) -) -_dynamically_resolved_patch_summaries = itertools.chain( - _dynamically_resolved_patch_summaries, - # some platforms e.g. Windows don't have `fork()` - [{'os': frozenset({'fork'})}], -) -_dynamically_resolved_patch_summaries = cast( # See `ty` issue #3428 - Iterable[_PatchSummary], - map(_filter_patches, _dynamically_resolved_patch_summaries), -) -_GLOBAL_MINIMAL_PATCHES = _get_patch_summary_union( - _GLOBAL_MINIMAL_PATCHES, *_dynamically_resolved_patch_summaries, -) - -# This is only patched if we called -# `_line_profiler_hooks.load_pth_hook()` -_HOOK_PATCHES = { - f'{load_pth_hook.__module__}.{load_pth_hook.__qualname__}': - frozenset({'called'}), -} -# Upper limit of what we could've patched -_GLOBAL_PATCHES = _get_patch_summary_union( - _GLOBAL_MINIMAL_PATCHES, - _HOOK_PATCHES, - get_patched_attributes([ - name for name in MP_PATCHES if not _mp_patch_is_internal(name) - ]), -) -# Actual patches using the default config -DEFAULT_GLOBAL_PATCHES = _get_patch_summary_union( - _GLOBAL_MINIMAL_PATCHES, get_patched_attributes(), -) - -_DEFAULT_MP_CONFIG = MPConfig.from_config(ConfigSource.from_default()) - - -@pytest.mark.parametrize(('run_profiled_code', 'label1'), - [(True, 'run-profiled'), (False, 'run-unrelated')]) -@pytest.mark.parametrize(('as_module', 'label2'), - [(True, 'run_module'), (False, 'run_path')]) -@pytest.mark.parametrize(('debug', 'label3'), - [(True, 'with-debug'), (False, 'no-debug')]) -def test_runpy_patches( - capsys: pytest.CaptureFixture[str], - ext_module: _ModuleFixture, - test_module: _ModuleFixture, - test_module_clone: _ModuleFixture, - create_cache: Callable[..., LineProfilingCache], - run_profiled_code: bool, - as_module: bool, - debug: bool, - label1: str, label2: str, label3: str, -) -> None: - """ - Test that the :py:mod:`runpy` clone created by - :py:func:`line_profiler._child_process_profiling\ -.create_runpy_wrapper` - correctly sets up profiling when its ``run_*()`` functions are - called. - """ - class restore_argv: - def __enter__(self) -> None: - self.argv = list(sys.argv) - - def __exit__(self, *_, **__) -> None: - sys.argv[:] = self.argv - - cache = create_cache( - rewrite_module=test_module.path, - profiling_targets=[str(ext_module.path)], - profile_imports=True, - debug=debug, - ) - assert cache.profiler is not None - runpy = create_runpy_wrapper(cache) - - nnums = 42 - nprocs = 2 - # If we're running some unrelated code, the profiler should not be - # involved - if run_profiled_code: - module = test_module - num_invocations, num_loops = 1, nprocs - expected_funcs: list[str] = ['my_external_sum'] - else: - module = test_module_clone - num_invocations, num_loops = 0, 0 - expected_funcs = [] - if as_module: - first_arg = module.name - runner = partial(runpy.run_module, alter_sys=True) - called_func = 'run_module' - else: - first_arg = str(module.path) - runner = runpy.run_path - called_func = 'run_path' - - # Check that the code is run - module.install(local=True, deps_only=not as_module) - with restore_argv(): - sys.argv[:] = [first_arg, f'--length={nnums}', '-n', str(nprocs)] - runner(first_arg, run_name='__main__') - stdout = capsys.readouterr().out - assert stdout.rstrip('\n') == str(nnums * (nnums + 1) // 2) - - # Check that profiler has received the appropriate targets - funcs = [func.__name__ for func in getattr(cache.profiler, 'functions')] - assert funcs == expected_funcs - - # Check that calls in the current process are profiled iif the - # correct file is executed - with StringIO() as sio: - cache.profiler.print_stats(sio) - stats = sio.getvalue() - _check_output(stats, 'EXT-INVOCATION', num_invocations) - _check_output(stats, 'EXT-LOOP', num_loops) - - # Check the debug-log entries are correctly gathered - _search_cache_logs( - cache, - debug, - { - rf'calling .*{called_func}\(': True, - r'calling .*exec\(': run_profiled_code, - }, - match_individual_messages=True, - flags=re.IGNORECASE, - ) - - -def test_cache_dump_load( - create_cache: Callable[..., LineProfilingCache], -) -> None: - """ - Test that: - - - We can round-trip the cache via :py:meth:`LineProfilingCache.dump` - and :py:meth:`LineProfilingCache.load` - - - The same instance is :py:meth:`LineProfilingCache.load`-ed in - subsequent calls - """ - original = create_cache( - profiling_targets=['foo', 'bar', 'baz'], main_pid=123456, - ) - cache_instances: list[LineProfilingCache] = [original] - envvars: set[str] = set(os.environ) - try: - original.inject_env_vars() # Needed for `.load()` - # Also test slipping stuff into the `._additional_data` - original._additional_data['foo'] = [1, 'string', None] - try: - # Env vars should be inserted - assert set(os.environ) == envvars.union(original.environ) > envvars - original.dump() - loaded = original.load() - cache_instances.append(loaded) - reloaded = original.load() - cache_instances.append(reloaded) - assert original is not loaded is reloaded - # Compare init fields - for field in dataclasses.fields(LineProfilingCache): - if not field.init: - continue - assert ( - getattr(original, field.name) - == getattr(loaded, field.name) - ) - # Compare `._additional_data` - assert original._additional_data == loaded._additional_data - finally: # Explicitly cleanup - for cache in cache_instances: - cache.cleanup() - finally: # Env vars restored after cleanup - assert set(os.environ) == envvars - - -@(_Params.new(('wrap_os_fork', 'label1'), - [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) - + _Params.new(('debug', 'label2'), - [(True, 'with-debug'), (False, 'no-debug')]) - + _Params.new(('patch_pool', 'patch_process', 'intercept_logs', 'label3'), - [(True, True, True, 'all-patches'), - (True, True, False, 'pool-and-process'), - (True, False, True, 'pool-and-logging'), - (True, False, False, 'pool-only'), - (False, True, True, 'process-and-logging'), - (False, True, False, 'process-only'), - (False, False, True, 'logging-only'), - (False, False, False, 'no-patches')])).sorted() -def test_cache_setup_main_process( - tmp_path_factory: pytest.TempPathFactory, - create_cache: Callable[..., LineProfilingCache], - wrap_os_fork: bool, - debug: bool, - patch_pool: bool, - patch_process: bool, - intercept_logs: bool, - label1: str, label2: str, label3: str, -) -> None: - """ - Test that :py:meth:`LineProfilingCache._setup_in_main_process` works - as expected. - """ - mp_patches: set[str] = set() - if patch_pool: - mp_patches.add('pool') - if patch_process: - mp_patches.add('process') - if intercept_logs: - mp_patches.add('logging') - - config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' - config.write_text(_get_toml_patches_section(mp_patches)) - cache = create_cache(debug=debug, config=config) - - # Check that only the requested patches are applied - patches = _summarize_patches([ - (True, _GLOBAL_MINIMAL_PATCHES), - *( - (name in mp_patches, _filter_patches(patch.summary)) - for name, patch in MP_PATCHES.items() - if not _mp_patch_is_internal(name) - ), - ]) - try: - patches['os']['fork'] = wrap_os_fork - except KeyError: - # `os.fork()` pruned because it doesn't exist on e.g. Windows - assert not hasattr(os, 'fork') - - with ExitStack() as stack: - patched = stack.enter_context(_preserve_attributes(patches)) - compare_patched = partial( - _preserve_attributes.compare_with_current_values, patched, - ) - original_pths = stack.enter_context(_preserve_pth_files()) - cache._setup_in_main_process(wrap_os_fork=wrap_os_fork) - # There should be exactly one extra `.pth` file - new_pth_hook, = _preserve_pth_files.get_pth_files() - original_pths - # Check whether the patches are applied - compare_patched(operator.is_not, assert_true=patches) - # Check that the instance is set as the `.load()`-ed one - assert cache is cache.load() - # Check whether the patches are reversed - cache.cleanup() - compare_patched() - - # Check the debug-log output - patterns: dict[str, bool] = dict.fromkeys( - [ - r'\(main process\)', - r'Injecting env var.*\$\{LINE_PROFILER_\w+\}', - re.escape(new_pth_hook), - ], - True, - ) - for target, maybe_patches in patches.items(): - patterns.update( - ('Patched.*' + re.escape(f'{target}.{attr}'), is_patched) - for attr, is_patched in maybe_patches.items() - ) - _search_cache_logs(cache, debug, patterns) - - -@pytest.mark.parametrize(('wrap_os_fork', 'label1'), - [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) -@pytest.mark.parametrize(('preimports', 'label2'), - [(True, 'with-preimports'), (False, 'no-preimports')]) -@pytest.mark.parametrize(('new_profiler', 'label3'), - [(True, 'no-profiler'), (False, 'with-profiler')]) -@pytest.mark.parametrize(('debug', 'label4'), - [(True, 'with-debug'), (False, 'no-debug')]) -@pytest.mark.parametrize('n', [100]) -@_preserve_attributes() -def test_cache_setup_child( - create_cache: Callable[..., LineProfilingCache], - ext_module_object: ModuleType, - another_pid: int, - wrap_os_fork: bool, - preimports: bool, - new_profiler: bool, - debug: bool, - n: int, - label1: str, label2: str, label3: str, label4: str, -) -> None: - """ - Test that :py:meth:`LineProfilingCache._setup_in_child_process` - works as expected. - """ - def list_profiled_funcs() -> list[str]: - return [ - f'{func.__module__}.{func.__qualname__}' - for func in getattr(cache.profiler, 'functions', []) - ] - - func = ext_module_object.my_external_sum - cache = create_cache( - profiling_targets=[f'{func.__module__}.{func.__qualname__}'], - preimports_module=preimports, - _use_curated_profiler=not new_profiler, - main_pid=another_pid, - debug=debug, - ) - assert (cache.profiler is None) == new_profiler - - seen_funcs = list_profiled_funcs() - if preimports: - preimport_targets = list(cache.profiling_targets) - else: - preimport_targets = [] - - with _preserve_obj_attributes(os, ['fork']) as preserved: - old_fork = preserved['fork'] - # Check that we're only setting up if there isn't already a - # profiler - assert cache._setup_in_child_process( - wrap_os_fork=wrap_os_fork, context='test_cache_setup_child', - ) == new_profiler - assert cache.profiler - if not new_profiler: - return - - # Check that the profiler has been presented with the profiling - # target - assert list_profiled_funcs() == (seen_funcs + preimport_targets) - - # Check that on cache cleanup: - # - Profiling data is collected - # - `os.fork()` is restored - # - The warning for empty profiling files is only issued when - # expected - assert func(range(1, n + 1)) == n * (n + 1) // 2 - stats = cache.profiler.get_stats() - for callback, has_nonempty_file, has_stats, fork_patched in [ - (lambda: None, False, False, wrap_os_fork), - (cache.cleanup, True, preimports, False), - ]: - callback() - with _check_warnings() as cw: - handle_warning: list[Callable[..., None]] = [] - if has_nonempty_file: - handle_warning.append(cw.forbid_warnings) - else: # Check for the warning but don't reissue it - handle_warning.extend([ - cw.expect_warnings, cw.suppress_warnings, - ]) - for handle in handle_warning: - handle(r'.* file\(s\) .* empty', module='line_profiler') - gathered = cache.gather_stats() - assert any(gathered.timings.values()) == has_stats, gathered - if hasattr(os, 'fork'): - assert (os.fork is not old_fork) == fork_patched - else: # E.g. Windows - assert old_fork == _NotSupplied.NOT_SUPPLIED - # Check that after cleaning up the profiler has been disabled - assert not getattr(cache.profiler, 'enable_count', 0) - - # Check that profiling results have been written to the cache - # directory - stats_file, = Path(cache.cache_dir).glob('*.lprof') - assert LineStats.from_files(stats_file) == stats == gathered - - # Check the debug-log output - patterns = { - f'Set up .*profiler.* {id(cache.profiler):#x}': True, - 'Loading preimports': preimports, - 'Created .*' + re.escape(stats_file.name): True, - 'Cleanup succeeded.*: .*dump_stats': True, - 'Loading results .*' + re.escape(stats_file.name): True, - } - _search_cache_logs(cache, debug, patterns) - - -@pytest.mark.parametrize('ppid_should_match', [True, False, None]) -@_preserve_attributes() -def test_load_pth_hook( - create_cache: Callable[..., LineProfilingCache], - another_pid: int, - ppid_should_match: bool | None, -) -> None: - """ - Simulate calling :py:func:`_line_profiler_hooks.load_pth_hook()` in - a child process. - - Notes: - - - The function is CALLED in the .pth file, but we don't actually - NEED a .pth file to call and test it. - - - The counterpart :py:meth:`line_profiler\ -._child_process_profiling.cache.LineProfilingCache.write_pth_hook()` - is implicitly tested in - :py:func:`test_cache_setup_main_process()`. - """ - # This test is mostly here to hack coverage; since the function is - # only to be called in child processes, `coverage` seems to have - # trouble getting data on it... - - # We basically only need this cache instance to set up the - # environment variables and the requisite files... - cache = create_cache(main_pid=another_pid) - if ppid_should_match is not None: - cache.inject_env_vars() - if ppid_should_match: - call_ppid = another_pid - else: # On a PPID mismatch, the function bails after checking - call_ppid = another_pid + 10 - else: - # Without the requisite envvars, the hook should bail very - # quickly (due to the `environ` lookup erroring out), regardless - # of the provided PPID - call_ppid = 0 - cache.dump() - - compare = _preserve_attributes.compare_with_current_values - patches = {**DEFAULT_GLOBAL_PATCHES, **_HOOK_PATCHES} - with _preserve_attributes(patches) as patched: - try: - # NOTE: this creates a cache instance that isn't - # automatically cleaned up by the `create_cache()` - # fixture!!! Hence the try-finally - load_pth_hook(call_ppid) - # Check that the patches are applied where appropriate - assert ( - getattr(load_pth_hook, 'called', False) - == bool(ppid_should_match) - ) - if ppid_should_match: - compare(patched, operator.is_not) - else: # no-op - compare(patched) - return - # Check that calling `load_pth_hook()` again is a no-op - with _preserve_attributes(patches) as re_patched: - load_pth_hook(call_ppid) - compare(re_patched) - finally: - try: - current_cache = LineProfilingCache.load() - except Exception: - pass - else: - current_cache.cleanup() - # Check that the patches are reversed - compare(patched) - - -@_preserve_pth_files() -@_preserve_attributes() -@_timeout -def _test_apply_mp_patches_inner( - tmp_path_factory: pytest.TempPathFactory, - create_cache: Callable[..., LineProfilingCache], - ext_module_object: ModuleType, - test_module_object: ModuleType, - start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], - mp_patches: Collection[str], - fail: bool, - n: int, - nprocs: int, -) -> None: - def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: - try: - LineStats.from_files(path, on_empty='error', on_defective='error') - except Exception: - return False - return True - - def get_lineno(path: os.PathLike[str] | str, query: str) -> int: - with Path(path).open() as fobj: - for i, line in enumerate(fobj): - if query in line: - return 1 + i - raise RuntimeError( - f'Did not find line containing {query!r} in {path!r}', - ) - - config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' - intercept_logs = 'logging' in mp_patches - patch_process = 'process' in mp_patches - cfg_chunks: list[str] = [ - _get_toml_patches_section(mp_patches), - # This is easier to debug than `ResultMismatch` - '[tool.line_profiler.child_processes.multiprocessing.polling]\n' - 'on_timeout = "error"', - ] - config.write_text('\n\n'.join(cfg_chunks)) - - # Note: no need to test the case for `my_local_sum()` separately, - # with `preimports_module=True`, both are just imported and added - # to the profiler, so the code paths are the same - profiled_func = ext_module_object.my_external_sum - called_func = partial( - test_module_object.sum_in_child_procs, - n=nprocs, - my_sum=profiled_func, - start_method=start_method, - fail=fail, - ) - - func_name = f'{profiled_func.__module__}.{profiled_func.__qualname__}' - cache = create_cache( - profiling_targets=[func_name], - preimports_module=True, - config=config, - debug=True, - ) - # Note: - # - The reversibility of the patches have already been tested in - # `test_cache_setup_main_process()`, so we just actually test the - # patched-in components themselves here. - # - `._setup_in_main_process()` doesn't include actually doing the - # preimports. To may the results more consistent between - # `start_method='dummy'` and the others, manually do them below. - cache._setup_in_main_process() # This calls `apply()` - assert cache.profiler is not None - assert cache.preimports_module is not None - run_path(str(cache.preimports_module), {'profile': cache.profiler}) - - timing_key = ( - inspect.getfile(profiled_func), - inspect.getsourcelines(profiled_func)[1], - profiled_func.__qualname__, - ) - assert ext_module_object.__file__ - loop_line = get_lineno(ext_module_object.__file__, 'EXT-LOOP') - - nloops_expected = n - if not fail: - # Counts from the one final sum over the parallel results - nloops_expected += nprocs - - fail_msg = 'forced failure' - try: - result = called_func(n) - except RuntimeError as e: - if not (fail and str(e) == fail_msg): - raise - else: - if fail: - msg = f"expected `RuntimeError({fail_msg!r})`, no error raised" - raise ValueError(msg) - else: # Check correctness of the results - assert result == n * (n + 1) // 2 - - # Check that calls in children are traced - cache.cleanup() - stats = cache.profiler.get_stats() - stats += cache.gather_stats() - entries = stats.timings[timing_key] - nloops = sum(nhits for ln, nhits, _ in entries if ln == loop_line) - ResultMismatch.compare(nloops_expected, nloops) - - # Check the debug logs to see if we have done everything right, esp. - # the logging interception part not covered by other tests - patterns: dict[str, bool] = {} - if patch_process: - # Note: if we're not using `Process`-based patch, there is no - # guaratee that the profiling result is written via cleanup - iter_stats: Iterable[Path] = Path(cache.cache_dir).glob('*.lprof') - iter_stats = cast( # See `ty` issue #3428 - Iterable[Path], filter(is_valid_stats_file, iter_stats), - ) - pat = 'Cleanup succeeded.*: .*dump_stats.*{}' - patterns.update({ - pat.format(re.escape(path.name)): True for path in iter_stats - }) - patterns[re.escape('`multiprocessing` logging (debug)')] = intercept_logs - _search_cache_logs(cache, True, patterns) - - -def _test_apply_mp_patches( - patch_pool: bool | None = None, - patch_process: bool | None = None, - intercept_logs: bool | None = None, - *, - start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], - **kwargs -) -> None: - if start_method not in ('dummy', *START_METHODS): - pytest.skip( - f'`multiprocessing` start method {start_method!r} ' - 'not available on the platform' - ) - - patches = cast(dict[str, bool], _DEFAULT_MP_CONFIG.patches.copy()) - for name, applied in { - 'pool': patch_pool, 'process': patch_process, - 'logging': intercept_logs, - }.items(): - if applied is not None: - patches[name] = applied - mp_patches = [name for name, applied in patches.items() if applied] - - with _check_warnings() as cw: - cw.forbid_warnings('.*resource_tracker', module='multiprocessing') - cw.forbid_warnings( - r'.* file\(s\) .* empty', - category=UserWarning, module='line_profiler', - ) - if start_method == 'fork': - # The `@_timeout` decorator spins up a new thread for - # executing `_test_apply_mp_patches_inner()`; explicitly - # ignore the associated warning when we use - # `start_method='fork'` - cw.suppress_warnings( - r'.*multi-?threaded.*fork\(\)', - category=DeprecationWarning, - ) - _test_apply_mp_patches_inner( - mp_patches=mp_patches, start_method=start_method, **kwargs, - ) - - -@(_Params.new('start_method', - ['fork', 'forkserver', 'spawn', 'dummy'], - defaults='dummy') - # We only need to check if `intercept_logs = logging` work, the other - # parametrizations don't matter - + _Params.new(('intercept_logs', 'label1'), - [(True, 'with-logging'), (False, 'no-logging')], - defaults=(None, 'default-logging'))).sorted() -@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label2'), - [(True, True, 'pool-and-process'), - (True, False, 'pool-only'), - (False, True, 'process-only')]) -@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) -def test_apply_mp_patches_success( - tmp_path_factory: pytest.TempPathFactory, - create_cache: Callable[..., LineProfilingCache], - ext_module_object: ModuleType, - test_module_object: ModuleType, - start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], - patch_pool: bool, - patch_process: bool, - intercept_logs: bool | None, - n: int, - nprocs: int, - label1: str, - label2: str, -) -> None: - """ - Test that :py:func:`line_profiler._child_process_profiling\ -.multiprocessing_patches.apply` - works as expected when the parallel workload does not error out. - - See also: - :py:func:`test_apply_mp_patches_failure` - """ - _test_apply_mp_patches( - patch_pool, - patch_process, - intercept_logs, - tmp_path_factory=tmp_path_factory, - create_cache=create_cache, - ext_module_object=ext_module_object, - test_module_object=test_module_object, - start_method=start_method, - fail=False, - n=n, - nprocs=nprocs, - ) - - -@pytest.mark.parametrize('start_method', - ['fork', 'forkserver', 'spawn', 'dummy']) -@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label'), - [(True, True, 'pool-and-process'), - (True, False, 'pool-only'), - (False, True, 'process-only')]) -@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) -def test_apply_mp_patches_failure( - tmp_path_factory: pytest.TempPathFactory, - create_cache: Callable[..., LineProfilingCache], - ext_module_object: ModuleType, - test_module_object: ModuleType, - start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], - patch_pool: bool, - patch_process: bool, - n: int, - nprocs: int, - label: str, -) -> None: - """ - Test that :py:func:`line_profiler._child_process_profiling\ -.multiprocessing_patches.apply` - works as expected when the parallel workload errors out. - - See also: - :py:func:`test_apply_mp_patches_success` - """ - _test_apply_mp_patches( - patch_pool, - patch_process, - tmp_path_factory=tmp_path_factory, - create_cache=create_cache, - ext_module_object=ext_module_object, - test_module_object=test_module_object, - start_method=start_method, - fail=True, - n=n, - nprocs=nprocs, - ) - - -# XXX: End of tests for implementation details - -# ========================= Integration tests ========================== - - -def _get_mp_start_method_fuzzer(label_name: str | None) -> _Params: - """ - Returns: - :py:class:`_Params` object which does a full Cartesian-product - fuzz between ``fail`` (true or false) and ``start_method`` - ('fork', 'forkserver', and 'spawn'; default :py:const:`None`) - """ - if label_name is None: - label_name, drop_label = '_', True - else: - drop_label = False - fuzz_fail = _Params.new(('fail', label_name), - [(True, 'failure'), (False, 'success')], - defaults=(False, 'success')) - if drop_label: - fuzz_fail = fuzz_fail.drop_params(label_name) - fuzz_start = _Params.new('start_method', ['fork', 'forkserver', 'spawn'], - defaults=None) - return fuzz_fail * fuzz_start - - -@(_Params.new(('run_func', 'label1'), - [(run_module, 'module'), (run_script, 'script')]) - * _Params.new(('use_local_func', 'label2'), - [(True, 'local'), (False, 'ext')]) - # Python can't pickle things unless they resided in a retrievable - # location (so not the script supplied by `python -c`) - + _Params.new(('run_func', 'label1', 'use_local_func', 'label2'), - [(run_literal_code, 'literal-code', False, 'ext')]) - # Also fuzz the parallelization-related stuff, esp. check what - # happens if an exception is raised inside the parallelly-run func - + _get_mp_start_method_fuzzer('label3') - + _Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], - defaults=(None, None))).sorted() -def test_multiproc_script_sanity_check( - run_func: Callable[..., subprocess.CompletedProcess], - request: pytest.FixtureRequest, - test_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - use_local_func: bool, - fail: bool, - start_method: Literal['fork', 'forkserver', 'spawn'] | None, - nnums: int | None, - nprocs: int | None, - # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, label3: str, -) -> None: - """ - Sanity check that the test module functions as expected when run - with vanilla Python. - """ - run_func( - request, test_module, tmp_path_factory, - runner=sys.executable, profile=False, - fail=fail, - use_local_func=use_local_func, - start_method=start_method, - nnums=nnums, nprocs=nprocs, - ) - - -@pytest.mark.parametrize( - ('run_func', 'label1'), - [(run_module, 'module'), - (run_script, 'script'), - (run_literal_code, 'literal-code')] -) -@pytest.mark.parametrize( - ('runner', 'outfile', 'profile', - 'label2'), # Dummy argument to make `pytest` output more legible - # This is essentially a no-op since it doesn't actually do - # line-profiling, but we check that code path for completeness - [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] - # Run line profiling with and w/o profiling targets - + [(['kernprof', '-q', '-l'], 'out.lprof', False, - 'line_profiler-inactive'), - (['kernprof', '-q', '-l'], 'out.lprof', True, - 'line_profiler-active')], -) -def test_running_multiproc_script( - run_func: Callable[..., subprocess.CompletedProcess], - request: pytest.FixtureRequest, - test_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - runner: str | list[str], - outfile: str | None, - profile: bool, - # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, -) -> None: - """ - Check that `kernprof` can RUN the test module in various contexts - (`kernprof [...] `, `kernprof [...] -m `, and - `kernprof [...] -c "code"`). - - Notes: - - See issue #422 for the original motivation. - - - This test does not test the actual profiling, just the - execution of the code and presence of profiling data - thereafter. - """ - run_func(request, test_module, tmp_path_factory, runner, outfile, profile) - - -_fuzz_prof_mp_run_func = _Params.new(('run_func', 'label1'), - [(run_module, 'module'), - (run_script, 'script'), - (run_literal_code, 'literal-code')], - defaults=(run_script, 'script')) -_fuzz_prof_mp_markers = ( - (_fuzz_prof_mp_run_func - + _Params.new(('prof_child_procs', 'label2'), - [(True, 'with-child-prof'), (False, 'no-child-prof')]) - + _get_mp_start_method_fuzzer(None)) - # Test all `multiproc` start methods with both locally- and - # externally-defined profiling targets - * (_Params.new(('preimports', 'label3'), [(False, 'no-preimports')]) - + _Params.new(('use_local_func', 'label4'), - [(True, 'local'), (False, 'external')], - defaults=(False, 'external'))) - # The 'with-preimports' case is already tested rather thoroughly in - # `test_apply_mp_patches()`, so exclude these from the above "main" - # param matrix and just test the different `kernprof` modes via the - # `run_func()`s - + (_fuzz_prof_mp_run_func - + _Params.new(('preimports', 'label3'), [(True, 'with-preimports')])) - # Just throw in a case where we actually run `kernprof` in a - # subprocess, otherwise it is more convenient to do so in-process - + _Params.new(('subproc', 'label5'), - [(True, 'subproc'), (False, 'in-proc')], - defaults=(False, 'in-proc')) -).sorted().split_on_params('fail') - - -def _test_profiling_multiproc_script( - run_func: Callable[..., subprocess.CompletedProcess], - request: pytest.FixtureRequest, - test_module: _ModuleFixture, - ext_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - prof_child_procs: bool, - preimports: bool, - use_local_func: bool, - fail: bool, - start_method: Literal['fork', 'forkserver', 'spawn'] | None, - nnums: int, - nprocs: int, - **kwargs -) -> None: - # How many calls do we expect? - nhits = dict.fromkeys( - ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, - ) - # Make sure we're profiling the right function - tag = 'LOCAL' if use_local_func else 'EXT' - tag_call = tag + '-INVOCATION' - tag_loop = tag + '-LOOP' - if not fail: - # The final sum in the parent process should always be profiled - # unless the child processes failed and we never returned from - # `Pool.starmap()` - nhits[tag_call] += 1 - nhits[tag_loop] += nprocs - if prof_child_procs: - # When profiling extends into child processes, each of them - # invokes the sum function once and when combined they loop thru - # all the items - nhits[tag_call] += nprocs - nhits[tag_loop] += nnums - - runner = ['kernprof', '-l'] - runner.extend([ - '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), - '--{}preimports'.format('' if preimports else 'no-'), - ]) - if not use_local_func: - # Also make sure to include the external module in `--prof-mod` - runner.append(f'--prof-mod={ext_module.name}') - kwargs.setdefault('timeout', _TEST_TIMEOUT) - if prof_child_procs and _DEBUG: - kwargs.setdefault('debug_log', 'debug.log') - run_func( - request, test_module, tmp_path_factory, - runner=runner, - outfile='out.lprof', - profile=True, - use_local_func=use_local_func, - fail=fail, - start_method=start_method, - nhits=nhits, - nnums=nnums, - nprocs=nprocs, - **kwargs, - ) - - -@(_fuzz_prof_mp_markers[False]) -@pytest.mark.parametrize( - # XXX: should we explicitly test the single-proc case? We already - # have quite a lot of subtests tho... - ('nnums', 'nprocs'), [(2000, 3)], -) -def test_profiling_multiproc_script_success( - run_func: Callable[..., subprocess.CompletedProcess], - request: pytest.FixtureRequest, - test_module: _ModuleFixture, - ext_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - prof_child_procs: bool, - preimports: bool, - use_local_func: bool, - start_method: Literal['fork', 'forkserver', 'spawn'] | None, - nnums: int, - nprocs: int, - subproc: bool, - # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, label3: str, label4: str, label5: str, -) -> None: - """ - Check that `kernprof` can PROFILE the test module in various - contexts when the parallel workload runs without errors, optionally - extending profiling into child processes. - - Note: - This test function is heavily parametrized. Here is why that is - necessary: - - - ``run_func`` tests the different :cmd:`kernprof` modes (see - :py:func:`~.test_running_multiproc_script`). - - - ``preimports`` tests that both mechanisms for setting up - profiling targets work: - - - :py:const:`True`: child processes import the module - generated by - :py:mod:`line_profiler.autoprofile.eager_preimports`, like - the main :py:mod:`kernprof` process does. - - - :py:const:`False`: child processes rewrite the executed code - before passing it to :py:mod:`runpy`, similar to what - :py:mod:`line_profiler.autoprofile.autoprofile` does. - - These code paths go through different - :py:mod:`multiprocessing` components that we have patched and - thus needs separate testing. - - - ``use_local_func`` tests that we can consistently set up - profiling in both functions locally-defined in the profiled - code and imported by it. - - - ``fail`` tests that our patches and hook doesn't choke when - exceptions occur in child processes, and profiling data can - still be collected. - - - ``start_method`` tests whether all available - :py:mod:`multiprocessing` start methods are covered. - - - ``prof_child_procs`` of course toggles whether to do the - patches to set up profiling in child processes. - - See also: - :py:func:`test_profiling_multiproc_script_failure` - """ - _test_profiling_multiproc_script( - run_func=run_func, - request=request, - test_module=test_module, - ext_module=ext_module, - tmp_path_factory=tmp_path_factory, - prof_child_procs=prof_child_procs, - preimports=preimports, - use_local_func=use_local_func, - fail=False, - start_method=start_method, - nnums=nnums, - nprocs=nprocs, - subproc=subproc, - ) - - -@(_fuzz_prof_mp_markers[True]) -@pytest.mark.parametrize(('nnums', 'nprocs'), [(2000, 3)]) -def test_profiling_multiproc_script_failure( - run_func: Callable[..., subprocess.CompletedProcess], - request: pytest.FixtureRequest, - test_module: _ModuleFixture, - ext_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - prof_child_procs: bool, - preimports: bool, - use_local_func: bool, - start_method: Literal['fork', 'forkserver', 'spawn'] | None, - nnums: int, - nprocs: int, - subproc: bool, - # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, label3: str, label4: str, label5: str, -) -> None: - """ - Check that `kernprof` can PROFILE the test module in various - contexts when the parallel workload errors out, optionally - extending profiling into child processes. - - See also: - :py:func:`test_profiling_multiproc_script_success` - """ - _test_profiling_multiproc_script( - run_func=run_func, - request=request, - test_module=test_module, - ext_module=ext_module, - tmp_path_factory=tmp_path_factory, - prof_child_procs=prof_child_procs, - preimports=preimports, - use_local_func=use_local_func, - fail=True, - start_method=start_method, - nnums=nnums, - nprocs=nprocs, - subproc=subproc, - ) - - -_fuzz_bare = ( - _Params.new(('use_subprocess', 'label1'), - [(True, 'subprocess.run'), (False, 'os.system')]) - * _Params.new(('prof_child_procs', 'label2'), - [(True, 'with-child-prof'), (False, 'no-child-prof')]) - * _Params.new('n', [200]) -) - - -def _test_profiling_bare_python( - tmp_path_factory: pytest.TempPathFactory, - ext_module: _ModuleFixture, - use_subprocess: bool, - prof_child_procs: bool, - fail: bool, - n: int, -) -> None: - ext_module.install(children=True) - temp_dir = tmp_path_factory.mktemp('mytemp') - - script_path = temp_dir / 'my-script.py' - script_content = strip(""" - from {EXT_MODULE} import my_external_sum - - - if __name__ == '__main__': - numbers = list(range(1, 1 + {N})) - result = my_external_sum(numbers, {FAIL}) - """.format( - EXT_MODULE=ext_module.name, - N=n, - FAIL=fail, - )) - script_path.write_text(script_content) - - out_file = temp_dir / 'out.lprof' - debug_log_file = temp_dir / 'debug.log' - write_debug = _DEBUG and prof_child_procs - cmd = [ - 'kernprof', '-lv', '--preimports', - f'--prof-mod={ext_module.name}', - f'--outfile={out_file}', - '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), - ] - if write_debug: - cmd.append(f'--debug-log={debug_log_file}') - sub_cmd = [sys.executable, str(script_path)] - if use_subprocess: - code = strip(f""" - import subprocess - - - subprocess.run({sub_cmd!r}, check=True) - """) - else: - code = strip(""" - import os - - - if os.system({!r}): - raise RuntimeError('called process failed') - """.format(concat_command_line(sub_cmd))) - cmd.extend(['-c', code]) - proc = _run_subproc( - cmd, text=True, capture_output=True, timeout=_TEST_TIMEOUT, - ) - - nhits = {'EXT-INVOCATION': 1, 'EXT-LOOP': n} - if not prof_child_procs: - for k in nhits: - nhits[k] = 0 - - try: - # Check that the code errors out when expected - assert bool(fail) == bool(proc.returncode) - # Check that the profiling output is as expected - for tag, num in nhits.items(): - _check_output(proc.stdout, tag, num) - finally: - if write_debug: - print('-- Combined debug logs --', file=sys.stderr) - print( - indent(debug_log_file.read_text(), ' '), - end='', file=sys.stderr, - ) - print('-- End of debug logs --', file=sys.stderr) - - -@_fuzz_bare -def test_profiling_bare_python_success( - tmp_path_factory: pytest.TempPathFactory, - ext_module: _ModuleFixture, - use_subprocess: bool, - prof_child_procs: bool, - n: int, - # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, -) -> None: - """ - Check that `kernprof` can profile the target functions if the code - invokes another bare Python process (via either :py:func:`os.system` - or :py:func:`subprocess.run`) that calls them and exits without - errors. - - See also: - :py:func:`test_profiling_bare_python_failure` - """ - _test_profiling_bare_python( - tmp_path_factory=tmp_path_factory, - ext_module=ext_module, - use_subprocess=use_subprocess, - prof_child_procs=prof_child_procs, - fail=False, - n=n, - ) - - -@_fuzz_bare -def test_profiling_bare_python_failure( - tmp_path_factory: pytest.TempPathFactory, - ext_module: _ModuleFixture, - use_subprocess: bool, - prof_child_procs: bool, - n: int, - label1: str, - label2: str, -) -> None: - """ - Check that `kernprof` can profile the target functions if the code - invokes another bare Python process (via either :py:func:`os.system` - or :py:func:`subprocess.run`) that calls them and exits with errors. - - See also: - :py:func:`test_profiling_bare_python_success` - """ - _test_profiling_bare_python( - tmp_path_factory=tmp_path_factory, - ext_module=ext_module, - use_subprocess=use_subprocess, - prof_child_procs=prof_child_procs, - fail=True, - n=n, - ) diff --git a/tests/test_child_procs/conftest.py b/tests/test_child_procs/conftest.py new file mode 100644 index 00000000..5d28a320 --- /dev/null +++ b/tests/test_child_procs/conftest.py @@ -0,0 +1,332 @@ +from __future__ import annotations + +import os +import shutil +from collections.abc import Callable, Collection, Generator +from functools import partial +from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory +from textwrap import indent +from types import ModuleType +from typing import Any + +import pytest + +from line_profiler._child_process_profiling.cache import LineProfilingCache +from line_profiler.curated_profiling import ( + CuratedProfilerContext, ClassifiedPreimportTargets, +) +from line_profiler.autoprofile.util_static import _static_parse +from line_profiler.line_profiler import LineProfiler + +from ._test_child_procs_utils import ( + preserve_object_attrs, ModuleFixture, StartMethod, ResultMismatch, +) + + +__all__ = ( + 'ext_module', 'pool_test_module', 'pool_test_module_clone', + 'ext_module_object', 'pool_test_module_object', + 'create_cache', 'curated_profiler', 'another_pid', +) + +# Only write the files once per test session + +_EXAMPLES_PATH = Path(__file__).parent / 'multiproc_examples' + + +@pytest.fixture(scope='session') +def _ext_module() -> Generator[Path, None, None]: + name = next(ModuleFixture.propose_name('my_ext_module')) + with TemporaryDirectory() as mydir_str: + my_dir = Path(mydir_str) + my_dir.mkdir(exist_ok=True) + my_module = my_dir / f'{name}.py' + shutil.copy(_EXAMPLES_PATH / 'external_module.py', my_module) + yield my_module + + +@pytest.fixture(scope='session') +def _pool_test_module(_ext_module: Path) -> Generator[Path, None, None]: + name = next(ModuleFixture.propose_name('my_pool_test_module')) + body = (_EXAMPLES_PATH / 'pool_test_module.py').read_text() + body = body.replace('external_module', _ext_module.stem) + with TemporaryDirectory() as mydir_str: + my_dir = Path(mydir_str) + my_dir.mkdir(exist_ok=True) + my_module = my_dir / f'{name}.py' + my_module.write_text(body) + yield my_module + + +@pytest.fixture +def ext_module( + _ext_module: Path, monkeypatch: pytest.MonkeyPatch, +) -> Generator[ModuleFixture, None, None]: + """ + Yields: + :py:class:`ModuleFixture` helper object containing the code at + ``./multiproc_examples/external_module.py`` + """ + yield ModuleFixture(_ext_module, monkeypatch) + + +@pytest.fixture +def pool_test_module( + _pool_test_module: Path, + ext_module: ModuleFixture, + monkeypatch: pytest.MonkeyPatch, +) -> Generator[ModuleFixture, None, None]: + """ + Yields: + :py:class:`ModuleFixture` helper object containing the code at + ``./multiproc_examples/pool_test_module.py`` + """ + def build_command_line( + fail: bool, start_method: StartMethod | None, *, + use_local_func: bool = False, + nnums: int | None = None, + nprocs: int | None = None, + ) -> list[str]: + args: list[str] = [] + if fail: + args.append('--force-failure') + if start_method: + args.extend(['-s', start_method]) + if use_local_func: + args.append('--local') + if nnums: + args.extend(['-l', str(nnums)]) + if nprocs: + args.extend(['-n', str(nprocs)]) + return args + + def get_output(*, nnums: int | None = None, **_) -> int: + if nnums is None: + nnums = default_nnums + return nnums * (nnums + 1) // 2 + + default_nnums = _static_parse('NUM_NUMBERS', _pool_test_module) + + yield ModuleFixture( + _pool_test_module, monkeypatch, [ext_module], + build_command_line, get_output, + ) + + +@pytest.fixture +def pool_test_module_clone( + tmp_path_factory: pytest.TempPathFactory, + monkeypatch: pytest.MonkeyPatch, + pool_test_module: ModuleFixture, + ext_module: ModuleFixture, +) -> Generator[ModuleFixture, None, None]: + """ + Yields: + :py:class:`ModuleFixture` helper object containing the same + code as :py:data:`pool_test_module` + """ + tmpdir = tmp_path_factory.mktemp('my_path') + name = next(ModuleFixture.propose_name('my_cloned_pool_test_module')) + path = tmpdir / f'{name}.py' + path.write_text(pool_test_module.path.read_text()) + yield ModuleFixture( + path, monkeypatch, [ext_module], + pool_test_module.build_command_line, pool_test_module.get_output, + ) + + +@pytest.fixture +def ext_module_object( + ext_module: ModuleFixture, +) -> Generator[ModuleType, None, None]: + """ + Yields: + :py:class:`ModuleType` object containing the code at + :py:data:`EXTERNAL_MODULE_BODY`, and is torn down at the end of + the test + """ + yield from ext_module._import_module_helper() + + +@pytest.fixture +def pool_test_module_object( + pool_test_module: ModuleFixture, ext_module_object: ModuleType, +) -> Generator[ModuleType, None, None]: + """ + Yields: + :py:class:`ModuleType` object containing the code at + :py:data:`TEST_MODULE_TEMPLATE`, and is torn down at the end of + the test + """ + yield from pool_test_module._import_module_helper() + + +@pytest.fixture +def create_cache( + tmp_path_factory: pytest.TempPathFactory, + request: pytest.FixtureRequest, +) -> Generator[Callable[..., LineProfilingCache], None, None]: + """ + Wrapper around the :py:class:`LineProfilingCache` instantiator + which: + + - Automatically creates a tempdir and provides it as the + :py:attr:`LineProfilingCache.cache_dir`, + + - Extends the argument ``preimports_module`` to allow for taking + boolean values: + + - ``True``: a temporary preimports module is automatically written + based on ``profiling_targets`` and supplied to the base + constructor. + + - ``False``: equivalent to ``None``. + + - Unless the argument ``_use_curated_profiler: bool = True`` is set + to :py:const:`False`, automatically creates an instance of + :py:class:`LineProfiler` that is curated by a + :py:class:`CuratedProfilerContext` and provides it as the + :py:attr:`LineProfilingCache.profiler`, and + + - At teardown: + + - Removes tempdirs and tempfiles generated. + + - Restores the value of the class' internal reference to the + :py:meth:`LineProfilingCache.load`-ed instance. + + - Calls the `.cleanup()` method of each instance created. + + - Prints these diagnostics for each instance: + + - The stats on the ``.profiler`` associated with each instance + (if any) + + - The stats gathered by + :py:meth:`LineProfilingCache.gather_stats()` + + - The debug logs (if ``.debug`` is true) + """ + def instantiate( + *, + profiling_targets: Collection[str] = (), + preimports_module: os.PathLike[str] | str | bool | None = None, + _use_curated_profiler: bool = True, + **kwargs + ) -> LineProfilingCache: + tmpdir = tmp_path_factory.mktemp('my_cache_dir') + pim: os.PathLike[str] | str | None + if preimports_module in (True, False): + if preimports_module: + targets = ( + ClassifiedPreimportTargets.from_targets(profiling_targets) + ) + if targets: + pim = tmpdir / 'preimports.py' + with pim.open(mode='w') as fobj: + targets.write_preimport_module(fobj) + else: + pim = None + else: + pim = None + else: + # The type checker needs some convincing... + assert not isinstance(preimports_module, bool) + pim = preimports_module + cache = LineProfilingCache( + tmpdir, + profiling_targets=profiling_targets, + preimports_module=pim, + **kwargs, + ) + if _use_curated_profiler: + cache.profiler = request.getfixturevalue('curated_profiler') + instances.append(cache) + return cache + + def print_result( + cache: LineProfilingCache, topic: str, result: str, *notes: str, + ) -> None: + header = '{} ({}):'.format( + topic, '; '.join([f'cache instance {id(cache):#x}', *notes]), + ) + print(header, indent(result, ' '), sep='\n') + + def print_profiler_stats(cache: LineProfilingCache) -> None: + if cache.profiler is None: + result = '' + notes = [] + else: + with StringIO() as sio: + cache.profiler.print_stats(sio) + result = sio.getvalue() + notes = [f'profiler instance {id(cache.profiler):#x}'] + print_result(cache, 'Native profiler stats', result, *notes) + + def print_gathered_stats(cache: LineProfilingCache) -> None: + with StringIO() as sio: + cache.gather_stats().print(sio) + result = sio.getvalue() + print_result(cache, 'Gathered profiler stats', result) + + def print_debug_logs(cache: LineProfilingCache) -> None: + if cache.debug: + result = '\n'.join( + entry.to_text() for entry in cache._gather_debug_log_entries() + ) + else: + result = '' + print_result(cache, 'Gathered debug logs', result) + + instances: list[LineProfilingCache] = [] + handlers: list[Callable[[LineProfilingCache], None]] + handlers = [print_profiler_stats, print_gathered_stats, print_debug_logs] + try: + with preserve_object_attrs(LineProfilingCache, ['_loaded_instance']): + yield instantiate + finally: + for cache in instances: + callbacks: list[Callable[[], Any]] = [cache.cleanup] + callbacks.extend(partial(func, cache) for func in handlers) + for callback in callbacks: + try: + callback() + except Exception: + pass + + +@pytest.fixture +def curated_profiler() -> Generator[LineProfiler, None, None]: + """ + Yields: + Fresh instance of :py:class:`LineProfiler` that is managed by a + :py:class:`CuratedProfilerContext` + """ + prof = LineProfiler() + with CuratedProfilerContext(prof, insert_builtin=True): + yield prof + + +@pytest.fixture +def another_pid() -> int: + """ + Get a PID which is distinct from the current one. + """ + curr_pid = os.getpid() + pid = (curr_pid - 42) % (2 * 16) + assert pid != curr_pid + return pid + + +@pytest.fixture(autouse=True) +def _trim_mismatch_traceback(pytestconfig: pytest.Config) -> None: + """ + Truncate the traceback of raised :py:class`ResultMismatch` for more + useful error attribution. + """ + try: + pytestconfig.pluginmanager.register(ResultMismatch) + except ValueError: # Already registered + pass diff --git a/tests/test_child_procs/multiproc_examples/external_module.py b/tests/test_child_procs/multiproc_examples/external_module.py new file mode 100644 index 00000000..cf104c9a --- /dev/null +++ b/tests/test_child_procs/multiproc_examples/external_module.py @@ -0,0 +1,10 @@ +from __future__ import annotations + + +def my_external_sum(x: list[int], fail: bool = False) -> int: + result: int = 0 # GREP_MARKER[EXT-INVOCATION] + for item in x: + result += item # GREP_MARKER[EXT-LOOP] + if fail: + raise RuntimeError('forced failure') + return result diff --git a/tests/test_child_procs/multiproc_examples/pool_test_module.py b/tests/test_child_procs/multiproc_examples/pool_test_module.py new file mode 100644 index 00000000..ecf6b87e --- /dev/null +++ b/tests/test_child_procs/multiproc_examples/pool_test_module.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from argparse import ArgumentParser +from collections.abc import Callable +from multiprocessing import dummy, get_context, Pool +from typing import Literal + +from external_module import my_external_sum + + +NUM_NUMBERS = 100 +NUM_PROCS = 4 + + +def my_local_sum(x: list[int], fail: bool = False) -> int: + result: int = 0 # GREP_MARKER[LOCAL-INVOCATION] + # The reversing is to prevent bytecode aliasing with + # `my_external_sum()` (see issue #424, PR #425) + for item in reversed(x): + result += item # GREP_MARKER[LOCAL-LOOP] + if fail: + raise RuntimeError('forced failure') + return result + + +def sum_in_child_procs( + length: int, n: int, my_sum: Callable[[list[int], bool], int], + start_method: Literal[ + 'fork', 'forkserver', 'spawn', 'dummy' + ] | None = None, + fail: bool = False, +) -> int: + my_list: list[int] = list(range(1, length + 1)) + sublists: list[list[int]] = [] + subsums: list[int] + sublength = length // n + if sublength * n < length: + sublength += 1 + while my_list: + sublist, my_list = my_list[:sublength], my_list[sublength:] + sublists.append(sublist) + if start_method == 'dummy': + pool = dummy.Pool(n) + elif start_method: + pool = get_context(start_method).Pool(n) + else: + pool = Pool(n) + with pool: + subsums = pool.starmap(my_sum, [(sl, fail) for sl in sublists]) + pool.close() + pool.join() + return my_sum(subsums, fail) + + +def main(args: list[str] | None = None) -> None: + parser = ArgumentParser() + parser.add_argument('-l', '--length', type=int, default=NUM_NUMBERS) + parser.add_argument('-n', type=int, default=NUM_PROCS) + parser.add_argument( + '-s', '--start-method', + choices=['fork', 'forkserver', 'spawn'], default=None, + ) + parser.add_argument('-f', '--force-failure', action='store_true') + parser.add_argument( + '--local', + action='store_const', + dest='my_sum', + default=my_external_sum, + const=my_local_sum, + ) + options = parser.parse_args(args) + print(sum_in_child_procs( + options.length, options.n, options.my_sum, + start_method=options.start_method, + fail=options.force_failure, + )) + + +if __name__ == '__main__': + main() diff --git a/tests/test_child_procs/test_child_procs.py b/tests/test_child_procs/test_child_procs.py new file mode 100644 index 00000000..f570421a --- /dev/null +++ b/tests/test_child_procs/test_child_procs.py @@ -0,0 +1,1205 @@ +from __future__ import annotations + +import dataclasses +import inspect +import operator +import os +import re +import sys +from collections.abc import Callable, Collection, Iterable +from contextlib import ExitStack +from functools import partial +from io import StringIO +from pathlib import Path +from runpy import run_path +from subprocess import CompletedProcess +from textwrap import indent +from types import ModuleType +from typing import cast + +import pytest + +from _line_profiler_hooks import load_pth_hook +from line_profiler._child_process_profiling.cache import LineProfilingCache +from line_profiler._child_process_profiling.runpy_patches import ( + create_runpy_wrapper, +) +from line_profiler._child_process_profiling.multiprocessing_patches import ( + MPConfig, _PATCHES as MP_PATCHES, +) +from line_profiler.line_profiler import LineStats +from line_profiler.toml_config import ConfigSource + +from ._test_child_procs_utils import ( + DEBUG, DEFAULT_TIMEOUT, NOT_SUPPLIED, START_METHODS, PATCH_SUMMARIES, + ModuleFixture, Params, ResultMismatch, StartMethod, + preserve_object_attrs, preserve_targets, + cleanup_extra_pth_files, add_timeout, + CheckWarnings, + mp_patch_is_internal, get_mp_patches_toml_text, + summarize_mp_patches, filter_mp_patch_summary, + concat_command_line, run_subproc, + run_module, run_script, run_literal_code, check_tagged_line_nhits, + strip, search_cache_logs, +) + + +# ============================= Unit tests ============================= + +# XXX: Tests in this section concern implementation details, and the +# tested APIs and behaviors MUST NOT be relied upon by end-users. + +_DEFAULT_MP_CONFIG = MPConfig.from_config(ConfigSource.from_default()) + + +@pytest.mark.parametrize(('run_profiled_code', 'label1'), + [(True, 'run-profiled'), (False, 'run-unrelated')]) +@pytest.mark.parametrize(('as_module', 'label2'), + [(True, 'run_module'), (False, 'run_path')]) +@pytest.mark.parametrize(('debug', 'label3'), + [(True, 'with-debug'), (False, 'no-debug')]) +def test_runpy_patches( + capsys: pytest.CaptureFixture[str], + ext_module: ModuleFixture, + pool_test_module: ModuleFixture, + pool_test_module_clone: ModuleFixture, + create_cache: Callable[..., LineProfilingCache], + run_profiled_code: bool, + as_module: bool, + debug: bool, + label1: str, label2: str, label3: str, +) -> None: + """ + Test that the :py:mod:`runpy` clone created by + :py:func:`line_profiler._child_process_profiling\ +.create_runpy_wrapper` + correctly sets up profiling when its ``run_*()`` functions are + called. + """ + class restore_argv: + def __enter__(self) -> None: + self.argv = list(sys.argv) + + def __exit__(self, *_, **__) -> None: + sys.argv[:] = self.argv + + cache = create_cache( + rewrite_module=pool_test_module.path, + profiling_targets=[str(ext_module.path)], + profile_imports=True, + debug=debug, + ) + assert cache.profiler is not None + runpy = create_runpy_wrapper(cache) + + nnums = 42 + nprocs = 2 + # If we're running some unrelated code, the profiler should not be + # involved + if run_profiled_code: + module = pool_test_module + num_invocations, num_loops = 1, nprocs + expected_funcs: list[str] = ['my_external_sum'] + else: + module = pool_test_module_clone + num_invocations, num_loops = 0, 0 + expected_funcs = [] + if as_module: + first_arg = module.name + runner = partial(runpy.run_module, alter_sys=True) + called_func = 'run_module' + else: + first_arg = str(module.path) + runner = runpy.run_path + called_func = 'run_path' + + # Check that the code is run + module.install(local=True, deps_only=not as_module) + with restore_argv(): + sys.argv[:] = [first_arg, f'--length={nnums}', '-n', str(nprocs)] + runner(first_arg, run_name='__main__') + stdout = capsys.readouterr().out + assert stdout.rstrip('\n') == str(nnums * (nnums + 1) // 2) + + # Check that profiler has received the appropriate targets + funcs = [func.__name__ for func in getattr(cache.profiler, 'functions')] + assert funcs == expected_funcs + + # Check that calls in the current process are profiled iif the + # correct file is executed + with StringIO() as sio: + cache.profiler.print_stats(sio) + stats = sio.getvalue() + check_tagged_line_nhits(stats, 'EXT-INVOCATION', num_invocations) + check_tagged_line_nhits(stats, 'EXT-LOOP', num_loops) + + # Check the debug-log entries are correctly gathered + search_cache_logs( + cache, + debug, + { + rf'calling .*{called_func}\(': True, + r'calling .*exec\(': run_profiled_code, + }, + match_individual_messages=True, + flags=re.IGNORECASE, + ) + + +def test_cache_dump_load( + create_cache: Callable[..., LineProfilingCache], +) -> None: + """ + Test that: + + - We can round-trip the cache via :py:meth:`LineProfilingCache.dump` + and :py:meth:`LineProfilingCache.load` + + - The same instance is :py:meth:`LineProfilingCache.load`-ed in + subsequent calls + """ + original = create_cache( + profiling_targets=['foo', 'bar', 'baz'], main_pid=123456, + ) + cache_instances: list[LineProfilingCache] = [original] + envvars: set[str] = set(os.environ) + try: + original.inject_env_vars() # Needed for `.load()` + # Also test slipping stuff into the `._additional_data` + original._additional_data['foo'] = [1, 'string', None] + try: + # Env vars should be inserted + assert set(os.environ) == envvars.union(original.environ) > envvars + original.dump() + loaded = original.load() + cache_instances.append(loaded) + reloaded = original.load() + cache_instances.append(reloaded) + assert original is not loaded is reloaded + # Compare init fields + for field in dataclasses.fields(LineProfilingCache): + if not field.init: + continue + assert ( + getattr(original, field.name) + == getattr(loaded, field.name) + ) + # Compare `._additional_data` + assert original._additional_data == loaded._additional_data + finally: # Explicitly cleanup + for cache in cache_instances: + cache.cleanup() + finally: # Env vars restored after cleanup + assert set(os.environ) == envvars + + +@(Params.new(('wrap_os_fork', 'label1'), + [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) + + Params.new(('debug', 'label2'), + [(True, 'with-debug'), (False, 'no-debug')]) + + Params.new(('patch_pool', 'patch_process', 'intercept_logs', 'label3'), + [(True, True, True, 'all-patches'), + (True, True, False, 'pool-and-process'), + (True, False, True, 'pool-and-logging'), + (True, False, False, 'pool-only'), + (False, True, True, 'process-and-logging'), + (False, True, False, 'process-only'), + (False, False, True, 'logging-only'), + (False, False, False, 'no-patches')])).sorted() +def test_cache_setup_main_process( + tmp_path_factory: pytest.TempPathFactory, + create_cache: Callable[..., LineProfilingCache], + wrap_os_fork: bool, + debug: bool, + patch_pool: bool, + patch_process: bool, + intercept_logs: bool, + label1: str, label2: str, label3: str, +) -> None: + """ + Test that :py:meth:`LineProfilingCache._setup_in_main_process` works + as expected. + """ + mp_patches: set[str] = set() + if patch_pool: + mp_patches.add('pool') + if patch_process: + mp_patches.add('process') + if intercept_logs: + mp_patches.add('logging') + + config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' + config.write_text(get_mp_patches_toml_text(mp_patches)) + cache = create_cache(debug=debug, config=config) + + # Check that only the requested patches are applied + patches = summarize_mp_patches([ + (True, PATCH_SUMMARIES['minimal']), + *( + (name in mp_patches, filter_mp_patch_summary(patch.summary)) + for name, patch in MP_PATCHES.items() + if not mp_patch_is_internal(name) + ), + ]) + try: + patches['os']['fork'] = wrap_os_fork + except KeyError: + # `os.fork()` pruned because it doesn't exist on e.g. Windows + assert not hasattr(os, 'fork') + + with ExitStack() as stack: + patched = stack.enter_context(preserve_targets(patches)) + compare_patched = partial( + preserve_targets.compare_with_current_values, patched, + ) + original_pths = stack.enter_context(cleanup_extra_pth_files()) + cache._setup_in_main_process(wrap_os_fork=wrap_os_fork) + # There should be exactly one extra `.pth` file + new_pth_hook, = cleanup_extra_pth_files.get_pth_files() - original_pths + # Check whether the patches are applied + compare_patched(operator.is_not, assert_true=patches) + # Check that the instance is set as the `.load()`-ed one + assert cache is cache.load() + # Check whether the patches are reversed + cache.cleanup() + compare_patched() + + # Check the debug-log output + patterns: dict[str, bool] = dict.fromkeys( + [ + r'\(main process\)', + r'Injecting env var.*\$\{LINE_PROFILER_\w+\}', + re.escape(new_pth_hook), + ], + True, + ) + for target, maybe_patches in patches.items(): + patterns.update( + ('Patched.*' + re.escape(f'{target}.{attr}'), is_patched) + for attr, is_patched in maybe_patches.items() + ) + search_cache_logs(cache, debug, patterns) + + +@pytest.mark.parametrize(('wrap_os_fork', 'label1'), + [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) +@pytest.mark.parametrize(('preimports', 'label2'), + [(True, 'with-preimports'), (False, 'no-preimports')]) +@pytest.mark.parametrize(('new_profiler', 'label3'), + [(True, 'no-profiler'), (False, 'with-profiler')]) +@pytest.mark.parametrize(('debug', 'label4'), + [(True, 'with-debug'), (False, 'no-debug')]) +@pytest.mark.parametrize('n', [100]) +@preserve_targets() +def test_cache_setup_child( + create_cache: Callable[..., LineProfilingCache], + ext_module_object: ModuleType, + another_pid: int, + wrap_os_fork: bool, + preimports: bool, + new_profiler: bool, + debug: bool, + n: int, + label1: str, label2: str, label3: str, label4: str, +) -> None: + """ + Test that :py:meth:`LineProfilingCache._setup_in_child_process` + works as expected. + """ + def list_profiled_funcs() -> list[str]: + return [ + f'{func.__module__}.{func.__qualname__}' + for func in getattr(cache.profiler, 'functions', []) + ] + + func = ext_module_object.my_external_sum + cache = create_cache( + profiling_targets=[f'{func.__module__}.{func.__qualname__}'], + preimports_module=preimports, + _use_curated_profiler=not new_profiler, + main_pid=another_pid, + debug=debug, + ) + assert (cache.profiler is None) == new_profiler + + seen_funcs = list_profiled_funcs() + if preimports: + preimport_targets = list(cache.profiling_targets) + else: + preimport_targets = [] + + with preserve_object_attrs(os, ['fork']) as preserved: + old_fork = preserved['fork'] + # Check that we're only setting up if there isn't already a + # profiler + assert cache._setup_in_child_process( + wrap_os_fork=wrap_os_fork, context='test_cache_setup_child', + ) == new_profiler + assert cache.profiler + if not new_profiler: + return + + # Check that the profiler has been presented with the profiling + # target + assert list_profiled_funcs() == (seen_funcs + preimport_targets) + + # Check that on cache cleanup: + # - Profiling data is collected + # - `os.fork()` is restored + # - The warning for empty profiling files is only issued when + # expected + assert func(range(1, n + 1)) == n * (n + 1) // 2 + stats = cache.profiler.get_stats() + for callback, has_nonempty_file, has_stats, fork_patched in [ + (lambda: None, False, False, wrap_os_fork), + (cache.cleanup, True, preimports, False), + ]: + callback() + with CheckWarnings() as cw: + handle_warning: list[Callable[..., None]] = [] + if has_nonempty_file: + handle_warning.append(cw.forbid_warnings) + else: # Check for the warning but don't reissue it + handle_warning.extend([ + cw.expect_warnings, cw.suppress_warnings, + ]) + for handle in handle_warning: + handle(r'.* file\(s\) .* empty', module='line_profiler') + gathered = cache.gather_stats() + assert any(gathered.timings.values()) == has_stats, gathered + if hasattr(os, 'fork'): + assert (os.fork is not old_fork) == fork_patched + else: # E.g. Windows + assert old_fork == NOT_SUPPLIED + # Check that after cleaning up the profiler has been disabled + assert not getattr(cache.profiler, 'enable_count', 0) + + # Check that profiling results have been written to the cache + # directory + stats_file, = Path(cache.cache_dir).glob('*.lprof') + assert LineStats.from_files(stats_file) == stats == gathered + + # Check the debug-log output + patterns = { + f'Set up .*profiler.* {id(cache.profiler):#x}': True, + 'Loading preimports': preimports, + 'Created .*' + re.escape(stats_file.name): True, + 'Cleanup succeeded.*: .*dump_stats': True, + 'Loading results .*' + re.escape(stats_file.name): True, + } + search_cache_logs(cache, debug, patterns) + + +@pytest.mark.parametrize('ppid_should_match', [True, False, None]) +@preserve_targets() +def test_load_pth_hook( + create_cache: Callable[..., LineProfilingCache], + another_pid: int, + ppid_should_match: bool | None, +) -> None: + """ + Simulate calling :py:func:`_line_profiler_hooks.load_pth_hook()` in + a child process. + + Notes: + + - The function is CALLED in the .pth file, but we don't actually + NEED a .pth file to call and test it. + + - The counterpart :py:meth:`line_profiler\ +._child_process_profiling.cache.LineProfilingCache.write_pth_hook()` + is implicitly tested in + :py:func:`test_cache_setup_main_process()`. + """ + # This test is mostly here to hack coverage; since the function is + # only to be called in child processes, `coverage` seems to have + # trouble getting data on it... + + # We basically only need this cache instance to set up the + # environment variables and the requisite files... + cache = create_cache(main_pid=another_pid) + if ppid_should_match is not None: + cache.inject_env_vars() + if ppid_should_match: + call_ppid = another_pid + else: # On a PPID mismatch, the function bails after checking + call_ppid = another_pid + 10 + else: + # Without the requisite envvars, the hook should bail very + # quickly (due to the `environ` lookup erroring out), regardless + # of the provided PPID + call_ppid = 0 + cache.dump() + + compare = preserve_targets.compare_with_current_values + patches = { + target: frozenset(attr for attr, patched in attrs.items() if patched) + for target, attrs in summarize_mp_patches([ + (True, PATCH_SUMMARIES['default']), + (True, PATCH_SUMMARIES['pth_hook']), + ]).items() + } + with preserve_targets(patches) as patched: + try: + # NOTE: this creates a cache instance that isn't + # automatically cleaned up by the `create_cache()` + # fixture!!! Hence the try-finally + load_pth_hook(call_ppid) + # Check that the patches are applied where appropriate + assert ( + getattr(load_pth_hook, 'called', False) + == bool(ppid_should_match) + ) + if ppid_should_match: + compare(patched, operator.is_not) + else: # no-op + compare(patched) + return + # Check that calling `load_pth_hook()` again is a no-op + with preserve_targets(patches) as re_patched: + load_pth_hook(call_ppid) + compare(re_patched) + finally: + try: + current_cache = LineProfilingCache.load() + except Exception: + pass + else: + current_cache.cleanup() + # Check that the patches are reversed + compare(patched) + + +@cleanup_extra_pth_files() +@preserve_targets() +@add_timeout +def _test_apply_mp_patches_inner( + tmp_path_factory: pytest.TempPathFactory, + create_cache: Callable[..., LineProfilingCache], + ext_module_object: ModuleType, + pool_test_module_object: ModuleType, + start_method: StartMethod, + mp_patches: Collection[str], + fail: bool, + n: int, + nprocs: int, +) -> None: + def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: + try: + LineStats.from_files(path, on_empty='error', on_defective='error') + except Exception: + return False + return True + + def get_lineno(path: os.PathLike[str] | str, query: str) -> int: + with Path(path).open() as fobj: + for i, line in enumerate(fobj): + if query in line: + return 1 + i + raise RuntimeError( + f'Did not find line containing {query!r} in {path!r}', + ) + + config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' + intercept_logs = 'logging' in mp_patches + patch_process = 'process' in mp_patches + cfg_chunks: list[str] = [ + get_mp_patches_toml_text(mp_patches), + # This is easier to debug than `ResultMismatch` + '[tool.line_profiler.child_processes.multiprocessing.polling]\n' + 'on_timeout = "error"', + ] + config.write_text('\n\n'.join(cfg_chunks)) + + # Note: no need to test the case for `my_local_sum()` separately, + # with `preimports_module=True`, both are just imported and added + # to the profiler, so the code paths are the same + profiled_func = ext_module_object.my_external_sum + called_func = partial( + pool_test_module_object.sum_in_child_procs, + n=nprocs, + my_sum=profiled_func, + start_method=start_method, + fail=fail, + ) + + func_name = f'{profiled_func.__module__}.{profiled_func.__qualname__}' + cache = create_cache( + profiling_targets=[func_name], + preimports_module=True, + config=config, + debug=True, + ) + # Note: + # - The reversibility of the patches have already been tested in + # `test_cache_setup_main_process()`, so we just actually test the + # patched-in components themselves here. + # - `._setup_in_main_process()` doesn't include actually doing the + # preimports. To may the results more consistent between + # `start_method='dummy'` and the others, manually do them below. + cache._setup_in_main_process() # This calls `apply()` + assert cache.profiler is not None + assert cache.preimports_module is not None + run_path(str(cache.preimports_module), {'profile': cache.profiler}) + + timing_key = ( + inspect.getfile(profiled_func), + inspect.getsourcelines(profiled_func)[1], + profiled_func.__qualname__, + ) + assert ext_module_object.__file__ + loop_line = get_lineno(ext_module_object.__file__, 'EXT-LOOP') + + nloops_expected = n + if not fail: + # Counts from the one final sum over the parallel results + nloops_expected += nprocs + + fail_msg = 'forced failure' + try: + result = called_func(n) + except RuntimeError as e: + if not (fail and str(e) == fail_msg): + raise + else: + if fail: + msg = f"expected `RuntimeError({fail_msg!r})`, no error raised" + raise ValueError(msg) + else: # Check correctness of the results + assert result == n * (n + 1) // 2 + + # Check that calls in children are traced + cache.cleanup() + stats = cache.profiler.get_stats() + stats += cache.gather_stats() + entries = stats.timings[timing_key] + nloops = sum(nhits for ln, nhits, _ in entries if ln == loop_line) + ResultMismatch.compare(nloops_expected, nloops) + + # Check the debug logs to see if we have done everything right, esp. + # the logging interception part not covered by other tests + patterns: dict[str, bool] = {} + if patch_process: + # Note: if we're not using `Process`-based patch, there is no + # guaratee that the profiling result is written via cleanup + iter_stats: Iterable[Path] = Path(cache.cache_dir).glob('*.lprof') + iter_stats = cast( # See `ty` issue #3428 + Iterable[Path], filter(is_valid_stats_file, iter_stats), + ) + pat = 'Cleanup succeeded.*: .*dump_stats.*{}' + patterns.update({ + pat.format(re.escape(path.name)): True for path in iter_stats + }) + patterns[re.escape('`multiprocessing` logging (debug)')] = intercept_logs + search_cache_logs(cache, True, patterns) + + +def _test_apply_mp_patches( + patch_pool: bool | None = None, + patch_process: bool | None = None, + intercept_logs: bool | None = None, + *, + start_method: StartMethod, + **kwargs +) -> None: + if start_method not in ('dummy', *START_METHODS): + pytest.skip( + f'`multiprocessing` start method {start_method!r} ' + 'not available on the platform' + ) + + patches = cast(dict[str, bool], _DEFAULT_MP_CONFIG.patches.copy()) + for name, applied in { + 'pool': patch_pool, 'process': patch_process, + 'logging': intercept_logs, + }.items(): + if applied is not None: + patches[name] = applied + mp_patches = [name for name, applied in patches.items() if applied] + + with CheckWarnings() as cw: + cw.forbid_warnings('.*resource_tracker', module='multiprocessing') + cw.forbid_warnings( + r'.* file\(s\) .* empty', + category=UserWarning, module='line_profiler', + ) + if start_method == 'fork': + # The `@add_timeout` decorator spins up a new thread for + # executing `_test_apply_mp_patches_inner()`; explicitly + # ignore the associated warning when we use + # `start_method='fork'` + cw.suppress_warnings( + r'.*multi-?threaded.*fork\(\)', category=DeprecationWarning, + ) + _test_apply_mp_patches_inner( + mp_patches=mp_patches, start_method=start_method, **kwargs, + ) + + +@(Params.new('start_method', ['fork', 'forkserver', 'spawn', 'dummy'], + defaults='dummy') + # We only need to check if `intercept_logs = logging` work, the other + # parametrizations don't matter + + Params.new(('intercept_logs', 'label1'), + [(True, 'with-logging'), (False, 'no-logging')], + defaults=(None, 'default-logging'))).sorted() +@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label2'), + [(True, True, 'pool-and-process'), + (True, False, 'pool-only'), + (False, True, 'process-only')]) +@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) +def test_apply_mp_patches_success( + tmp_path_factory: pytest.TempPathFactory, + create_cache: Callable[..., LineProfilingCache], + ext_module_object: ModuleType, + pool_test_module_object: ModuleType, + start_method: StartMethod, + patch_pool: bool, + patch_process: bool, + intercept_logs: bool | None, + n: int, + nprocs: int, + label1: str, + label2: str, +) -> None: + """ + Test that :py:func:`line_profiler._child_process_profiling\ +.multiprocessing_patches.apply` + works as expected when the parallel workload does not error out. + + See also: + :py:func:`test_apply_mp_patches_failure` + """ + _test_apply_mp_patches( + patch_pool, + patch_process, + intercept_logs, + tmp_path_factory=tmp_path_factory, + create_cache=create_cache, + ext_module_object=ext_module_object, + pool_test_module_object=pool_test_module_object, + start_method=start_method, + fail=False, + n=n, + nprocs=nprocs, + ) + + +@pytest.mark.parametrize('start_method', + ['fork', 'forkserver', 'spawn', 'dummy']) +@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label'), + [(True, True, 'pool-and-process'), + (True, False, 'pool-only'), + (False, True, 'process-only')]) +@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) +def test_apply_mp_patches_failure( + tmp_path_factory: pytest.TempPathFactory, + create_cache: Callable[..., LineProfilingCache], + ext_module_object: ModuleType, + pool_test_module_object: ModuleType, + start_method: StartMethod, + patch_pool: bool, + patch_process: bool, + n: int, + nprocs: int, + label: str, +) -> None: + """ + Test that :py:func:`line_profiler._child_process_profiling\ +.multiprocessing_patches.apply` + works as expected when the parallel workload errors out. + + See also: + :py:func:`test_apply_mp_patches_success` + """ + _test_apply_mp_patches( + patch_pool, + patch_process, + tmp_path_factory=tmp_path_factory, + create_cache=create_cache, + ext_module_object=ext_module_object, + pool_test_module_object=pool_test_module_object, + start_method=start_method, + fail=True, + n=n, + nprocs=nprocs, + ) + + +# XXX: End of tests for implementation details + +# ========================= Integration tests ========================== + + +def _get_mp_start_method_fuzzer(label_name: str | None) -> Params: + """ + Returns: + :py:class:`Params` object which does a full Cartesian-product + fuzz between ``fail`` (true or false) and ``start_method`` + ('fork', 'forkserver', and 'spawn'; default :py:const:`None`) + """ + if label_name is None: + label_name, drop_label = '_', True + else: + drop_label = False + fuzz_fail = Params.new(('fail', label_name), + [(True, 'failure'), (False, 'success')], + defaults=(False, 'success')) + if drop_label: + fuzz_fail = fuzz_fail.drop_params(label_name) + fuzz_start = Params.new('start_method', ['fork', 'forkserver', 'spawn'], + defaults=None) + return fuzz_fail * fuzz_start + + +@(Params.new(('run_func', 'label1'), + [(run_module, 'module'), (run_script, 'script')]) + * Params.new(('use_local_func', 'label2'), + [(True, 'local'), (False, 'ext')]) + # Python can't pickle things unless they resided in a retrievable + # location (so not the script supplied by `python -c`) + + Params.new(('run_func', 'label1', 'use_local_func', 'label2'), + [(run_literal_code, 'literal-code', False, 'ext')]) + # Also fuzz the parallelization-related stuff, esp. check what + # happens if an exception is raised inside the parallelly-run func + + _get_mp_start_method_fuzzer('label3') + + Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], + defaults=(None, None))).sorted() +def test_multiproc_script_sanity_check( + run_func: Callable[..., CompletedProcess], + request: pytest.FixtureRequest, + pool_test_module: ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + use_local_func: bool, + fail: bool, + start_method: StartMethod | None, + nnums: int | None, + nprocs: int | None, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, label3: str, +) -> None: + """ + Sanity check that the test module functions as expected when run + with vanilla Python. + """ + run_func( + request, pool_test_module, tmp_path_factory, + runner=sys.executable, profile=False, + fail=fail, + start_method=start_method, + test_module_kwargs=dict( + use_local_func=use_local_func, nnums=nnums, nprocs=nprocs, + ), + ) + + +@pytest.mark.parametrize( + ('run_func', 'label1'), + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')] +) +@pytest.mark.parametrize( + ('runner', 'outfile', 'profile', + 'label2'), # Dummy argument to make `pytest` output more legible + # This is essentially a no-op since it doesn't actually do + # line-profiling, but we check that code path for completeness + [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] + # Run line profiling with and w/o profiling targets + + [(['kernprof', '-q', '-l'], 'out.lprof', False, + 'line_profiler-inactive'), + (['kernprof', '-q', '-l'], 'out.lprof', True, + 'line_profiler-active')], +) +def test_running_multiproc_script( + run_func: Callable[..., CompletedProcess], + request: pytest.FixtureRequest, + pool_test_module: ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + runner: str | list[str], + outfile: str | None, + profile: bool, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, +) -> None: + """ + Check that `kernprof` can RUN the test module in various contexts + (`kernprof [...] `, `kernprof [...] -m `, and + `kernprof [...] -c "code"`). + + Notes: + - See issue #422 for the original motivation. + + - This test does not test the actual profiling, just the + execution of the code and presence of profiling data + thereafter. + """ + run_func( + request, pool_test_module, tmp_path_factory, runner, outfile, profile, + ) + + +_fuzz_prof_mp_run_func = Params.new(('run_func', 'label1'), + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')], + defaults=(run_script, 'script')) +_fuzz_prof_mp_markers = ( + (_fuzz_prof_mp_run_func + + Params.new(('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')]) + + _get_mp_start_method_fuzzer(None)) + # Test all `multiproc` start methods with both locally- and + # externally-defined profiling targets + * (Params.new(('preimports', 'label3'), [(False, 'no-preimports')]) + + Params.new(('use_local_func', 'label4'), + [(True, 'local'), (False, 'external')], + defaults=(False, 'external'))) + # The 'with-preimports' case is already tested rather thoroughly in + # `test_apply_mp_patches()`, so exclude these from the above "main" + # param matrix and just test the different `kernprof` modes via the + # `run_func()`s + + (_fuzz_prof_mp_run_func + + Params.new(('preimports', 'label3'), [(True, 'with-preimports')])) + # Just throw in a case where we actually run `kernprof` in a + # subprocess, otherwise it is more convenient to do so in-process + + Params.new(('subproc', 'label5'), + [(True, 'subproc'), (False, 'in-proc')], + defaults=(False, 'in-proc')) +).sorted().split_on_params('fail') + + +def _test_profiling_multiproc_script( + run_func: Callable[..., CompletedProcess], + request: pytest.FixtureRequest, + pool_test_module: ModuleFixture, + ext_module: ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + prof_child_procs: bool, + preimports: bool, + use_local_func: bool, + fail: bool, + start_method: StartMethod | None, + nnums: int, + nprocs: int, + **kwargs +) -> None: + # How many calls do we expect? + nhits = dict.fromkeys( + ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, + ) + # Make sure we're profiling the right function + tag = 'LOCAL' if use_local_func else 'EXT' + tag_call = tag + '-INVOCATION' + tag_loop = tag + '-LOOP' + if not fail: + # The final sum in the parent process should always be profiled + # unless the child processes failed and we never returned from + # `Pool.starmap()` + nhits[tag_call] += 1 + nhits[tag_loop] += nprocs + if prof_child_procs: + # When profiling extends into child processes, each of them + # invokes the sum function once and when combined they loop thru + # all the items + nhits[tag_call] += nprocs + nhits[tag_loop] += nnums + + runner = ['kernprof', '-l'] + runner.extend([ + '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), + '--{}preimports'.format('' if preimports else 'no-'), + ]) + if not use_local_func: + # Also make sure to include the external module in `--prof-mod` + runner.append(f'--prof-mod={ext_module.name}') + kwargs.setdefault('timeout', DEFAULT_TIMEOUT) + if prof_child_procs and DEBUG: + kwargs.setdefault('debug_log', 'debug.log') + run_func( + request, pool_test_module, tmp_path_factory, + runner=runner, + outfile='out.lprof', + profile=True, + fail=fail, + start_method=start_method, + nhits=nhits, + test_module_kwargs=dict( + use_local_func=use_local_func, + nnums=nnums, + nprocs=nprocs, + ), + **kwargs, + ) + + +@(_fuzz_prof_mp_markers[False]) +@pytest.mark.parametrize( + # XXX: should we explicitly test the single-proc case? We already + # have quite a lot of subtests tho... + ('nnums', 'nprocs'), [(2000, 3)], +) +def test_profiling_multiproc_script_success( + run_func: Callable[..., CompletedProcess], + request: pytest.FixtureRequest, + pool_test_module: ModuleFixture, + ext_module: ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + prof_child_procs: bool, + preimports: bool, + use_local_func: bool, + start_method: StartMethod | None, + nnums: int, + nprocs: int, + subproc: bool, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, label3: str, label4: str, label5: str, +) -> None: + """ + Check that `kernprof` can PROFILE the test module in various + contexts when the parallel workload runs without errors, optionally + extending profiling into child processes. + + Note: + This test function is heavily parametrized. Here is why that is + necessary: + + - ``run_func`` tests the different :cmd:`kernprof` modes (see + :py:func:`~.test_running_multiproc_script`). + + - ``preimports`` tests that both mechanisms for setting up + profiling targets work: + + - :py:const:`True`: child processes import the module + generated by + :py:mod:`line_profiler.autoprofile.eager_preimports`, like + the main :py:mod:`kernprof` process does. + + - :py:const:`False`: child processes rewrite the executed code + before passing it to :py:mod:`runpy`, similar to what + :py:mod:`line_profiler.autoprofile.autoprofile` does. + + These code paths go through different + :py:mod:`multiprocessing` components that we have patched and + thus needs separate testing. + + - ``use_local_func`` tests that we can consistently set up + profiling in both functions locally-defined in the profiled + code and imported by it. + + - ``fail`` tests that our patches and hook doesn't choke when + exceptions occur in child processes, and profiling data can + still be collected. + + - ``start_method`` tests whether all available + :py:mod:`multiprocessing` start methods are covered. + + - ``prof_child_procs`` of course toggles whether to do the + patches to set up profiling in child processes. + + See also: + :py:func:`test_profiling_multiproc_script_failure` + """ + _test_profiling_multiproc_script( + run_func=run_func, + request=request, + pool_test_module=pool_test_module, + ext_module=ext_module, + tmp_path_factory=tmp_path_factory, + prof_child_procs=prof_child_procs, + preimports=preimports, + use_local_func=use_local_func, + fail=False, + start_method=start_method, + nnums=nnums, + nprocs=nprocs, + subproc=subproc, + ) + + +@(_fuzz_prof_mp_markers[True]) +@pytest.mark.parametrize(('nnums', 'nprocs'), [(2000, 3)]) +def test_profiling_multiproc_script_failure( + run_func: Callable[..., CompletedProcess], + request: pytest.FixtureRequest, + pool_test_module: ModuleFixture, + ext_module: ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + prof_child_procs: bool, + preimports: bool, + use_local_func: bool, + start_method: StartMethod | None, + nnums: int, + nprocs: int, + subproc: bool, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, label3: str, label4: str, label5: str, +) -> None: + """ + Check that `kernprof` can PROFILE the test module in various + contexts when the parallel workload errors out, optionally + extending profiling into child processes. + + See also: + :py:func:`test_profiling_multiproc_script_success` + """ + _test_profiling_multiproc_script( + run_func=run_func, + request=request, + pool_test_module=pool_test_module, + ext_module=ext_module, + tmp_path_factory=tmp_path_factory, + prof_child_procs=prof_child_procs, + preimports=preimports, + use_local_func=use_local_func, + fail=True, + start_method=start_method, + nnums=nnums, + nprocs=nprocs, + subproc=subproc, + ) + + +_fuzz_bare = ( + Params.new(('use_subprocess', 'label1'), + [(True, 'subprocess.run'), (False, 'os.system')]) + * Params.new(('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')]) + * Params.new('n', [200]) +) + + +def _test_profiling_bare_python( + tmp_path_factory: pytest.TempPathFactory, + ext_module: ModuleFixture, + use_subprocess: bool, + prof_child_procs: bool, + fail: bool, + n: int, +) -> None: + ext_module.install(children=True) + temp_dir = tmp_path_factory.mktemp('mytemp') + + script_path = temp_dir / 'my-script.py' + script_content = strip(""" + from {EXT_MODULE} import my_external_sum + + + if __name__ == '__main__': + numbers = list(range(1, 1 + {N})) + result = my_external_sum(numbers, {FAIL}) + """.format( + EXT_MODULE=ext_module.name, + N=n, + FAIL=fail, + )) + script_path.write_text(script_content) + + out_file = temp_dir / 'out.lprof' + debug_log_file = temp_dir / 'debug.log' + write_debug = DEBUG and prof_child_procs + cmd = [ + 'kernprof', '-lv', '--preimports', + f'--prof-mod={ext_module.name}', + f'--outfile={out_file}', + '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), + ] + if write_debug: + cmd.append(f'--debug-log={debug_log_file}') + sub_cmd = [sys.executable, str(script_path)] + if use_subprocess: + code = strip(f""" + import subprocess + + + subprocess.run({sub_cmd!r}, check=True) + """) + else: + code = strip(""" + import os + + + if os.system({!r}): + raise RuntimeError('called process failed') + """.format(concat_command_line(sub_cmd))) + cmd.extend(['-c', code]) + proc = run_subproc( + cmd, text=True, capture_output=True, timeout=DEFAULT_TIMEOUT, + ) + + nhits = {'EXT-INVOCATION': 1, 'EXT-LOOP': n} + if not prof_child_procs: + for k in nhits: + nhits[k] = 0 + + try: + # Check that the code errors out when expected + assert bool(fail) == bool(proc.returncode) + # Check that the profiling output is as expected + for tag, num in nhits.items(): + check_tagged_line_nhits(proc.stdout, tag, num) + finally: + if write_debug: + print('-- Combined debug logs --', file=sys.stderr) + print( + indent(debug_log_file.read_text(), ' '), + end='', file=sys.stderr, + ) + print('-- End of debug logs --', file=sys.stderr) + + +@_fuzz_bare +def test_profiling_bare_python_success( + tmp_path_factory: pytest.TempPathFactory, + ext_module: ModuleFixture, + use_subprocess: bool, + prof_child_procs: bool, + n: int, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, +) -> None: + """ + Check that `kernprof` can profile the target functions if the code + invokes another bare Python process (via either :py:func:`os.system` + or :py:func:`subprocess.run`) that calls them and exits without + errors. + + See also: + :py:func:`test_profiling_bare_python_failure` + """ + _test_profiling_bare_python( + tmp_path_factory=tmp_path_factory, + ext_module=ext_module, + use_subprocess=use_subprocess, + prof_child_procs=prof_child_procs, + fail=False, + n=n, + ) + + +@_fuzz_bare +def test_profiling_bare_python_failure( + tmp_path_factory: pytest.TempPathFactory, + ext_module: ModuleFixture, + use_subprocess: bool, + prof_child_procs: bool, + n: int, + label1: str, + label2: str, +) -> None: + """ + Check that `kernprof` can profile the target functions if the code + invokes another bare Python process (via either :py:func:`os.system` + or :py:func:`subprocess.run`) that calls them and exits with errors. + + See also: + :py:func:`test_profiling_bare_python_success` + """ + _test_profiling_bare_python( + tmp_path_factory=tmp_path_factory, + ext_module=ext_module, + use_subprocess=use_subprocess, + prof_child_procs=prof_child_procs, + fail=True, + n=n, + ) From eb59171730c9f4d9052c0cc75b0ec461c65fa124 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 29 Jun 2026 12:23:57 +0200 Subject: [PATCH 141/165] Fix `multiprocessing` profiling patch conflict line_profiler/_child_process_profiling/multiprocessing_patches /_profiling_patches.py wrap_process() New wrapper for `Pool.Process` that marks a worker process wrap_bootstrap() Now avoiding calling end-of function cleanup for marked pool-worker processes, because they may be terminated mid-write to the profiling-stats file (which should already be written to on a per-task basis), causing data loss/corruption _get_terminate_poller(), _process_has_returned() Removed dead code --- .../_profiling_patches.py | 81 +++++++++++-------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py index e97c8e2a..2e0c379e 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py @@ -3,23 +3,25 @@ from collections.abc import Callable from functools import partial from multiprocessing.process import BaseProcess -from typing import TypeVar, cast +from typing import TypeVar from typing_extensions import Concatenate, ParamSpec from ..cache import LineProfilingCache from ._infrastructure import SingleModulePatch from ._queue import Queue, PutWrapper -from .mp_config import MPConfig -from .poller import Poller, OnTimeout __all__ = ( - 'POOL_PATCH', 'PROCESS_PATCH', 'wrap_bootstrap', 'wrap_worker', + 'POOL_PATCH', 'PROCESS_PATCH', + 'wrap_bootstrap', 'wrap_process', 'wrap_worker', ) T = TypeVar('T') +P = TypeVar('P', bound=BaseProcess) PS = ParamSpec('PS') +_POOL_WORKER_MARKER = '__line_profiler_multiprocessing_is_pool_worker__' + # ------------------------------ Helpers ------------------------------- @@ -44,6 +46,15 @@ def dump_stats_quick( stats_dumper() +def _mark_worker(worker: P) -> P: + setattr(worker, _POOL_WORKER_MARKER, True) + return worker + + +def _is_marked_worker(proc: BaseProcess) -> bool: + return getattr(proc, _POOL_WORKER_MARKER, False) + + # ---------------- `multiprocessing.pool.Pool` patches ----------------- @@ -93,8 +104,31 @@ def wrap_worker( return vanilla_impl(inqueue, outqueue, *args, **kwargs) +@LineProfilingCache._method_wrapper +def wrap_process( + _, vanilla_impl: Callable[PS, P], *args: PS.args, **kwargs: PS.kwargs +) -> P: + """ + Wrap around :py:meth:`multiprocessing.pool.Pool.Process` so that the + worker processes created by the pool are marked and can be + distinguished from processes otherwise managed. + + Notes: + + - :py:meth:`multiprocessing.pool.Pool.Process` is a static + method. + + - Technically one can inspect the :py:attr:`.BaseProcess.name` + of the process to see that it is a ``PoolWorker``, but since + said attribute is writable it may be more robust to set up a + separate marker. + """ + return _mark_worker(vanilla_impl(*args, **kwargs)) + + POOL_PATCH = SingleModulePatch('pool') POOL_PATCH.add_method('', 'worker', wrap_worker) +POOL_PATCH.add_method('Pool', 'Process', wrap_process, 'static') # ----------- `multiprocessing.process.BaseProcess` patches ------------ @@ -124,42 +158,19 @@ def wrap_bootstrap( terminated prematurely (e.g. via :py:meth:`.BaseProcess.terminate`), profiling data may be missing. + + - To prevent data corruption/loss, the end-of-function write to + the temporary profiling-stat file only happens for + non-pool-managed :py:class:`BaseProcess` objects, because they + are regularly :py:meth:`.BaseProcess.terminate`-ed by their + managing pool. """ try: return vanilla_impl(self, *args, **kwargs) finally: reason = 'exiting `multiprocessing.process.BaseProcess._bootstrap`' - dump_stats_quick(cache, reason=reason) - - -def _get_terminate_poller( - cache: LineProfilingCache, process: BaseProcess, -) -> Poller: - config = MPConfig.from_cache(cache) - cd, timeout, on_timeout = config.polling - if on_timeout not in ('ignore', 'warn', 'error'): - on_timeout = config.get_defaults().polling.on_timeout - # `_process_has_returned()` takes a `timeout` which it passes to - # `popen.wait()`; said timeout is essentially a limit as to how - # often the function is called, hence our cooldown - poller = Poller.poll_until(_process_has_returned, process, cache, cd) - return poller.with_timeout(timeout, cast(OnTimeout, on_timeout)) - - -def _process_has_returned( - proc: BaseProcess, cache: LineProfilingCache, timeout: float, -) -> bool: - popen = getattr(proc, '_popen', None) - if popen is None: - msg, result = 'No associated process', True - else: - result = popen.wait(timeout) is not None - if result: - msg = f'Process {popen.pid} has returned' - else: - msg = f'Waiting for process {popen.pid} to return...' - cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') - return result + if not _is_marked_worker(self): + dump_stats_quick(cache, reason=reason) PROCESS_PATCH = SingleModulePatch('process') From de85fad777d3eb72e41657090311fd718e866491 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 29 Jun 2026 12:41:36 +0200 Subject: [PATCH 142/165] WIP: separate tests for the `process` patch tests/test_child_procs/test_child_procs.py test_apply_mp_patches_pool_{failure,success}() - Renamed from `test_apply_mp_patches_*()` - Removed subtests for when only the `process` patch is applied test_pool_multiproc_script_sanity_check() test_running_pool_multiproc_script() test_profiling_pool_multiproc_script_{success,failure}() Renamed from resp. tests without the `pool_` infix TODO: new tests for `BaseProcess`-based parallelism --- tests/test_child_procs/test_child_procs.py | 83 ++++++++++------------ 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/tests/test_child_procs/test_child_procs.py b/tests/test_child_procs/test_child_procs.py index f570421a..1f7aab16 100644 --- a/tests/test_child_procs/test_child_procs.py +++ b/tests/test_child_procs/test_child_procs.py @@ -473,7 +473,7 @@ def test_load_pth_hook( @cleanup_extra_pth_files() @preserve_targets() @add_timeout -def _test_apply_mp_patches_inner( +def _test_apply_mp_patches_pool_inner( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, @@ -502,7 +502,7 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' intercept_logs = 'logging' in mp_patches - patch_process = 'process' in mp_patches + assert {'pool', 'process'} & set(mp_patches) # Needed for profiling cfg_chunks: list[str] = [ get_mp_patches_toml_text(mp_patches), # This is easier to debug than `ResultMismatch` @@ -579,23 +579,19 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests patterns: dict[str, bool] = {} - if patch_process: - # Note: if we're not using `Process`-based patch, there is no - # guaratee that the profiling result is written via cleanup - iter_stats: Iterable[Path] = Path(cache.cache_dir).glob('*.lprof') - iter_stats = cast( # See `ty` issue #3428 - Iterable[Path], filter(is_valid_stats_file, iter_stats), - ) - pat = 'Cleanup succeeded.*: .*dump_stats.*{}' - patterns.update({ - pat.format(re.escape(path.name)): True for path in iter_stats - }) + iter_stats: Iterable[Path] = Path(cache.cache_dir).glob('*.lprof') + iter_stats = cast( # See `ty` issue #3428 + Iterable[Path], filter(is_valid_stats_file, iter_stats), + ) + pat = 'Cleanup succeeded.*: .*dump_stats.*{}' + patterns.update({ + pat.format(re.escape(path.name)): True for path in iter_stats + }) patterns[re.escape('`multiprocessing` logging (debug)')] = intercept_logs search_cache_logs(cache, True, patterns) -def _test_apply_mp_patches( - patch_pool: bool | None = None, +def _test_apply_mp_patches_pool( patch_process: bool | None = None, intercept_logs: bool | None = None, *, @@ -610,8 +606,7 @@ def _test_apply_mp_patches( patches = cast(dict[str, bool], _DEFAULT_MP_CONFIG.patches.copy()) for name, applied in { - 'pool': patch_pool, 'process': patch_process, - 'logging': intercept_logs, + 'pool': True, 'process': patch_process, 'logging': intercept_logs, }.items(): if applied is not None: patches[name] = applied @@ -625,13 +620,13 @@ def _test_apply_mp_patches( ) if start_method == 'fork': # The `@add_timeout` decorator spins up a new thread for - # executing `_test_apply_mp_patches_inner()`; explicitly - # ignore the associated warning when we use + # executing `_test_apply_mp_patches_pool_inner()`; + # explicitly ignore the associated warning when we use # `start_method='fork'` cw.suppress_warnings( r'.*multi-?threaded.*fork\(\)', category=DeprecationWarning, ) - _test_apply_mp_patches_inner( + _test_apply_mp_patches_pool_inner( mp_patches=mp_patches, start_method=start_method, **kwargs, ) @@ -643,18 +638,16 @@ def _test_apply_mp_patches( + Params.new(('intercept_logs', 'label1'), [(True, 'with-logging'), (False, 'no-logging')], defaults=(None, 'default-logging'))).sorted() -@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label2'), - [(True, True, 'pool-and-process'), - (True, False, 'pool-only'), - (False, True, 'process-only')]) +@pytest.mark.parametrize(('patch_process', 'label2'), + [(True, 'with-process-patch'), + (False, 'no-process-patch')]) @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) -def test_apply_mp_patches_success( +def test_apply_mp_patches_pool_success( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, pool_test_module_object: ModuleType, start_method: StartMethod, - patch_pool: bool, patch_process: bool, intercept_logs: bool | None, n: int, @@ -665,13 +658,14 @@ def test_apply_mp_patches_success( """ Test that :py:func:`line_profiler._child_process_profiling\ .multiprocessing_patches.apply` - works as expected when the parallel workload does not error out. + works as expected for Python code using parallelism based on + :py:class:`multiprocessing.pool.Pool`, when the parallel workload + DOES NOT error out. See also: :py:func:`test_apply_mp_patches_failure` """ - _test_apply_mp_patches( - patch_pool, + _test_apply_mp_patches_pool( patch_process, intercept_logs, tmp_path_factory=tmp_path_factory, @@ -687,18 +681,16 @@ def test_apply_mp_patches_success( @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) -@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label'), - [(True, True, 'pool-and-process'), - (True, False, 'pool-only'), - (False, True, 'process-only')]) +@pytest.mark.parametrize(('patch_process', 'label'), + [(True, 'with-process-patch'), + (False, 'no-process-patch')]) @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) -def test_apply_mp_patches_failure( +def test_apply_mp_patches_pool_failure( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, pool_test_module_object: ModuleType, start_method: StartMethod, - patch_pool: bool, patch_process: bool, n: int, nprocs: int, @@ -707,13 +699,14 @@ def test_apply_mp_patches_failure( """ Test that :py:func:`line_profiler._child_process_profiling\ .multiprocessing_patches.apply` - works as expected when the parallel workload errors out. + works as expected for Python code using parallelism based on + :py:class:`multiprocessing.pool.Pool`, when the parallel workload + DOES error out. See also: :py:func:`test_apply_mp_patches_success` """ - _test_apply_mp_patches( - patch_pool, + _test_apply_mp_patches_pool( patch_process, tmp_path_factory=tmp_path_factory, create_cache=create_cache, @@ -765,7 +758,7 @@ def _get_mp_start_method_fuzzer(label_name: str | None) -> Params: + _get_mp_start_method_fuzzer('label3') + Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], defaults=(None, None))).sorted() -def test_multiproc_script_sanity_check( +def test_pool_multiproc_script_sanity_check( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, pool_test_module: ModuleFixture, @@ -811,7 +804,7 @@ def test_multiproc_script_sanity_check( (['kernprof', '-q', '-l'], 'out.lprof', True, 'line_profiler-active')], ) -def test_running_multiproc_script( +def test_running_pool_multiproc_script( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, pool_test_module: ModuleFixture, @@ -869,7 +862,7 @@ def test_running_multiproc_script( ).sorted().split_on_params('fail') -def _test_profiling_multiproc_script( +def _test_profiling_pool_multiproc_script( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, pool_test_module: ModuleFixture, @@ -939,7 +932,7 @@ def _test_profiling_multiproc_script( # have quite a lot of subtests tho... ('nnums', 'nprocs'), [(2000, 3)], ) -def test_profiling_multiproc_script_success( +def test_profiling_pool_multiproc_script_success( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, pool_test_module: ModuleFixture, @@ -1000,7 +993,7 @@ def test_profiling_multiproc_script_success( See also: :py:func:`test_profiling_multiproc_script_failure` """ - _test_profiling_multiproc_script( + _test_profiling_pool_multiproc_script( run_func=run_func, request=request, pool_test_module=pool_test_module, @@ -1019,7 +1012,7 @@ def test_profiling_multiproc_script_success( @(_fuzz_prof_mp_markers[True]) @pytest.mark.parametrize(('nnums', 'nprocs'), [(2000, 3)]) -def test_profiling_multiproc_script_failure( +def test_profiling_pool_multiproc_script_failure( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, pool_test_module: ModuleFixture, @@ -1043,7 +1036,7 @@ def test_profiling_multiproc_script_failure( See also: :py:func:`test_profiling_multiproc_script_success` """ - _test_profiling_multiproc_script( + _test_profiling_pool_multiproc_script( run_func=run_func, request=request, pool_test_module=pool_test_module, From 92263ab2d2d0ecc35e815a4e934d5879a21c02d0 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 29 Jun 2026 16:42:32 +0200 Subject: [PATCH 143/165] WIP: test script for `Process`-based parallelism tests/test_child_procs/multiproc_examples/ external_module.py::split_workload() New function for generating parallel workloads (lists of consecutive numbers) pool_test_module.py my_local_sum() Removed superfluous call to `reverse()` (because #424 has been fixed) sum_in_child_procs() Now using `split_workload()` main() Now permitting `--start-method=dummy` process_test_module.py New test module for testing directly managing `multiprocessing.process.BaseProcess` instances and eschewing `multiprocessing.pool.Pool` Worker Wrapper around a `multiprocessing.process.BaseProcess` which allows for executing a callable in another process and retrieving the result my_local_sum(), sum_in_child_procs(), main() Functions and CLI with the same interface as those in `pool_test_module.py` --- .../multiproc_examples/external_module.py | 23 ++ .../multiproc_examples/pool_test_module.py | 18 +- .../multiproc_examples/process_test_module.py | 271 ++++++++++++++++++ 3 files changed, 298 insertions(+), 14 deletions(-) create mode 100644 tests/test_child_procs/multiproc_examples/process_test_module.py diff --git a/tests/test_child_procs/multiproc_examples/external_module.py b/tests/test_child_procs/multiproc_examples/external_module.py index cf104c9a..66fce759 100644 --- a/tests/test_child_procs/multiproc_examples/external_module.py +++ b/tests/test_child_procs/multiproc_examples/external_module.py @@ -1,5 +1,7 @@ from __future__ import annotations +from itertools import islice + def my_external_sum(x: list[int], fail: bool = False) -> int: result: int = 0 # GREP_MARKER[EXT-INVOCATION] @@ -8,3 +10,24 @@ def my_external_sum(x: list[int], fail: bool = False) -> int: if fail: raise RuntimeError('forced failure') return result + + +def split_workload(length: int, n: int) -> list[list[int]]: + """ + Example: + >>> split_workload(10, 3) + [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10]] + >>> split_workload(8, 4) + [[1, 2], [3, 4], [5, 6], [7, 8]] + """ + iter_entries = iter(range(1, length + 1)) + result: list[list[int]] = [] + sublength = length // n + if sublength * n < length: + sublength += 1 + while True: + sublist = list(islice(iter_entries, sublength)) + if not sublist: + break + result.append(sublist) + return result diff --git a/tests/test_child_procs/multiproc_examples/pool_test_module.py b/tests/test_child_procs/multiproc_examples/pool_test_module.py index ecf6b87e..ffc4d5ec 100644 --- a/tests/test_child_procs/multiproc_examples/pool_test_module.py +++ b/tests/test_child_procs/multiproc_examples/pool_test_module.py @@ -5,7 +5,7 @@ from multiprocessing import dummy, get_context, Pool from typing import Literal -from external_module import my_external_sum +from external_module import my_external_sum, split_workload NUM_NUMBERS = 100 @@ -14,9 +14,7 @@ def my_local_sum(x: list[int], fail: bool = False) -> int: result: int = 0 # GREP_MARKER[LOCAL-INVOCATION] - # The reversing is to prevent bytecode aliasing with - # `my_external_sum()` (see issue #424, PR #425) - for item in reversed(x): + for item in x: result += item # GREP_MARKER[LOCAL-LOOP] if fail: raise RuntimeError('forced failure') @@ -30,15 +28,7 @@ def sum_in_child_procs( ] | None = None, fail: bool = False, ) -> int: - my_list: list[int] = list(range(1, length + 1)) - sublists: list[list[int]] = [] - subsums: list[int] - sublength = length // n - if sublength * n < length: - sublength += 1 - while my_list: - sublist, my_list = my_list[:sublength], my_list[sublength:] - sublists.append(sublist) + sublists = split_workload(length, n) if start_method == 'dummy': pool = dummy.Pool(n) elif start_method: @@ -58,7 +48,7 @@ def main(args: list[str] | None = None) -> None: parser.add_argument('-n', type=int, default=NUM_PROCS) parser.add_argument( '-s', '--start-method', - choices=['fork', 'forkserver', 'spawn'], default=None, + choices=['fork', 'forkserver', 'spawn', 'dummy'], default=None, ) parser.add_argument('-f', '--force-failure', action='store_true') parser.add_argument( diff --git a/tests/test_child_procs/multiproc_examples/process_test_module.py b/tests/test_child_procs/multiproc_examples/process_test_module.py new file mode 100644 index 00000000..3b5fdab2 --- /dev/null +++ b/tests/test_child_procs/multiproc_examples/process_test_module.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import atexit +import os +import pickle +import shutil +from argparse import ArgumentParser +from collections.abc import Callable, Mapping, Sequence +from contextlib import ExitStack +from dataclasses import dataclass, field +from functools import partial +from itertools import count +from multiprocessing import dummy, get_context +from multiprocessing.process import BaseProcess +from pathlib import Path +from tempfile import mkdtemp, mkstemp, TemporaryDirectory +from time import sleep +from typing import Any, ClassVar, Generic, Literal, TypeVar, cast, final +from typing_extensions import ParamSpec, Self + +from external_module import my_external_sum, split_workload + + +NUM_NUMBERS = 100 +NUM_PROCS = 4 + +T = TypeVar('T') +PS = ParamSpec('PS') +StartMethod = Literal['fork', 'forkserver', 'spawn', 'dummy'] + + +def my_local_sum(x: list[int], fail: bool = False) -> int: + result: int = 0 # GREP_MARKER[LOCAL-INVOCATION] + for item in x: + result += item # GREP_MARKER[LOCAL-LOOP] + if fail: + raise RuntimeError('forced failure') + return result + + +def _format_attrs(attrs: Mapping[str, Any]) -> str: + return ', '.join(f'{attr}={value!r}' for attr, value in attrs.items()) + + +@dataclass(init=False) +class Timeout(RuntimeError): + worker: Worker + timeout: float + + def __init__(self, worker: Worker, timeout: float) -> None: + super().__init__(worker, timeout) + self.worker = worker + self.timeout = timeout + + def _get_repr_attrs(self) -> dict[str, Any]: + return {**self.worker._get_repr_attrs(), 'timeout': self.timeout} + + def __str__(self) -> str: + return 'timed out in {:.2g} s ({})'.format( + self.timeout, _format_attrs(self.worker._get_repr_attrs()), + ) + + def __repr__(self) -> str: + attrs = _format_attrs({ + 'timeout': self.timeout, **self.worker._get_repr_attrs(), + }) + return '<{} @ {:#x} ({})>'.format(type(self).__name__, id(self), attrs) + + +@final +@dataclass +class Worker(Generic[T]): + """ + Example: + >>> import os + >>> from time import sleep + + >>> with Worker.new(sum, args=(range(10),)) as worker: + ... assert os.getpid() != worker.process.pid is not None + ... assert worker.get_result() == 45 + >>> with Worker.new(sleep, args=(10,), daemon=True) as worker: + ... worker.get_result(timeout=.125) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + process_test_module.Timeout: timed out in 0.12 s (...) + """ + process: BaseProcess + tmpdir: Path + result_callback: Callable[[], T] + _result: T = field(init=False) + _counter: ClassVar[count] = count() + + def get_result(self, timeout: float | None = None) -> T: + try: + return self._result + except AttributeError: + pass + self.process.join(timeout) + if self.process.is_alive(): + assert timeout is not None + raise Timeout(self, timeout) + self._result = self.result_callback() + return self._result + + def _get_repr_attrs(self) -> dict[str, Any]: + attrs: dict[str, Any] = { + 'name': self.process.name, + 'pid': self.process.pid, + 'exitcode': self.process.exitcode, + } + try: + attrs['result'] = self._result + except AttributeError: + pass + return attrs + + def __repr__(self) -> str: + attrs = _format_attrs(self._get_repr_attrs()) + return '<{} @ {:#x} ({})>'.format(type(self).__name__, id(self), attrs) + + def __enter__(self) -> Self: + self.process.start() + return self + + def __exit__(self, *_, **__) -> None: + try: + self._end_process('terminate') + finally: + self._end_process('join') + self._end_process('close') + + def _end_process( + self, method: Literal['terminate', 'kill', 'join', 'close'], /, + *args, **kwargs + ) -> None: + if self.process.is_alive(): + getattr(self.process, method)(*args, **kwargs) + sleep(0) + + @classmethod + def new( + cls, + target: Callable[..., T], + *, + args: Sequence[Any] | None = None, + kwargs: Mapping[str, Any] | None = None, + start_method: StartMethod | None = None, + tmpdir: Path | None = None, + **kw + ) -> Worker[T]: + if tmpdir is None: + tmpdir = Path(mkdtemp(prefix='worker-tmpdir-')) + rmdir = partial(shutil.rmtree, tmpdir, ignore_errors=True) + atexit.register(rmdir) + get_process: Callable[..., BaseProcess] + if start_method == 'dummy': + # Type-checkers don't seem to like a subclass as a + # base-class constructor... + get_process = cast(Callable[..., BaseProcess], dummy.Process) + else: + get_process = get_context(start_method).Process + handle, fname = mkstemp(suffix='.pkl', dir=tmpdir) + outfile = Path(fname) + os.close(handle) + proc = get_process( + target=cls._write_result, + args=(outfile, target, *(args or ())), + kwargs=(kwargs or {}), + name=f'MyWorker-{cls._get_count()}', + **kw, + ) + result_callback = cast( + Callable[[], T], partial(cls._read_result, outfile), + ) + return cls(proc, tmpdir, result_callback) + + @classmethod + def _get_count(cls) -> int: + return next(cls._counter) + + @staticmethod + def _read_result(out: Path) -> Any: + with out.open(mode='rb') as fobj: + ok, result = pickle.load(fobj) + if ok: + return result + else: + assert isinstance(result, BaseException) + raise result from None + + @staticmethod + def _write_result( + out: Path, func: Callable[PS, Any], /, + *args: PS.args, **kwargs: PS.kwargs + ) -> None: + try: + result = True, func(*args, **kwargs) + except BaseException as e: + result = False, type(e)(*e.args) # Truncate traceback + raise + finally: + with out.open(mode='wb') as fobj: + pickle.dump(result, fobj) + + +def gather_results( + workers: Sequence[Worker[T]], timeout: float | None = None, +) -> list[T]: + """ + Attempt to gather all partial results from all workers even if some + workers errored out. + + Note: + The ``timeout`` is applied on a per-worker basis. + """ + result: list[T] = [] + xc: BaseException | None = None + for worker in workers: + try: + result.append(worker.get_result(timeout)) + except BaseException as e: + xc = e + if xc is None: + return result + raise xc + + +def sum_in_child_procs( + length: int, n: int, my_sum: Callable[[list[int], bool], int], + start_method: StartMethod | None = None, + fail: bool = False, + timeout: float | None = None, +) -> int: + with ExitStack() as stack: + tmpdir = Path(stack.enter_context(TemporaryDirectory())) + new_worker = partial( + Worker.new, start_method=start_method, tmpdir=tmpdir, daemon=True, + ) + workers: list[Worker] = [] + for entries in split_workload(length, n): + worker = new_worker(my_sum, args=(entries, fail)) + workers.append(stack.enter_context(worker)) + return my_sum(gather_results(workers, timeout=timeout), fail) + + +def main(args: list[str] | None = None) -> None: + parser = ArgumentParser() + parser.add_argument('-l', '--length', type=int, default=NUM_NUMBERS) + parser.add_argument('-n', type=int, default=NUM_PROCS) + parser.add_argument( + '-s', '--start-method', + choices=['fork', 'forkserver', 'spawn', 'dummy'], default=None, + ) + parser.add_argument('-f', '--force-failure', action='store_true') + parser.add_argument( + '--local', + action='store_const', + dest='my_sum', + default=my_external_sum, + const=my_local_sum, + ) + options = parser.parse_args(args) + print(sum_in_child_procs( + options.length, options.n, options.my_sum, + start_method=options.start_method, + fail=options.force_failure, + )) + + +if __name__ == '__main__': + main() From 85f17c88b4a925b29d2f6a6e4d7695030dd83690 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 29 Jun 2026 20:23:54 +0200 Subject: [PATCH 144/165] New tests for `Process`-based parallelism tests/test_child_procs/conftest.py process_test_module[_object] New fixture representing `tests/test_child_procs/multiproc_examples/process_test_module.py` ext_module_object, pool_test_module_object Updated docstrings tests/test_child_procs/multiproc_examples/pool_test_module.py Fixed imports of `external_module` utilities, placing individual import targets in their own import statements (see issue #433) tests/test_child_procs/multiproc_examples/process_test_module.py - Fixed import... (ditto) - Fixes to `Worker` when `start_method='dummy'`: - No longer errors out when `daemon` is passed to `Worker.new()` (see GH issue python/cpython#85716) - No longer re-raising caught exceptions (in dummy worker backed by thread objects) so that we don't get a `PytestUnhandledThreadExceptionWarning` tests/test_child_procs/test_child_procs.py test_runpy_patches() Updated because of the additonal import in `pool_test_module.py` test_apply_mp_patches_{success,failure}() test_multiproc_script_sanity_check() test_profiling_multiproc_script_{success,failure}() Refactored from the resp. tests infixed with `pool_`, so as to also test `process_test_module.py` --- tests/test_child_procs/conftest.py | 136 +++++++++++----- .../multiproc_examples/pool_test_module.py | 3 +- .../multiproc_examples/process_test_module.py | 27 +++- tests/test_child_procs/test_child_procs.py | 149 +++++++++++------- 4 files changed, 219 insertions(+), 96 deletions(-) diff --git a/tests/test_child_procs/conftest.py b/tests/test_child_procs/conftest.py index 5d28a320..00c0f9f7 100644 --- a/tests/test_child_procs/conftest.py +++ b/tests/test_child_procs/conftest.py @@ -27,10 +27,14 @@ __all__ = ( 'ext_module', 'pool_test_module', 'pool_test_module_clone', + 'process_test_module', 'ext_module_object', 'pool_test_module_object', + 'process_test_module_object', 'create_cache', 'curated_profiler', 'another_pid', ) +# ========================== Module fixtures =========================== + # Only write the files once per test session _EXAMPLES_PATH = Path(__file__).parent / 'multiproc_examples' @@ -49,8 +53,23 @@ def _ext_module() -> Generator[Path, None, None]: @pytest.fixture(scope='session') def _pool_test_module(_ext_module: Path) -> Generator[Path, None, None]: - name = next(ModuleFixture.propose_name('my_pool_test_module')) - body = (_EXAMPLES_PATH / 'pool_test_module.py').read_text() + yield from _dependent_module_path( + _ext_module, 'pool_test_module.py', 'my_pool_test_module', + ) + + +@pytest.fixture(scope='session') +def _process_test_module(_ext_module: Path) -> Generator[Path, None, None]: + yield from _dependent_module_path( + _ext_module, 'process_test_module.py', 'my_process_test_module', + ) + + +def _dependent_module_path( + _ext_module: Path, basename: str, module_name: str, +) -> Generator[Path, None, None]: + name = next(ModuleFixture.propose_name(module_name)) + body = (_EXAMPLES_PATH / basename).read_text() body = body.replace('external_module', _ext_module.stem) with TemporaryDirectory() as mydir_str: my_dir = Path(mydir_str) @@ -60,6 +79,35 @@ def _pool_test_module(_ext_module: Path) -> Generator[Path, None, None]: yield my_module +def _build_command_line( + fail: bool, start_method: StartMethod | None, *, + use_local_func: bool = False, + nnums: int | None = None, + nprocs: int | None = None, +) -> list[str]: + args: list[str] = [] + if fail: + args.append('--force-failure') + if start_method: + args.extend(['-s', start_method]) + if use_local_func: + args.append('--local') + if nnums: + args.extend(['-l', str(nnums)]) + if nprocs: + args.extend(['-n', str(nprocs)]) + return args + + +def _get_output(nnums_: int, /) -> Callable[..., int]: + def get_output(nnums: int | None = None, **_) -> int: + if nnums is None: + nnums = nnums_ + return nnums * (nnums + 1) // 2 + + return get_output + + @pytest.fixture def ext_module( _ext_module: Path, monkeypatch: pytest.MonkeyPatch, @@ -83,36 +131,7 @@ def pool_test_module( :py:class:`ModuleFixture` helper object containing the code at ``./multiproc_examples/pool_test_module.py`` """ - def build_command_line( - fail: bool, start_method: StartMethod | None, *, - use_local_func: bool = False, - nnums: int | None = None, - nprocs: int | None = None, - ) -> list[str]: - args: list[str] = [] - if fail: - args.append('--force-failure') - if start_method: - args.extend(['-s', start_method]) - if use_local_func: - args.append('--local') - if nnums: - args.extend(['-l', str(nnums)]) - if nprocs: - args.extend(['-n', str(nprocs)]) - return args - - def get_output(*, nnums: int | None = None, **_) -> int: - if nnums is None: - nnums = default_nnums - return nnums * (nnums + 1) // 2 - - default_nnums = _static_parse('NUM_NUMBERS', _pool_test_module) - - yield ModuleFixture( - _pool_test_module, monkeypatch, [ext_module], - build_command_line, get_output, - ) + yield from _yield_test_module(_pool_test_module, ext_module, monkeypatch) @pytest.fixture @@ -137,6 +156,35 @@ def pool_test_module_clone( ) +@pytest.fixture +def process_test_module( + _process_test_module: Path, + ext_module: ModuleFixture, + monkeypatch: pytest.MonkeyPatch, +) -> Generator[ModuleFixture, None, None]: + """ + Yields: + :py:class:`ModuleFixture` helper object containing the code at + ``./multiproc_examples/process_test_module.py`` + """ + yield from _yield_test_module( + _process_test_module, ext_module, monkeypatch, + ) + + +def _yield_test_module( + test_module: Path, + ext_module: ModuleFixture, + monkeypatch: pytest.MonkeyPatch, +) -> Generator[ModuleFixture, None, None]: + default_nnums = _static_parse('NUM_NUMBERS', test_module) + assert isinstance(default_nnums, int) + yield ModuleFixture( + test_module, monkeypatch, [ext_module], + _build_command_line, _get_output(default_nnums), + ) + + @pytest.fixture def ext_module_object( ext_module: ModuleFixture, @@ -144,8 +192,8 @@ def ext_module_object( """ Yields: :py:class:`ModuleType` object containing the code at - :py:data:`EXTERNAL_MODULE_BODY`, and is torn down at the end of - the test + ``./multiproc_examples/external_module.py``, and is torn down at + the end of the test """ yield from ext_module._import_module_helper() @@ -157,12 +205,28 @@ def pool_test_module_object( """ Yields: :py:class:`ModuleType` object containing the code at - :py:data:`TEST_MODULE_TEMPLATE`, and is torn down at the end of - the test + ``./multiproc_examples/pool_test_module.py``, and is torn down + at the end of the test """ yield from pool_test_module._import_module_helper() +@pytest.fixture +def process_test_module_object( + process_test_module: ModuleFixture, ext_module_object: ModuleType, +) -> Generator[ModuleType, None, None]: + """ + Yields: + :py:class:`ModuleType` object containing the code at + ``./multiproc_examples/process_test_module.py``, and is torn + down at the end of the test + """ + yield from process_test_module._import_module_helper() + + +# =========================== Misc. fixtures =========================== + + @pytest.fixture def create_cache( tmp_path_factory: pytest.TempPathFactory, diff --git a/tests/test_child_procs/multiproc_examples/pool_test_module.py b/tests/test_child_procs/multiproc_examples/pool_test_module.py index ffc4d5ec..71c60654 100644 --- a/tests/test_child_procs/multiproc_examples/pool_test_module.py +++ b/tests/test_child_procs/multiproc_examples/pool_test_module.py @@ -5,7 +5,8 @@ from multiprocessing import dummy, get_context, Pool from typing import Literal -from external_module import my_external_sum, split_workload +from external_module import my_external_sum +from external_module import split_workload # See issue #433 NUM_NUMBERS = 100 diff --git a/tests/test_child_procs/multiproc_examples/process_test_module.py b/tests/test_child_procs/multiproc_examples/process_test_module.py index 3b5fdab2..81e5076c 100644 --- a/tests/test_child_procs/multiproc_examples/process_test_module.py +++ b/tests/test_child_procs/multiproc_examples/process_test_module.py @@ -18,7 +18,8 @@ from typing import Any, ClassVar, Generic, Literal, TypeVar, cast, final from typing_extensions import ParamSpec, Self -from external_module import my_external_sum, split_workload +from external_module import my_external_sum +from external_module import split_workload # See issue #433 NUM_NUMBERS = 100 @@ -153,22 +154,35 @@ def new( rmdir = partial(shutil.rmtree, tmpdir, ignore_errors=True) atexit.register(rmdir) get_process: Callable[..., BaseProcess] - if start_method == 'dummy': + daemon: bool | None = None + use_threads = start_method == 'dummy' + if use_threads: # Type-checkers don't seem to like a subclass as a # base-class constructor... get_process = cast(Callable[..., BaseProcess], dummy.Process) + # Note: `DummyProcess` somehow doesn't take the `daemon` + # parameter, despite `threading.Thread` doing so... + # (see cpython issue GH-#85716, PR GH-#21869) + daemon = kw.pop('daemon', None) else: - get_process = get_context(start_method).Process + mp_context = get_context(start_method) + assert hasattr(mp_context, 'Process') # Assure `mypy` + get_process = mp_context.Process handle, fname = mkstemp(suffix='.pkl', dir=tmpdir) outfile = Path(fname) os.close(handle) proc = get_process( target=cls._write_result, - args=(outfile, target, *(args or ())), + # Suppress (i.e. don't reraise) errors on threads, or + # `pytest` will complain + args=(outfile, target, use_threads, *(args or ())), kwargs=(kwargs or {}), name=f'MyWorker-{cls._get_count()}', **kw, ) + if daemon is not None: + # Should've been set except for `DummyProcess` + proc.daemon = daemon result_callback = cast( Callable[[], T], partial(cls._read_result, outfile), ) @@ -190,14 +204,15 @@ def _read_result(out: Path) -> Any: @staticmethod def _write_result( - out: Path, func: Callable[PS, Any], /, + out: Path, func: Callable[PS, Any], suppress_errors: bool, /, *args: PS.args, **kwargs: PS.kwargs ) -> None: try: result = True, func(*args, **kwargs) except BaseException as e: result = False, type(e)(*e.args) # Truncate traceback - raise + if not suppress_errors: + raise finally: with out.open(mode='wb') as fobj: pickle.dump(result, fobj) diff --git a/tests/test_child_procs/test_child_procs.py b/tests/test_child_procs/test_child_procs.py index 1f7aab16..8a435553 100644 --- a/tests/test_child_procs/test_child_procs.py +++ b/tests/test_child_procs/test_child_procs.py @@ -99,11 +99,11 @@ def __exit__(self, *_, **__) -> None: if run_profiled_code: module = pool_test_module num_invocations, num_loops = 1, nprocs - expected_funcs: list[str] = ['my_external_sum'] + expected_funcs: set[str] = {'my_external_sum', 'split_workload'} else: module = pool_test_module_clone num_invocations, num_loops = 0, 0 - expected_funcs = [] + expected_funcs = set() if as_module: first_arg = module.name runner = partial(runpy.run_module, alter_sys=True) @@ -122,7 +122,7 @@ def __exit__(self, *_, **__) -> None: assert stdout.rstrip('\n') == str(nnums * (nnums + 1) // 2) # Check that profiler has received the appropriate targets - funcs = [func.__name__ for func in getattr(cache.profiler, 'functions')] + funcs = {func.__name__ for func in getattr(cache.profiler, 'functions')} assert funcs == expected_funcs # Check that calls in the current process are profiled iif the @@ -473,11 +473,11 @@ def test_load_pth_hook( @cleanup_extra_pth_files() @preserve_targets() @add_timeout -def _test_apply_mp_patches_pool_inner( +def _test_apply_mp_patches_inner( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, - pool_test_module_object: ModuleType, + test_module_object: ModuleType, start_method: StartMethod, mp_patches: Collection[str], fail: bool, @@ -516,7 +516,7 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # to the profiler, so the code paths are the same profiled_func = ext_module_object.my_external_sum called_func = partial( - pool_test_module_object.sum_in_child_procs, + test_module_object.sum_in_child_procs, n=nprocs, my_sum=profiled_func, start_method=start_method, @@ -587,12 +587,17 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: patterns.update({ pat.format(re.escape(path.name)): True for path in iter_stats }) - patterns[re.escape('`multiprocessing` logging (debug)')] = intercept_logs + logger_pat = '{} {}'.format( + re.escape('`multiprocessing` logging'), + r'\((sub_)?debug|info|sub_warning|warn\)', + ) + patterns[logger_pat] = intercept_logs search_cache_logs(cache, True, patterns) -def _test_apply_mp_patches_pool( +def _test_apply_mp_patches( patch_process: bool | None = None, + patch_pool: bool | None = None, intercept_logs: bool | None = None, *, start_method: StartMethod, @@ -606,7 +611,8 @@ def _test_apply_mp_patches_pool( patches = cast(dict[str, bool], _DEFAULT_MP_CONFIG.patches.copy()) for name, applied in { - 'pool': True, 'process': patch_process, 'logging': intercept_logs, + 'pool': patch_pool, 'process': patch_process, + 'logging': intercept_logs, }.items(): if applied is not None: patches[name] = applied @@ -626,7 +632,7 @@ def _test_apply_mp_patches_pool( cw.suppress_warnings( r'.*multi-?threaded.*fork\(\)', category=DeprecationWarning, ) - _test_apply_mp_patches_pool_inner( + _test_apply_mp_patches_inner( mp_patches=mp_patches, start_method=start_method, **kwargs, ) @@ -638,17 +644,22 @@ def _test_apply_mp_patches_pool( + Params.new(('intercept_logs', 'label1'), [(True, 'with-logging'), (False, 'no-logging')], defaults=(None, 'default-logging'))).sorted() -@pytest.mark.parametrize(('patch_process', 'label2'), - [(True, 'with-process-patch'), - (False, 'no-process-patch')]) +@pytest.mark.parametrize( + ('test_module', 'patch_process', 'patch_pool', 'label2'), + [('pool_test_module', True, True, 'patch-pool-and-process'), + ('pool_test_module', False, True, 'patch-pool-only'), + ('process_test_module', True, True, 'patch-pool-and-process'), + ('process_test_module', True, False, 'patch-process-only')]) @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) -def test_apply_mp_patches_pool_success( +def test_apply_mp_patches_success( + request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, - pool_test_module_object: ModuleType, + test_module: str, start_method: StartMethod, patch_process: bool, + patch_pool: bool, intercept_logs: bool | None, n: int, nprocs: int, @@ -659,19 +670,23 @@ def test_apply_mp_patches_pool_success( Test that :py:func:`line_profiler._child_process_profiling\ .multiprocessing_patches.apply` works as expected for Python code using parallelism based on - :py:class:`multiprocessing.pool.Pool`, when the parallel workload - DOES NOT error out. + :py:class:`multiprocessing.pool.Pool` and + :py:class:`multiprocessing.process.BaseProcess`, when the parallel + workload DOES NOT error out. See also: :py:func:`test_apply_mp_patches_failure` """ - _test_apply_mp_patches_pool( + test_module_object = request.getfixturevalue(test_module + '_object') + assert isinstance(test_module_object, ModuleType) + _test_apply_mp_patches( patch_process, + patch_pool, intercept_logs, tmp_path_factory=tmp_path_factory, create_cache=create_cache, ext_module_object=ext_module_object, - pool_test_module_object=pool_test_module_object, + test_module_object=test_module_object, start_method=start_method, fail=False, n=n, @@ -681,17 +696,22 @@ def test_apply_mp_patches_pool_success( @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) -@pytest.mark.parametrize(('patch_process', 'label'), - [(True, 'with-process-patch'), - (False, 'no-process-patch')]) +@pytest.mark.parametrize( + ('test_module', 'patch_process', 'patch_pool', 'label'), + [('pool_test_module', True, True, 'patch-pool-and-process'), + ('pool_test_module', False, True, 'patch-pool-only'), + ('process_test_module', True, True, 'patch-pool-and-process'), + ('process_test_module', True, False, 'patch-process-only')]) @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) -def test_apply_mp_patches_pool_failure( +def test_apply_mp_patches_failure( + request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, - pool_test_module_object: ModuleType, + test_module: str, start_method: StartMethod, patch_process: bool, + patch_pool: bool, n: int, nprocs: int, label: str, @@ -700,18 +720,22 @@ def test_apply_mp_patches_pool_failure( Test that :py:func:`line_profiler._child_process_profiling\ .multiprocessing_patches.apply` works as expected for Python code using parallelism based on - :py:class:`multiprocessing.pool.Pool`, when the parallel workload - DOES error out. + :py:class:`multiprocessing.pool.Pool` and + :py:class:`multiprocessing.process.BaseProcess`, when the parallel + workload DOES error out. See also: :py:func:`test_apply_mp_patches_success` """ - _test_apply_mp_patches_pool( + test_module_object = request.getfixturevalue(test_module + '_object') + assert isinstance(test_module_object, ModuleType) + _test_apply_mp_patches( patch_process, + patch_pool, tmp_path_factory=tmp_path_factory, create_cache=create_cache, ext_module_object=ext_module_object, - pool_test_module_object=pool_test_module_object, + test_module_object=test_module_object, start_method=start_method, fail=True, n=n, @@ -745,23 +769,27 @@ def _get_mp_start_method_fuzzer(label_name: str | None) -> Params: return fuzz_fail * fuzz_start -@(Params.new(('run_func', 'label1'), - [(run_module, 'module'), (run_script, 'script')]) +@(Params.new('test_module', ['pool_test_module', 'process_test_module']) + * Params.new(('run_func', 'label1'), + [(run_module, 'module'), (run_script, 'script')]) * Params.new(('use_local_func', 'label2'), [(True, 'local'), (False, 'ext')]) # Python can't pickle things unless they resided in a retrievable - # location (so not the script supplied by `python -c`) - + Params.new(('run_func', 'label1', 'use_local_func', 'label2'), - [(run_literal_code, 'literal-code', False, 'ext')]) + # location (so not the script supplied by `python -c`); this also + # means that `process_test_module` cannot be `python -c`-ed at all, + # because `Worker` is locally-defined + + Params.new( + ('test_module', 'run_func', 'label1', 'use_local_func', 'label2'), + [('pool_test_module', run_literal_code, 'literal-code', False, 'ext')]) # Also fuzz the parallelization-related stuff, esp. check what # happens if an exception is raised inside the parallelly-run func + _get_mp_start_method_fuzzer('label3') + Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], defaults=(None, None))).sorted() -def test_pool_multiproc_script_sanity_check( +def test_multiproc_script_sanity_check( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, - pool_test_module: ModuleFixture, + test_module: str, tmp_path_factory: pytest.TempPathFactory, use_local_func: bool, fail: bool, @@ -772,11 +800,13 @@ def test_pool_multiproc_script_sanity_check( label1: str, label2: str, label3: str, ) -> None: """ - Sanity check that the test module functions as expected when run + Sanity check that the test modules function as expected when run with vanilla Python. """ + module_fixture = request.getfixturevalue(test_module) + assert isinstance(module_fixture, ModuleFixture) run_func( - request, pool_test_module, tmp_path_factory, + request, module_fixture, tmp_path_factory, runner=sys.executable, profile=False, fail=fail, start_method=start_method, @@ -804,7 +834,7 @@ def test_pool_multiproc_script_sanity_check( (['kernprof', '-q', '-l'], 'out.lprof', True, 'line_profiler-active')], ) -def test_running_pool_multiproc_script( +def test_running_multiproc_script( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, pool_test_module: ModuleFixture, @@ -816,7 +846,7 @@ def test_running_pool_multiproc_script( label1: str, label2: str, ) -> None: """ - Check that `kernprof` can RUN the test module in various contexts + Check that `kernprof` can RUN a test module in various contexts (`kernprof [...] `, `kernprof [...] -m `, and `kernprof [...] -c "code"`). @@ -826,6 +856,9 @@ def test_running_pool_multiproc_script( - This test does not test the actual profiling, just the execution of the code and presence of profiling data thereafter. + + - Because this is mostly a sanity check, we only run one of the + test modules. """ run_func( request, pool_test_module, tmp_path_factory, runner, outfile, profile, @@ -848,6 +881,8 @@ def test_running_pool_multiproc_script( + Params.new(('use_local_func', 'label4'), [(True, 'local'), (False, 'external')], defaults=(False, 'external'))) + # Test all of the above with both test modules + * Params.new('test_module', ['pool_test_module', 'process_test_module']) # The 'with-preimports' case is already tested rather thoroughly in # `test_apply_mp_patches()`, so exclude these from the above "main" # param matrix and just test the different `kernprof` modes via the @@ -862,10 +897,10 @@ def test_running_pool_multiproc_script( ).sorted().split_on_params('fail') -def _test_profiling_pool_multiproc_script( +def _test_profiling_multiproc_script( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, - pool_test_module: ModuleFixture, + test_module: ModuleFixture, ext_module: ModuleFixture, tmp_path_factory: pytest.TempPathFactory, prof_child_procs: bool, @@ -888,7 +923,7 @@ def _test_profiling_pool_multiproc_script( if not fail: # The final sum in the parent process should always be profiled # unless the child processes failed and we never returned from - # `Pool.starmap()` + # `Pool.starmap()` (or `gather_results()`) nhits[tag_call] += 1 nhits[tag_loop] += nprocs if prof_child_procs: @@ -910,7 +945,7 @@ def _test_profiling_pool_multiproc_script( if prof_child_procs and DEBUG: kwargs.setdefault('debug_log', 'debug.log') run_func( - request, pool_test_module, tmp_path_factory, + request, test_module, tmp_path_factory, runner=runner, outfile='out.lprof', profile=True, @@ -932,10 +967,10 @@ def _test_profiling_pool_multiproc_script( # have quite a lot of subtests tho... ('nnums', 'nprocs'), [(2000, 3)], ) -def test_profiling_pool_multiproc_script_success( +def test_profiling_multiproc_script_success( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, - pool_test_module: ModuleFixture, + test_module: str, ext_module: ModuleFixture, tmp_path_factory: pytest.TempPathFactory, prof_child_procs: bool, @@ -949,7 +984,7 @@ def test_profiling_pool_multiproc_script_success( label1: str, label2: str, label3: str, label4: str, label5: str, ) -> None: """ - Check that `kernprof` can PROFILE the test module in various + Check that `kernprof` can PROFILE the test modules in various contexts when the parallel workload runs without errors, optionally extending profiling into child processes. @@ -960,6 +995,10 @@ def test_profiling_pool_multiproc_script_success( - ``run_func`` tests the different :cmd:`kernprof` modes (see :py:func:`~.test_running_multiproc_script`). + - ``test_module`` chooses what kind of parallelism the test + module should use (:py:class:`multiprocessing.pool.Pool` vs + :py:class:`multiprocessing.process.BaseProcess`). + - ``preimports`` tests that both mechanisms for setting up profiling targets work: @@ -993,10 +1032,12 @@ def test_profiling_pool_multiproc_script_success( See also: :py:func:`test_profiling_multiproc_script_failure` """ - _test_profiling_pool_multiproc_script( + module_fixture = request.getfixturevalue(test_module) + assert isinstance(module_fixture, ModuleFixture) + _test_profiling_multiproc_script( run_func=run_func, request=request, - pool_test_module=pool_test_module, + test_module=module_fixture, ext_module=ext_module, tmp_path_factory=tmp_path_factory, prof_child_procs=prof_child_procs, @@ -1012,10 +1053,10 @@ def test_profiling_pool_multiproc_script_success( @(_fuzz_prof_mp_markers[True]) @pytest.mark.parametrize(('nnums', 'nprocs'), [(2000, 3)]) -def test_profiling_pool_multiproc_script_failure( +def test_profiling_multiproc_script_failure( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, - pool_test_module: ModuleFixture, + test_module: str, ext_module: ModuleFixture, tmp_path_factory: pytest.TempPathFactory, prof_child_procs: bool, @@ -1029,17 +1070,19 @@ def test_profiling_pool_multiproc_script_failure( label1: str, label2: str, label3: str, label4: str, label5: str, ) -> None: """ - Check that `kernprof` can PROFILE the test module in various + Check that `kernprof` can PROFILE the test modules in various contexts when the parallel workload errors out, optionally extending profiling into child processes. See also: :py:func:`test_profiling_multiproc_script_success` """ - _test_profiling_pool_multiproc_script( + module_fixture = request.getfixturevalue(test_module) + assert isinstance(module_fixture, ModuleFixture) + _test_profiling_multiproc_script( run_func=run_func, request=request, - pool_test_module=pool_test_module, + test_module=module_fixture, ext_module=ext_module, tmp_path_factory=tmp_path_factory, prof_child_procs=prof_child_procs, From ad70668df725d8cf7189e2dc4a5508af84e37bcc Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 30 Jun 2026 03:32:18 +0200 Subject: [PATCH 145/165] Minor fixes for `test_child_procs` tests tests/test_child_procs/_test_child_procs_utils.py search_cache_logs() Updated error messages to include info about the `re.Match` objects (where appropriate) add_timeout(), CheckWarnings Fixed error-type representations in doctests tests/test_child_procs/multiproc_examples/process_test_module.py ::Worker Added `xdoctest` skip directive to the doctest, because it gives a pickling error (of unknown reasons) when run with `xdoctest` but not `doctest` XXX: Try to replicate and mitigate the last test failure, where `search_cache_logs()` seemed to be complaining about a nonexistent regex match --- .../test_child_procs/_test_child_procs_utils.py | 17 ++++++++++++----- .../multiproc_examples/process_test_module.py | 7 +++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/test_child_procs/_test_child_procs_utils.py b/tests/test_child_procs/_test_child_procs_utils.py index cdeeb4ed..b433c380 100644 --- a/tests/test_child_procs/_test_child_procs_utils.py +++ b/tests/test_child_procs/_test_child_procs_utils.py @@ -267,12 +267,17 @@ def search_cache_logs( to_match = dict.fromkeys(patterns, True) for pat, should_match in to_match.items(): pattern = re.compile(pat, flags) - if any(pattern.search(chunk) for chunk in text_chunks) == should_match: + matches = [pattern.search(chunk) for chunk in text_chunks] + if any(matches) == should_match: continue - raise ResultMismatch( + error_msg = ( f'pattern {pattern!r} to {"" if should_match else "not "}match ' f'{cache!r}\'s logs: {text_chunks!r}' ) + if should_match: + raise ResultMismatch(error_msg, 'no matches') + else: + raise ResultMismatch(error_msg, [m for m in matches if m]) # ================ `pytest` stuff: fixtures and markers ================ @@ -1068,7 +1073,7 @@ def add_timeout( >>> my_func(4, delay=5) # doctest: +NORMALIZE_WHITESPACE Traceback (most recent call last): ... - test_child_procs.TestTimeout: + test_child_procs._test_child_procs_utils.TestTimeout: my_func(4, delay=5): timed out after 0.5 s """ if func is None: @@ -1384,7 +1389,8 @@ class CheckWarnings(Sequence[_WarningInfo]): This is printed before the error Traceback (most recent call last): ... - test_child_procs.ResultMismatch: expected no warnings matching + test_child_procs._test_child_procs_utils.ResultMismatch: \ +expected no warnings matching _WarningMatcher(message='foo', category=), got 1 ([...]) @@ -1397,7 +1403,8 @@ class CheckWarnings(Sequence[_WarningInfo]): This is printed before the error Traceback (most recent call last): ... - test_child_procs.ResultMismatch: expected warnings matching + test_child_procs._test_child_procs_utils.ResultMismatch: \ +expected warnings matching _WarningMatcher(category=), got none out of 1 ([...]) >>> assert len(cw) == 1 diff --git a/tests/test_child_procs/multiproc_examples/process_test_module.py b/tests/test_child_procs/multiproc_examples/process_test_module.py index 81e5076c..95e8b05f 100644 --- a/tests/test_child_procs/multiproc_examples/process_test_module.py +++ b/tests/test_child_procs/multiproc_examples/process_test_module.py @@ -73,6 +73,8 @@ def __repr__(self) -> str: class Worker(Generic[T]): """ Example: + >>> # xdoctest: +SKIP + >>> import os >>> from time import sleep @@ -84,6 +86,11 @@ class Worker(Generic[T]): Traceback (most recent call last): ... process_test_module.Timeout: timed out in 0.12 s (...) + + Note: + For currently unknown reasons, the above doctest results in + pickling errors when run with :py:mod:`xdoctest` but not vanilla + :py:mod:`doctest`; hence the ``SKIP`` directive. """ process: BaseProcess tmpdir: Path From 7737fbb1d23f0a2a13107de7899c94e5c32e7c73 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 2 Jul 2026 02:45:57 +0200 Subject: [PATCH 146/165] No longer isolating a test in a timed thread tests/test_child_procs/test_child_procs.py ::test_apply_mp_patches_{success,failure}() - Removed `add_timeout()` decorator around inner function because the code hindering `BaseProcess.terminate()` is no longer used - Removed warning suppression because we no longer expect the warning about forking in a multithreaded process --- tests/test_child_procs/_test_child_procs_utils.py | 6 +++++- tests/test_child_procs/test_child_procs.py | 11 +---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/tests/test_child_procs/_test_child_procs_utils.py b/tests/test_child_procs/_test_child_procs_utils.py index b433c380..c4398950 100644 --- a/tests/test_child_procs/_test_child_procs_utils.py +++ b/tests/test_child_procs/_test_child_procs_utils.py @@ -1912,13 +1912,17 @@ def get_streams( with ExitStack() as stack: if check_warnings: # See similar indictions against warnings in - # `_test_apply_mp_patches()` + # `test_child_procs.py::_test_apply_mp_patches()` cw = stack.enter_context(CheckWarnings()) cw.forbid_warnings('.*resource_tracker', module='multiprocessing') cw.forbid_warnings( r'.* file\(s\) .* empty', category=UserWarning, module='line_profiler', ) + # The `@add_timeout` decorator spins up a new thread for + # executing `kernprof.main()`; + # explicitly ignore the associated warning when we use + # `start_method='fork'` if timeout: cw.suppress_warnings( r'.*multi-?threaded.*fork\(\)', diff --git a/tests/test_child_procs/test_child_procs.py b/tests/test_child_procs/test_child_procs.py index 8a435553..f5adc193 100644 --- a/tests/test_child_procs/test_child_procs.py +++ b/tests/test_child_procs/test_child_procs.py @@ -34,7 +34,7 @@ DEBUG, DEFAULT_TIMEOUT, NOT_SUPPLIED, START_METHODS, PATCH_SUMMARIES, ModuleFixture, Params, ResultMismatch, StartMethod, preserve_object_attrs, preserve_targets, - cleanup_extra_pth_files, add_timeout, + cleanup_extra_pth_files, CheckWarnings, mp_patch_is_internal, get_mp_patches_toml_text, summarize_mp_patches, filter_mp_patch_summary, @@ -472,7 +472,6 @@ def test_load_pth_hook( @cleanup_extra_pth_files() @preserve_targets() -@add_timeout def _test_apply_mp_patches_inner( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], @@ -624,14 +623,6 @@ def _test_apply_mp_patches( r'.* file\(s\) .* empty', category=UserWarning, module='line_profiler', ) - if start_method == 'fork': - # The `@add_timeout` decorator spins up a new thread for - # executing `_test_apply_mp_patches_pool_inner()`; - # explicitly ignore the associated warning when we use - # `start_method='fork'` - cw.suppress_warnings( - r'.*multi-?threaded.*fork\(\)', category=DeprecationWarning, - ) _test_apply_mp_patches_inner( mp_patches=mp_patches, start_method=start_method, **kwargs, ) From dcc89050c3d5e92a70c83d626e4e5f4023425ace Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 2 Jul 2026 22:41:55 +0200 Subject: [PATCH 147/165] Removed polling stuff entirely line_profiler/_child_process_profiling/multiprocessing_patches/ __init__.py No longer exposing the `Poller` and `Timeout` names (as they no longer exist) _mandatory_patches.py PROCESS_SETUP_PATCH Renamed and refactored from `PROCESS_TERMINATION_PATCH`: no longer going through the polling-for-lock-file routine because it should not be necessary (setup code should have been run before moving onto the parallel workload thanks to the .pth file anyway) wrap_start(), wrap_terminate() Removed mp_config.py::MPConfig Removed the `.polling` field poller.py Removed line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes.multiprocessing.polling] Removed subtable setup.py Updated `line_profiler._multiproc_patches` plugin paths --- .../multiprocessing_patches/__init__.py | 5 +- .../_mandatory_patches.py | 132 +----------- .../multiprocessing_patches/mp_config.py | 39 ++-- .../multiprocessing_patches/poller.py | 198 ------------------ line_profiler/rc/line_profiler.toml | 23 -- setup.py | 4 +- 6 files changed, 21 insertions(+), 380 deletions(-) delete mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches/poller.py diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py index 3fb62409..4e87c6ba 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py @@ -21,18 +21,15 @@ from ..cache import LineProfilingCache from ._infrastructure import Registry from .mp_config import MPConfig -from .poller import Poller -__all__ = ('MPConfig', 'Poller', 'Registry', 'Timeout', 'apply') +__all__ = ('MPConfig', 'Registry', 'apply') PublicPatch = Literal['pool', 'process', 'logging'] _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' _PATCHES = Registry.from_entry_point() -Timeout = Poller.Timeout - def apply( cache: LineProfilingCache, diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py index 8830bc7e..9a56f966 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -7,7 +7,6 @@ from functools import partial from multiprocessing.pool import Pool from multiprocessing.process import BaseProcess -from pathlib import Path from types import MappingProxyType as mappingproxy, MethodType from typing import Any, ClassVar, TypeVar, cast from typing_extensions import Concatenate, ParamSpec @@ -37,14 +36,12 @@ from ..runpy_patches import create_runpy_wrapper from ._infrastructure import SingleModulePatch from ._queue import Queue, PutWrapper -from .mp_config import MPConfig -from .poller import Poller, OnTimeout __all__ = ( - 'POOL_WORKER_PID_PATCH', 'PROCESS_TERMINATION_PATCH', + 'POOL_WORKER_PID_PATCH', 'PROCESS_SETUP_PATCH', 'RebootForkserverPatch', 'ResourceTrackerPatch', 'RunpyPatch', - 'wrap_terminate', 'wrap_start', 'wrap_bootstrap', + 'wrap_bootstrap', 'wrap_handle_results', 'wrap_terminate_pool', 'wrap_worker', ) @@ -67,10 +64,6 @@ def setup_mp_child( # nocover - Unregister the :py:mod:`atexit` hook associated with ``cache`` to avoid possible clashes with the profiling-file writing managed by this module. - - - Remove the per-child-process lock file which prevents the parent - from :py:meth:`.BaseProcess.terminate`-ing it before it can be - properly set up and causing a hang. """ if cache.main_pid == os.getpid(): # Not in a child process return @@ -78,7 +71,7 @@ def setup_mp_child( # nocover msg = 'Performing setup for `multiprocessing` child processes...' cache._debug_output(msg) setup: Callable[[LineProfilingCache, BaseProcess], Any] - for setup in [_unregister_atexit_hook, _remove_lock_file]: + for setup in [_unregister_atexit_hook]: try: setup(cache, proc) except Exception as e: @@ -100,78 +93,9 @@ def _unregister_atexit_hook( # nocover atexit.unregister(cache._atexit_hook) -def _remove_lock_file( # nocover - cache: LineProfilingCache, proc: BaseProcess, -) -> None: - lock_file = _get_lock_file(proc) - if lock_file is None: - return - lock_file.unlink(missing_ok=True) - cache._debug_output(f'Removed lock file {lock_file.name!r}') - - -def _get_lock_file(proc: BaseProcess) -> Path | None: - return getattr(proc, _LOCK_FILE_LOC, None) - - -def _set_lock_file(proc: BaseProcess, lock_file: Path) -> None: - setattr(proc, _LOCK_FILE_LOC, lock_file) - - # ----------- `multiprocessing.process.BaseProcess` patches ------------ -@LineProfilingCache._method_wrapper -def wrap_terminate( - cache: LineProfilingCache, - vanilla_impl: Callable[[BaseProcess], None], - self: BaseProcess, -) -> None: - """ - Wrap around :py:meth:`.BaseProcess.terminate` to make sure that we - don't attempt to kill the child (OS-level) process before it has - been set up, by polling for when the child process has completed - setup and deleted its lock file. - - See also: - :py:func:`line_profiler._child_process_profiling.\ -multiprocessing_patches._profiling_patches.wrap_terminate` - """ - lock_file = _get_lock_file(self) - if lock_file is None: - cache._debug_output(f'no lock file associated with {self!r}') - vanilla_impl(self) - return - try: - with _get_terminate_poller(cache, self, lock_file): - pass - except Poller.Timeout as e: # Also handles `~.TimeoutWarning` - cache._debug_output(f'{type(e).__qualname__}: {e}') - raise - finally: # Always call `Process.terminate()` to avoid orphans - vanilla_impl(self) - - -@LineProfilingCache._method_wrapper -def wrap_start( - cache: LineProfilingCache, - vanilla_impl: Callable[[BaseProcess], None], - self: BaseProcess, -) -> None: - """ - Wrap around :py:meth:`.BaseProcess.start` to make sure that we - don't attempt to kill the child (OS-level) process before it has - been set up, by setting up a lock file which the child process - should delete upon completing setup. - """ - prefix = f'process-termination-lock-{os.getpid()}-{id(self):#x}-' - # This assigns the tempfile to the instance dict, which should be - # pickled along with the rest of the instance and sent to the child - # process - _set_lock_file(self, cache.make_tempfile(prefix=prefix, suffix='.lock')) - vanilla_impl(self) - - @LineProfilingCache._method_wrapper # nocover def wrap_bootstrap( cache: LineProfilingCache, @@ -182,58 +106,14 @@ def wrap_bootstrap( ) -> T: """ Wrap around :py:meth:`.BaseProcess._bootstrap` to perform setups - and signal to the parent process thereafter. - - Notes: - This is only invoked in child processes, and - :py:mod:`coverage` seems to be having trouble with them in the - current setup, probably due to issues with .pth file - precendence causing :py:mod:`line_profiler` to be loaded - before it. Hence the ``# nocover``. + specific to :py:mod:`multiprocessing`-managed processes. """ setup_mp_child(cache, self) return vanilla_impl(self, *args, **kwargs) -def _get_terminate_poller( - cache: LineProfilingCache, process: BaseProcess, lock_file: Path, -) -> Poller: - config = MPConfig.from_cache(cache) - cd, timeout, on_timeout = config.polling - if on_timeout not in ('ignore', 'warn', 'error'): - on_timeout = config.get_defaults().polling.on_timeout - return ( - Poller.poll_until(_lock_file_removed, cache, process, lock_file) - .with_cooldown(cd) - .with_timeout(timeout, cast(OnTimeout, on_timeout)) - ) - - -def _lock_file_removed( - cache: LineProfilingCache, proc: BaseProcess, path: Path, -) -> bool: - exists = path.exists() - if exists: - msg = ( - f'Waiting for process {proc.ident} to set up and ' - f'delete the lock file {path.name!r}...' - ) - else: - msg = f'Process {proc.ident} has been set up' - cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') - return not exists - - -PROCESS_TERMINATION_PATCH = SingleModulePatch( - 'process', priority=1, -).add_target( - 'BaseProcess', - { - 'terminate': wrap_terminate, - 'start': wrap_start, - '_bootstrap': wrap_bootstrap, - }, -) +PROCESS_SETUP_PATCH = SingleModulePatch('process', priority=1) +PROCESS_SETUP_PATCH.add_method('BaseProcess', '_bootstrap', wrap_bootstrap) # ---------------------- PID bookkeeping patches ----------------------- diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py b/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py index cd2a19ce..92ef2230 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/mp_config.py @@ -1,7 +1,7 @@ from __future__ import annotations import dataclasses -from typing import Any, NamedTuple, final +from typing import final from typing_extensions import Self from ...toml_config import ConfigSource @@ -11,37 +11,23 @@ __all__ = ('MPConfig',) -@final -class _PollerArgs(NamedTuple): - cooldown: float - timeout: float - on_timeout: str | None - - @classmethod - def new(cls, cooldown: Any, timeout: Any, on_timeout: Any) -> Self: - try: - cd = max(float(cooldown), 0) - except (TypeError, ValueError): - cd = 0 - try: - to = max(float(timeout), 0) - except (TypeError, ValueError): - to = 0 - try: - ot: str | None = on_timeout.lower() - except Exception: # Fallback (use `_Poller`'s default) - ot = None - return cls(cd, to, ot) - - @final @dataclasses.dataclass class MPConfig: """ Consolidate the config options into a structured object. + + Notes: + + - This corresponds to the + ``tool.child_processes.multiprocessing`` table in + ``line_profiler.toml``. + + - While the object has currently only a single field, it is + intentionally kept this way to make the config scheme + extensible. """ patches: dict[str, bool] - polling: _PollerArgs @classmethod def from_config(cls, config: ConfigSource) -> Self: @@ -50,8 +36,7 @@ def from_config(cls, config: ConfigSource) -> Self: .get_subconfig('child_processes', 'multiprocessing') .conf_dict ) - polling = _PollerArgs.new(**loaded['polling']) - return cls(patches=dict(loaded['patches']), polling=polling) + return cls(patches=dict(loaded['patches'])) @classmethod def from_cache(cls, cache: LineProfilingCache) -> Self: diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/poller.py b/line_profiler/_child_process_profiling/multiprocessing_patches/poller.py deleted file mode 100644 index cb8b60ff..00000000 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/poller.py +++ /dev/null @@ -1,198 +0,0 @@ -from __future__ import annotations - -import warnings -from collections.abc import Callable -from functools import partial -from time import sleep, monotonic -from typing import Any, Literal, NoReturn -from typing_extensions import ParamSpec, Self - -from ... import _diagnostics as diagnostics - - -__all__ = ('Poller', 'OnTimeout') - -PS = ParamSpec('PS') -OnTimeout = Literal['ignore', 'warn', 'error'] - - -class Poller: - """ - Poll a callable until it returns true-y. - - Example: - >>> import warnings - >>> from contextlib import ExitStack - >>> from functools import partial - >>> from itertools import count - >>> from typing import Iterator, Literal - - >>> def count_until( - ... limit: int, mode: Literal['until', 'while'] = 'until', - ... ) -> bool: - ... def counter_is_big_enough( - ... counter: Iterator[int], limit: int, - ... ) -> bool: - ... return next(counter) >= limit - ... - ... def counter_is_small_enough( - ... counter: Iterator[int], limit: int, - ... ) -> bool: - ... return next(counter) < limit - ... - ... # The branches are ultimately equal in results, but we - ... # want to explicitly test both `.poll_until()` and - ... # `.poll_while()` - ... if mode == 'until': - ... get_poller = partial( - ... Poller.poll_until, counter_is_big_enough, - ... ) - ... else: - ... get_poller = partial( - ... Poller.poll_while, counter_is_small_enough, - ... ) - ... return get_poller(count(), limit) - - >>> with count_until(10).with_cooldown(.01).with_timeout(1): - ... # Note: we shouldn't really need that much time, but - ... # something in CI seems to be slowing down the polling - ... # loop... - ... print('We counted up to 10') - We counted up to 10 - - >>> with ( - ... count_until(100) - ... .with_cooldown(.01) - ... .with_timeout(.5) # `[on_]timeout` separately supplied - ... .with_timeout(on_timeout='ignore') - ... ): - ... print("We probably didn't count up to 100 but whatever") - We probably didn't count up to 100 but whatever - - >>> with ( # doctest: +NORMALIZE_WHITESPACE - ... count_until(30).with_cooldown(.01).with_timeout(.25) - ... ): - ... print('We counted up to 30') - Traceback (most recent call last): - ... - line_profiler...Poller.Timeout: ... - timed out (... s >= 0.25 s) waiting for - callback ...counter_is_big_enough... to return true - - >>> with ExitStack() as stack: # doctest: +NORMALIZE_WHITESPACE - ... enter = stack.enter_context - ... enter(warnings.catch_warnings()) - ... warnings.simplefilter('error', Poller.TimeoutWarning) - ... enter( - ... count_until(30, 'while') - ... .with_cooldown(.01) - ... .with_timeout(.25, 'warn') - ... ) - ... print('We counted up to 30 again') - Traceback (most recent call last): - ... - line_profiler...Poller.TimeoutWarning: ... - timed out (... s >= 0.25 s) waiting for - callback ...counter_is_small_enough... to return true - """ - def __init__( - self, - func: Callable[[], Any], - cooldown: float = 0, - timeout: float = 0, - on_timeout: OnTimeout = 'error', - ) -> None: - self._func: Callable[[], Any] = func - self._cooldown = max(0, cooldown) - self._timeout = max(0, timeout) - self._on_timeout = on_timeout - - def sleep(self): - cd = self._cooldown - if cd > 0: - sleep(cd) - - def with_cooldown(self, cooldown: float) -> Self: - return type(self)( - self._func, cooldown, self._timeout, self._on_timeout, - ) - - def with_timeout( - self, - timeout: float | None = None, - on_timeout: OnTimeout | None = None, - ) -> Self: - if timeout is None: - timeout = self._timeout - if on_timeout is None: - on_timeout = self._on_timeout - return type(self)(self._func, self._cooldown, timeout, on_timeout) - - @classmethod - def poll_until( - cls, func: Callable[PS, Any], /, *args: PS.args, **kwargs: PS.kwargs - ) -> Self: - if args or kwargs: - func = partial(func, *args, **kwargs) - return cls(func) - - @classmethod - def poll_while( - cls, func: Callable[PS, Any], /, *args: PS.args, **kwargs: PS.kwargs - ) -> Self: - def negated( - func: Callable[PS, Any], *a: PS.args, **k: PS.kwargs - ) -> bool: - return not func(*a, **k) - - return cls(partial(negated, func, *args, **kwargs)) - - def __enter__(self) -> Self: - def error(msg: str) -> NoReturn: - raise type(self).Timeout(msg) - - def warn(msg: str) -> None: - # Write log before issuing the warning because that may be - # promoted to an exception - diagnostics.log.warning(msg) - warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) - - def no_op(_) -> None: - pass - - timeout = self._timeout - callback = self._func - - handle_timeout: Callable[[str], Any] = { - 'error': error, 'warn': warn, 'ignore': no_op, - }[self._on_timeout] - fmt = '.3g' - timeout_msg_header = f'{type(self).__name__} at {id(self):#x}' - - start = monotonic() - while not callback(): - elapsed = monotonic() - start - if timeout and elapsed >= timeout: - handle_timeout( - f'{timeout_msg_header}: ' - f'timed out ({elapsed:{fmt}} s >= {timeout:{fmt}} s) ' - f'waiting for callback {callback!r} to return true' - ) - break - self.sleep() - return self - - def __exit__(self, *_, **__) -> None: - pass - - class Timeout(RuntimeError): - """ - Raised when a :py:class:`Poller` is timed out when polling. - """ - pass - - class TimeoutWarning(Timeout, UserWarning): - """ - Issued when a :py:class:`Poller` is timed out when polling. - """ - pass diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index f7776531..69c4a4e6 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -242,27 +242,4 @@ process = true # cache's debug logs logging = false -# Polling controls, used for the `pool` patch - -[tool.line_profiler.child_processes.multiprocessing.polling] - -# - `multiprocessing.polling.cooldown` (float): -# Cooldown time (seconds) before successive polls on child processes -# (set to <= 0 to disable cooldowns) -cooldown = 0.03125 # 1/32-nd of a second -# - `multiprocessing.polling.timeout` (float): -# Time (seconds) before the main process disregards the alive-ness of -# child processes/existence of lock files, and unblocks calls to -# `.terminate()` a (most probably errored-out) child anyway (set to -# <= 0 to disable timeouts) -timeout = 0.25 -# - `multiprocessing.polling.on_timeout` -# (Literal['error', 'warn', 'ignore']): -# What to do when the above timeout is exhausted, before actually -# `.terminate()`-ing the child process: -# - 'error': raise an error -# - `warn`: issue a warning -# - `ignore`: nothing -on_timeout = 'warn' - # XXX: --- End of implementation details --- diff --git a/setup.py b/setup.py index 05e3fdc2..ae2f7a1d 100755 --- a/setup.py +++ b/setup.py @@ -304,10 +304,10 @@ def run_cythonize(force=False): 'kernprof=kernprof:main', ], 'line_profiler._multiproc_patches': [ - '__process_termination' + '__process_setup' '=line_profiler._child_process_profiling' '.multiprocessing_patches._mandatory_patches' - ':PROCESS_TERMINATION_PATCH', + ':PROCESS_SETUP_PATCH', '__reboot_forkserver' '=line_profiler._child_process_profiling' '.multiprocessing_patches._mandatory_patches' From c818cc3b50e24b8f2ebec5e450c78ccbc6b41346 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 3 Jul 2026 00:53:20 +0200 Subject: [PATCH 148/165] Fix `@add_timeout` doctest which leaked a thread tests/test_child_procs/_test_child_procs_utils.py::@add_timeout - Added optional `name` parameter so that the background thread can be retrieved for debug purposes - Fixed doctest so that it doesn't create a long-lived background thread which pollutes the thread pool and causes subsequent tests using `os.fork()` to issue the forking-multithreaded-process `DeprecationWarning` --- .../_test_child_procs_utils.py | 49 ++++++++++++++++--- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/tests/test_child_procs/_test_child_procs_utils.py b/tests/test_child_procs/_test_child_procs_utils.py index c4398950..1a0a3793 100644 --- a/tests/test_child_procs/_test_child_procs_utils.py +++ b/tests/test_child_procs/_test_child_procs_utils.py @@ -1024,14 +1024,18 @@ def active_instances(self) -> set[LineProfiler]: @overload def add_timeout( - func: Callable[PS, T], *, timeout: float = DEFAULT_TIMEOUT, + func: Callable[PS, T], *, + timeout: float = DEFAULT_TIMEOUT, + name: str | None = None, ) -> Callable[PS, T]: ... @overload def add_timeout( - func: None = None, *, timeout: float = DEFAULT_TIMEOUT, + func: None = None, *, + timeout: float = DEFAULT_TIMEOUT, + name: str | None = None, ) -> Callable[[Callable[PS, T]], Callable[PS, T]]: ... @@ -1039,19 +1043,36 @@ def add_timeout( def add_timeout( func: Callable[PS, T] | None = None, *, timeout: float = DEFAULT_TIMEOUT, + name: str | None = None, ) -> Callable[PS, T] | Callable[[Callable[PS, T]], Callable[PS, T]]: """ Decorate the test function so that it is run in another thread and - can be timed out. + can be timed out; if a ``name`` is provided, it will be set to the + thread. Example: >>> from time import sleep + >>> from threading import enumerate as enum_threads + >>> from uuid import uuid4 - >>> @add_timeout(timeout=.5) + >>> thread_name = f'my_func-{uuid4()}' + >>> sleep_loop_interval = .0625 + >>> test_is_over = False + + >>> @add_timeout(timeout=.5, name=thread_name) ... def my_func( ... n: int, delay: float = 1, error: bool = False, ... ) -> list[int]: - ... sleep(delay) + ... # Sleep in pulses here so the doctest (+ cleanup) + ... # doesn't take 10s if we have `delay=10` + ... nloops, remainder = divmod(delay, sleep_loop_interval) + ... intervals = [sleep_loop_interval] * int(nloops) + ... if remainder: + ... intervals.append(remainder) + ... for interval in intervals: + ... if test_is_over: + ... break + ... sleep(interval) ... if error: ... raise RuntimeError('my error message') ... return list(range(n)) @@ -1075,11 +1096,23 @@ def add_timeout( ... test_child_procs._test_child_procs_utils.TestTimeout: my_func(4, delay=5): timed out after 0.5 s + + Note: + When a call is timed out, the background (daemonized) thread + handling it may still be running; it is the caller's + responsibility to locate said thread and SOMEHOW terminate it. + The ``name`` parameter is to facilitate that. + + >>> test_is_over = True # Tell `my_func()` to stop sleeping... + >>> my_thread = next( + ... t for t in enum_threads() if t.name == thread_name + ... ) + >>> my_thread.join() # ... or it may pollute the thread pool """ if func is None: return cast( Callable[[Callable[PS, T]], Callable[PS, T]], - partial(add_timeout, timeout=timeout), + partial(add_timeout, timeout=timeout, name=name), ) @wraps(func) @@ -1128,7 +1161,9 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: msg = f'{call_repr}: timed out after {timeout:.2g} s' raise TestTimeout(msg) - new_thread = partial(threading.Thread, target=worker, daemon=True) + new_thread = partial( + threading.Thread, target=worker, name=name, daemon=True, + ) return wrapper From e2dd75109632caaf8e3e1c690e4ab17ed8af4581 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 6 Jul 2026 21:56:09 +0200 Subject: [PATCH 149/165] Fix doctest for `Worker` wrapper around MP procs (See `fable-review-pr431-2026-07-05.md::D.11` in TTsangSC#5) tests/test_child_procs/multiproc_examples/process_test_module.py::Worker __doc__ Removed `# xdoctest: +SKIP` directive .__enter__(), .__exit__() Now temporarily patching the module's parent directory into `sys.path` to ensure that `Worker` and its members can be pickled by `multiprocessing`; see `xdoctest` issue #207 --- .../multiproc_examples/process_test_module.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/test_child_procs/multiproc_examples/process_test_module.py b/tests/test_child_procs/multiproc_examples/process_test_module.py index 95e8b05f..6416c179 100644 --- a/tests/test_child_procs/multiproc_examples/process_test_module.py +++ b/tests/test_child_procs/multiproc_examples/process_test_module.py @@ -4,6 +4,7 @@ import os import pickle import shutil +import sys from argparse import ArgumentParser from collections.abc import Callable, Mapping, Sequence from contextlib import ExitStack @@ -73,8 +74,6 @@ def __repr__(self) -> str: class Worker(Generic[T]): """ Example: - >>> # xdoctest: +SKIP - >>> import os >>> from time import sleep @@ -96,7 +95,9 @@ class Worker(Generic[T]): tmpdir: Path result_callback: Callable[[], T] _result: T = field(init=False) + _should_interpolate: bool = field(default=False, init=False) _counter: ClassVar[count] = count() + _module_path: ClassVar[str] = str(Path(__file__).parent) def get_result(self, timeout: float | None = None) -> T: try: @@ -127,10 +128,24 @@ def __repr__(self) -> str: return '<{} @ {:#x} ({})>'.format(type(self).__name__, id(self), attrs) def __enter__(self) -> Self: + # Ensure that this module is import-able; see + # Erotemic/xdoctest#207 + path = self._module_path + self._should_interpolate = interpolate = not any( + os.path.samefile(path, p) for p in sys.path if os.path.exists(p) + ) + if interpolate: + sys.path.insert(0, path) self.process.start() return self def __exit__(self, *_, **__) -> None: + if self._should_interpolate: + self._should_interpolate = False + try: + sys.path.remove(self._module_path) + except ValueError: + pass try: self._end_process('terminate') finally: From 14e9ee7251b64672bb5719bab52c7d1f687655a2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 6 Jul 2026 22:41:57 +0200 Subject: [PATCH 150/165] Lint and misc. fixes (See `fable-review-pr431-2026-07-05.md::P9`, `P13`, `D.2` in TTsangSC#5) Fixed typoes in docs and comments _line_profiler_hooks.py::load_pth_hook() Moved load-once check to before the imports (overhead of re-importing already-imported modules should be minimal, but anyway) line_profiler/line_profiler_utils.py::make_tempfile() Now erroring out and removing the tempfile if we can't close the handle created by `tempfile.mkstemp()` line_profiler/_child_process_profiling/multiprocessing_patches /__init__.py::apply() Removed outdated references to Windows-specific behaviors tests/test_child_procs/test_child_procs.py ::_test_apply_mp_patches_inner() Removed outdated config for polling in `line_profiler._child_process_profiling.multiprocessing_patches` --- _line_profiler_hooks.py | 22 +++++++++--------- kernprof.py | 2 +- .../_child_process_profiling/cache.py | 8 +++---- .../multiprocessing_patches/__init__.py | 23 ++++++++----------- .../_mandatory_patches.py | 4 ++-- line_profiler/cleanup.py | 5 ++-- line_profiler/curated_profiling.py | 2 +- line_profiler/line_profiler_utils.py | 13 ++++++++--- .../_test_child_procs_utils.py | 12 +++++----- tests/test_child_procs/test_child_procs.py | 10 ++------ 10 files changed, 48 insertions(+), 53 deletions(-) diff --git a/_line_profiler_hooks.py b/_line_profiler_hooks.py index 5b5c888a..cb52e9e1 100644 --- a/_line_profiler_hooks.py +++ b/_line_profiler_hooks.py @@ -2,16 +2,16 @@ Additional hooks installed by :py:mod:`line_profiler`. Notes: - - This file and its content should be considered an implmentation + - This file and its content should be considered an implementation detail of :py:mod:`line_profiler`; currently we just use this to set up shop in a child Python process, and extend profiling to therein. - This current implementation writes temporary .pth files to the site-packages directory, which are executed for all Python - processes referring to the same :path:`lib/`. However, only - processes originating from a parent which set the requisite - environment variables will execute to the profiling code. + processes referring to the same ``lib/``. However, only processes + originating from a parent which set the requisite environment + variables will execute to the profiling code. - Said .pth file always import this module; hence, this file is kept intentionally lean and separate from the main @@ -43,6 +43,12 @@ def load_pth_hook(ppid: int) -> None: return if env_ppid != ppid: return + # Note: .pth files may be double-loaded in a virtual environment + # (see https://stackoverflow.com/questions/58807569), so work around + # that; + # also see similar check in `coverage.control.process_startup()` + if getattr(load_pth_hook, 'called', False): + return # If we're here, we're most probably in a descendent process of a # profiled Python process, so we can be more liberal with the @@ -51,19 +57,13 @@ def load_pth_hook(ppid: int) -> None: from line_profiler._diagnostics import DEBUG, log from line_profiler._child_process_profiling.cache import LineProfilingCache - # Note: .pth files may be double-loaded in a virtual environment - # (see https://stackoverflow.com/questions/58807569), so work around - # that; - # also see similar check in `coverage.control.process_startup()` - if getattr(load_pth_hook, 'called', False): - return try: cache = LineProfilingCache.load() cache._setup_in_child_process(True, 'pth') except Exception as e: # nocover if DEBUG: msg = f'{type(e)}: {e}' - # Write log befor issuing warning, in case the warning is + # Write log before issuing warning, in case the warning is # promoted to an exception log.warning(msg) warnings.warn(msg) diff --git a/kernprof.py b/kernprof.py index 625d9035..8317a994 100755 --- a/kernprof.py +++ b/kernprof.py @@ -789,7 +789,7 @@ def _add_core_parser_arguments(parser): 'Minimum value (and the value implied if the bare option ' f'is given) is 1 s. (Default: {def_out_int})', ) - # Hidden option for dumping the debug logs to a desinated location + # Hidden option for dumping the debug logs to a designated location add_argument(out_opts, '--debug-log', help=SUPPRESS) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index aeaedcdc..33b9ccf1 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -38,8 +38,6 @@ T = TypeVar('T') PS = ParamSpec('PS') -# Note: `typing.AnyStr` deprecated since 3.13 -AnyStr = TypeVar('AnyStr', str, bytes) _THIS_SUBPACKAGE, *_ = (lambda: None).__module__.rpartition('.') INHERITED_CACHE_ENV_VARNAME_PREFIX = ( @@ -146,8 +144,8 @@ def load(cls) -> Self: If a previously :py:meth:`.~.load`-ed instance exists, it is returned instead of a new instance. """ - # `ty` needs some help here, evenif we've marked the class to be - # `@final` + # `ty` needs some help here, even if we've marked the class to + # be `@final` instance = cast(Self | None, cls._loaded_instance) if instance is None: pid = os.environ[INHERITED_PID_ENV_VARNAME] @@ -401,7 +399,7 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: (see :py:meth:`~.inject_env_vars()`) - A ``.pth`` file written so that child processes - automaticaly runs setup code (see + automatically runs setup code (see :py:meth:`.write_pth_hook`) - :py:func:`os.fork` wrapped so that profiling set up in diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py index 4e87c6ba..0a2f66e5 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py @@ -56,19 +56,14 @@ def apply( Patches: ``'pool'``: - On Windows - Patch :py:class:`multiprocessing.pool.Pool`'s - ``._get_tasks()`` and ``._guarded_task_generation()`` - methods so that parallel tasks write profiling output. - Else - Patch :py:func:`multiprocessing.pool.worker` so that - profiling output is written as each child process runs - out of task. + Patch :py:func:`multiprocessing.pool.worker` so that + profiling output is written as each pool-worker child + process completes a task. ``'process'``: - Patch :py:class:`multiprocessing.process.BaseProcess`'s - ``._bootstrap()`` method (and ``.terminate()`` on Windows) - so that child processes write profiling output on exit and - are given enough time for that. + Patch + :py:meth:`multiprocessing.process.BaseProcess._bootstrap` + so that non-pool-worker child processes write profiling + output on exit. ``'logging'``: Patch :py:mod:`multiprocessing.util`'s logging methods (e.g. ``debug()`` and ``info()``) so that their messages are teed @@ -87,12 +82,12 @@ def apply( Note: Rebooting the fork server is necessary because its process - staticly inherits the environment when it is first spun up + statically inherits the environment when it is first spun up (see :py:func:`multiprocessing.forkserver.ensure_running`). Thus, without the reboots: - If in the same Python process we ever start up two separate - profliing sessions managed by different caches, the child + profiling sessions managed by different caches, the child processes forked from the server will fail to inherit the updated environment variables injected by the newer cache instance, leading to the setup code in this subpackage not diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py index 9a56f966..310eea09 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -172,7 +172,7 @@ def wrap_terminate_pool( try: vanilla_impl(cls, taskqueue, inqueue, outqueue, pool, *args, **kwargs) finally: - # Guard against dummy ppol; see similar code in + # Guard against dummy pool; see similar code in # `multiprocessing.pool` if pool and hasattr(pool[0], 'terminate'): for worker in pool: @@ -280,7 +280,7 @@ class RebootForkserverPatch: Note: This uses - :py:method:`multiprocessing.forkserver.ForkServer._stop()` which + :py:meth:`multiprocessing.forkserver.ForkServer._stop()` which is private API, but it's the same hack used in Python's own test suite -- see the comment to said method. """ diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index 11d21890..396c7d5a 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -177,7 +177,7 @@ def add_cleanup_with_priority( Callback to be called at cleanup priority (float): Numeric priority value; callbacks with a HIGHER value - are invoked BEFORE those with bigger values + are invoked BEFORE those with lower values *args, **kwargs: Arguments ``callback`` should be called with @@ -404,7 +404,8 @@ def _debug_output(self, msg: str, /) -> None: Write debugging output. Note: - This default implementation just writes to the logger. + This default implementation just writes to the logger at the + ``DEBUG`` level. """ diagnostics.log.debug(msg) diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index e87dec60..8f2fd935 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -109,7 +109,7 @@ def from_targets( ) -> Self: """ Create an instance based on a collection of targets - (like what is supplied to :cmd:`kernprof --prof-mod=...`). + (like what is supplied to ``kernprof --prof-mod=...``). Args: targets (Collection[str]) diff --git a/line_profiler/line_profiler_utils.py b/line_profiler/line_profiler_utils.py index eae5d27f..73b83693 100644 --- a/line_profiler/line_profiler_utils.py +++ b/line_profiler/line_profiler_utils.py @@ -266,7 +266,7 @@ def repr_partial(self, ptl: partial, level: int) -> str: def format_call(self, /, *args, **kwargs) -> str: """ - Convenience method for Formating a call a la + Convenience method for Formatting a call a la :py:meth:`inspect.BoundArguments.__str__`. Example: @@ -344,9 +344,16 @@ def make_tempfile(**kwargs) -> Path: Convenience wrapper around :py:func:`tempfile.mkstemp`, discarding and closing the integer handle (which if left unattended causes problems on some platforms). + + Note: + If for whatever reason the handle cannot be closed, the function + errors out and the tempfile is deleted. """ handle, fname = mkstemp(**kwargs) + path = Path(fname) try: - return Path(fname) - finally: os.close(handle) + return path + except Exception: + path.unlink() + raise diff --git a/tests/test_child_procs/_test_child_procs_utils.py b/tests/test_child_procs/_test_child_procs_utils.py index 1a0a3793..04cc6de6 100644 --- a/tests/test_child_procs/_test_child_procs_utils.py +++ b/tests/test_child_procs/_test_child_procs_utils.py @@ -1718,7 +1718,7 @@ def filter_mp_patch_summary(summary: _PatchSummary) -> dict[str, set[str]]: } # Get patches that are dynamically resolved: while these patches are # always applied, some of the patch targets are -# platform-/Pyhon-version-specific and may not always exist +# platform-/Python-version-specific and may not always exist _dynamically_resolved_patch_summaries: Iterable[_PatchSummary] = ( patch.summary for name, patch in _MP_PATCHES.items() # Basic `multiprocessing` patches are always applied @@ -1907,7 +1907,7 @@ def _run_kernprof_main_in_process( **_kwargs ) -> subprocess.CompletedProcess: """ - Emulate running :cmd:`kernprof` in a subprocess with in-process + Emulate running ``kernprof`` in a subprocess with in-process machineries as best as we can, so that we can retrieve more debugging output when things do go south. """ @@ -1946,7 +1946,7 @@ def get_streams( main = kernprof_main with ExitStack() as stack: if check_warnings: - # See similar indictions against warnings in + # See similar indictments against warnings in # `test_child_procs.py::_test_apply_mp_patches()` cw = stack.enter_context(CheckWarnings()) cw.forbid_warnings('.*resource_tracker', module='multiprocessing') @@ -2054,7 +2054,7 @@ def _run_test_module( Returns: process_running_the_test_module (subprocess.CompletedProcess): Process object - profliing_stats (LineStats | None): + profiling_stats (LineStats | None): Line-profiling stats (where available) """ if isinstance(runner, str): @@ -2068,7 +2068,7 @@ def _run_test_module( if profile and not profiled_code_is_tempfile: runner_cli_args.extend(['--prof-mod', str(test_module.path)]) if nhits is not None: - # We need `kernprof` to write the profliing results immediately + # We need `kernprof` to write the profiling results immediately # to preserve data from tempfiles (see note below) runner_cli_args.append('--view') @@ -2166,7 +2166,7 @@ def check_tagged_line_nhits(output: str, tag: str, nhits: int) -> None: Check the output of :py:meth:`LineStats.print` for the number of hits on the line tagged with the comment ``# GREP_MARKER[<...>]``. """ - # The line should be preixed with 5 numbers: + # The line should be prefixed with 5 numbers: # lineno, nhits, time, time-per-hit, % time actual_nhits = 0 for line in output.splitlines(): diff --git a/tests/test_child_procs/test_child_procs.py b/tests/test_child_procs/test_child_procs.py index f5adc193..87036aad 100644 --- a/tests/test_child_procs/test_child_procs.py +++ b/tests/test_child_procs/test_child_procs.py @@ -502,13 +502,7 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' intercept_logs = 'logging' in mp_patches assert {'pool', 'process'} & set(mp_patches) # Needed for profiling - cfg_chunks: list[str] = [ - get_mp_patches_toml_text(mp_patches), - # This is easier to debug than `ResultMismatch` - '[tool.line_profiler.child_processes.multiprocessing.polling]\n' - 'on_timeout = "error"', - ] - config.write_text('\n\n'.join(cfg_chunks)) + config.write_text(get_mp_patches_toml_text(mp_patches)) # Note: no need to test the case for `my_local_sum()` separately, # with `preimports_module=True`, both are just imported and added @@ -983,7 +977,7 @@ def test_profiling_multiproc_script_success( This test function is heavily parametrized. Here is why that is necessary: - - ``run_func`` tests the different :cmd:`kernprof` modes (see + - ``run_func`` tests the different ``kernprof`` modes (see :py:func:`~.test_running_multiproc_script`). - ``test_module`` chooses what kind of parallelism the test From d8dfcd9854274a065b8ae21f8cbe43d2d61483ab Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 6 Jul 2026 23:51:47 +0200 Subject: [PATCH 151/165] Log failing cleanups as warnings (See `fable-review-pr431-2026-07-05.md::P13` in TTsangSC#5) line_profiler/_child_process_profiling/_cache_logging.py TIMESTAMP_PATTERN Added formatting field `level` add_timestamp() Added optional argument `level` CacheLoggingEntry .__doc__ Updated instantiations in doctest .level New field (2nd item) .to_text(), .from_text(), .write() Added handling of the `.level` field .new() Added optional argument `level` line_profiler/_child_process_profiling/cache.py::LineProfilingCache _debug_output() - Added optional argument `level` in keeping with the base class - Moved implementation of `._make_debug_entry` (used nowhere else) inside _setup_in_child_process() Updated patching of `CuratedProfilerContext._debug_output` _make_debug_entry Removed line_profiler/cleanup.py::Cleanup _debug_output() Added optional argument `level` for setting the logging level _cleanup() If a callback fails, it is now logged with `Logger.warning()` instead of `Logger.debug()` --- .../_cache_logging.py | 69 +++++++++++++++---- .../_child_process_profiling/cache.py | 16 ++--- line_profiler/cleanup.py | 34 ++++++--- 3 files changed, 86 insertions(+), 33 deletions(-) diff --git a/line_profiler/_child_process_profiling/_cache_logging.py b/line_profiler/_child_process_profiling/_cache_logging.py index 8190059a..94a5c867 100644 --- a/line_profiler/_child_process_profiling/_cache_logging.py +++ b/line_profiler/_child_process_profiling/_cache_logging.py @@ -7,6 +7,7 @@ import re from collections.abc import Generator from datetime import datetime +from enum import auto from itertools import pairwise from pathlib import Path from string import Formatter as StringParser @@ -15,14 +16,13 @@ from typing_extensions import Self from .. import _diagnostics as diagnostics -from ..line_profiler_utils import block_indent +from ..line_profiler_utils import block_indent, StringEnum __all__ = ('CacheLoggingEntry',) - FILENAME_PATTERN = 'debug_log_{main_pid}_{current_pid}.log' -TIMESTAMP_PATTERN = '[cache-debug-log {timestamp} DEBUG]' +TIMESTAMP_PATTERN = '[cache-debug-log {timestamp} {level}]' HEADER_PATTERN = 'PID {current_pid} ({main_pid}): Cache {obj_id:#x}' TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S' @@ -34,6 +34,14 @@ HEADER_MAIN_INDICATOR = 'main process' +class LogLevel(StringEnum): + DEBUG = auto() + INFO = auto() + WARNING = auto() + ERROR = auto() + CRITICAL = auto() + + def get_logger_header(current_pid: int, main_pid: int, obj_id: int) -> str: """ Returns: @@ -88,7 +96,11 @@ def parse_timestamp(ts: str) -> datetime: return datetime.strptime(ts, parse_format) -def add_timestamp(msg: str, timestamp: datetime | None = None) -> str: +def add_timestamp( + msg: str, + timestamp: datetime | None = None, + level: str | LogLevel = LogLevel.DEBUG, +) -> str: """ Returns: msg_with_timestamp (str): @@ -99,6 +111,7 @@ def add_timestamp(msg: str, timestamp: datetime | None = None) -> str: timestamp = datetime.now() ts_formatted = TIMESTAMP_PATTERN.format( timestamp=format_timestamp(timestamp), + level=str(level).upper(), ) return block_indent(msg, ts_formatted + TIMESTAMP_SPACING) @@ -205,6 +218,7 @@ class CacheLoggingEntry(NamedTuple): >>> >>> entry = CacheLoggingEntry( ... datetime(1900, 1, 1, 0, 0, 0, 0), + ... LogLevel.DEBUG, ... 12345, ... 12345, ... 12345678, @@ -217,13 +231,14 @@ class CacheLoggingEntry(NamedTuple): multiple lines >>> another_entry = CacheLoggingEntry( ... datetime(2000, 12, 31, 12, 34, 56, 789000), + ... LogLevel.INFO, ... 12345, ... 54321, ... 87654321, ... 'FOO BAR BAZ', ... ) >>> print(another_entry.to_text()) - [cache-debug-log 2000-12-31 12:34:56,789 DEBUG] PID 54321 \ + [cache-debug-log 2000-12-31 12:34:56,789 INFO] PID 54321 \ (12345): Cache 0x5397fb1: FOO BAR BAZ >>> log_text = '\\n'.join([ ... e.to_text() for e in [entry, another_entry] @@ -233,13 +248,16 @@ class CacheLoggingEntry(NamedTuple): ... ] """ timestamp: datetime + level: LogLevel main_pid: int current_pid: int cache_id: int msg: str def to_text(self) -> str: - return add_timestamp(self._get_header() + self.msg, self.timestamp) + return add_timestamp( + self._get_header() + self.msg, self.timestamp, self.level, + ) def _get_header(self) -> str: return get_logger_header( @@ -247,16 +265,29 @@ def _get_header(self) -> str: ) + HEADER_SEP def write(self, tee: os.PathLike[str] | str | None = None) -> None: + """ + Write the log message using + :py:mod:`line_profiler._diagnostics.log`. If ``tee`` is a path, + also tee thereto with an appropriate timestamp. + """ log_msg = self._get_header() + self.msg - diagnostics.log.debug(log_msg) + log_func = getattr(diagnostics.log, self.level.lower()) + log_func(log_msg) if tee is None: return with Path(tee).open(mode='a') as fobj: - print(add_timestamp(log_msg, self.timestamp), file=fobj) + full_msg = add_timestamp(log_msg, self.timestamp, self.level) + print(full_msg, file=fobj) @classmethod - def new(cls, main_pid: int, cache_id: int, msg: str) -> Self: - return cls(datetime.now(), main_pid, os.getpid(), cache_id, msg) + def new( + cls, main_pid: int, cache_id: int, msg: str, + level: str | LogLevel = LogLevel.DEBUG, + ) -> Self: + return cls( + datetime.now(), LogLevel(level), main_pid, + os.getpid(), cache_id, msg, + ) @classmethod def from_file(cls, file: os.PathLike[str] | str | TextIO) -> list[Self]: @@ -286,7 +317,7 @@ def gen_timestamps(text: str) -> Generator[re.Match, None, None]: return def gen_message_blocks(text: str) -> Generator[ - tuple[datetime, re.Match, str], None, None + tuple[datetime, LogLevel, re.Match, str], None, None ]: timestamps = list(gen_timestamps(text)) if not timestamps: @@ -295,18 +326,22 @@ def gen_message_blocks(text: str) -> Generator[ # Handle all the entries up till the 2nd-to-last one for this_match, next_match in pairwise(timestamps): ts = parse_timestamp(this_match.group('timestamp')) + level = LogLevel(this_match.group('level')) text_block = text[this_match.start():next_match.start()] - yield (ts, this_match, text_block.rstrip('\n')) + yield (ts, level, this_match, text_block.rstrip('\n')) # Handle the last entry last_match = timestamps[-1] yield ( parse_timestamp(last_match.group('timestamp')), + LogLevel(last_match.group('level')), last_match, text[last_match.start():].rstrip('\n'), ) def get_entries(text: str) -> Generator[Self, None, None]: - for timestamp, ts_match, text_block in gen_message_blocks(text): + for ( + timestamp, level, ts_match, text_block, + ) in gen_message_blocks(text): # Strip the block indent ts_text = ts_match.group(0) assert text_block.startswith(ts_text), ( @@ -326,10 +361,14 @@ def get_entries(text: str) -> Generator[Self, None, None]: cache_id = parse_id(header_match.group('obj_id')) # The rest of the block is the message proper msg = text_block[header_match.end():] - yield cls(timestamp, main_pid, current_pid, cache_id, msg) + yield cls( + timestamp, level, main_pid, current_pid, cache_id, msg, + ) timestamp_pattern = fmt_to_regex( - f'{TIMESTAMP_PATTERN}{TIMESTAMP_SPACING}', timestamp='.+?', + f'{TIMESTAMP_PATTERN}{TIMESTAMP_SPACING}', + timestamp='.+?', + level='({})'.format('|'.join(LogLevel.__members__)), ) timestamp_regex = re.compile('^' + timestamp_pattern, re.MULTILINE) header_regex = re.compile(fmt_to_regex( diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 33b9ccf1..fb3f90ee 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -26,7 +26,7 @@ from _line_profiler_hooks import INHERITED_PID_ENV_VARNAME, load_pth_hook from .. import _diagnostics as diagnostics -from ..cleanup import Cleanup, _CALLBACK_REPR_HELPER +from ..cleanup import Cleanup, LogLevel, _CALLBACK_REPR_HELPER from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats from ..toml_config import ConfigSource @@ -371,13 +371,14 @@ def get_pth_config() -> Mapping[str, Any]: return fpath - def _debug_output(self, msg: str) -> None: + def _debug_output(self, msg: str, /, level: LogLevel = 'debug') -> None: """ Beside writing to the logger, also write to the :py:attr:`~._debug_log`. """ + entry = CacheLoggingEntry.new(self.main_pid, id(self), msg, level) try: - self._make_debug_entry(msg).write(self._debug_log) + entry.write(self._debug_log) except OSError: # Cache dir may have been rm-ed during cleanup pass @@ -446,9 +447,10 @@ def _setup_in_child_process( calling this function, true otherwise """ def wrap_ctx_debug( - ctx: CuratedProfilerContext, msg: str, + ctx: CuratedProfilerContext, msg: str, /, + level: LogLevel = 'debug', ) -> None: - self._debug_output(f' Context {id(ctx):#x}: {msg}') + self._debug_output(f' Context {id(ctx):#x}: {msg}', level) if not context: context = '...' @@ -820,10 +822,6 @@ def _debug_log(self) -> Path | None: ) return Path(self.cache_dir) / fname - @cached_property - def _make_debug_entry(self) -> Callable[[str], CacheLoggingEntry]: - return partial(CacheLoggingEntry.new, self.main_pid, id(self)) - @cached_property def _consistent_with_loaded_instance(self) -> bool: cls = type(self) diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index 396c7d5a..b35024ff 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -10,7 +10,7 @@ from inspect import getattr_static from operator import setitem from pathlib import Path -from typing import Any, TypeVar, cast +from typing import Any, Literal, Protocol, TypeVar, cast, overload from typing_extensions import Concatenate, ParamSpec, Self from .line_profiler_utils import CallbackRepr, make_tempfile @@ -24,12 +24,26 @@ V = TypeVar('V') _Stacks = dict[float, list[Callable[[], Any]]] _StackContexts = list[_Stacks] +LogLevel = Literal['debug', 'info', 'warning', 'error', 'critical'] _CALLBACK_REPR_HELPER = CallbackRepr(maxother=cast(int, float('inf'))) _CALLBACK_REPR = _CALLBACK_REPR_HELPER.repr +class _LoggingCallback(Protocol): + @overload + def __call__(self, msg: str, /) -> Any: + ... + + @overload + def __call__(self, msg: str, /, level: LogLevel) -> Any: + ... + + def __call__(self, *_, **__): + ... + + class Cleanup: """ Object which holds cleanup callbacks. Also provides convenience @@ -124,7 +138,7 @@ def pop_n_levels_of_contexts( @staticmethod def _cleanup( - log: Callable[[str], Any], stacks: _Stacks, reason: str | None, + log: _LoggingCallback, stacks: _Stacks, reason: str | None, ) -> None: ncallbacks_total = sum(len(stack) for stack in stacks.values()) note = f'{ncallbacks_total} callback(s)' @@ -146,14 +160,15 @@ def _cleanup( try: callback() except Exception as e: - state = 'failed' + success, state = False, 'failed' msg = f'{callback_repr}: {type(e).__name__}: {e}' else: - state, msg = 'succeeded', f'{callback_repr}' - log( + success, state, msg = True, 'succeeded', f'{callback_repr}' + msg = ( f'- Cleanup {state} ' - f'({ncallbacks_run}/{ncallbacks_total}): {msg}', + f'({ncallbacks_run}/{ncallbacks_total}): {msg}' ) + log(msg, 'debug' if success else 'warning') log(f'... cleanup completed ({note})') def add_cleanup( @@ -399,15 +414,16 @@ def _get_name(obj: Any, /) -> str: name = f'{obj.__module__}.{name}' return str(name) - def _debug_output(self, msg: str, /) -> None: + def _debug_output(self, msg: str, /, level: LogLevel = 'debug') -> None: """ Write debugging output. Note: This default implementation just writes to the logger at the - ``DEBUG`` level. + specified level. """ - diagnostics.log.debug(msg) + log_func = getattr(diagnostics.log, level) + log_func(msg) @property def _current_context(self) -> _Stacks: From c3b72a9d2517f8c0e62b827fe447d1a87f05470c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 7 Jul 2026 00:46:44 +0200 Subject: [PATCH 152/165] Fix `LineStats.from_files()` warning messages (See `fable-review-pr431-2026-07-05.md::P10` in TTsangSC#5) line_profiler/line_profiler.py::LineStats.from_files() - Added new private params `_note_on_empty` and `_note_on_defective` so that calling methods can bolster the emitted warning messages with relevant notes - Improved formatting for errors without arguments in warnings - Fixed formatting of the empty-file warning messages - Reworded defective-file warning messages line_profiler/_child_process_profiling/cache.py ::LineProfilingCache.gather_stats() Now calling `LineStats.from_files()` with notes for more helpful warning messages --- .../_child_process_profiling/cache.py | 9 ++++++- line_profiler/line_profiler.py | 25 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index fb3f90ee..1bcf7b37 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -226,8 +226,15 @@ def is_empty(path: Path) -> bool: ) if not fnames: return LineStats.get_empty_instance() + caveat = ( + 'note that empty/malformed profiling files may be produced ' + 'when child processes exits uncleanly (e.g. `.terminate()`-ed ' + 'or `.kill()`-ed by the parent)' + ) return LineStats.from_files( - *fnames, on_empty=on_empty, on_defective=on_defective, + *fnames, + on_empty=on_empty, on_defective=on_defective, + _note_on_empty=caveat, _note_on_defective=caveat, ) def _dump_debug_logs(self) -> None: diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 539f8703..f79570bd 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -387,6 +387,8 @@ def from_files( *files: PathLike[str] | str, on_empty: Literal['ignore', 'warn', 'error'] = 'warn', on_defective: Literal['ignore', 'warn', 'error'] = 'error', + _note_on_empty: str | None = None, + _note_on_defective: str | None = None, ) -> Self: """ Utility function to load an instance from the given filenames. @@ -424,18 +426,33 @@ def from_files( except Exception as e: if on_defective == 'error': raise - failures[str(file)] = f'{type(e).__name__}: {e}' + failure = type(e).__name__ + if str(e): + failure = f'{failure}: {e}' + failures[str(file)] = failure problems: Collection[Any] - for problems, description, behavior in [ - (list(empty_files), 'is/are empty and thus skipped:', on_empty), - (failures, 'failed to load and is/are skipped', on_defective), + for problems, description, behavior, note in [ + ( + list(empty_files), + 'is/are empty and thus skipped', + on_empty, + _note_on_empty, + ), + ( + failures, + 'cannot be loaded and thus is/are skipped', + on_defective, + _note_on_defective, + ), ]: if not problems: continue msg = '{} file(s) out of {} {}: {!r}'.format( len(problems), len(all_files), description, problems, ) + if note: + msg = f'{msg}; {note}' if behavior == 'warn': # Log before warning because warnings may be promoted to # errors From f6894f43d3f2a73134cbc838569810338de58e29 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Jul 2026 04:26:51 +0200 Subject: [PATCH 153/165] .pth-file-related fixes (See `fable-review-pr431-2026-07-05.md::P3` and `P12` in TTsangSC#5) line_profiler/_child_process_profiling/_retrieve_pids.py New helper submodule for checking which PIDs are active Caveats: - There is no Python-native method for checking the process tables so the implementations are platform-specific and relies on parsing command output (`ps` on POSIX, `tasklist` on Windows) - Said parsing esp. on Windows may be locale-sensitive - TODO: we haven't even tested whether Windows implemention works yet... line_profiler/_child_process_profiling/cache.py::LineProfilingCache _setup_in_main_process() Now allowing for `.write_pth_hook()` to error out, emitting a warning that profiling cannot continue in children except for forked ones write_pth_hook() - Added param `clean_stale` to optionally run cleanup for .pth files associated with stale main PIDs; during normal execution they should've been cleaned up by the owning process, but if it is e.g. killed this may be necessary to prevent .pth files from piling up - Now including info about the main PID in the filename of the .pth file, along with other normalizations of the affixes; note that this means the filename stem is no longer guaranteed to start with `prefix` and end with `suffix` - Added search order for the default `dir` to write the .pth file to: - Location of `_line_profiler_hooks` if a `site-packages` directory - `sysconfig.get_path('purelib')` (existing behavior) - `site.getusersitepackages()` if `site.ENABLE_USER_SITE` is true The .pth file is written to the first writable location - The .pth file written now fails gracefully in case the function (`_line_profiler_hooks.load_pth_hook()`) to be imported and called cannot be found for whatever reason _cleanup_stale_pth_files(), _find_pth_files() New helper methods for locating and pruning .pth files associated with stale PIDs _get_graceful_oneline_call() New (doctest-ed) helper method for writing a .pth file calling a function, which gracefully/silently fails in case of module/function lookup errors; this prevents .pth files installed to shared locations from tripping up other interpreters, esp. those without `_line_profiler_hooks` installed _enumerate_pth_installation_locations() New helper method for enumerating the possible locations to install a .pth file to line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes.pth_files] Updated docs --- .../_retrieve_pids.py | 88 ++++++ .../_child_process_profiling/cache.py | 294 +++++++++++++++++- line_profiler/rc/line_profiler.toml | 10 +- 3 files changed, 376 insertions(+), 16 deletions(-) create mode 100644 line_profiler/_child_process_profiling/_retrieve_pids.py diff --git a/line_profiler/_child_process_profiling/_retrieve_pids.py b/line_profiler/_child_process_profiling/_retrieve_pids.py new file mode 100644 index 00000000..27c76af2 --- /dev/null +++ b/line_profiler/_child_process_profiling/_retrieve_pids.py @@ -0,0 +1,88 @@ +""" +Retrieve active PIDs; uses platform-specific solutions. +""" +import csv +import subprocess +import sys +from collections.abc import Callable, Collection +from shutil import which +from typing import NoReturn + + +__all__ = ( + 'CAN_RETRIEVE_PIDS', 'processes_are_alive', 'get_active_processes', +) + +CAN_RETRIEVE_PIDS: bool +get_active_processes: Callable[[], set[int]] + + +def _find_executable(name: str) -> str: + maybe_path = which(name) + if maybe_path: + return maybe_path + raise RuntimeError(f'cannot find required executable for `{name}`') + + +def processes_are_alive(pids: Collection[int]) -> dict[int, bool]: + """ + Returns: + procs_alive (dict[int, bool]): + Dictionary mapping each PID to whether it is alive + """ + all_pids = get_active_processes() + return {p: p in all_pids for p in set(pids)} + + +def get_active_processes_posix() -> set[int]: + """ + Get the active PIDs using ``ps`` in POSIX(-like) environments. + """ + # `a`: processes from all users + # `u`: excluded, because we don't want usernames clogging up the 1st + # column + # `x`: include detached processes + cmd = [_find_executable('ps'), 'ax', '-o', 'pid'] + ps = subprocess.run(cmd, capture_output=True, text=True, check=True) + _, *procs = ps.stdout.splitlines() # Drop the header line + return {int(p.strip()) for p in procs} + + +def get_active_processes_windows() -> set[int]: + """ + Get the active PIDs using ``tasklist`` on Windows. + """ + # CSV formatted output + cmd = [_find_executable('tasklist'), '/fo', 'csv'] + tasklist = subprocess.run(cmd, capture_output=True, text=True, check=True) + header_line, *rows = tasklist.stdout.splitlines() + headers = {h.strip() for h in header_line.lower().split(',')} + # XXX: this is *probably* locale-safe given that even JP has PID as + # a header... but who knows + if 'pid' not in headers: + raise RuntimeError( + 'cannot find PID column in `tasklist` output, ' + f'got CSV header: {header_line!r}' + ) + return { + int(row['pid']) for row in csv.DictReader([header_line.lower(), *rows]) + } + + +def get_active_processes_not_supported() -> NoReturn: # nocover + """ + Unsupported platform; calling this results in a + :py:class:`NotImplementedError`. + """ + raise NotImplementedError(f'unsupported platform: `{sys.platform}`') + + +if sys.platform.startswith('win32'): + CAN_RETRIEVE_PIDS = True + get_active_processes = get_active_processes_windows +elif sys.platform.startswith('wasi'): # nocover + CAN_RETRIEVE_PIDS = False + get_active_processes = get_active_processes_not_supported +else: + CAN_RETRIEVE_PIDS = True + get_active_processes = get_active_processes_posix diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 1bcf7b37..0c0261ea 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -7,30 +7,34 @@ import atexit import dataclasses import os +import site +import sys import sysconfig +import warnings try: import _pickle as pickle except ImportError: import pickle # type: ignore[assignment,no-redef] from collections.abc import ( - Collection, Callable, Iterable, Mapping, MutableMapping, + Collection, Callable, Generator, Iterable, Mapping, MutableMapping, ) from functools import partial, cached_property, wraps from importlib import import_module from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent -from types import ModuleType +from types import ModuleType, TracebackType from typing import Any, ClassVar, Literal, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self -from _line_profiler_hooks import INHERITED_PID_ENV_VARNAME, load_pth_hook +import _line_profiler_hooks as _lp_hooks from .. import _diagnostics as diagnostics from ..cleanup import Cleanup, LogLevel, _CALLBACK_REPR_HELPER from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats from ..toml_config import ConfigSource from ._cache_logging import CacheLoggingEntry +from ._retrieve_pids import CAN_RETRIEVE_PIDS, processes_are_alive __all__ = ('LineProfilingCache',) @@ -148,7 +152,7 @@ def load(cls) -> Self: # be `@final` instance = cast(Self | None, cls._loaded_instance) if instance is None: - pid = os.environ[INHERITED_PID_ENV_VARNAME] + pid = os.environ[_lp_hooks.INHERITED_PID_ENV_VARNAME] cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}' cache_dir = os.environ[cache_varname] msg = ( @@ -324,6 +328,7 @@ def write_pth_hook( dir: os.PathLike[str] | str | None = None, # Get rid of the .pth file ASAP so as to be the least disruptive priority: float = 1, + clean_stale: bool = True, **kwargs ) -> Path: """ @@ -333,16 +338,33 @@ def write_pth_hook( Args: prefix, suffix (str | None): Optional filename-stem affixes of the .pth file; default - is to use default values loaded from :py:attr:`.config` + is to use default values loaded from :py:attr:`.config`. dir (os.PathLike[str] | str | None): Optional directory to create the .pth file in; default - is to use ``sysconfig.get_path('purelib')`` + is to look up a list of directories where .pth files can + usually be installed to, depending on the environment + and interpreter state. + clean_stale (bool): + Whether to look at the directory where the .pth file + has been written to and prune stale .pth files. priority, **kwargs: Passed to :py:meth:`.make_tempfile`. Returns: fpath (Path): Path to the written .pth file + + Notes: + - Due to normalizations and attachment of extra data, users + should NOT count on the affixes bracketing the filename + stem of the created file. + + - During normal execution, the .pth file should be cleaned + up, as with all other tempfiles created with + :py:meth:`.make_tempfile` created without + ``delete=False``. However, if the Python control flow is + broken (e.g. process killed), cleanup can fail to occur. + Hence the option to ``clean_stale``. """ def get_pth_config() -> Mapping[str, Any]: # Note: the only keys in it should be `prefix` and `suffix` @@ -362,22 +384,256 @@ def get_pth_config() -> Mapping[str, Any]: prefix = str(get_pth_config()['prefix']) if suffix is None: suffix = str(get_pth_config()['suffix']) + prefix, suffix_template = self._normalize_pth_affixes(prefix, suffix) + suffix = suffix_template.format(self.main_pid) + if dir is None: - dir = sysconfig.get_path('purelib') + dirs: list[Path] + dirs = list(self._enumerate_pth_installation_locations()) + else: + dirs = [Path(dir)] + + failures: dict[Path, str] = {} + tempfile_msg_template = 'Created tempfile {0.name!r} at {0.parent}' + for dir in dirs: + try: + fpath = self.make_tempfile( + prefix=prefix, suffix=suffix, + dir=dir, priority=priority, + _format_debug_msg=tempfile_msg_template.format, + **kwargs + ) + except OSError as e: + failures[dir] = self._format_exception(e) + else: + break + else: + raise RuntimeError( + 'cannot create .pth file in any of the following directories ' + f'({{path: error}}): {failures!r}' + ) - template = 'import {0.__module__}; {0.__module__}.{0.__name__}({1})' - fpath = self.make_tempfile( - prefix=prefix, suffix=suffix + '.pth', dir=dir, priority=priority, - **kwargs, + content = self._get_graceful_oneline_call( + _lp_hooks.__name__, + _lp_hooks.load_pth_hook.__name__, + self.main_pid, ) try: - fpath.write_text(template.format(load_pth_hook, self.main_pid)) + fpath.write_text(content) except Exception: fpath.unlink(missing_ok=True) raise + if clean_stale: + self._cleanup_stale_pth_files( + prefix, suffix_template, fpath.parent, + ) return fpath + def _cleanup_stale_pth_files( + self, prefix: str, suffix_template: str, dir: Path, + ) -> None: + def log(msg: str, level: LogLevel = 'debug') -> None: + msg = f'._cleanup_stale_pth_files(): {msg}' + self._debug_output(msg, level) + + def warn(msg: str) -> None: + log(msg, 'warning') + # 3: code calling `._cleanup_stale_pth_files()` + warnings.warn(msg, stacklevel=3) + + if not CAN_RETRIEVE_PIDS: # nocover + warn( + 'lookup for stale .pth files not currently possible ' + f'on this platform (`{sys.platform}`); cleanup aborted', + ) + return + + try: + pth_files = self._find_pth_files(prefix, suffix_template, dir) + is_alive = processes_are_alive(pth_files) + except Exception as e: # nocover + try: + frame = cast(TracebackType, e.__traceback__).tb_frame + context = f'{frame.f_code.co_filename}:{frame.f_lineno}' + except Exception: # E.g. no traceback + context = '' + xc = self._format_exception(e) + if context: + xc = f'{xc} ({context})' + warn( + f'lookup for stale .pth files failed ({xc}); ' + 'cleanup aborted', + ) + return + + nfiles = sum(len(p) for p in pth_files.values()) + nstale = 0 + npruned = 0 + log(f'found {nfiles} matching .pth file(s)') + for ppid, pths in pth_files.items(): + if is_alive[ppid]: + log( + f'skipping over {len(pths)} .pth file(s) ' + f'associated with main PID {ppid} (do not seem stale)' + ) + continue + for pth in pths: + nstale += 1 + try: + pth.unlink() + except Exception as e: + success = False + status = f'failed ({self._format_exception(e)})' + else: + success, status = True, 'succeeded' + npruned += 1 + report: Callable[[str], Any] = log if success else warn + report(f'cleanup for stale .pth file {pth.name!r} {status}') + log(f'summary: {nfiles} found, {nstale} stale, {npruned} pruned') + + @staticmethod + def _find_pth_files( + prefix: str, suffix_template: str, dir: Path, + ) -> dict[int, set[Path]]: + result: dict[int, set[Path]] = {} + assert suffix_template.endswith('ppid-{}.pth') + glob_pattern = f'{prefix}?*{suffix_template.format("?*")}' + for pth in dir.glob(glob_pattern): + *_, suffix = pth.name.rpartition('ppid-') + assert suffix.endswith('.pth') + ppid = int(suffix[:-len('.pth')]) + result.setdefault(ppid, set()).add(pth) + return result + + @staticmethod + def _normalize_pth_affixes(prefix: str, suffix: str) -> tuple[str, str]: + prefix = prefix.rstrip('-') + '-' + # Escape braces because we'll use the suffix as a formatting + # template + suffix = suffix.strip('-').replace('{', '{{').replace('}', '}}') + suffix_chunks: list[str] = ['ppid', '{}'] + if suffix: + suffix_chunks.insert(0, suffix) + suffix_template = ''.join('-' + chunk for chunk in suffix_chunks) + suffix_template += '.pth' + return prefix, suffix_template + + @staticmethod + def _get_graceful_oneline_call( + module: str, func: str, /, *args, **kwargs, + ) -> str: + r""" + Get the content of a one-line .pth file which imports a + function and calls it with the supplied arguments, and fails + gracefully (e.g. due to failed imports) as far as possible. + + Example: + >>> get_pth = LineProfilingCache._get_graceful_oneline_call + >>> run = lambda stmts: exec(stmts, {}) # Isolate side-fxs + + >>> good_stmts = get_pth( + ... 'builtins', 'print', [1, 2], 'b', None, sep='\n', + ... ) + >>> run(good_stmts) + [1, 2] + b + None + + Check that non-frozen modules are correctly handled: + + >>> pprint_stmts = get_pth( + ... 'pprint', 'pprint', [1, 2, 3], width=5, + ... ) + >>> run(pprint_stmts) + [1, + 2, + 3] + + Note that it satisfies the requirements for .pth files: + + >>> assert good_stmts.startswith('import') + >>> assert len(good_stmts.splitlines()) == 1 + + If the import/module-spec lookup fails or the if the + attribute lookup on the module fails, the code is a no-op: + + >>> from contextlib import ExitStack, redirect_stdout + >>> from io import StringIO + + >>> with ExitStack() as stack: + ... fobj = stack.enter_context(StringIO()) + ... stack.enter_context(redirect_stdout(fobj)) + ... bad_module_stmts = get_pth( + ... 'buuiltins', 'print', 'foo', + ... ) + ... run(bad_module_stmts) # Import fails -> no-op + ... bad_func_stmts = get_pth( + ... 'builtins', 'priint', 'foo', + ... ) + ... run(bad_module_stmts) # Attr lookup fails -> no-op + ... assert not (stdout := fobj.getvalue()), stdout + + (Note: when running with :py:mod:`xdoctest`, leaving the + expected output empty does not by default check AGAINST + output; hence the context contraption.) + """ + assert module and not module.isspace() + assert all(chunk.isidentifier() for chunk in module.split()) + assert func.isidentifier() + + call_args = [repr(a) for a in args] + call_args.extend(f'{k}={v!r}' for k, v in kwargs.items()) + call_args = ['mod', repr(func)] + call_args + + statements = [ + 'import importlib.util as iu', + 'dummy = lambda *_, **__: None', + 'call = lambda obj, attr, /, *a, **k: ' + '(func if callable(func := getattr(obj, attr, None)) else dummy)' + '(*a, **k)', + # Import the module + f'spec = iu.find_spec({module!r})', + 'mod = iu.module_from_spec(spec) if spec else None', + "call(spec.loader, 'exec_module', mod) if mod else None", + # Retrieve and call the function + f'call({", ".join(call_args)}) if mod else None', + ] + return '; '.join(statements) + + @staticmethod + def _enumerate_pth_installation_locations() -> Generator[Path, None, None]: + """ + Enumerate locations where the .pth file can potentially be + installed to; the lookup order is: + + - Directory where :py:mod:`_line_profiler_hooks` is installed to + (if among the output of :py:func:`site.getsitepackages`) + + - ``sysconfig.get_path('purelib')`` + + - The output of :py:func:`site.getusersitepackages` + """ + def filter_dirs( + maybe_dirs: Iterable[os.PathLike[str] | str], + ) -> Generator[Path, None, None]: + for path in maybe_dirs: + if os.path.isdir(path): + yield Path(path) + + try: + path = Path(_lp_hooks.__file__).parent + except Exception: + pass + else: + if any( + path.samefile(p) for p in filter_dirs(site.getsitepackages()) + ): + yield path + yield Path(sysconfig.get_path('purelib')) + if site.ENABLE_USER_SITE: + yield Path(site.getusersitepackages()) + def _debug_output(self, msg: str, /, level: LogLevel = 'debug') -> None: """ Beside writing to the logger, also write to the @@ -422,7 +678,17 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: """ self.dump() self.inject_env_vars() - self.write_pth_hook() + try: + self.write_pth_hook() + except Exception as e: + xc = self._format_exception(e) + msg = ( + 'cannot write a .pth file for setting up profiling ' + f'in child processes ({xc}); profiling data cannot be ' + 'collected for non-`fork()`ed children' + ) + self._debug_output(msg, 'warning') + warnings.warn(msg, stacklevel=2) self._setup_common(wrap_os_fork, {'reboot_forkserver': True}) self._replace_loaded_instance() @@ -812,7 +1078,7 @@ def environ(self) -> dict[str, str]: """ cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{self.main_pid}' return { - INHERITED_PID_ENV_VARNAME: str(self.main_pid), + _lp_hooks.INHERITED_PID_ENV_VARNAME: str(self.main_pid), cache_varname: str(self.cache_dir), } diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 69c4a4e6..e8ea381c 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -217,8 +217,14 @@ percent = 8 # - `pth_files.prefix`, `.suffix` (str): # Affixes to use for the stem of the name of the .pth file created -# Note: may be useful to tweak this in case of issues with .pth file -# precedence. +# Notes: +# - May be useful to tweak this in case of issues with .pth file +# precedence. +# - The PID of the main process is always suffixed on top of this +# suffix; this is so that .pth files can be identified by their +# "owning" processes and stale files can be cleaned up. In general +# end users should NOT count on the affixes appearing as-is in the +# final filename. prefix = '_line_profiler-profiling-hook-' suffix = '' From 841976a4fa35b76e0460cea0bb731a98e522ada3 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 5 Jul 2026 00:20:15 +0000 Subject: [PATCH 154/165] Fix double-counting of pre-fork stats in forked children (PR431-1) Forked children reuse the parent's profiler object (necessary: it holds the already-wrapped functions), so their stats dumps contained all data the parent had accumulated before the fork. The parent dumps that same data itself, and gather_stats() merges by summation, so every line executed before a fork was counted once per forked child: a 100k-loop parent with three fork children reported 400,004 hits instead of 100,001, for hits and times alike. fork is the default start method on Linux <= 3.13, so merged --prof-child-procs numbers were wrong in the most common configuration. (Evidence: dev/planning/ fable-review-pr431-2026-07-05.md, finding P1.) Fix: snapshot the profiler's stats at the top of the os.fork wrapper's child branch and pass them down to _DumpStatsHelper as a subtractive baseline; every dump now writes get_stats() - baseline via the new LineStats.__sub__/__isub__ (the exact inverse of __add__, raising ValueError when the subtrahend is not a prefix of the minuend). The delta is re-derived on every dump since pool workers dump per task. Nested forks fall out naturally: each fork snapshots the state at its own fork point. New tests: tests/test_child_procs/test_fork_stats_attribution.py runs the real CLI in a real subprocess over fork/spawn/forkserver x Process/Pool/ProcessPoolExecutor and asserts exact hit counts, plus a multi-fork variant asserting counts do not scale with child count. (These also add the first ProcessPoolExecutor coverage.) LineStats subtraction is covered by doctests. Co-Authored-By: Claude Fable 5 --- .../_child_process_profiling/cache.py | 54 ++++++- line_profiler/line_profiler.py | 108 ++++++++++++++ .../test_fork_stats_attribution.py | 141 ++++++++++++++++++ 3 files changed, 299 insertions(+), 4 deletions(-) create mode 100644 tests/test_child_procs/test_fork_stats_attribution.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 0c0261ea..eaf8e70d 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -64,12 +64,36 @@ def _import_sibling(submodule: str) -> ModuleType: _private_field = partial(dataclasses.field, init=False, repr=False) +def _dump_stats( + prof: LineProfiler, + outfile: os.PathLike[str] | str, + baseline: LineStats | None = None, +) -> None: + """ + Write ``prof``'s stats to ``outfile``; if a ``baseline`` is given + (e.g. the stats state inherited across an :py:func:`os.fork`), + subtract it first so that only work done in *this* process is + written (the process the baseline was inherited from writes the + baseline's contents itself). + """ + if baseline is None: + prof.dump_stats(outfile) + return + stats = prof.get_stats() - baseline + stats.to_file(outfile) + + class _DumpStatsHelper(Cleanup): def __init__( - self, prof: LineProfiler, outfile: os.PathLike[str] | str, + self, + prof: LineProfiler, + outfile: os.PathLike[str] | str, + baseline: LineStats | None = None, ) -> None: super().__init__() - callback = self._callback = partial(prof.dump_stats, outfile) + callback = self._callback = partial( + _dump_stats, prof, outfile, baseline, + ) self.add_cleanup(callback) def __repr__(self) -> str: @@ -697,6 +721,7 @@ def _setup_in_child_process( wrap_os_fork: bool = False, context: str = '', prof: LineProfiler | None = None, + baseline: LineStats | None = None, ) -> bool: """ Set up shop in a forked/spawned child process so that @@ -713,6 +738,14 @@ def _setup_in_child_process( prof (LineProfiler | None): Optional profiler instance to associate with the cache; if not provided, an instance is created + baseline (LineStats | None): + Stats which ``prof`` already held when this process + came into existence (only meaningful for forked + processes, which inherit the parent's profiler state); + they are subtracted from every stats dump so that the + pre-existing data isn't double-counted when the parent + gathers and merges the *separately-dumped* child and + parent stats Returns: has_set_up (bool): @@ -766,7 +799,9 @@ def wrap_ctx_debug( suffix='.lprof', delete=False, ) - self._stats_dumper = dumper = _DumpStatsHelper(prof, prof_outfile) + self._stats_dumper = dumper = _DumpStatsHelper( + prof, prof_outfile, baseline, + ) self.patch( # If we call `dumper.cleanup()` instead of `dumper` (e.g. # in some `multiprocessing` patches), the subsequent @@ -823,6 +858,15 @@ def wrapper() -> int: return result # If we're here, we are in the fork pid = os.getpid() + # Snapshot the profiler state inherited from the parent + # BEFORE any further code runs in the fork; it is used as a + # subtractive baseline for this process's stats dumps so + # that pre-fork data (which the parent dumps itself) isn't + # double-counted at gathering time + if self.profiler is None: + baseline = None + else: + baseline = self.profiler.get_stats() forked = self.copy() # Ditch inherited cleanups forked._debug_output(f'Forked: {ppid} -> {pid}') if forked._replace_loaded_instance(): @@ -838,7 +882,9 @@ def wrapper() -> int: # Note: we can reuse the profiler instance in the fork, but # it needs to go through setup so that the separate # profiling results are dumped into another output file - forked._setup_in_child_process(False, 'fork', self.profiler) + forked._setup_in_child_process( + False, 'fork', self.profiler, baseline, + ) return result self.patch(os, 'fork', wrapper, name='os') diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index f79570bd..718c10e4 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -337,6 +337,114 @@ def __iadd__(self, other: _StatsLike) -> Self: self.timings, self.unit = self._get_aggregated_timings([self, other]) return self + def __sub__(self, other: _StatsLike) -> Self: + """ + Subtract a "baseline" from this instance; the inverse of + :py:meth:`~.__add__`. Entries which reach zero hits and zero + time are dropped. The result is expressed in ``self``'s + :py:attr:`~.unit`. + + Raises: + ValueError: + If ``other`` contains an entry absent from (or larger + than the corresponding entry in) ``self``; + a valid baseline must be a "prefix" of ``self``. + + Example: + >>> baseline = LineStats( + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): + ... [(11, 2, 1000), (12, 1, 500)]}, + ... 1E-6) + >>> new = LineStats( + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): + ... [(11, 12, 3000), (12, 6, 600)], + ... ('eggs.py', 5, 'baz'): [(5, 2, 500)]}, + ... 1E-6) + >>> new - baseline + LineStats({('spam.py', 10, 'bar'): [(11, 10, 2000), \ +(12, 5, 100)], ('eggs.py', 5, 'baz'): [(5, 2, 500)]}, 1E-06) + >>> assert (new - baseline) + baseline == new + >>> new - new + LineStats({}, 1E-06) + >>> baseline - new + Traceback (most recent call last): + ... + ValueError: ...not a prefix... + """ + timings, unit = self._get_subtracted_timings(self, other) + return type(self)(timings, unit) + + def __isub__(self, other: _StatsLike) -> Self: + """ + In-place version of :py:meth:`~.__sub__`. + + Example: + >>> baseline = LineStats( + ... {('spam.py', 1, 'foo'): [(2, 10, 300)]}, 1E-6) + >>> stats = LineStats( + ... {('spam.py', 1, 'foo'): [(2, 15, 450)]}, 1E-6) + >>> address = id(stats) + >>> stats -= baseline + >>> assert id(stats) == address + >>> stats + LineStats({('spam.py', 1, 'foo'): [(2, 5, 150)]}, 1E-06) + """ + self.timings, self.unit = self._get_subtracted_timings(self, other) + return self + + @staticmethod + def _get_subtracted_timings(minuend, subtrahend): + """ + Compute ``minuend - subtrahend`` timings, expressed in + ``minuend.unit``; see :py:meth:`~.__sub__`. + """ + def prefix_error(reason): + return ValueError( + 'subtrahend is not a prefix of the minuend ' + f'(cannot subtract): {reason}' + ) + + unit = minuend.unit + factor = subtrahend.unit / unit + timings = { + key: {lineno: (nhits, time) for lineno, nhits, time in entries} + for key, entries in minuend.timings.items() + } + for key, entries in subtrahend.timings.items(): + try: + min_entries = timings[key] + except KeyError: + raise prefix_error(f'{key!r} not in the minuend') from None + for lineno, nhits, time in entries: + try: + prev_nhits, prev_time = min_entries[lineno] + except KeyError: + raise prefix_error( + f'line {lineno} of {key!r} not in the minuend' + ) from None + new_nhits = prev_nhits - nhits + new_time = int(round(prev_time - factor * time, 0)) + if new_nhits < 0 or new_time < 0: + raise prefix_error( + f'line {lineno} of {key!r}: ' + f'({prev_nhits}, {prev_time}) - ({nhits}, {time})' + + ('' if factor == 1 else f' * {factor}') + + f' = ({new_nhits}, {new_time})' + ) + if new_nhits or new_time: + min_entries[lineno] = new_nhits, new_time + else: + del min_entries[lineno] + return { + key: [ + (lineno, nhits, time) + for lineno, (nhits, time) in sorted(entries.items()) + ] + for key, entries in timings.items() if entries + }, unit + def print( self, stream: io.TextIOBase | None = None, diff --git a/tests/test_child_procs/test_fork_stats_attribution.py b/tests/test_child_procs/test_fork_stats_attribution.py new file mode 100644 index 00000000..ace55a36 --- /dev/null +++ b/tests/test_child_procs/test_fork_stats_attribution.py @@ -0,0 +1,141 @@ +""" +Regression tests: profiling data must be attributed to the process that +produced it exactly once. + +Forked children inherit the parent profiler's accumulated stats; if they +dump them verbatim, every line the parent executed before the fork is +counted once per forked child when the results are merged (the parent +dumps the same data itself). See +``LineProfilingCache._wrap_os_fork`` and ``_dump_profiler_stats`` for +the subtractive-baseline fix these tests pin down. +""" +import multiprocessing +import subprocess +import sys + +import pytest + +from line_profiler import LineStats + + +LOOP_COUNT = 5000 +NUM_TASKS = 4 +TIMEOUT = 120 + +_API_SNIPPETS = { + 'process': """ + procs = [ctx.Process(target=child_work, args=(i,)) + for i in range(num_children)] + for p in procs: + p.start() + for p in procs: + p.join() +""", + 'pool': """ + with ctx.Pool(2) as pool: + results = pool.map(child_work, range(num_tasks)) + assert results == [i * 2 for i in range(num_tasks)] +""", + 'cfut': """ + import concurrent.futures as cf + + with cf.ProcessPoolExecutor(max_workers=2, mp_context=ctx) as ex: + results = list(ex.map(child_work, range(num_tasks))) + assert results == [i * 2 for i in range(num_tasks)] +""", +} + + +def _write_workload(path, method, api, num_children=1): + source = f""" +import multiprocessing as mp + + +def child_work(x): + doubled = x * 2 + return doubled + + +def main(): + num_tasks = {NUM_TASKS} + num_children = {num_children} + acc = 0 + for i in range({LOOP_COUNT}): + acc += i + ctx = mp.get_context({method!r}) +{_API_SNIPPETS[api]} + +if __name__ == '__main__': + main() +""" + path.write_text(source) + lines = source.splitlines() + return { + 'acc': 1 + lines.index(' acc += i'), + 'child_work': 1 + lines.index(' doubled = x * 2'), + } + + +def _run_kernprof(tmp_path, script): + """ + Run the real CLI in a real subprocess (the .pth/env machinery only + fully engages for a fresh interpreter) and load the merged stats. + """ + outfile = tmp_path / 'out.lprof' + proc = subprocess.run( + [sys.executable, '-m', 'kernprof', + '-l', f'--prof-mod={script}', '--prof-child-procs', + f'--outfile={outfile}', str(script)], + cwd=tmp_path, capture_output=True, text=True, timeout=TIMEOUT, + ) + assert proc.returncode == 0, (proc.stdout, proc.stderr) + return LineStats.from_files(outfile) + + +def _nhits(stats, func_name, lineno): + matches = [ + entry + for (_, _, func), entries in stats.timings.items() + if func == func_name + for entry in entries + if entry[0] == lineno + ] + assert matches, ( + f'no timings recorded for line {lineno} of {func_name}(); ' + f'keys = {sorted(stats.timings)!r}' + ) + return sum(nhits for _, nhits, _ in matches) + + +@pytest.mark.parametrize('api', sorted(_API_SNIPPETS)) +@pytest.mark.parametrize('method', ['fork', 'forkserver', 'spawn']) +def test_exact_stats_across_start_methods(tmp_path, method, api): + """ + The merged profile must show the parent's loop exactly once and the + children's work exactly ``NUM_TASKS`` times, no matter how the + children came into existence. + """ + if method not in multiprocessing.get_all_start_methods(): + pytest.skip(f'start method {method!r} unavailable') + script = tmp_path / 'workload.py' + expected_child_hits = 1 if api == 'process' else NUM_TASKS + linenos = _write_workload(script, method, api) + stats = _run_kernprof(tmp_path, script) + assert _nhits(stats, 'main', linenos['acc']) == LOOP_COUNT + assert ( + _nhits(stats, 'child_work', linenos['child_work']) + == expected_child_hits + ) + + +def test_stats_do_not_scale_with_fork_children(tmp_path): + """ + Pre-fork parent data must not be re-contributed once per child. + """ + if 'fork' not in multiprocessing.get_all_start_methods(): + pytest.skip("start method 'fork' unavailable") + script = tmp_path / 'workload.py' + linenos = _write_workload(script, 'fork', 'process', num_children=3) + stats = _run_kernprof(tmp_path, script) + assert _nhits(stats, 'main', linenos['acc']) == LOOP_COUNT + assert _nhits(stats, 'child_work', linenos['child_work']) == 3 From 2176893b8ae9ba7a73d5d6c52246b99f7bff7f63 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 5 Jul 2026 00:32:27 +0000 Subject: [PATCH 155/165] Make the pool result protocol tolerate unpatched workers (PR431-2) The mandatory pool patch changes the parent<->worker result protocol (workers push their PID with each result), and the patched parent unconditionally 2-way-unpacked every result. A worker that was never patched -- e.g. its interpreter never loaded the profiling .pth hook, or multiprocessing.set_executable() points at a different Python -- pushes vanilla 3-tuples, so the unpack raised ValueError inside the pool's result-handler thread; the thread died and every AsyncResult.get() blocked forever. --prof-child-procs turned a working program into a deadlock (reproduced; evidence: dev/planning/ fable-review-pr431-2026-07-05.md, finding P2), and load_pth_hook swallowing all exceptions made every silent hook failure take this path. Fix: - Tagged protocol: patched workers now push (PID_TAG, pid, obj) triplets, where PID_TAG is a distinctive module constant. - Tolerant parent: _wrap_outqueue_quick_get() strips tagged results and passes anything else through untouched with a once-per-session UserWarning that the worker's profile data will be missing. - Loud hook failure: load_pth_hook() now always warns when child setup fails (previously only under DEBUG) while still letting the child run unprofiled. Tests: unit tests for the tagging/unwrapping, an integration test running a patched parent against genuinely vanilla spawn workers (hard-timeout-guarded: a regression fails in seconds instead of hanging CI), and a fork control case asserting no warning when the workers are patched. Co-Authored-By: Claude Fable 5 --- _line_profiler_hooks.py | 21 ++-- .../_mandatory_patches.py | 55 +++++++-- .../multiprocessing_patches/_queue.py | 19 ++- tests/test_child_procs/test_pool_protocol.py | 112 ++++++++++++++++++ 4 files changed, 187 insertions(+), 20 deletions(-) create mode 100644 tests/test_child_procs/test_pool_protocol.py diff --git a/_line_profiler_hooks.py b/_line_profiler_hooks.py index cb52e9e1..8418e4a4 100644 --- a/_line_profiler_hooks.py +++ b/_line_profiler_hooks.py @@ -54,19 +54,26 @@ def load_pth_hook(ppid: int) -> None: # profiled Python process, so we can be more liberal with the # imports without worrying about overhead import warnings - from line_profiler._diagnostics import DEBUG, log + from line_profiler._diagnostics import log from line_profiler._child_process_profiling.cache import LineProfilingCache try: cache = LineProfilingCache.load() cache._setup_in_child_process(True, 'pth') except Exception as e: # nocover - if DEBUG: - msg = f'{type(e)}: {e}' - # Write log before issuing warning, in case the warning is - # promoted to an exception - log.warning(msg) - warnings.warn(msg) + # A child which has the profiling environment but cannot set + # up profiling is an abnormal condition the user asked to know + # about, so always warn (but still let the process run + # unprofiled) + msg = ( + f'line_profiler child-process profiling setup failed in ' + f'PID {os.getpid()} ({type(e).__name__}: {e}); ' + f'this process runs unprofiled' + ) + # Write log before issuing the warning, in case the warning is + # promoted to an exception + log.warning(msg) + warnings.warn(msg) load_pth_hook.called = True # type: ignore else: cache.patch(load_pth_hook, 'called', True) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py index 310eea09..e0926a46 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -3,6 +3,7 @@ import atexit import os import multiprocessing +import warnings from collections.abc import Callable from functools import partial from multiprocessing.pool import Pool @@ -32,10 +33,11 @@ else: _CAN_USE_RESOURCE_TRACKER = True +from ... import _diagnostics as diagnostics from ..cache import LineProfilingCache from ..runpy_patches import create_runpy_wrapper from ._infrastructure import SingleModulePatch -from ._queue import Queue, PutWrapper +from ._queue import Queue, PutWrapper, PID_TAG __all__ = ( @@ -127,8 +129,8 @@ def wrap_handle_results( ], outqueue: Queue, # Since we patched `outqueue.put()` in the child process, the result - # tuple pushed to the parent has an extra item (the child PID) - get: Callable[[], tuple[int, tuple[Any, ...]] | None], + # pushed to the parent is (normally) a `(PID_TAG, pid, obj)` triplet + get: Callable[[], tuple[Any, ...] | None], *args: PS.args, **kwargs: PS.kwargs ) -> None: @@ -225,22 +227,59 @@ def _get_worker_ntasks(worker: BaseProcess, cache: LineProfilingCache) -> int: return ntasks_finalized.setdefault(key, ntasks) +_UNTAGGED_RESULT_WARNING = ( + 'received a pool-task result without the profiling worker-PID tag; ' + 'the worker process appears not to have been set up for profiling ' + '(e.g. its interpreter never loaded the profiling startup hook), ' + 'so its profiling data will be missing from the output' +) + + def _wrap_outqueue_quick_get( cache: LineProfilingCache, - vanilla_impl: Callable[PS, tuple[int, tuple[Any, ...]] | None], + vanilla_impl: Callable[PS, tuple[Any, ...] | None], *args: PS.args, **kwargs: PS.kwargs ) -> tuple[Any, ...] | None: """ Take and process the PID of the child process completing the task. + + Note: + A worker which was never patched (its interpreter didn't run + the profiling startup hook) pushes vanilla un-tagged results; + those are passed through untouched, with a once-per-session + warning, so that a mixed patched-parent/vanilla-worker setup + degrades to missing profile data instead of killing the pool's + result-handler thread (and thereby deadlocking every + ``AsyncResult.get()``). """ result = vanilla_impl(*args, **kwargs) if result is None: return None - pid, orig_result = result - ntasks = _get_ntasks(cache) - ntasks[pid] = ntasks.get(pid, 0) + 1 - return orig_result + if ( + isinstance(result, tuple) + and len(result) == 3 + and result[0] == PID_TAG + ): + _, pid, orig_result = result + ntasks = _get_ntasks(cache) + ntasks[pid] = ntasks.get(pid, 0) + 1 + return orig_result + _warn_untagged_result_once(cache) + return result + + +def _warn_untagged_result_once(cache: LineProfilingCache) -> None: + key = 'warned_untagged_pool_result' + # No lock: a race just means an extra warning, and this runs on the + # pool's single result-handler thread anyway + if cache._additional_data.get(key): + return + cache._additional_data[key] = True + # Log before warning in case the warning is promoted to an error + diagnostics.log.warning(_UNTAGGED_RESULT_WARNING) + cache._debug_output(_UNTAGGED_RESULT_WARNING) + warnings.warn(_UNTAGGED_RESULT_WARNING) def _get_ntasks(cache: LineProfilingCache) -> dict[int, int]: diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py index f2eac0b8..3e901b21 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py @@ -4,7 +4,16 @@ from typing import Any, Protocol -__all__ = ('Queue', 'PutWrapper') +__all__ = ('Queue', 'PutWrapper', 'PID_TAG') + +#: First element of the result triplets pushed to the parent by patched +#: pool workers (see ``PutWrapper`` with ``push_to_parent=True``); lets +#: the parent-side result handler distinguish tagged results from +#: vanilla ones coming from workers which were never patched (e.g. +#: because their interpreter never loaded the profiling startup hook), +#: so that a mixed setup degrades to missing profile data instead of +#: crashing the pool's result-handler thread. +PID_TAG = '__line_profiler_pool_worker_pid__' class Queue(Protocol): @@ -23,9 +32,9 @@ class PutWrapper: """ Wrap around a queue (the ``outqueue`` argument to :py:func:`multiprocessing.pool.worker`) so that each call to its - ``.put()`` is preceded by calling a ``callback()``; its result is - optionally attached to the tuple pushed back to the parent if - ``push_to_parent`` is true. + ``.put()`` is preceded by calling a ``callback()``; if + ``push_to_parent`` is true, the object pushed to the parent is + replaced with the triplet ``(PID_TAG, callback(), obj)``. """ def __init__( self, @@ -43,7 +52,7 @@ def __getattr__(self, attr: str) -> Any: def put(self, obj: Any) -> None: data = self._callback() if self._push: - obj = data, obj + obj = PID_TAG, data, obj self._queue.put(obj) def get(self) -> Any: diff --git a/tests/test_child_procs/test_pool_protocol.py b/tests/test_child_procs/test_pool_protocol.py new file mode 100644 index 00000000..4d2d860f --- /dev/null +++ b/tests/test_child_procs/test_pool_protocol.py @@ -0,0 +1,112 @@ +""" +Tests for the tagged pool result protocol. + +Patched pool workers push ``(PID_TAG, pid, result)`` triplets so the +patched parent can attribute results to worker PIDs. A worker that was +never patched (e.g. its interpreter never loaded the profiling startup +hook, or it runs via ``multiprocessing.set_executable()`` pointing at a +different Python) pushes vanilla results; the parent must pass those +through with a warning rather than crash its result-handler thread — +the crash presents to the user as ``pool.map()`` hanging forever. +""" +import math +import multiprocessing + +import pytest + +from line_profiler._child_process_profiling import ( + multiprocessing_patches as mp_patches, +) +from line_profiler._child_process_profiling.multiprocessing_patches import ( + _queue, +) +from line_profiler._child_process_profiling.multiprocessing_patches import ( + _mandatory_patches, +) + + +GET_TIMEOUT = 60 + + +def test_put_wrapper_tags_results(): + pushed = [] + + class FakeQueue: + put = staticmethod(pushed.append) + + wrapper = _queue.PutWrapper( + FakeQueue(), lambda: 1234, push_to_parent=True, + ) + wrapper.put(('job', 0, 'result')) + assert pushed == [(_queue.PID_TAG, 1234, ('job', 0, 'result'))] + + +@pytest.mark.parametrize( + 'incoming, expected, is_tagged', + [ + ((_queue.PID_TAG, 1234, ('job', 0, 'ok')), ('job', 0, 'ok'), True), + (('job', 0, 'ok'), ('job', 0, 'ok'), False), # vanilla worker + (None, None, True), # queue sentinel, never warns + ], +) +def test_quick_get_unwrapping(create_cache, incoming, expected, is_tagged): + cache = create_cache(_use_curated_profiler=False) + unwrap = _mandatory_patches._wrap_outqueue_quick_get + if is_tagged: + assert unwrap(cache, lambda: incoming) == expected + else: + with pytest.warns(UserWarning, match='without the profiling'): + assert unwrap(cache, lambda: incoming) == expected + # ... but only once per session + assert unwrap(cache, lambda: incoming) == expected + + +def _run_pool_map(method): + import os + + ctx = multiprocessing.get_context(method) + with ctx.Pool(1) as pool: + result = pool.map_async(math.sqrt, [0, 1, 4, 9]) + # A hard timeout so that a protocol regression fails fast + # instead of hanging the suite (the historical failure mode) + values = result.get(timeout=GET_TIMEOUT) + # Guard against environments where the pool silently degrades: + # the values must really come from another process + worker_pid = pool.apply_async(os.getpid).get(timeout=GET_TIMEOUT) + assert worker_pid != os.getpid() + return values + + +def test_patched_parent_with_vanilla_spawn_worker(create_cache): + """ + The parent is patched, but the spawn children know nothing of the + profiling session (no env vars are injected, no .pth hook exists + for them), so they push vanilla un-tagged results: the map must + still complete, with a warning. + """ + if 'spawn' not in multiprocessing.get_all_start_methods(): + pytest.skip("start method 'spawn' unavailable") + cache = create_cache(_use_curated_profiler=False) + # Make the wrapped methods resolve `LineProfilingCache.load()` to + # this instance without injecting env vars (children stay vanilla) + cache._replace_loaded_instance(force=True) + mp_patches.apply(cache) + with pytest.warns(UserWarning, match='without the profiling'): + assert _run_pool_map('spawn') == [0.0, 1.0, 2.0, 3.0] + + +def test_patched_parent_with_patched_fork_worker(create_cache): + """ + Control case: fork children inherit the parent's patches, results + arrive tagged, and no warning fires. + """ + if 'fork' not in multiprocessing.get_all_start_methods(): + pytest.skip("start method 'fork' unavailable") + cache = create_cache() + cache._replace_loaded_instance(force=True) + mp_patches.apply(cache) + import warnings as warnings_mod + + with warnings_mod.catch_warnings(): + warnings_mod.simplefilter('error') + assert _run_pool_map('fork') == [0.0, 1.0, 2.0, 3.0] From d771ada6f08249a06e3001dc9acc1978dad0c30e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 10 Jul 2026 02:14:12 +0200 Subject: [PATCH 156/165] Refactor the pool-working protocol line_profiler/_child_process_profiling/multiprocessing_patches /_mandatory_patches.py _LOCK_FILE_LOC Removed unused constant wrap_handle_results(), wrap_worker() No longer exposed (now automatically generated from more basic callbacks) _wrap_outqueue_quick_get(), _warn_untagged_result_once() Refactored into `._queue.QuickGetWrapper` POOL_WORKER_PID_PATCH Now using the new `get_mp_pool_patch()` to generate the patching wrappers for `Pool._handle_results()` and `worker()` line_profiler/_child_process_profiling/multiprocessing_patches /_queue.py PID_TAG Removed PutWrapper.__init__() Updated call signature: instead of `push_to_parent`, it now takes an optional `tag: str | None = None`, where the callback's return value is `.put()` along with the provided `tag` where available QuickGetWrapper New helper object generalizing the old `._mandatory_patches._wrap_outqueue_quick_get()` get_mp_pool_patch() New helper function for generating the patch for `multiprocessing.pool`, pairing up the callbacks which resp. returns the extra data in the child process and processes said data in the parent tests/test_child_procs/test_pool_protocol.py Added type annotations test_put_wrapper_tags_results() Refactored because `PID_TAG` no longer exists test_quick_get_wrapper_unwrapping_results() - Refactored from the old `test_quick_get_unwrapping()`, because (1) `_wrap_outqueue_quick_get()` no longer exists, and (2) `QuickGetWrapper` allows for more directly testing the desired behavior - Added missing check against repeated warning - Added check for the data-processing callback test_patched_parent_with_vanilla_spawn_worker() Updated warning check because of the changed warning messgae emitted --- .../_mandatory_patches.py | 124 +---------- .../multiprocessing_patches/_queue.py | 197 ++++++++++++++++-- tests/test_child_procs/test_pool_protocol.py | 125 +++++++---- 3 files changed, 280 insertions(+), 166 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py index e0926a46..5ec3127b 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -3,7 +3,6 @@ import atexit import os import multiprocessing -import warnings from collections.abc import Callable from functools import partial from multiprocessing.pool import Pool @@ -33,22 +32,18 @@ else: _CAN_USE_RESOURCE_TRACKER = True -from ... import _diagnostics as diagnostics from ..cache import LineProfilingCache from ..runpy_patches import create_runpy_wrapper from ._infrastructure import SingleModulePatch -from ._queue import Queue, PutWrapper, PID_TAG +from ._queue import Queue, get_mp_pool_patch __all__ = ( 'POOL_WORKER_PID_PATCH', 'PROCESS_SETUP_PATCH', 'RebootForkserverPatch', 'ResourceTrackerPatch', 'RunpyPatch', - 'wrap_bootstrap', - 'wrap_handle_results', 'wrap_terminate_pool', 'wrap_worker', + 'wrap_bootstrap', 'wrap_terminate_pool', ) -_LOCK_FILE_LOC = '__line_profiler_multiprocessing_process_lock_file__' - T = TypeVar('T') P = TypeVar('P', bound=BaseProcess) PS = ParamSpec('PS') @@ -120,35 +115,6 @@ def wrap_bootstrap( # ---------------------- PID bookkeeping patches ----------------------- -@LineProfilingCache._method_wrapper -def wrap_handle_results( - cache: LineProfilingCache, - vanilla_impl: Callable[ - Concatenate[Queue, Callable[[], tuple[Any, ...] | None], PS], - None - ], - outqueue: Queue, - # Since we patched `outqueue.put()` in the child process, the result - # pushed to the parent is (normally) a `(PID_TAG, pid, obj)` triplet - get: Callable[[], tuple[Any, ...] | None], - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around :py:meth:`.Pool._handle_results` so that it handles the - extra info (PID of child process handling the task) included by - :py:func:`.wrap_worker`. - - Note: - :py:meth:`.Pool._handle_results` is a static method. - """ - # Somehow this doesn't type-check with either `mypy` or `ty` when - # we use a `TypeVar` instead of `Any` with the tuple items... - # (see `ty` issue #3467) - wrapped_get = partial(_wrap_outqueue_quick_get, cache, get) - vanilla_impl(outqueue, wrapped_get, *args, **kwargs) - - @LineProfilingCache._method_wrapper def wrap_terminate_pool( cache: LineProfilingCache, @@ -182,29 +148,6 @@ def wrap_terminate_pool( _get_worker_ntasks(worker, cache) -@LineProfilingCache._method_wrapper # nocover -def wrap_worker( - _, # We don't need the cache instance, but `@_method_wrapper` does - vanilla_impl: Callable[Concatenate[Queue, Queue, PS], None], - inqueue: Queue, - outqueue: Queue, - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around :py:func:`multiprocessing.pool.worker` so that child - processes report their PIDs as they pass the task results back to - the parent. - - Note: - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. - """ - outqueue = PutWrapper(outqueue, os.getpid, push_to_parent=True) - return vanilla_impl(inqueue, outqueue, *args, **kwargs) - - def _get_worker_ntasks(worker: BaseProcess, cache: LineProfilingCache) -> int: """ Check if the process has run any tasks; if not, report to the cache. @@ -227,59 +170,12 @@ def _get_worker_ntasks(worker: BaseProcess, cache: LineProfilingCache) -> int: return ntasks_finalized.setdefault(key, ntasks) -_UNTAGGED_RESULT_WARNING = ( - 'received a pool-task result without the profiling worker-PID tag; ' - 'the worker process appears not to have been set up for profiling ' - '(e.g. its interpreter never loaded the profiling startup hook), ' - 'so its profiling data will be missing from the output' -) - - -def _wrap_outqueue_quick_get( - cache: LineProfilingCache, - vanilla_impl: Callable[PS, tuple[Any, ...] | None], - *args: PS.args, - **kwargs: PS.kwargs -) -> tuple[Any, ...] | None: +def _increment_ntasks(cache: LineProfilingCache, pid: int) -> None: """ Take and process the PID of the child process completing the task. - - Note: - A worker which was never patched (its interpreter didn't run - the profiling startup hook) pushes vanilla un-tagged results; - those are passed through untouched, with a once-per-session - warning, so that a mixed patched-parent/vanilla-worker setup - degrades to missing profile data instead of killing the pool's - result-handler thread (and thereby deadlocking every - ``AsyncResult.get()``). """ - result = vanilla_impl(*args, **kwargs) - if result is None: - return None - if ( - isinstance(result, tuple) - and len(result) == 3 - and result[0] == PID_TAG - ): - _, pid, orig_result = result - ntasks = _get_ntasks(cache) - ntasks[pid] = ntasks.get(pid, 0) + 1 - return orig_result - _warn_untagged_result_once(cache) - return result - - -def _warn_untagged_result_once(cache: LineProfilingCache) -> None: - key = 'warned_untagged_pool_result' - # No lock: a race just means an extra warning, and this runs on the - # pool's single result-handler thread anyway - if cache._additional_data.get(key): - return - cache._additional_data[key] = True - # Log before warning in case the warning is promoted to an error - diagnostics.log.warning(_UNTAGGED_RESULT_WARNING) - cache._debug_output(_UNTAGGED_RESULT_WARNING) - warnings.warn(_UNTAGGED_RESULT_WARNING) + ntasks = _get_ntasks(cache) + ntasks[pid] = ntasks.get(pid, 0) + 1 def _get_ntasks(cache: LineProfilingCache) -> dict[int, int]: @@ -296,11 +192,11 @@ def _get_ntasks_finalized( ) -POOL_WORKER_PID_PATCH = ( - SingleModulePatch('pool') - .add_method('', 'worker', wrap_worker) - .add_method('Pool', '_handle_results', wrap_handle_results, 'static') - .add_method('Pool', '_terminate_pool', wrap_terminate_pool, 'class') +POOL_WORKER_PID_PATCH = get_mp_pool_patch( + os.getpid, _increment_ntasks, '__line_profiler_pool_worker_pid__', +) +POOL_WORKER_PID_PATCH.add_method( + 'Pool', '_terminate_pool', wrap_terminate_pool, 'class', ) # --------------------------- Misc. patches ---------------------------- diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py index 3e901b21..b2ccefc8 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py @@ -1,19 +1,28 @@ from __future__ import annotations +import warnings from collections.abc import Callable -from typing import Any, Protocol +from functools import partial +from typing import Any, Generic, Protocol, TypeVar, cast +from typing_extensions import Concatenate, ParamSpec +from ...cleanup import _CALLBACK_REPR +from ..cache import LineProfilingCache +from ._infrastructure import SingleModulePatch -__all__ = ('Queue', 'PutWrapper', 'PID_TAG') -#: First element of the result triplets pushed to the parent by patched -#: pool workers (see ``PutWrapper`` with ``push_to_parent=True``); lets -#: the parent-side result handler distinguish tagged results from -#: vanilla ones coming from workers which were never patched (e.g. -#: because their interpreter never loaded the profiling startup hook), -#: so that a mixed setup degrades to missing profile data instead of -#: crashing the pool's result-handler thread. -PID_TAG = '__line_profiler_pool_worker_pid__' +__all__ = ('Queue', 'PutWrapper', 'QuickGetWrapper', 'get_mp_pool_patch') + +T = TypeVar('T') +PS = ParamSpec('PS') + +_POOL_PATCHES: set[str] = set() +_UNTAGGED_RESULT_WARNING_TEMPLATE = ( + 'received a pool-task result `{result}` without the expected tag {tag!r}; ' + 'the worker process appears not to have been set up for profiling ' + '(e.g. its interpreter never loaded the profiling startup hook), ' + 'so its profiling data will be missing from the output' +) class Queue(Protocol): @@ -33,27 +42,183 @@ class PutWrapper: Wrap around a queue (the ``outqueue`` argument to :py:func:`multiprocessing.pool.worker`) so that each call to its ``.put()`` is preceded by calling a ``callback()``; if - ``push_to_parent`` is true, the object pushed to the parent is - replaced with the triplet ``(PID_TAG, callback(), obj)``. + ``tag`` is given, the object pushed to the parent is replaced with + the triplet ``(tag, callback(), obj)``. """ def __init__( self, queue: Queue, callback: Callable[[], Any], - push_to_parent: bool = False, + tag: str | None = None, ) -> None: self._queue = queue self._callback = callback - self._push = push_to_parent + self._tag = tag def __getattr__(self, attr: str) -> Any: return getattr(self._queue, attr) def put(self, obj: Any) -> None: data = self._callback() - if self._push: - obj = PID_TAG, data, obj + if self._tag is not None: + obj = self._tag, data, obj self._queue.put(obj) def get(self) -> Any: return self._queue.get() + + +class QuickGetWrapper(Generic[PS, T]): + """ + Wrap around a :py:attr:`multiprocessing.pool.Pool._quick_get` to + intercept and process data slipped into the queue by a + :py:class:`PutWrapper`. + + - If the result of the get is :py:const:`None` (i.e. a sentinel + value), it is a no-op. + + - If the result is a 3-tuple consisting of the ``tag``, some + ``data``, followed by the original result, the ``data`` is + processed with + ``callback(cache: LineProfilingCache, data: T) -> Any`` and the + original result is returned. + + - Otherwise, a warning (once per ``tag``) is emitted and the result + is returned as-is. + """ + def __init__( + self, + cache: LineProfilingCache, + get: Callable[PS, tuple[Any, ...] | None], + callback: Callable[[LineProfilingCache, T], Any], + tag: str, + ) -> None: + self._impl = get + self._callback = partial(callback, cache) + self._warn = partial(self._warn_untagged_result_once, cache, tag) + self._tag = tag + + def __call__( + self, /, *args: PS.args, **kwargs: PS.kwargs + ) -> tuple[Any, ...] | None: + """ + Note: + A worker which was never patched (its interpreter didn't run + the profiling startup hook) pushes vanilla un-tagged + results; those are passed through untouched, with a + once-per-session warning, so that a mixed + patched-parent/vanilla-worker setup degrades to missing + profile data instead of killing the pool's result-handler + thread (and thereby deadlocking every + :py:meth:`multiprocessing.pool.AsyncResult.get`). + """ + result = self._impl(*args, **kwargs) + if result is None: + return None + if ( + isinstance(result, tuple) + and len(result) == 3 + and result[0] == self._tag + ): + _, data, orig_result = result + self._callback(cast(T, data)) + return orig_result + self._warn(result) + return result + + @staticmethod + def _warn_untagged_result_once( + cache: LineProfilingCache, tag: str, result: Any, + ) -> None: + key = 'mp_result_handler_untagged_result_warnings' + # No lock: a race just means an extra warning, and this runs on + # the pool's single result-handler thread anyway + has_warned = cache._additional_data.setdefault(key, {}) + if has_warned.get(tag): + return + has_warned[tag] = True + # Log before warning in case the warning is promoted to an error + msg = _UNTAGGED_RESULT_WARNING_TEMPLATE.format( + result=_CALLBACK_REPR(result), tag=tag, + ) + cache._debug_output(msg, 'warning') + warnings.warn(msg) + + +def get_mp_pool_patch( + get_data: Callable[[], T], + process_data: Callable[[LineProfilingCache, T], Any], + tag: str, +) -> SingleModulePatch: + """ + Create a patch for :py:mod:`multiprocessing.pool` which: + + - Patches :py:func:`multiprocessing.pool.worker` so that extra data + are created in child/worker processes after running EACH task by + ``get_data()``, and pushed back to parent process alongside said + task's result. + + - Patches :py:meth:`multiprocessing.pool.Pool._handle_results` so + that said extra data is, where possible, retrieved from + interprocess communication and processed by the parent's active + :py:class:`.LineProfilingCache` instance. + """ + if tag in _POOL_PATCHES: + raise RuntimeError(f'tag {tag!r} already in use') + _POOL_PATCHES.add(tag) + + wrap_outqueue = partial(PutWrapper, callback=get_data, tag=tag) + wrap_quick_get = partial(QuickGetWrapper, callback=process_data, tag=tag) + + @LineProfilingCache._method_wrapper + def wrap_handle_results( + cache: LineProfilingCache, + vanilla_impl: Callable[ + Concatenate[Queue, Callable[[], tuple[Any, ...] | None], PS], + None + ], + outqueue: Queue, + # Since we patched `outqueue.put()` in the child process, the + # result pushed to the parent is (normally) a `(tag, data, obj)` + # triplet + get: Callable[[], tuple[Any, ...] | None], + *args: PS.args, + **kwargs: PS.kwargs + ) -> None: + """ + Wrap around :py:meth:`multiprocessing.pool.Pool._handle_results` + so that it handles the extra info (result of calling + ``get_data()`` in a child process after each task) included by + ``wrap_worker()`` with ``process_data(cache, data)``. + + Note: + :py:meth:`multiprocessing.pool.Pool._handle_results` is a + static method. + """ + vanilla_impl(outqueue, wrap_quick_get(cache, get), *args, **kwargs) + + @LineProfilingCache._method_wrapper # nocover + def wrap_worker( + # We don't need the cache instance, but `@_method_wrapper` does + _, + vanilla_impl: Callable[Concatenate[Queue, Queue, PS], None], + inqueue: Queue, + outqueue: Queue, + *args: PS.args, + **kwargs: PS.kwargs + ) -> None: + """ + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes attach the result of ``get_data()`` as they pass the + task results back to the parent. + + Note: + This is only called in child processes and thus we can't + reliably measure coverage thereon, hence the ``# nocover``. + """ + return vanilla_impl(inqueue, wrap_outqueue(outqueue), *args, **kwargs) + + patch = SingleModulePatch('pool') + patch.add_method('', 'worker', wrap_worker) + patch.add_method('Pool', '_handle_results', wrap_handle_results, 'static') + return patch diff --git a/tests/test_child_procs/test_pool_protocol.py b/tests/test_child_procs/test_pool_protocol.py index 4d2d860f..4db4d0e7 100644 --- a/tests/test_child_procs/test_pool_protocol.py +++ b/tests/test_child_procs/test_pool_protocol.py @@ -1,8 +1,8 @@ """ Tests for the tagged pool result protocol. -Patched pool workers push ``(PID_TAG, pid, result)`` triplets so the -patched parent can attribute results to worker PIDs. A worker that was +Patched pool workers push ``(tag, pid, result)`` triplets so th patched +parent can e.g. attribute results to worker PIDs. A worker that was never patched (e.g. its interpreter never loaded the profiling startup hook, or it runs via ``multiprocessing.set_executable()`` pointing at a different Python) pushes vanilla results; the parent must pass those @@ -11,57 +11,108 @@ """ import math import multiprocessing +import warnings +from collections.abc import Callable +from typing import Any, Literal +from typing_extensions import Never import pytest -from line_profiler._child_process_profiling import ( - multiprocessing_patches as mp_patches, -) -from line_profiler._child_process_profiling.multiprocessing_patches import ( - _queue, -) +from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._child_process_profiling.multiprocessing_patches import ( - _mandatory_patches, + _queue, apply as mp_apply, ) +from ._test_child_procs_utils import CheckWarnings + GET_TIMEOUT = 60 +TEST_TAG = '__test_pool_protocol_test_tag__' -def test_put_wrapper_tags_results(): - pushed = [] +def test_put_wrapper_tags_results() -> None: + """ + Check that :py:class:`line_profiler._child_process_profiling.\ +multiprocessing_patches._queue.PutWrapper` + behaves as expected, inserting a tag and the return value of a + callback into the pushed tuple. + """ + pushed: list[Any] = [] class FakeQueue: - put = staticmethod(pushed.append) + @staticmethod + def put(obj: Any) -> None: + pushed.append(obj) + + @staticmethod + def get() -> Never: # Just here to comply with the interface + raise NotImplementedError - wrapper = _queue.PutWrapper( - FakeQueue(), lambda: 1234, push_to_parent=True, - ) + wrapper = _queue.PutWrapper(FakeQueue(), lambda: 1234, tag=TEST_TAG) wrapper.put(('job', 0, 'result')) - assert pushed == [(_queue.PID_TAG, 1234, ('job', 0, 'result'))] + assert pushed == [(TEST_TAG, 1234, ('job', 0, 'result'))] @pytest.mark.parametrize( - 'incoming, expected, is_tagged', + 'incoming, expected_result, expected_data, is_tagged, is_sentinel', [ - ((_queue.PID_TAG, 1234, ('job', 0, 'ok')), ('job', 0, 'ok'), True), - (('job', 0, 'ok'), ('job', 0, 'ok'), False), # vanilla worker - (None, None, True), # queue sentinel, never warns + ( + (TEST_TAG, 1234, ('job', 0, 'ok')), + ('job', 0, 'ok'), 1234, True, False, + ), + # vanilla worker + (('job', 0, 'ok'), ('job', 0, 'ok'), None, False, False), + (None, None, None, True, True), # queue sentinel, never warns ], ) -def test_quick_get_unwrapping(create_cache, incoming, expected, is_tagged): +def test_quick_get_wrapper_unwrapping_results( + create_cache: Callable[..., LineProfilingCache], + incoming: tuple[Any, ...] | None, + expected_result: tuple[Any, ...] | None, + expected_data: Any, + is_tagged: bool, + is_sentinel: bool, +) -> None: + """ + Check that :py:class:`line_profiler._child_process_profiling.\ +multiprocessing_patches._queue.QuickGetWrapper` + behaves as expected, pulling a tag and the extra data from the + pulled tuple (where appropriate) and returning the original result. + """ + def get() -> tuple[Any, ...] | None: + return incoming + + def store(_, data: int) -> None: + stored.append(data) + cache = create_cache(_use_curated_profiler=False) - unwrap = _mandatory_patches._wrap_outqueue_quick_get + stored: list[Any] = [] + quick_get = _queue.QuickGetWrapper(cache, get, store, TEST_TAG) + checked_warning = { + 'category': UserWarning, + 'message': f'.*without the expected tag.*{TEST_TAG}', + } if is_tagged: - assert unwrap(cache, lambda: incoming) == expected + with CheckWarnings() as cw: + cw.forbid_warnings(**checked_warning) + assert quick_get() == expected_result else: - with pytest.warns(UserWarning, match='without the profiling'): - assert unwrap(cache, lambda: incoming) == expected + with CheckWarnings(reissue_warnings=False) as cw: + cw.expect_warnings(**checked_warning) + assert quick_get() == expected_result # ... but only once per session - assert unwrap(cache, lambda: incoming) == expected + with CheckWarnings() as cw: + cw.forbid_warnings(**checked_warning) + assert quick_get() == expected_result + if is_tagged and not is_sentinel: + assert stored == [expected_data] + else: + assert not stored -def _run_pool_map(method): +def _run_pool_map( + method: Literal['spawn', 'fork', 'forkserver'], +) -> list[float]: import os ctx = multiprocessing.get_context(method) @@ -77,7 +128,9 @@ def _run_pool_map(method): return values -def test_patched_parent_with_vanilla_spawn_worker(create_cache): +def test_patched_parent_with_vanilla_spawn_worker( + create_cache: Callable[..., LineProfilingCache], +) -> None: """ The parent is patched, but the spawn children know nothing of the profiling session (no env vars are injected, no .pth hook exists @@ -90,12 +143,14 @@ def test_patched_parent_with_vanilla_spawn_worker(create_cache): # Make the wrapped methods resolve `LineProfilingCache.load()` to # this instance without injecting env vars (children stay vanilla) cache._replace_loaded_instance(force=True) - mp_patches.apply(cache) - with pytest.warns(UserWarning, match='without the profiling'): + mp_apply(cache) + with pytest.warns(UserWarning, match='without the expected'): assert _run_pool_map('spawn') == [0.0, 1.0, 2.0, 3.0] -def test_patched_parent_with_patched_fork_worker(create_cache): +def test_patched_parent_with_patched_fork_worker( + create_cache: Callable[..., LineProfilingCache], +) -> None: """ Control case: fork children inherit the parent's patches, results arrive tagged, and no warning fires. @@ -104,9 +159,7 @@ def test_patched_parent_with_patched_fork_worker(create_cache): pytest.skip("start method 'fork' unavailable") cache = create_cache() cache._replace_loaded_instance(force=True) - mp_patches.apply(cache) - import warnings as warnings_mod - - with warnings_mod.catch_warnings(): - warnings_mod.simplefilter('error') + mp_apply(cache) + with warnings.catch_warnings(): + warnings.simplefilter('error') assert _run_pool_map('fork') == [0.0, 1.0, 2.0, 3.0] From c28cc150b95b754eec844e91e3af128b3418e3cd Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 10 Jul 2026 03:21:43 +0200 Subject: [PATCH 157/165] Ditch the entry-point system for multiproc patches (See `fable-pr431-plan-2026-07-05.md::P8` in TTsangSC#5) line_profiler/_child_process_profiling/multiprocessing_patches /__init__.py _PATCHES Now loaded from `Registry.get_default()` apply() Replaced assertion with `RuntimeError` line_profiler/_child_process_profiling/multiprocessing_patches /_infrastructure.py::Registry DEFAULT_ENTRY_POINT, from_entry_point() Removed (we don't actually expect to load patches from 3rd party packages, so just hard-code the patch locations) get_default() New instantiator refactored from `.from_entry_point()` loading patches from the hard-coded sibling submodules setup.py Removed the now-unused `line_profiler._multiproc_patches` entry point --- .../multiprocessing_patches/__init__.py | 5 +- .../_infrastructure.py | 53 +++++++++++-------- setup.py | 36 ------------- 3 files changed, 35 insertions(+), 59 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py index 0a2f66e5..65c6f321 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py @@ -28,7 +28,7 @@ PublicPatch = Literal['pool', 'process', 'logging'] _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' -_PATCHES = Registry.from_entry_point() +_PATCHES = Registry.get_default() def apply( @@ -112,7 +112,8 @@ def apply( patches_ = {p.lower() for p in patches} # Sanity check on `_PATCHES` for patch in get_args(PublicPatch): - assert patch in _PATCHES + if patch not in _PATCHES: + raise RuntimeError(f'Cannot load patch `{patch}`') for name, patch in _PATCHES.select(patches_).items(): if name == '__reboot_forkserver' and not reboot_forkserver: continue diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py index b2973b05..808ff1b3 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_infrastructure.py @@ -7,7 +7,6 @@ ) from functools import partial from importlib import import_module -from importlib.metadata import entry_points from inspect import getattr_static from operator import attrgetter from types import MappingProxyType as mappingproxy, ModuleType @@ -283,8 +282,7 @@ class Registry(Mapping[str, Patch]): """ Mapping subclass for managing patches. """ - _loaded_instances: ClassVar[dict[str, Registry]] = {} - DEFAULT_ENTRY_POINT: ClassVar[str] = 'line_profiler._multiproc_patches' + _default: ClassVar[Registry] def __init__(self) -> None: self._patches: dict[str, tuple[float, Patch]] = {} @@ -425,21 +423,18 @@ def _prioritized(self) -> dict[float, dict[str, Patch]]: return result @classmethod - def from_entry_point(cls, entry_point: str | None = None) -> Registry: + def get_default(cls) -> Registry: """ - Args: - entry_point (str | None): - Entry point to load patches from; Returns: - Instance representing the patches loaded from the provided - ``entry_point``; default is :py:attr:`.DEFAULT_ENTRY_POINT` + Instance summarizing the patches loaded from the various + ``.*_patches`` sibling modules Note: This method does NOT create a copy. Example: - >>> reg = Registry.from_entry_point() - >>> assert reg.from_entry_point() is reg + >>> reg = Registry.get_default() + >>> assert reg.get_default() is reg Check for the default plugins that should be installed and their contents: @@ -471,26 +466,42 @@ def check(patch: P) -> P: raise TypeError(f'patch `{patch!r}`: {error}') return patch - if entry_point is None: - entry_point = cls.DEFAULT_ENTRY_POINT try: - return cls._loaded_instances[entry_point] - except KeyError: + return cls._default + except AttributeError: pass + instance = Registry() - for ep_obj in entry_points(group=entry_point): + subpkg, *_ = cls.__module__.rpartition('.') + for name, (sibling, patch_loc) in { + '__process_setup': ('_mandatory_patches', 'PROCESS_SETUP_PATCH'), + '__pool_worker_pid': + ('_mandatory_patches', 'POOL_WORKER_PID_PATCH'), + '__reboot_forkserver': + ('_mandatory_patches', 'RebootForkserverPatch'), + '__resource_tracker': + ('_mandatory_patches', 'ResourceTrackerPatch'), + '__spawn_runpy': ('_mandatory_patches', 'RunpyPatch'), + + 'logging': ('_optional_patches', 'LOGGING_PATCH'), + + 'pool': ('_profiling_patches', 'POOL_PATCH'), + 'process': ('_profiling_patches', 'PROCESS_PATCH'), + }.items(): try: - patch = check(cast(Patch, ep_obj.load())) + mod = import_module(f'{subpkg}.{sibling}') + patch = check(cast(Patch, getattr(mod, patch_loc))) except Exception as e: error = type(e).__name__ if str(error): error = f'{error}: {e}' msg = ( - f'failed to load patch {ep_obj.name!r} ' - f'from entry point {ep_obj!r}: {error}' + f'failed to load patch {name!r} ' + f'from sibling submodule `{subpkg}.{sibling}`: {error}' ) diagnostics.log.warning(msg) warnings.warn(msg) else: - instance.register(ep_obj.name, patch) - return cls._loaded_instances.setdefault(entry_point, instance) + instance.register(name, patch) + cls._default = instance + return instance diff --git a/setup.py b/setup.py index ae2f7a1d..a0810cc6 100755 --- a/setup.py +++ b/setup.py @@ -303,42 +303,6 @@ def run_cythonize(force=False): 'console_scripts': [ 'kernprof=kernprof:main', ], - 'line_profiler._multiproc_patches': [ - '__process_setup' - '=line_profiler._child_process_profiling' - '.multiprocessing_patches._mandatory_patches' - ':PROCESS_SETUP_PATCH', - '__reboot_forkserver' - '=line_profiler._child_process_profiling' - '.multiprocessing_patches._mandatory_patches' - ':RebootForkserverPatch', - '__resource_tracker' - '=line_profiler._child_process_profiling' - '.multiprocessing_patches._mandatory_patches' - ':ResourceTrackerPatch', - '__spawn_runpy' - '=line_profiler._child_process_profiling' - '.multiprocessing_patches._mandatory_patches' - ':RunpyPatch', - '__pool_worker_pid' - '=line_profiler._child_process_profiling' - '.multiprocessing_patches._mandatory_patches' - ':POOL_WORKER_PID_PATCH', - - 'logging' - '=line_profiler._child_process_profiling' - '.multiprocessing_patches._optional_patches' - ':LOGGING_PATCH', - - 'pool' - '=line_profiler._child_process_profiling' - '.multiprocessing_patches._profiling_patches' - ':POOL_PATCH', - 'process' - '=line_profiler._child_process_profiling' - '.multiprocessing_patches._profiling_patches' - ':PROCESS_PATCH', - ], } setupkw['name'] = NAME setupkw['version'] = VERSION From c902c29656915458c1b4528540db07f930fe75ee Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 10 Jul 2026 07:29:11 +0200 Subject: [PATCH 158/165] Assertion + end-of-process management fixes (See `fable-review-pr431-2026-07-05.md::P4` in TTsangSC#5) line_profiler/_child_process_profiling/multiprocessing_patches /_mandatory_patches.py wrap_join_exited_workers() New wrapper around `Pool._join_exited_workers()` which mirrors the bookkeeping that `wrap_terminate_pool()` does for workers, except instead it is for that exited early during the pool's lifetime wrap_terminate_pool() Replaced assertion in the `finally` clause with a warning that some worker processes remains alive after calling `Pool.terminate()` RebootForkserverPatch.reboot() Rewrote bare assertion to be more informative (shouldn't happen though) --- .../_mandatory_patches.py | 60 ++++++++++++++++--- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py index 5ec3127b..3c64beee 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -3,7 +3,8 @@ import atexit import os import multiprocessing -from collections.abc import Callable +import warnings +from collections.abc import Callable, Sequence from functools import partial from multiprocessing.pool import Pool from multiprocessing.process import BaseProcess @@ -41,7 +42,7 @@ __all__ = ( 'POOL_WORKER_PID_PATCH', 'PROCESS_SETUP_PATCH', 'RebootForkserverPatch', 'ResourceTrackerPatch', 'RunpyPatch', - 'wrap_bootstrap', 'wrap_terminate_pool', + 'wrap_bootstrap', 'wrap_terminate_pool', 'wrap_join_exited_workers', ) T = TypeVar('T') @@ -140,12 +141,49 @@ def wrap_terminate_pool( try: vanilla_impl(cls, taskqueue, inqueue, outqueue, pool, *args, **kwargs) finally: - # Guard against dummy pool; see similar code in - # `multiprocessing.pool` - if pool and hasattr(pool[0], 'terminate'): + if _is_nonempty_normal_pool(pool): + failures: list[P] = [] for worker in pool: - assert not worker.is_alive() + if worker.is_alive(): # nocover + failures.append(worker) + continue _get_worker_ntasks(worker, cache) + if failures: # nocover + msg = ( + f'{len(failures)} worker(s) still alive after ' + f'`Pool.terminate()`: {failures!r}' + ) + cache._debug_output(msg, 'warning') + warnings.warn(msg) + + +@LineProfilingCache._method_wrapper +def wrap_join_exited_workers( + cache: LineProfilingCache, + vanilla_impl: Callable[[list[P]], bool], + pool: list[P], +) -> bool: + """ + Wrap around :py:meth:`.Pool._join_exited_workers` so that we recover + task info from worker processes which have already exited within the + pool's lifetime. If a worker was idle and hasn't processed any task, + it is reported to the cache. + + Note: + :py:meth:`.Pool._join_exited_workers` is a static method. + """ + if _is_nonempty_normal_pool(pool): + for worker in pool: + if worker.exitcode is not None: + _get_worker_ntasks(worker, cache) + return vanilla_impl(pool) + + +def _is_nonempty_normal_pool(pool: Sequence[BaseProcess]) -> bool: + """ + Guard against dummy pool; see similar code in `multiprocessing.pool` + """ + return bool(pool and hasattr(pool[0], 'terminate')) def _get_worker_ntasks(worker: BaseProcess, cache: LineProfilingCache) -> int: @@ -198,6 +236,9 @@ def _get_ntasks_finalized( POOL_WORKER_PID_PATCH.add_method( 'Pool', '_terminate_pool', wrap_terminate_pool, 'class', ) +POOL_WORKER_PID_PATCH.add_method( + 'Pool', '_join_exited_workers', wrap_join_exited_workers, 'static', +) # --------------------------- Misc. patches ---------------------------- @@ -231,9 +272,12 @@ def apply(cls, cache: LineProfilingCache, **_) -> None: @staticmethod def reboot() -> None: + fs_obj = forkserver._forkserver + stop = getattr(fs_obj, '_stop', None) # Appease the type-checker since `._stop()` is not public API - stop = getattr(forkserver._forkserver, '_stop', None) - assert callable(stop) + if not callable(stop): # nocover + msg = f'ForkServer._stop() (= {stop!r}) not callable' + raise AssertionError(msg) # Shouldn't happen stop() From ac287679784334abe1f745316c0baf535da882c4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 11 Jul 2026 01:21:16 +0200 Subject: [PATCH 159/165] Reduce per-multiproc-task IO overhead (See `fable-pr431-plan-2026-07-05.md::P11` in TTsangSC#5) line_profiler/_child_process_profiling/cache.py _StatsHelper - Refactored from `_DumpStatsHelper` - Added new methods `.get()` and `.dump()`, paralleling `LineProfiler.get_stats()` and `.dump_stats()` LineProfilingCache._stats_helper - Renamed from `._stats_dumper` - Now a `_StatsHelper | None` line_profiler/_child_process_profiling/multiprocessing_patches /__init__.py::apply.__doc__ line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes.multiprocessing.patches] Updated descriptions of the `pool` patch line_profiler/_child_process_profiling/multiprocessing_patches /_mandatory_patches.py wrap_terminate_pool(), wrap_join_exited_workers() Removed (refactored into `.._pool_patch_helpers.get_worker_finalization_patch()`) POOL_WORKER_PID_PATCH - Updated data-processing callback to be compatible with `get_per_task_callback_patch()`, and to emit debug-log messages for the # tasks handled by the worker - Now also constructed with `get_worker_finalization_patch()` line_profiler/_child_process_profiling/multiprocessing_patches /_pool_patch_helpers.py get_per_task_callback_patch() - Refactored from `.._queue.get_mp_pool_patch()` - Updarted call signature of `get_data`: `(LineProfilingCache) -> T` - Added optional param `patch` so that we can stack the `get_*_patch()` functions get_worker_finalization_patch() New helper function Rrefactored from `.._mandatory_patches.wrap_terminate_pool()` and `.wrap_join_exited_workers()`, returning a single `SingleModulePatch` object patching the resp. methods to execute callbacks on finalized worker processes line_profiler/_child_process_profiling/multiprocessing_patches /_profiling_patches.py wrap_worker() Removed POOL_PATCH - Reworked functionality: instead of writing the accumulated stats to disk on each task processed, worker child processes now send the stats back to the parent process, which is then responsible for the writing when the worker objects are "finalized" (joined or terminated); this should reduce the per-task overhead, particularly in terms of IO - Now also constructed with `get_worker_finalization_patch()` line_profiler/_child_process_profiling/multiprocessing_patches /_queue.py::get_mp_pool_patch() Removed (superseded by `.._pool_patch_helpers.get_per_task_callback_patch()`) tests/test_child_procs/test_child_procs.py ::test_cache_setup_child(), _test_apply_mp_patches_inner() Updated debug-log messages to be matched, because the writing of profiling stats is no longer necessarily a result of a `Cleanup.cleanup()` call --- .../_child_process_profiling/cache.py | 73 +++---- .../multiprocessing_patches/__init__.py | 8 +- .../_mandatory_patches.py | 99 ++------- .../_pool_patch_helpers.py | 197 ++++++++++++++++++ .../_profiling_patches.py | 147 +++++++++---- .../multiprocessing_patches/_queue.py | 85 +------- line_profiler/rc/line_profiler.toml | 5 +- tests/test_child_procs/test_child_procs.py | 4 +- 8 files changed, 361 insertions(+), 257 deletions(-) create mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches/_pool_patch_helpers.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index eaf8e70d..e217090f 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -64,26 +64,7 @@ def _import_sibling(submodule: str) -> ModuleType: _private_field = partial(dataclasses.field, init=False, repr=False) -def _dump_stats( - prof: LineProfiler, - outfile: os.PathLike[str] | str, - baseline: LineStats | None = None, -) -> None: - """ - Write ``prof``'s stats to ``outfile``; if a ``baseline`` is given - (e.g. the stats state inherited across an :py:func:`os.fork`), - subtract it first so that only work done in *this* process is - written (the process the baseline was inherited from writes the - baseline's contents itself). - """ - if baseline is None: - prof.dump_stats(outfile) - return - stats = prof.get_stats() - baseline - stats.to_file(outfile) - - -class _DumpStatsHelper(Cleanup): +class _StatsHelper(Cleanup): def __init__( self, prof: LineProfiler, @@ -91,22 +72,37 @@ def __init__( baseline: LineStats | None = None, ) -> None: super().__init__() - callback = self._callback = partial( - _dump_stats, prof, outfile, baseline, - ) - self.add_cleanup(callback) + self._prof = prof + self.outfile = outfile + self._baseline = baseline + self.add_cleanup(self.dump) def __repr__(self) -> str: name = type(self).__name__ - get_repr = _CALLBACK_REPR_HELPER.repr - return f'<{name} @ {hex(id(self))}: {get_repr(self._callback)}>' + func = partial(self._prof.dump_stats, self.outfile) + repr_callback = _CALLBACK_REPR_HELPER.repr(func) + return f'<{name} @ {hex(id(self))}: {repr_callback}>' - def __call__(self) -> None: - self._callback() + def get(self) -> LineStats: + """ + If a ``baseline`` is given (e.g. the stats state inherited + across an :py:func:`os.fork`), subtract it first so that only + work done in *this* process is written; the process the baseline + was inherited from writes the baseline's contents itself. + """ + stats = self._prof.get_stats() + if self._baseline is not None: + stats -= self._baseline + return stats + + def dump(self) -> None: + self.get().to_file(self.outfile) + + __call__ = dump def cleanup(self, *args, force: bool = False, **kwargs) -> None: if force and not any(self._current_context.values()): - self.add_cleanup(self._callback) + self.add_cleanup(self.dump) super().cleanup(*args, **kwargs) @@ -132,7 +128,7 @@ class LineProfilingCache(Cleanup): debug: bool = diagnostics.DEBUG profiler: LineProfiler | None = _private_field(default=None) - _stats_dumper: _DumpStatsHelper | None = _private_field(default=None) + _stats_helper: _StatsHelper | None = _private_field(default=None) # These are unstructured fields; other components can decide on what # to put in them. They are also pickled by `.dump()`, and are thus # retrievable in `.load()`-ed instances. @@ -799,18 +795,15 @@ def wrap_ctx_debug( suffix='.lprof', delete=False, ) - self._stats_dumper = dumper = _DumpStatsHelper( - prof, prof_outfile, baseline, - ) + self._stats_helper = sh = _StatsHelper(prof, prof_outfile, baseline) self.patch( - # If we call `dumper.cleanup()` instead of `dumper` (e.g. - # in some `multiprocessing` patches), the subsequent - # debug-log messages are attributed to and handled by this - # cache instance - dumper, '_debug_output', self._debug_output, - cleanup=False, name='._stats_dumper', + # If we call `sh.cleanup()` instead of `sh` (e.g. in some + # `multiprocessing` patches), the subsequent debug-log msgs + # are attributed to and handled by this cache instance + sh, '_debug_output', self._debug_output, + cleanup=False, name='._stats_helper', ) - self.add_cleanup_with_priority(self._stats_dumper, 1) + self.add_cleanup_with_priority(sh, 1) # Various setups self._setup_common(wrap_os_fork, {'reboot_forkserver': False}) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py index 65c6f321..2fd06fcb 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/__init__.py @@ -56,9 +56,11 @@ def apply( Patches: ``'pool'``: - Patch :py:func:`multiprocessing.pool.worker` so that - profiling output is written as each pool-worker child - process completes a task. + Patch :py:class:`multiprocessing.pool.Pool` and + :py:func:`multiprocessing.pool.worker` so that profiling + output is recorded as each pool-worker child process + completes a task, and written to disk as the pool is + terminated. ``'process'``: Patch :py:meth:`multiprocessing.process.BaseProcess._bootstrap` diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py index 3c64beee..165738f4 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_mandatory_patches.py @@ -3,10 +3,8 @@ import atexit import os import multiprocessing -import warnings -from collections.abc import Callable, Sequence +from collections.abc import Callable from functools import partial -from multiprocessing.pool import Pool from multiprocessing.process import BaseProcess from types import MappingProxyType as mappingproxy, MethodType from typing import Any, ClassVar, TypeVar, cast @@ -36,13 +34,15 @@ from ..cache import LineProfilingCache from ..runpy_patches import create_runpy_wrapper from ._infrastructure import SingleModulePatch -from ._queue import Queue, get_mp_pool_patch +from ._pool_patch_helpers import ( + get_per_task_callback_patch, get_worker_finalization_patch, +) __all__ = ( 'POOL_WORKER_PID_PATCH', 'PROCESS_SETUP_PATCH', 'RebootForkserverPatch', 'ResourceTrackerPatch', 'RunpyPatch', - 'wrap_bootstrap', 'wrap_terminate_pool', 'wrap_join_exited_workers', + 'wrap_bootstrap', ) T = TypeVar('T') @@ -116,77 +116,7 @@ def wrap_bootstrap( # ---------------------- PID bookkeeping patches ----------------------- -@LineProfilingCache._method_wrapper -def wrap_terminate_pool( - cache: LineProfilingCache, - vanilla_impl: Callable[ - Concatenate[type[Pool], Queue, Queue, Queue, list[P], PS], None - ], - cls: type[Pool], - taskqueue: Queue, - inqueue: Queue, - outqueue: Queue, - pool: list[P], - *args: PS.args, - **kwargs: PS.kwargs -) -> None: - """ - Wrap around :py:meth:`.Pool._terminate_pool` so that we recover task - info from the worker processes. If a worker was idle and hasn't - processed any task, it is reported to the cache. - - Note: - :py:meth:`.Pool._terminate_pool` is a class method. - """ - try: - vanilla_impl(cls, taskqueue, inqueue, outqueue, pool, *args, **kwargs) - finally: - if _is_nonempty_normal_pool(pool): - failures: list[P] = [] - for worker in pool: - if worker.is_alive(): # nocover - failures.append(worker) - continue - _get_worker_ntasks(worker, cache) - if failures: # nocover - msg = ( - f'{len(failures)} worker(s) still alive after ' - f'`Pool.terminate()`: {failures!r}' - ) - cache._debug_output(msg, 'warning') - warnings.warn(msg) - - -@LineProfilingCache._method_wrapper -def wrap_join_exited_workers( - cache: LineProfilingCache, - vanilla_impl: Callable[[list[P]], bool], - pool: list[P], -) -> bool: - """ - Wrap around :py:meth:`.Pool._join_exited_workers` so that we recover - task info from worker processes which have already exited within the - pool's lifetime. If a worker was idle and hasn't processed any task, - it is reported to the cache. - - Note: - :py:meth:`.Pool._join_exited_workers` is a static method. - """ - if _is_nonempty_normal_pool(pool): - for worker in pool: - if worker.exitcode is not None: - _get_worker_ntasks(worker, cache) - return vanilla_impl(pool) - - -def _is_nonempty_normal_pool(pool: Sequence[BaseProcess]) -> bool: - """ - Guard against dummy pool; see similar code in `multiprocessing.pool` - """ - return bool(pool and hasattr(pool[0], 'terminate')) - - -def _get_worker_ntasks(worker: BaseProcess, cache: LineProfilingCache) -> int: +def _get_worker_ntasks(cache: LineProfilingCache, worker: BaseProcess) -> int: """ Check if the process has run any tasks; if not, report to the cache. @@ -203,6 +133,8 @@ def _get_worker_ntasks(worker: BaseProcess, cache: LineProfilingCache) -> int: except KeyError: pass ntasks = _get_ntasks(cache).pop(pid, 0) + msg = 'Worker {0.name!r} (PID: {0.pid}) ran {1} task(s)' + cache._debug_output(msg.format(worker, ntasks)) if not ntasks: cache._warn_possible_lack_of_stats(pid) return ntasks_finalized.setdefault(key, ntasks) @@ -230,15 +162,14 @@ def _get_ntasks_finalized( ) -POOL_WORKER_PID_PATCH = get_mp_pool_patch( - os.getpid, _increment_ntasks, '__line_profiler_pool_worker_pid__', -) -POOL_WORKER_PID_PATCH.add_method( - 'Pool', '_terminate_pool', wrap_terminate_pool, 'class', -) -POOL_WORKER_PID_PATCH.add_method( - 'Pool', '_join_exited_workers', wrap_join_exited_workers, 'static', +def _get_pid(_) -> int: + return os.getpid() + + +POOL_WORKER_PID_PATCH = get_per_task_callback_patch( + _get_pid, _increment_ntasks, '__line_profiler_pool_worker_pid__', ) +get_worker_finalization_patch(_get_worker_ntasks, POOL_WORKER_PID_PATCH) # --------------------------- Misc. patches ---------------------------- diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_pool_patch_helpers.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_pool_patch_helpers.py new file mode 100644 index 00000000..f719aab7 --- /dev/null +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_pool_patch_helpers.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import warnings +from collections.abc import Callable +from functools import partial +from multiprocessing.pool import Pool +from multiprocessing.process import BaseProcess +from typing import Any, TypeVar +from typing_extensions import Concatenate, ParamSpec + +from ..cache import LineProfilingCache +from ._infrastructure import SingleModulePatch +from ._queue import Queue, PutWrapper, QuickGetWrapper + + +__all__ = ('get_per_task_callback_patch', 'get_worker_finalization_patch') + +T = TypeVar('T') +P = TypeVar('P', bound=BaseProcess) +PS = ParamSpec('PS') + +_POOL_PATCHES: set[str] = set() + + +def get_per_task_callback_patch( + get_data: Callable[[LineProfilingCache], T], + process_data: Callable[[LineProfilingCache, T], Any], + tag: str, + patch: SingleModulePatch | None = None, +) -> SingleModulePatch: + """ + Create a patch for :py:mod:`multiprocessing.pool` which: + + - Patches :py:func:`multiprocessing.pool.worker` so that extra data + are created in child/worker processes after running EACH task by + ``get_data(cache)``, and pushed back to parent process alongside + said task's result. + + - Patches :py:meth:`multiprocessing.pool.Pool._handle_results` so + that said extra data is, where possible, retrieved from + interprocess communication and processed by the parent's active + :py:class:`.LineProfilingCache` instance. + """ + if tag in _POOL_PATCHES: + raise RuntimeError(f'tag {tag!r} already in use') + _POOL_PATCHES.add(tag) + + wrap_quick_get = partial(QuickGetWrapper, callback=process_data, tag=tag) + + @LineProfilingCache._method_wrapper + def wrap_handle_results( + cache: LineProfilingCache, + vanilla_impl: Callable[ + Concatenate[Queue, Callable[[], tuple[Any, ...] | None], PS], + None + ], + outqueue: Queue, + # Since we patched `outqueue.put()` in the child process, the + # result pushed to the parent is (normally) a `(tag, data, obj)` + # triplet + get: Callable[[], tuple[Any, ...] | None], + *args: PS.args, + **kwargs: PS.kwargs + ) -> None: + """ + Wrap around :py:meth:`multiprocessing.pool.Pool._handle_results` + so that it handles the extra info (result of calling + ``get_data()`` in a child process after each task) included by + ``wrap_worker()`` with ``process_data(cache, data)``. + + Note: + :py:meth:`multiprocessing.pool.Pool._handle_results` is a + static method. + """ + vanilla_impl(outqueue, wrap_quick_get(cache, get), *args, **kwargs) + + @LineProfilingCache._method_wrapper # nocover + def wrap_worker( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[Queue, Queue, PS], None], + inqueue: Queue, + outqueue: Queue, + *args: PS.args, + **kwargs: PS.kwargs + ) -> None: + """ + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes attach the result of ``get_data()`` as they pass the + task results back to the parent. + + Note: + This is only called in child processes and thus we can't + reliably measure coverage thereon, hence the ``# nocover``. + """ + outqueue = PutWrapper(outqueue, partial(get_data, cache), tag) + return vanilla_impl(inqueue, outqueue, *args, **kwargs) + + if patch is None: + patch = SingleModulePatch('pool') + _check_patch(patch) + patch.add_method('', 'worker', wrap_worker) + patch.add_method('Pool', '_handle_results', wrap_handle_results, 'static') + return patch + + +def get_worker_finalization_patch( + process_worker: Callable[[LineProfilingCache, BaseProcess], Any], + patch: SingleModulePatch | None = None, +) -> SingleModulePatch: + """ + Create a patch for :py:mod:`multiprocessing.pool` which patches + :py:meth:`multiprocessing.pool.Pool._terminate_pool` and + :py:meth:`multiprocessing.pool.Pool._join_exited_workers` so that + when worker processes are finalized a callback is run. + """ + @LineProfilingCache._method_wrapper + def wrap_terminate_pool( + cache: LineProfilingCache, + vanilla_impl: Callable[ + Concatenate[type[Pool], Queue, Queue, Queue, list[P], PS], None + ], + cls: type[Pool], + taskqueue: Queue, + inqueue: Queue, + outqueue: Queue, + pool: list[P], + *args: PS.args, + **kwargs: PS.kwargs + ) -> None: + """ + Wrap around :py:meth:`.Pool._terminate_pool` so that we run + ``process_worker()`` on finished worker processes. + + Note: + :py:meth:`.Pool._terminate_pool` is a class method. + """ + try: + vanilla_impl( + cls, taskqueue, inqueue, outqueue, pool, *args, **kwargs, + ) + finally: + # Guard against dummy pool; see similar code in + # `multiprocessing.pool` + if pool and hasattr(pool[0], 'terminate'): + failures: list[P] = [] + for worker in pool: + if worker.is_alive(): # nocover + failures.append(worker) + continue + process_worker(cache, worker) + if failures: # nocover + msg = ( + f'{len(failures)} worker(s) still alive after ' + f'`Pool.terminate()`: {failures!r}' + ) + cache._debug_output(msg, 'warning') + warnings.warn(msg) + + @LineProfilingCache._method_wrapper + def wrap_join_exited_workers( + cache: LineProfilingCache, + vanilla_impl: Callable[[list[P]], bool], + pool: list[P], + ) -> bool: + """ + Wrap around :py:meth:`.Pool._join_exited_workers` so that we run + ``process_worker()`` on finished worker processes. + + Note: + :py:meth:`.Pool._join_exited_workers` is a static method. + """ + before: dict[int, P] = {id(p): p for p in pool} + if not vanilla_impl(pool): # No workers cleaned up + return False + after: dict[int, P] = {id(p): p for p in pool} + for i, worker in before.items(): + if after.get(i) is not worker: # Worker removed from `pool` + process_worker(cache, worker) + return True + + if patch is None: + patch = SingleModulePatch('pool') + _check_patch(patch) + add = partial(patch.add_method, 'Pool') + add('_terminate_pool', wrap_terminate_pool, 'class') + add('_join_exited_workers', wrap_join_exited_workers, 'static') + return patch + + +def _check_patch(patch: SingleModulePatch) -> None: + if patch.submodule == 'pool': + return + msg = ( + 'patch = {0!r}, .submodule = {0.submodule!r}: ' + 'expected patch target to be `multiprocessing.pool`' + ) + raise AssertionError(msg.format(patch)) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py index 2e0c379e..6cd8f2fc 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_profiling_patches.py @@ -1,19 +1,22 @@ from __future__ import annotations from collections.abc import Callable -from functools import partial from multiprocessing.process import BaseProcess -from typing import TypeVar +from pathlib import Path +from typing import TypeVar, cast from typing_extensions import Concatenate, ParamSpec +from ...line_profiler import LineStats from ..cache import LineProfilingCache from ._infrastructure import SingleModulePatch -from ._queue import Queue, PutWrapper +from ._pool_patch_helpers import ( + get_per_task_callback_patch, get_worker_finalization_patch, +) __all__ = ( 'POOL_PATCH', 'PROCESS_PATCH', - 'wrap_bootstrap', 'wrap_process', 'wrap_worker', + 'wrap_bootstrap', 'wrap_process', ) T = TypeVar('T') @@ -33,17 +36,17 @@ def dump_stats_quick( We don't really care about cleanup in the child process, so just dump the stats and bail to reduce the chance of end-of-process shenanigans causing a deadlock... - but do use ``._stats_dumper.cleanup()`` instead of + but do use ``._stats_helper.cleanup()`` instead of ``.__call__()`` so that we get debugging output (if ``debug`` is true) """ - stats_dumper = cache._stats_dumper - if stats_dumper is None: + stats_helper = cache._stats_helper + if stats_helper is None: return if cache.debug: - stats_dumper.cleanup(force=True, reason=reason) + stats_helper.cleanup(force=True, reason=reason) else: - stats_dumper() + stats_helper() def _mark_worker(worker: P) -> P: @@ -58,20 +61,31 @@ def _is_marked_worker(proc: BaseProcess) -> bool: # ---------------- `multiprocessing.pool.Pool` patches ----------------- -@LineProfilingCache._method_wrapper # nocover -def wrap_worker( - cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[Queue, Queue, PS], None], - inqueue: Queue, - outqueue: Queue, - *args: PS.args, - **kwargs: PS.kwargs -) -> None: +@LineProfilingCache._method_wrapper +def wrap_process( + _, vanilla_impl: Callable[PS, P], *args: PS.args, **kwargs: PS.kwargs +) -> P: """ - Wrap around :py:func:`multiprocessing.pool.worker` so that child - processes can write profiling output before pushing the result of - each task back to the parent. + Wrap around :py:meth:`.Pool.Process` so that the worker processes + created by the pool are marked and can be distinguished from + processes otherwise managed. + + Notes: + + - :py:meth:`.Pool.Process` is a static method. + - Technically one can inspect the :py:attr:`.BaseProcess.name` + of the process to see that it is a ``PoolWorker``, but since + said attribute is writable it may be more robust to set up a + separate marker. + """ + return _mark_worker(vanilla_impl(*args, **kwargs)) + + +def _report_stats_and_dest( + cache: LineProfilingCache, +) -> tuple[LineStats, Path] | None: # nocover + """ Notes: - This is only called in child processes and thus we can't @@ -95,39 +109,86 @@ def wrap_worker( So this is about as good as we can do. + - Instead of dumping the stats to disk every task, it should be + less overhead for us to just send them back to the parent via + the preexisting connection. + + - Unless the code paths varied significantly between tasks, the + physical size of the stats should not have changed too much – + running the same code more only increment the timing entries, + but do not generate more thereof. Thus, calculating the + delta-stats between tasks and only sending those would have + been a waste of time here; on top of that, the parent would + also have to perform additional processing to accumulate the + deltas. So we just send the entire stats object. + .. _1: https://github.com/python/cpython/issues/73945 .. _2: https://github.com/python/cpython/issues/82408 .. _3: https://github.com/coveragepy/coveragepy/issues/1310 """ - dump = partial(dump_stats_quick, cache, reason='processed task') - outqueue = PutWrapper(outqueue, dump) - return vanilla_impl(inqueue, outqueue, *args, **kwargs) + stats_helper = cache._stats_helper + if stats_helper is None: + return None + return stats_helper.get(), Path(stats_helper.outfile) -@LineProfilingCache._method_wrapper -def wrap_process( - _, vanilla_impl: Callable[PS, P], *args: PS.args, **kwargs: PS.kwargs -) -> P: +def _record_stats( + cache: LineProfilingCache, + stats_and_dest: tuple[LineStats, Path] | None, +) -> None: """ - Wrap around :py:meth:`multiprocessing.pool.Pool.Process` so that the - worker processes created by the pool are marked and can be - distinguished from processes otherwise managed. - - Notes: - - - :py:meth:`multiprocessing.pool.Pool.Process` is a static - method. + Record the stats gathered from workers so that they can be dealt + with when the process pool is terminated. - - Technically one can inspect the :py:attr:`.BaseProcess.name` - of the process to see that it is a ``PoolWorker``, but since - said attribute is writable it may be more robust to set up a - separate marker. + See also: + :py:func:`_report_stats_and_dest` """ - return _mark_worker(vanilla_impl(*args, **kwargs)) + if stats_and_dest is None: + return # No-op + stats, dest = stats_and_dest + _get_worker_stats(cache)[dest] = stats -POOL_PATCH = SingleModulePatch('pool') -POOL_PATCH.add_method('', 'worker', wrap_worker) +def _write_recorded_stats( + cache: LineProfilingCache, worker: BaseProcess, +) -> None: + """ + Write the gathered stats associated with ``worker``. + """ + pid = getattr(worker, 'pid', None) + if pid is None: + return + worker_stats = _get_worker_stats(cache) + xc: Exception | None = None + msg = '{0} (centralized `.dump_stats()`): {1.name!r} (PID: {1.pid}) -> {2}' + for outfile in cache._get_profiling_outfiles(pid): + stats = worker_stats.pop(outfile, None) + if stats is None: + continue + try: + stats.to_file(outfile) + except Exception as e: + xc = e + state, outcome = 'Failed', cache._format_exception(e) + else: + state, outcome = 'Succeeded', repr(outfile.name) + cache._debug_output(msg.format(state, worker, outcome)) + if xc is not None: + raise xc + + +def _get_worker_stats(cache: LineProfilingCache) -> dict[Path, LineStats]: + key = 'mp_pool_worker_stats' + return cache._additional_data.setdefault( + key, cast(dict[Path, LineStats], {}), + ) + + +POOL_PATCH = get_per_task_callback_patch( + _report_stats_and_dest, _record_stats, + '__line_profiler_pool_worker_stats__', +) +get_worker_finalization_patch(_write_recorded_stats, POOL_PATCH) POOL_PATCH.add_method('Pool', 'Process', wrap_process, 'static') # ----------- `multiprocessing.process.BaseProcess` patches ------------ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py b/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py index b2ccefc8..20238ef4 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches/_queue.py @@ -4,19 +4,17 @@ from collections.abc import Callable from functools import partial from typing import Any, Generic, Protocol, TypeVar, cast -from typing_extensions import Concatenate, ParamSpec +from typing_extensions import ParamSpec from ...cleanup import _CALLBACK_REPR from ..cache import LineProfilingCache -from ._infrastructure import SingleModulePatch -__all__ = ('Queue', 'PutWrapper', 'QuickGetWrapper', 'get_mp_pool_patch') +__all__ = ('Queue', 'PutWrapper', 'QuickGetWrapper') T = TypeVar('T') PS = ParamSpec('PS') -_POOL_PATCHES: set[str] = set() _UNTAGGED_RESULT_WARNING_TEMPLATE = ( 'received a pool-task result `{result}` without the expected tag {tag!r}; ' 'the worker process appears not to have been set up for profiling ' @@ -143,82 +141,3 @@ def _warn_untagged_result_once( ) cache._debug_output(msg, 'warning') warnings.warn(msg) - - -def get_mp_pool_patch( - get_data: Callable[[], T], - process_data: Callable[[LineProfilingCache, T], Any], - tag: str, -) -> SingleModulePatch: - """ - Create a patch for :py:mod:`multiprocessing.pool` which: - - - Patches :py:func:`multiprocessing.pool.worker` so that extra data - are created in child/worker processes after running EACH task by - ``get_data()``, and pushed back to parent process alongside said - task's result. - - - Patches :py:meth:`multiprocessing.pool.Pool._handle_results` so - that said extra data is, where possible, retrieved from - interprocess communication and processed by the parent's active - :py:class:`.LineProfilingCache` instance. - """ - if tag in _POOL_PATCHES: - raise RuntimeError(f'tag {tag!r} already in use') - _POOL_PATCHES.add(tag) - - wrap_outqueue = partial(PutWrapper, callback=get_data, tag=tag) - wrap_quick_get = partial(QuickGetWrapper, callback=process_data, tag=tag) - - @LineProfilingCache._method_wrapper - def wrap_handle_results( - cache: LineProfilingCache, - vanilla_impl: Callable[ - Concatenate[Queue, Callable[[], tuple[Any, ...] | None], PS], - None - ], - outqueue: Queue, - # Since we patched `outqueue.put()` in the child process, the - # result pushed to the parent is (normally) a `(tag, data, obj)` - # triplet - get: Callable[[], tuple[Any, ...] | None], - *args: PS.args, - **kwargs: PS.kwargs - ) -> None: - """ - Wrap around :py:meth:`multiprocessing.pool.Pool._handle_results` - so that it handles the extra info (result of calling - ``get_data()`` in a child process after each task) included by - ``wrap_worker()`` with ``process_data(cache, data)``. - - Note: - :py:meth:`multiprocessing.pool.Pool._handle_results` is a - static method. - """ - vanilla_impl(outqueue, wrap_quick_get(cache, get), *args, **kwargs) - - @LineProfilingCache._method_wrapper # nocover - def wrap_worker( - # We don't need the cache instance, but `@_method_wrapper` does - _, - vanilla_impl: Callable[Concatenate[Queue, Queue, PS], None], - inqueue: Queue, - outqueue: Queue, - *args: PS.args, - **kwargs: PS.kwargs - ) -> None: - """ - Wrap around :py:func:`multiprocessing.pool.worker` so that child - processes attach the result of ``get_data()`` as they pass the - task results back to the parent. - - Note: - This is only called in child processes and thus we can't - reliably measure coverage thereon, hence the ``# nocover``. - """ - return vanilla_impl(inqueue, wrap_outqueue(outqueue), *args, **kwargs) - - patch = SingleModulePatch('pool') - patch.add_method('', 'worker', wrap_worker) - patch.add_method('Pool', '_handle_results', wrap_handle_results, 'static') - return patch diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index e8ea381c..7a7e3f23 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -233,8 +233,9 @@ suffix = '' [tool.line_profiler.child_processes.multiprocessing.patches] # - `multiprocessing.patches.pool` (bool): -# Whether to patch `multiprocessing.pool.worker()` so that each child -# process write profiling output as each task is completed +# Whether to patch `multiprocessing.pool.worker()` and `Pool` so that +# child processes report profiling stats to the parent as each task is +# completed pool = true # - `multiprocessing.patches.process` (bool): # Whether to patch `multiprocessing.process.BaseProcess`, so that each diff --git a/tests/test_child_procs/test_child_procs.py b/tests/test_child_procs/test_child_procs.py index 87036aad..988ccedd 100644 --- a/tests/test_child_procs/test_child_procs.py +++ b/tests/test_child_procs/test_child_procs.py @@ -384,7 +384,7 @@ def list_profiled_funcs() -> list[str]: f'Set up .*profiler.* {id(cache.profiler):#x}': True, 'Loading preimports': preimports, 'Created .*' + re.escape(stats_file.name): True, - 'Cleanup succeeded.*: .*dump_stats': True, + '[Ss]ucceeded.*dump_stats': True, 'Loading results .*' + re.escape(stats_file.name): True, } search_cache_logs(cache, debug, patterns) @@ -576,7 +576,7 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: iter_stats = cast( # See `ty` issue #3428 Iterable[Path], filter(is_valid_stats_file, iter_stats), ) - pat = 'Cleanup succeeded.*: .*dump_stats.*{}' + pat = '[Ss]ucceeded.*dump_stats.*{}' patterns.update({ pat.format(re.escape(path.name)): True for path in iter_stats }) From e7152a60c8103da504a454e1e9f781ce6a6f2633 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 11 Jul 2026 02:42:46 +0200 Subject: [PATCH 160/165] `kernprof.py::_manage_profiler` cleanup (See `fable-pr431-plan-2026-07-05.md::P5` and `P6` in TTsangSC#5) kernprof.py::_manage_profiler .__enter__() Fixed bug where if the setups for the executed script and/or the session cache fails, the `CuratedProfilerContext` isn't cleaned up .__exit__() Fixed bug where if the gathering of child-process profiling stats fails, the process-local stats are not written --- kernprof.py | 79 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/kernprof.py b/kernprof.py index 8317a994..c3ad0037 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1199,42 +1199,29 @@ def __enter__(self): self._ctx.install() # Keep the generated pre-imports file to be reused in child # processes - script_file, preimports_file = _prepare_exec_script( - self.options, self.module, self.prof, - exit_on_error=self.exit_on_error, - keep_preimports_file=self.set_up_child_profiling, - ) - if self.set_up_child_profiling: - self.cache = cache = _prepare_child_profiling_cache( - self.options, self._ctx, self.prof, - preimports_file, script_file, + try: + script_file, preimports_file = _prepare_exec_script( + self.options, self.module, self.prof, + exit_on_error=self.exit_on_error, + keep_preimports_file=self.set_up_child_profiling, ) - # Add deferred callbacks for gathering debug logfiles - # (should run right before `.cache.cache_dir` is wiped): - # - Write the debug logs to the `._diagnostics` logger - if cache.debug: - self._ctx.add_cleanup_with_priority( - cache._dump_debug_logs, CLEANUP_PRIORITIES['gather_logs'], - ) - # - Write the debug logs to a specific file - if self.options.debug_log: - self._ctx.add_cleanup_with_priority( - self._gather_debug_log, - CLEANUP_PRIORITIES['gather_logs'], - self.options.debug_log, - ) - return self.prof, script_file + self._set_up_session_cache(script_file, preimports_file) + return self.prof, script_file + except BaseException: + # Make sure that we don't leak the changes made by `._ctx` + # before we've gotten out of `.__enter__()` + self._ctx.uninstall() + raise def __exit__(self, *_, **__): try: extra_stats = None - if self.set_up_child_profiling: - # Cleaning up here ensures the `multiprocessing` - # fork-server process is rebooted, thus any profiling - # data on it will be properly collected - self.cache.cleanup() - extra_stats = self.cache.gather_stats() - _post_profile(self.options, self.prof, extra_stats) + try: + extra_stats = self._gather_child_prof_stats() + finally: + # Write process-local stats regardless of whether + # child-process stats can be successfully gathered + _post_profile(self.options, self.prof, extra_stats) finally: self._ctx.uninstall() @@ -1243,6 +1230,36 @@ def _gather_debug_log(self, logfile): for entry in self.cache._gather_debug_log_entries(): print(entry.to_text(), file=fobj) + def _set_up_session_cache(self, script_file, preimports_file): + if not self.set_up_child_profiling: + return + self.cache = cache = _prepare_child_profiling_cache( + self.options, self._ctx, self.prof, preimports_file, script_file, + ) + # Add deferred callbacks for gathering debug logfiles + # (should run right before `.cache.cache_dir` is wiped): + # - Write the debug logs to the `._diagnostics` logger + if cache.debug: + self._ctx.add_cleanup_with_priority( + cache._dump_debug_logs, CLEANUP_PRIORITIES['gather_logs'], + ) + # - Write the debug logs to a specific file + if self.options.debug_log: + self._ctx.add_cleanup_with_priority( + self._gather_debug_log, + CLEANUP_PRIORITIES['gather_logs'], + self.options.debug_log, + ) + + def _gather_child_prof_stats(self): + if not self.set_up_child_profiling: + return None + # Cleaning up here ensures the `multiprocessing` fork-server + # process is rebooted, thus any profiling data on it will be + # properly collected + self.cache.cleanup() + return self.cache.gather_stats() + @property def set_up_child_profiling(self): return bool( From 4c4d2c538e4eeb00f91f0221ca8ed980f91502f7 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 11 Jul 2026 09:28:43 +0200 Subject: [PATCH 161/165] WIP: various `test_child_procs` fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (See `fable-pr431-plan-2026-07-05.md::D.1`, `D.3`, `D.4`, `D.8–.10` in TTsangSC#5) tests/test_child_procs/_test_child_procs_utils.py FuncCallTimeout Renamed from `TestTimeout` so that `pytest` doesn't try to collect it as a unit-test class and emit a warning _run_kernprof_main_in_process() Now accepting `cmd` which begins with e.g. `['python', '-m', 'kernprof']` and other similar forms, instead of only `['kernprof']` (`D.1`) @cleanup_extra_pth_files - Refactored to only process .pth files which is consistent with the `LineProfilingCache.write_pth_hook()` naming scheme (`D.3`) - Added doctest @preserve_object_attrs Now emitting debug messages for teardown-time errors and re-raising the last (`D.9`) ResultMismatch.rich_message Removed unused property (`D.10`) CheckWarnings - Extended doctest to cover `.suppress_warnings()` and `.propagate_warnings()`, because it makes more sense to keep both methods for symmetry (`D.10`) - Removed implementations of `Sequence` mixin methods (the inherited impls are more inefficient, but we don't really use any of them in the code anyway) (`D.10`) tests/test_child_procs/conftest.py check_purelib_dir_writable() New fixture to be requested by tests, which tests for the writability of `sysconfig.get_path('purelib')` and skips the test if it isn't (`D.4`) another_pid() Replaced a (botched) subtract-and-mod with a bitwise XOR to ensure that the "PID" is both different and positive (`D.8`) tests/test_child_procs/test_child_procs.py test_running_multiproc_script() _test_profiling_multiproc_script() _test_profiling_bare_python() Replaced unqualified `kernprof` commands with ` -m kernprof` (`D.1`) test_cache_setup_main_process() test_apply_mp_patches_{success,failure}() test_profiling_multiproc_script_{success,failure}() test_profiling_bare_python_{success,failure}() Now requesting `check_purelib_dir_writable`, skipping the tests when the pure-lib path directory is non-writable (`D.4`) tests/test_child_procs/test_fork_stats_attribution.py - Added type annotations - Refactored to avoid awkward indentations (which trip up the syntax highlighter) - Tests now requesting `check_purelib_dir_writable`, skipping the tests when the pure-lib path directory is non-writable (`D.4`) --- .../_test_child_procs_utils.py | 189 ++++++++++++++---- tests/test_child_procs/conftest.py | 28 ++- tests/test_child_procs/test_child_procs.py | 18 +- .../test_fork_stats_attribution.py | 96 +++++---- 4 files changed, 249 insertions(+), 82 deletions(-) diff --git a/tests/test_child_procs/_test_child_procs_utils.py b/tests/test_child_procs/_test_child_procs_utils.py index 04cc6de6..e30adb83 100644 --- a/tests/test_child_procs/_test_child_procs_utils.py +++ b/tests/test_child_procs/_test_child_procs_utils.py @@ -19,7 +19,7 @@ from abc import ABC, abstractmethod from argparse import ArgumentError from collections.abc import ( - Callable, Collection, Generator, Iterable, Iterator, Mapping, + Callable, Collection, Generator, Iterable, Mapping, Sequence, Set, ) from contextlib import ExitStack @@ -60,7 +60,7 @@ 'DEBUG', 'DEFAULT_TIMEOUT', 'NOT_SUPPLIED', 'START_METHODS', 'PATCH_SUMMARIES', 'StartMethod', 'ModuleFixture', 'Params', - 'ResultMismatch', 'TestTimeout', + 'ResultMismatch', 'FuncCallTimeout', 'preserve_object_attrs', 'preserve_targets', 'cleanup_extra_pth_files', 'add_timeout', 'CheckWarnings', @@ -176,18 +176,8 @@ def pytest_runtest_makereport( call.excinfo = xc.from_exception(xc.value.with_traceback(xc.tb)) return make_report(call=call) - @property - def rich_message(self) -> str: - msg = '{}: {}'.format(type(self).__name__, self.args[0]) - if self.__traceback__ is not None: - tb = self.__traceback__ - msg = '{}:{}: {}'.format( - tb.tb_frame.f_code.co_filename, tb.tb_lineno, msg, - ) - return msg - -class TestTimeout(RuntimeError): +class FuncCallTimeout(RuntimeError): """ Error raised by the :py:func:`add_timeout` decorator. """ @@ -789,11 +779,18 @@ def reset(attr: str, value: Any) -> None: return result def __exit__(self, *_, **__) -> None: + xc: Exception | None = None for callback in self._callbacks[::-1]: try: callback() - except Exception: - pass + except Exception as e: + xc = e + rep_xc = type(xc).__name__ + if str(xc): + rep_xc = f'{rep_xc}: {xc}' + self._debug(f'Callback {callback} failed: {rep_xc}') + if xc is not None: + raise xc class preserve_targets(_CallableContextManager[dict[str, dict[str, Any]]]): @@ -960,30 +957,121 @@ def targets(self, targets: Mapping[str, Collection[str]] | None) -> None: class cleanup_extra_pth_files(_CallableContextManager[frozenset[str]]): - def __init__(self, debug: bool = DEBUG) -> None: + r""" + Protect the venv where the test is run from temporary .pth files + created by :py:class:`.LineProfilingCache`. + + Example: + >>> from collections.abc import Generator + >>> from os import getpid + >>> from pathlib import Path + >>> from sysconfig import get_path + >>> from uuid import uuid4 + >>> import pytest + + >>> def _propose_alnum() -> Generator[str, None, None]: + ... while True: + ... uuid = str(uuid4()) + ... yield from uuid.split('-') + + >>> def _propose_names( + ... glob_pattern: str, + ... ) -> Generator[str, None, None]: + ... gen_alnum = _propose_alnum() + ... chunks = glob_pattern.split('*') + ... while True: + ... result: list[str] = [chunks[0]] + ... for chunk in chunks[1:]: + ... result.append(next(gen_alnum)) + ... result.append(chunk) + ... yield ''.join(result) + + >>> def get_new_pth(glob_pattern: str) -> Path: + ... gen_names = _propose_names(glob_pattern) + ... while True: + ... path = pth_loc / next(gen_names) + ... if not path.exists(): + ... return path + + >>> pth_loc = Path(get_path('purelib')) + >>> pth_from_cache = get_new_pth( + ... f'_line_profiler-profiling-hook-*-ppid-{getpid()}.pth', + ... ) + >>> pth_unrelated = get_new_pth(f'my-pth-*.pth') + + >>> try: + ... with cleanup_extra_pth_files(config=False, debug=False): + ... pth_from_cache.write_text('import sys') and None + ... pth_unrelated.write_text('import sys') and None + ... assert pth_from_cache.exists() + ... assert pth_unrelated.exists() + ... # This matches the pattern of filenames created by the + ... # cache and is thus dealt with + ... assert not pth_from_cache.exists() + ... # This is unrealated and is thus left alone + ... assert pth_unrelated.exists() + ... except PermissionError: + ... pytest.skip('Can\'t write to the pure-lib directory') + ... finally: # Cleanup + ... for pth in pth_from_cache, pth_unrelated: + ... pth.unlink(missing_ok=True) + """ + def __init__( + self, + config: os.PathLike[str] | str | bool | None = None, + debug: bool = DEBUG, + ) -> None: self.debug = debug + pattern = self._get_glob_pattern(config) + self._get_files = partial(self.get_pth_files, pattern) def __enter__(self) -> frozenset[str]: - self.old = self.get_pth_files() + self.old = self._get_files() return self.old def __exit__(self, *_, **__) -> None: - for new_pth_file in self.get_pth_files() - self.old: + for new_pth_file in self._get_files() - self.old: self._debug(f'Deleting stray .pth file: {new_pth_file!r}') (self._get_path() / new_pth_file).unlink(missing_ok=True) del self.old @classmethod - def get_pth_files(cls, name_only: bool = True) -> frozenset[str]: + def get_pth_files( + cls, pattern: str | None = None, name_only: bool = True, + ) -> frozenset[str]: + if pattern is None: + pattern = cls._get_glob_pattern() return frozenset( pth.name if name_only else str(pth) - for pth in cls._get_path().glob('*.pth') + for pth in cls._get_path().glob(pattern) ) @staticmethod def _get_path() -> Path: + """ + Note: + This doesn't go through the entire .pth-location search path + (see :py:meth:`.LineProfilingCache.write_pth_hook`), but + suffices for the testing environment. + """ return Path(sysconfig.get_path('purelib')) + @staticmethod + def _get_glob_pattern( + config: os.PathLike[str] | str | bool | None = None, + ) -> str: + cmap = ( + ConfigSource.from_config(config) + .get_subconfig('child_processes', 'pth_files') + .conf_dict + ) + prefix, suffix_template = LineProfilingCache._normalize_pth_affixes( + cmap['prefix'], cmap['suffix'], + ) + pattern = '{}*{}'.format(prefix, suffix_template.format(os.getpid())) + assert pattern.endswith('.pth') + return pattern + def _cleanup_profiling_in_current_thread() -> None: """ @@ -1094,7 +1182,7 @@ def add_timeout( >>> my_func(4, delay=5) # doctest: +NORMALIZE_WHITESPACE Traceback (most recent call last): ... - test_child_procs._test_child_procs_utils.TestTimeout: + test_child_procs._test_child_procs_utils.FuncCallTimeout: my_func(4, delay=5): timed out after 0.5 s Note: @@ -1159,7 +1247,7 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: name = getattr(func, '__name__', repr(func)) call_repr = f'{name}({", ".join(args_repr)})' msg = f'{call_repr}: timed out after {timeout:.2g} s' - raise TestTimeout(msg) + raise FuncCallTimeout(msg) new_thread = partial( threading.Thread, target=worker, name=name, daemon=True, @@ -1412,6 +1500,8 @@ class CheckWarnings(Sequence[_WarningInfo]): Example: >>> import warnings + Checking for/against warnings: + >>> cw = CheckWarnings( ... reissue_warnings=False, format_warnings=False, ... ) @@ -1444,6 +1534,24 @@ class CheckWarnings(Sequence[_WarningInfo]): got none out of 1 ([...]) >>> assert len(cw) == 1 >>> assert str(cw[0].message) == 'foobar' + + Controlling warning propagation: + + >>> with warnings.catch_warnings(record=True) as cw_outer: + ... with cw: + ... cw.suppress_warnings('foo') + ... warnings.warn('foo') + ... warnings.warn('bar') # Implicitly propagated + ... assert len(cw_outer) == 1 + ... assert str(cw_outer[0].message) == 'bar' + + >>> with warnings.catch_warnings(record=True) as cw_outer: + ... with cw: + ... cw.propagate_warnings('foo') + ... warnings.warn('foo') + ... warnings.warn('bar') # Implicitly suppressed + ... assert len(cw_outer) == 1 + ... assert str(cw_outer[0].message) == 'foo' """ def __init__( self, /, *, @@ -1552,21 +1660,6 @@ def __getitem__( def __len__(self) -> int: return len(self._current_warnings) - def __iter__(self) -> Iterator[_WarningInfo]: - return iter(self._current_warnings) - - def __reversed__(self) -> Iterator[_WarningInfo]: - return iter(reversed(self._current_warnings)) - - def __contains__(self, item: Any, /) -> bool: - return item in self._current_warnings - - def index(self, *args, **kwargs) -> int: - return self._current_warnings.index(*args, **kwargs) - - def count(self, *args, **kwargs) -> int: - return self._current_warnings.count(*args, **kwargs) - @property def _current_context( self, @@ -1925,7 +2018,23 @@ def get_streams( return encode(stdout), encode(stderr) assert not _kwargs - assert cmd[0] == 'kernprof' + if cmd[0] == 'kernprof': + args = cmd[1:] + elif any( + list(cmd[:3]) == [python, '-m', 'kernprof'] + for python in ( + 'python', + 'python3', + f'python3.{sys.version_info.minor}', + sys.executable, + ) + ): + args = cmd[3:] + else: + raise AssertionError( + 'Expected commands like `kernprof ...` or ' + f'`python -m kernprof ...`, got {cmd!r}' + ) cap: pytest.CaptureFixture | None = None stdout: str | bytes | None = None @@ -1968,8 +2077,8 @@ def get_streams( cleanup = stack.enter_context(Cleanup()) if env is not None: cleanup.update_mapping(os.environ, env) - main(cmd[1:], exit_on_error=False) - except TestTimeout: # `subprocess` uses `SIGKILL` + main(args, exit_on_error=False) + except FuncCallTimeout: # `subprocess` uses `SIGKILL` returncode, timed_out = -9, True status = f'error ({returncode})' except Exception as e: diff --git a/tests/test_child_procs/conftest.py b/tests/test_child_procs/conftest.py index 00c0f9f7..f92009f6 100644 --- a/tests/test_child_procs/conftest.py +++ b/tests/test_child_procs/conftest.py @@ -2,10 +2,12 @@ import os import shutil +import sysconfig from collections.abc import Callable, Collection, Generator from functools import partial from io import StringIO from pathlib import Path +from uuid import uuid4 from tempfile import TemporaryDirectory from textwrap import indent from types import ModuleType @@ -379,7 +381,7 @@ def another_pid() -> int: Get a PID which is distinct from the current one. """ curr_pid = os.getpid() - pid = (curr_pid - 42) % (2 * 16) + pid = curr_pid ^ 42 assert pid != curr_pid return pid @@ -394,3 +396,27 @@ def _trim_mismatch_traceback(pytestconfig: pytest.Config) -> None: pytestconfig.pluginmanager.register(ResultMismatch) except ValueError: # Already registered pass + + +@pytest.fixture() +def check_purelib_dir_writable() -> Generator[None, None, None]: + """ + Check that ``sysconfig.get_path('purelib')`` can be written to; if + not, the test is skipped. + """ + def propose_names() -> Generator[Path, None, None]: + while True: + pth = purelib_path / f'_temp-pth-{uuid4()}.pth' + if not pth.exists(): + yield pth + + purelib_path = Path(sysconfig.get_path('purelib')) + pth = next(propose_names()) + try: + pth.touch() + yield + except OSError: + msg = f"sysconfig.get_path('purelib') ({purelib_path}) not writable" + pytest.skip(msg) + finally: + pth.unlink(missing_ok=True) diff --git a/tests/test_child_procs/test_child_procs.py b/tests/test_child_procs/test_child_procs.py index 988ccedd..2a836cdc 100644 --- a/tests/test_child_procs/test_child_procs.py +++ b/tests/test_child_procs/test_child_procs.py @@ -206,6 +206,7 @@ def test_cache_dump_load( (False, True, False, 'process-only'), (False, False, True, 'logging-only'), (False, False, False, 'no-patches')])).sorted() +@pytest.mark.usefixtures('check_purelib_dir_writable') def test_cache_setup_main_process( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], @@ -636,6 +637,7 @@ def _test_apply_mp_patches( ('process_test_module', True, True, 'patch-pool-and-process'), ('process_test_module', True, False, 'patch-process-only')]) @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) +@pytest.mark.usefixtures('check_purelib_dir_writable') def test_apply_mp_patches_success( request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, @@ -688,6 +690,7 @@ def test_apply_mp_patches_success( ('process_test_module', True, True, 'patch-pool-and-process'), ('process_test_module', True, False, 'patch-process-only')]) @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) +@pytest.mark.usefixtures('check_purelib_dir_writable') def test_apply_mp_patches_failure( request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, @@ -812,11 +815,12 @@ def test_multiproc_script_sanity_check( 'label2'), # Dummy argument to make `pytest` output more legible # This is essentially a no-op since it doesn't actually do # line-profiling, but we check that code path for completeness - [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] + [([sys.executable, '-m', 'kernprof', '-q', '--no-line'], + 'out.prof', False, 'cProfile')] # Run line profiling with and w/o profiling targets - + [(['kernprof', '-q', '-l'], 'out.lprof', False, + + [([sys.executable, '-m', 'kernprof', '-q', '-l'], 'out.lprof', False, 'line_profiler-inactive'), - (['kernprof', '-q', '-l'], 'out.lprof', True, + ([sys.executable, '-m', 'kernprof', '-q', '-l'], 'out.lprof', True, 'line_profiler-active')], ) def test_running_multiproc_script( @@ -918,7 +922,7 @@ def _test_profiling_multiproc_script( nhits[tag_call] += nprocs nhits[tag_loop] += nnums - runner = ['kernprof', '-l'] + runner = [sys.executable, '-m', 'kernprof', '-l'] runner.extend([ '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), '--{}preimports'.format('' if preimports else 'no-'), @@ -952,6 +956,7 @@ def _test_profiling_multiproc_script( # have quite a lot of subtests tho... ('nnums', 'nprocs'), [(2000, 3)], ) +@pytest.mark.usefixtures('check_purelib_dir_writable') def test_profiling_multiproc_script_success( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, @@ -1038,6 +1043,7 @@ def test_profiling_multiproc_script_success( @(_fuzz_prof_mp_markers[True]) @pytest.mark.parametrize(('nnums', 'nprocs'), [(2000, 3)]) +@pytest.mark.usefixtures('check_purelib_dir_writable') def test_profiling_multiproc_script_failure( run_func: Callable[..., CompletedProcess], request: pytest.FixtureRequest, @@ -1120,7 +1126,7 @@ def _test_profiling_bare_python( debug_log_file = temp_dir / 'debug.log' write_debug = DEBUG and prof_child_procs cmd = [ - 'kernprof', '-lv', '--preimports', + sys.executable, '-m', 'kernprof', '-lv', '--preimports', f'--prof-mod={ext_module.name}', f'--outfile={out_file}', '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), @@ -1170,6 +1176,7 @@ def _test_profiling_bare_python( @_fuzz_bare +@pytest.mark.usefixtures('check_purelib_dir_writable') def test_profiling_bare_python_success( tmp_path_factory: pytest.TempPathFactory, ext_module: ModuleFixture, @@ -1199,6 +1206,7 @@ def test_profiling_bare_python_success( @_fuzz_bare +@pytest.mark.usefixtures('check_purelib_dir_writable') def test_profiling_bare_python_failure( tmp_path_factory: pytest.TempPathFactory, ext_module: ModuleFixture, diff --git a/tests/test_child_procs/test_fork_stats_attribution.py b/tests/test_child_procs/test_fork_stats_attribution.py index ace55a36..cbf2901c 100644 --- a/tests/test_child_procs/test_fork_stats_attribution.py +++ b/tests/test_child_procs/test_fork_stats_attribution.py @@ -5,70 +5,90 @@ Forked children inherit the parent profiler's accumulated stats; if they dump them verbatim, every line the parent executed before the fork is counted once per forked child when the results are merged (the parent -dumps the same data itself). See -``LineProfilingCache._wrap_os_fork`` and ``_dump_profiler_stats`` for -the subtractive-baseline fix these tests pin down. +dumps the same data itself). See ``LineProfilingCache._wrap_os_fork()`` +and ``_StatsHelper.dump()`` for the subtractive-baseline fix these tests +pin down. """ import multiprocessing +import os import subprocess import sys +from pathlib import Path +from textwrap import indent +from typing import Literal import pytest from line_profiler import LineStats +from ._test_child_procs_utils import strip as code_block + + +StartMethod = Literal['spawn', 'forkserver', 'fork'] +API = Literal['process', 'pool', 'cfut'] LOOP_COUNT = 5000 NUM_TASKS = 4 TIMEOUT = 120 -_API_SNIPPETS = { - 'process': """ +_API_SNIPPETS: dict[API, str] = { + 'process': code_block(""" procs = [ctx.Process(target=child_work, args=(i,)) for i in range(num_children)] for p in procs: p.start() for p in procs: p.join() -""", - 'pool': """ + """), + 'pool': code_block(""" with ctx.Pool(2) as pool: results = pool.map(child_work, range(num_tasks)) assert results == [i * 2 for i in range(num_tasks)] -""", - 'cfut': """ + """), + 'cfut': code_block(""" import concurrent.futures as cf with cf.ProcessPoolExecutor(max_workers=2, mp_context=ctx) as ex: results = list(ex.map(child_work, range(num_tasks))) assert results == [i * 2 for i in range(num_tasks)] -""", + """), } -def _write_workload(path, method, api, num_children=1): - source = f""" -import multiprocessing as mp - - -def child_work(x): - doubled = x * 2 - return doubled - - -def main(): - num_tasks = {NUM_TASKS} - num_children = {num_children} - acc = 0 - for i in range({LOOP_COUNT}): - acc += i - ctx = mp.get_context({method!r}) -{_API_SNIPPETS[api]} - -if __name__ == '__main__': - main() -""" +def _write_workload( + path: Path, method: StartMethod, api: API, num_children: int = 1, +) -> dict[str, int]: + blocks = { + 'import': 'import multiprocessing as mp', + 'child_work': code_block(""" + def child_work(x): + doubled = x * 2 + return doubled + """), + 'main': code_block(f""" + def main(): + num_tasks = {NUM_TASKS} + num_children = {num_children} + acc = 0 + for i in range({LOOP_COUNT}): + acc += i + ctx = mp.get_context({method!r}) + """), + 'guard': code_block(""" + if __name__ == '__main__': + main() + """), + } + blocks['main'] = '{}\n{}'.format( + blocks['main'], indent(_API_SNIPPETS[api], ' '), + ) + source = '\n\n\n'.join(blocks.values()) path.write_text(source) + print( + 'Current test: {}'.format(os.environ.get('PYTEST_CURRENT_TEST')), + 'Workload script:\n{}'.format(indent(source, ' ')), + sep='\n\n', + ) lines = source.splitlines() return { 'acc': 1 + lines.index(' acc += i'), @@ -76,7 +96,7 @@ def main(): } -def _run_kernprof(tmp_path, script): +def _run_kernprof(tmp_path: Path, script: Path) -> LineStats: """ Run the real CLI in a real subprocess (the .pth/env machinery only fully engages for a fresh interpreter) and load the merged stats. @@ -92,7 +112,7 @@ def _run_kernprof(tmp_path, script): return LineStats.from_files(outfile) -def _nhits(stats, func_name, lineno): +def _nhits(stats: LineStats, func_name: str, lineno: int) -> int: matches = [ entry for (_, _, func), entries in stats.timings.items() @@ -109,7 +129,10 @@ def _nhits(stats, func_name, lineno): @pytest.mark.parametrize('api', sorted(_API_SNIPPETS)) @pytest.mark.parametrize('method', ['fork', 'forkserver', 'spawn']) -def test_exact_stats_across_start_methods(tmp_path, method, api): +@pytest.mark.usefixtures('check_purelib_dir_writable') +def test_exact_stats_across_start_methods( + tmp_path: Path, method: StartMethod, api: API, +) -> None: """ The merged profile must show the parent's loop exactly once and the children's work exactly ``NUM_TASKS`` times, no matter how the @@ -128,7 +151,8 @@ def test_exact_stats_across_start_methods(tmp_path, method, api): ) -def test_stats_do_not_scale_with_fork_children(tmp_path): +@pytest.mark.usefixtures('check_purelib_dir_writable') +def test_stats_do_not_scale_with_fork_children(tmp_path: Path) -> None: """ Pre-fork parent data must not be re-contributed once per child. """ From 1476d5ab426befe8e4e30f88ad526d980b4496a7 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 11 Jul 2026 16:43:28 +0200 Subject: [PATCH 162/165] More test fixes + `kernprof` fixes in forking code (See `fable-pr431-plan-2026-07-05.md::D.5-6` in TTsangSC#5) kernprof.py _write_preimports() Now returning `None` if the tempfile has been immediately used and deleted main(), _manage_profiler.__exit__() Added checks to file-cleanup/IO routines so that they are no-ops if we're in a forked child created inside the executed code; all such interactions with the "outside world" should be handled by the calling parent process tests/test_child_procs/_test_child_procs_utils.py preserve_targets .__doc__ Extended doctest to test `verify` .__init__() Added parameter `verify` .__enter__(), .__exit__() Updated implementation to store the dict of old values and compare them with the values at context-exit with `.compare_with_current_values()` if `.verify` is true _run_kernprof_main_in_process() - Updated decorator: `@preserve_targets(verify=True)` - Added check against `os.environ` pollution (`D.6`) tests/test_child_procs/test_child_procs.py _test_applymp_patches_inner() Updated decorator: `@preserve_targets(verify=True)` test_profiling_multiproc_script_{success,failure}() Reworked parametrization to: - Remove unintentional default (`start_method=None`) cases - Ensure that we run at least one subtest with `subprocess.run(['kernprof', ...])` for each combination of `start_method` and `fail` (`D.5`) test_profiling_bare_python_{success,failure}() - Updated call signature - Added subtests which test when the new Python process is created via `os.fork()` (`D.6`) --- kernprof.py | 43 +++-- .../_test_child_procs_utils.py | 126 +++++++++++--- tests/test_child_procs/test_child_procs.py | 159 +++++++++++------- 3 files changed, 225 insertions(+), 103 deletions(-) diff --git a/kernprof.py b/kernprof.py index c3ad0037..82a4f031 100755 --- a/kernprof.py +++ b/kernprof.py @@ -954,6 +954,15 @@ def main(args=None, *, exit_on_error=True): files created during execution may be deferred to when the interpreter exits. """ + def rmdir_with_pid_guard(pid, dir, *, defer=False, **kwargs): + if os.getpid() != pid: # Leave cleanup to the main process + return + callback = functools.partial(_remove, dir, **kwargs) + if defer: + atexit.register(callback) + else: + callback() + real_parser, help_parser, special_info = _build_parsers(args=args) args = special_info['args'] module = special_info['module'] @@ -988,7 +997,8 @@ def main(args=None, *, exit_on_error=True): cleanup = no_op else: cleanup = functools.partial( - _remove, + rmdir_with_pid_guard, + os.getpid(), tmpdir, recursive=True, missing_ok=True, @@ -1005,10 +1015,7 @@ def main(args=None, *, exit_on_error=True): except BaseException: # Defer deletion to after the traceback has been formatted # if needs be - if os.listdir(tmpdir): - atexit.register(cleanup) - else: # Empty tempdir, just delete it - cleanup() + cleanup(defer=bool(os.path.isdir(tmpdir) and os.listdir(tmpdir))) raise else: # Execution succeeded, delete the tempdir ASAP cleanup() @@ -1072,9 +1079,9 @@ def _write_preimports(prof, options, exclude, keep=False): ns = {} # Use a fresh namespace execfile(temp_mod_path, ns, ns) # Delete the tempfile ASAP if its execution succeeded - if not (keep or diagnostics.KEEP_TEMPDIRS): - _remove(temp_mod_path) - return temp_mod_path + if keep or diagnostics.KEEP_TEMPDIRS: + return temp_mod_path + _remove(temp_mod_path) def _remove(path, *, recursive=False, missing_ok=False): @@ -1186,6 +1193,7 @@ def __init__(self, options, module, exit_on_error): self.options = options self.module = module self.exit_on_error = exit_on_error + self._pid = os.getpid() def __enter__(self): from line_profiler.curated_profiling import CuratedProfilerContext @@ -1215,13 +1223,18 @@ def __enter__(self): def __exit__(self, *_, **__): try: - extra_stats = None - try: - extra_stats = self._gather_child_prof_stats() - finally: - # Write process-local stats regardless of whether - # child-process stats can be successfully gathered - _post_profile(self.options, self.prof, extra_stats) + # Guard against when we've forked inside the executed and + # profiled code... + if os.getpid() == self._pid: + extra_stats = None + try: + extra_stats = self._gather_child_prof_stats() + finally: + # Write process-local stats regardless of whether + # child-process stats can be successfully gathered... + # but watch out for when we've forked inside the + # executed code + _post_profile(self.options, self.prof, extra_stats) finally: self._ctx.uninstall() diff --git a/tests/test_child_procs/_test_child_procs_utils.py b/tests/test_child_procs/_test_child_procs_utils.py index e30adb83..e888cd15 100644 --- a/tests/test_child_procs/_test_child_procs_utils.py +++ b/tests/test_child_procs/_test_child_procs_utils.py @@ -796,10 +796,14 @@ def __exit__(self, *_, **__) -> None: class preserve_targets(_CallableContextManager[dict[str, dict[str, Any]]]): """ Protect attributes on multiple target objects, which are resolved at - context entry. + context entry. If ``verify`` is true, check that the targets have + all been restored to their original value on context exit, and raise + an :py:class:`AssertionError` if that is not the case (before + restoring them anyways). Example: - >>> from functools import wraps + >>> import builtins + >>> from functools import partial, wraps >>> from line_profiler.curated_profiling import ( ... CuratedProfilerContext, ... ) @@ -807,23 +811,23 @@ class preserve_targets(_CallableContextManager[dict[str, dict[str, Any]]]): >>> assert not hasattr(CuratedProfilerContext, 'foo') >>> old_main = line_profiler.main - >>> - >>> + >>> new_context = partial(preserve_targets, debug=False) + >>> def foo(_) -> None: ... pass - ... - >>> + >>> @wraps(old_main) ... def main(*a, **k): ... return old_main(*a, **k) - ... - >>> + + Basic use: + >>> preserved = { ... 'line_profiler.curated_profiling' ... '.CuratedProfilerContext': {'foo'}, ... 'line_profiler.line_profiler': {'main'}, ... } - >>> with preserve_targets(preserved, debug=False) as old: + >>> with new_context(preserved) as old: ... assert old == { ... 'line_profiler.curated_profiling' ... '.CuratedProfilerContext': {'foo': NOT_SUPPLIED}, @@ -839,29 +843,67 @@ class preserve_targets(_CallableContextManager[dict[str, dict[str, Any]]]): old['line_profiler.line_profiler']['main'] >>> assert old_main is line_profiler.main >>> assert main is not line_profiler.main + + With ``verify``: + + >>> preserved = {'builtins': {'foo', 'bar'}} + >>> expected = { + ... 'builtins': {'foo': NOT_SUPPLIED, 'bar': NOT_SUPPLIED}, + ... } + >>> ctx = new_context(preserved, verify=True) + >>> with ctx as old: + ... assert old == expected + ... builtins.foo = 1 + ... del builtins.foo + ... print('ok') + ok + >>> with ctx as old: # doctest: +ELLIPSIS + ... assert old == expected + ... builtins.foo = 1 + ... # `bar` not restored -> `.__exit__()` error out + ... builtins.bar = 2 + ... del builtins.foo + ... print('ok, but...') + ok, but... + Traceback (most recent call last): + ... + AssertionError: Compared `builtins.bar` \ +(old:...NOT_SUPPLIED...; new: 2...)... + + >>> assert not hasattr(builtins, 'foo') + >>> assert not hasattr(builtins, 'bar') # Still cleaned up """ def __init__( self, targets: Mapping[str, Collection[str]] | None = None, *, - static: bool = True, debug: bool = DEBUG, + static: bool = True, debug: bool = DEBUG, verify: bool = False, ) -> None: self.targets = targets - self._stacks: list[ExitStack] = [] + self._ctxs: list[tuple[ExitStack, dict[str, dict[str, Any]]]] = [] self.static = static self.debug = debug + self.verify = verify def __enter__(self) -> dict[str, dict[str, Any]]: stack = ExitStack() - self._stacks.append(stack) - result: dict[str, Any] = {} + old: dict[str, Any] = {} for target, attrs in self.targets.items(): - result[target] = stack.enter_context(preserve_object_attrs( + old[target] = stack.enter_context(preserve_object_attrs( import_target(target), attrs, debug=self.debug, static=self.static, )) - return result + self._ctxs.append((stack, old)) + # Decouple the stored and returned maps for the original values + # in case the user messes around with the latter + old_copy = {key: value.copy() for key, value in old.items()} + return old_copy def __exit__(self, *_, **__) -> None: - self._stacks.pop().close() + stack, old = self._ctxs.pop() + try: + if self.verify: + self.compare_with_current_values(old, verbose=self.debug) + finally: + stack.close() @staticmethod def fetch_current_values( @@ -885,6 +927,7 @@ def compare_with_current_values( comparator: Callable[[Any, Any], bool] = operator.is_, assert_true: bool | Mapping[str, Mapping[str, bool]] = True, static: bool = True, + verbose: bool = True, ) -> dict[str, dict[str, bool]]: def get_from_mapping(target: str, attr: str) -> bool: if TYPE_CHECKING: @@ -906,7 +949,8 @@ def get_from_boolean(*_, **__) -> bool: new_values = new[target] cmp_results = result[target] = {} for attr, old_value in old_values.items(): - print(f'Checking: {target}.{attr}') + if verbose: + print(f'Checking: {target}.{attr}') new_value = new_values[attr] cmp_results[attr] = cmp_result = comparator( new_value, old_value, @@ -934,7 +978,8 @@ def get_from_boolean(*_, **__) -> bool: message = format_msg( f'comparison result with {comparator!r}: {cmp_result}' ) - print(message) + if verbose: + print(message) assert (not failures), '\n'.join(failures) return result @@ -1885,7 +1930,7 @@ def stderr(self) -> str | bytes | None: cmd_str = concat_command_line(cmd) print('Command:', cmd_str) - _print_env_deltas(kwargs.get('env')) + _get_env_deltas(kwargs.get('env')) print('-- Process start --') # Note: somehow `mypy` doesn't agree with simply unpacking the # `*args` into `subprocess.run()`... @@ -1987,7 +2032,7 @@ def _run_as_literal_code( @cleanup_extra_pth_files() -@preserve_targets() +@preserve_targets(verify=True) def _run_kernprof_main_in_process( request: pytest.FixtureRequest, check_warnings: bool, cmd: Sequence[str], *, @@ -2002,7 +2047,16 @@ def _run_kernprof_main_in_process( """ Emulate running ``kernprof`` in a subprocess with in-process machineries as best as we can, so that we can retrieve more - debugging output when things do go south. + debugging output when things do go south. Additional safety + railings: + + - `preserve_targets(verify=True)` errors out if any of the patches + is not reversed by :py:func:`kernprof.main` + + - Same goes for the key-value pairs in :py:data:`os.environ`. + + In both cases, if we don't get a match, an + :py:class:`AssertionError` is raised. """ def get_streams( ) -> tuple[str, str] | tuple[bytes, bytes] | tuple[None, None]: @@ -2042,7 +2096,7 @@ def get_streams( if capture_output: cap = request.getfixturevalue('capsys') print('Command:', concat_command_line(cmd)) - _print_env_deltas(env) + _get_env_deltas(env) print('-- Emulated process start --') get_streams() # Don't include the above in the captured outputs @@ -2077,7 +2131,16 @@ def get_streams( cleanup = stack.enter_context(Cleanup()) if env is not None: cleanup.update_mapping(os.environ, env) - main(args, exit_on_error=False) + old_environ = dict(os.environ) + try: + main(args, exit_on_error=False) + finally: + env_diff = _get_env_deltas(os.environ, old_environ, False) + if env_diff: + raise AssertionError( + f'{len(env_diff)} `os.environ` pollution(s): ' + f'{env_diff!r}' + ) except FuncCallTimeout: # `subprocess` uses `SIGKILL` returncode, timed_out = -9, True status = f'error ({returncode})' @@ -2118,12 +2181,18 @@ def get_streams( ) -def _print_env_deltas(env: Mapping[str, str] | None = None) -> None: +def _get_env_deltas( + env: Mapping[str, str] | None = None, + original: Mapping[str, str] | None = None, + verbose: bool = True, +) -> list[str]: if env is None: - return + return [] diff: list[str] = [] - for key in set(os.environ).union(env): - old = os.environ.get(key) + if original is None: + original = os.environ + for key in set(original).union(env): + old = original.get(key) new = env.get(key) if old is not None is new: item = f'{old!r} -> (deleted)' @@ -2134,8 +2203,9 @@ def _print_env_deltas(env: Mapping[str, str] | None = None) -> None: continue item = f'{old!r} -> {new!r}' diff.append(f'${{{key}}}: {item}') - if diff: + if verbose and diff: print('Env:', indent('\n'.join(diff), ' '), sep='\n') + return diff @cleanup_extra_pth_files() diff --git a/tests/test_child_procs/test_child_procs.py b/tests/test_child_procs/test_child_procs.py index 2a836cdc..0c693df1 100644 --- a/tests/test_child_procs/test_child_procs.py +++ b/tests/test_child_procs/test_child_procs.py @@ -15,7 +15,7 @@ from subprocess import CompletedProcess from textwrap import indent from types import ModuleType -from typing import cast +from typing import Literal, cast import pytest @@ -472,7 +472,7 @@ def test_load_pth_hook( @cleanup_extra_pth_files() -@preserve_targets() +@preserve_targets(verify=True) def _test_apply_mp_patches_inner( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], @@ -866,23 +866,24 @@ def test_running_multiproc_script( + _get_mp_start_method_fuzzer(None)) # Test all `multiproc` start methods with both locally- and # externally-defined profiling targets - * (Params.new(('preimports', 'label3'), [(False, 'no-preimports')]) - + Params.new(('use_local_func', 'label4'), - [(True, 'local'), (False, 'external')], - defaults=(False, 'external'))) + * Params.new(('use_local_func', 'label3'), + [(True, 'local'), (False, 'external')], + defaults=(False, 'external')) # Test all of the above with both test modules * Params.new('test_module', ['pool_test_module', 'process_test_module']) + # Add missing params + * Params.new(('preimports', 'label4'), [(False, 'no-preimports')]) + * Params.new(('subproc', 'label5'), [(False, 'in-proc')]) # The 'with-preimports' case is already tested rather thoroughly in # `test_apply_mp_patches()`, so exclude these from the above "main" # param matrix and just test the different `kernprof` modes via the # `run_func()`s + (_fuzz_prof_mp_run_func - + Params.new(('preimports', 'label3'), [(True, 'with-preimports')])) - # Just throw in a case where we actually run `kernprof` in a - # subprocess, otherwise it is more convenient to do so in-process - + Params.new(('subproc', 'label5'), - [(True, 'subproc'), (False, 'in-proc')], - defaults=(False, 'in-proc')) + * Params.new(('preimports', 'label4'), [(True, 'with-preimports')])) + # Similar to the above, we also run at least 1 subtest with + # `subproc=True` for each `(start_method, fail)` + + (_get_mp_start_method_fuzzer(None) + * Params.new(('subproc', 'label5'), [(True, 'subproc')])) ).sorted().split_on_params('fail') @@ -1087,10 +1088,10 @@ def test_profiling_multiproc_script_failure( ) +_TestBarePythonCase = Literal['subprocess.run', 'os.system', 'os.fork'] _fuzz_bare = ( - Params.new(('use_subprocess', 'label1'), - [(True, 'subprocess.run'), (False, 'os.system')]) - * Params.new(('prof_child_procs', 'label2'), + Params.new('case', ['subprocess.run', 'os.system', 'os.fork']) + * Params.new(('prof_child_procs', 'label'), [(True, 'with-child-prof'), (False, 'no-child-prof')]) * Params.new('n', [200]) ) @@ -1099,7 +1100,7 @@ def test_profiling_multiproc_script_failure( def _test_profiling_bare_python( tmp_path_factory: pytest.TempPathFactory, ext_module: ModuleFixture, - use_subprocess: bool, + case: _TestBarePythonCase, prof_child_procs: bool, fail: bool, n: int, @@ -1107,21 +1108,6 @@ def _test_profiling_bare_python( ext_module.install(children=True) temp_dir = tmp_path_factory.mktemp('mytemp') - script_path = temp_dir / 'my-script.py' - script_content = strip(""" - from {EXT_MODULE} import my_external_sum - - - if __name__ == '__main__': - numbers = list(range(1, 1 + {N})) - result = my_external_sum(numbers, {FAIL}) - """.format( - EXT_MODULE=ext_module.name, - N=n, - FAIL=fail, - )) - script_path.write_text(script_content) - out_file = temp_dir / 'out.lprof' debug_log_file = temp_dir / 'debug.log' write_debug = DEBUG and prof_child_procs @@ -1133,32 +1119,85 @@ def _test_profiling_bare_python( ] if write_debug: cmd.append(f'--debug-log={debug_log_file}') - sub_cmd = [sys.executable, str(script_path)] - if use_subprocess: - code = strip(f""" - import subprocess + if case == 'os.fork': + if not callable(getattr(os, 'fork', None)): + pytest.skip(f'Cannot fork on {sys.platform}') - subprocess.run({sub_cmd!r}, check=True) - """) - else: code = strip(""" import os + from {EXT_MODULE} import my_external_sum + + + def main(n: int, fail: bool) -> None: + numbers = list(range(1, 1 + n)) + child_pid = os.fork() + offset = int(not child_pid) # Literal[0, 1] + try: + my_external_sum(numbers[offset::2], fail) + finally: + if child_pid: # Always join the forked child + os.waitpid(child_pid, 0) + + + if __name__ == '__main__': + main({N}, {FAIL}) + """.format( + EXT_MODULE=ext_module.name, + N=n, + FAIL=fail, + )) + cmd.extend(['-c', code]) + nhits = {'EXT-INVOCATION': 2, 'EXT-LOOP': n} + if not prof_child_procs: + for k in nhits: # We still have the data in the main proc + nhits[k] //= 2 + else: + script_path = temp_dir / 'my-script.py' + script_content = strip(""" + from {EXT_MODULE} import my_external_sum + + + if __name__ == '__main__': + numbers = list(range(1, 1 + {N})) + result = my_external_sum(numbers, {FAIL}) + """.format( + EXT_MODULE=ext_module.name, + N=n, + FAIL=fail, + )) + script_path.write_text(script_content) + + sub_cmd = [sys.executable, str(script_path)] # FIXME + if case == 'subprocess.run': + code = strip(f""" + import subprocess + + + subprocess.run({sub_cmd!r}, check=True) + """) + elif case == 'os.system': + code = strip(""" + import os + + + if os.system({!r}): + raise RuntimeError('called process failed') + """.format(concat_command_line(sub_cmd))) + else: + msg = f'case = {case!r}: expected {_TestBarePythonCase}' + raise AssertionError(msg) + cmd.extend(['-c', code]) + nhits = {'EXT-INVOCATION': 1, 'EXT-LOOP': n} + if not prof_child_procs: + for k in nhits: + nhits[k] = 0 - if os.system({!r}): - raise RuntimeError('called process failed') - """.format(concat_command_line(sub_cmd))) - cmd.extend(['-c', code]) proc = run_subproc( cmd, text=True, capture_output=True, timeout=DEFAULT_TIMEOUT, ) - nhits = {'EXT-INVOCATION': 1, 'EXT-LOOP': n} - if not prof_child_procs: - for k in nhits: - nhits[k] = 0 - try: # Check that the code errors out when expected assert bool(fail) == bool(proc.returncode) @@ -1180,17 +1219,17 @@ def _test_profiling_bare_python( def test_profiling_bare_python_success( tmp_path_factory: pytest.TempPathFactory, ext_module: ModuleFixture, - use_subprocess: bool, + case: _TestBarePythonCase, prof_child_procs: bool, n: int, - # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, + # Dummy argument to make `pytest` output more legible + label: str, ) -> None: """ Check that `kernprof` can profile the target functions if the code - invokes another bare Python process (via either :py:func:`os.system` - or :py:func:`subprocess.run`) that calls them and exits without - errors. + invokes another bare Python process (via either + :py:func:`os.system`, :py:func:`subprocess.run`, or forking) that + calls them and exits WITHOUT errors. See also: :py:func:`test_profiling_bare_python_failure` @@ -1198,7 +1237,7 @@ def test_profiling_bare_python_success( _test_profiling_bare_python( tmp_path_factory=tmp_path_factory, ext_module=ext_module, - use_subprocess=use_subprocess, + case=case, prof_child_procs=prof_child_procs, fail=False, n=n, @@ -1210,16 +1249,16 @@ def test_profiling_bare_python_success( def test_profiling_bare_python_failure( tmp_path_factory: pytest.TempPathFactory, ext_module: ModuleFixture, - use_subprocess: bool, + case: _TestBarePythonCase, prof_child_procs: bool, n: int, - label1: str, - label2: str, + label: str, ) -> None: """ Check that `kernprof` can profile the target functions if the code - invokes another bare Python process (via either :py:func:`os.system` - or :py:func:`subprocess.run`) that calls them and exits with errors. + invokes another bare Python process (via either + :py:func:`os.system`, :py:func:`subprocess.run`, or forking) that + calls them and exits WITH errors. See also: :py:func:`test_profiling_bare_python_success` @@ -1227,7 +1266,7 @@ def test_profiling_bare_python_failure( _test_profiling_bare_python( tmp_path_factory=tmp_path_factory, ext_module=ext_module, - use_subprocess=use_subprocess, + case=case, prof_child_procs=prof_child_procs, fail=True, n=n, From ee21829851324587680325e6bb84e20ed586afb4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 11 Jul 2026 22:16:42 +0200 Subject: [PATCH 163/165] Test more edge cases (See `fable-review-pr431-2026-07-05.md::D.6` in TTsangSC#5) tests/test_child_procs/_test_child_procs_utils.py ::check_tagged_line_nhits() Added new optional parameter `comparator` so that comparisons other than `operator.eq` can be done tests/test_child_procs/test_abnormal_execution.py New test module for various edge cases (`D.6`) test_prematurely_terminated_process() Test that if a process is `.terminate()`-ed, it impairs collecting profiling data therein but doesn't affect the rest of the profiling session test_corrupted_child_stats_file() Test that if a child-process profiling-data file is corrupted, the profiling data therein are lost, but that doesn't affect the rest of the profiling session test_unwritable_purelib_path() Test if we can't write a .pth file, while we cannot set up profiling in non-forked children, it doesn't affect the rest of the profiling session Note: we omit tackling nested `kernprof` sessions (`P13.8`), because there isn't a(n easy) way to make it consistent. --- .../_test_child_procs_utils.py | 14 +- .../test_abnormal_execution.py | 315 ++++++++++++++++++ 2 files changed, 326 insertions(+), 3 deletions(-) create mode 100644 tests/test_child_procs/test_abnormal_execution.py diff --git a/tests/test_child_procs/_test_child_procs_utils.py b/tests/test_child_procs/_test_child_procs_utils.py index e888cd15..2978b85b 100644 --- a/tests/test_child_procs/_test_child_procs_utils.py +++ b/tests/test_child_procs/_test_child_procs_utils.py @@ -2049,7 +2049,7 @@ def _run_kernprof_main_in_process( machineries as best as we can, so that we can retrieve more debugging output when things do go south. Additional safety railings: - + - `preserve_targets(verify=True)` errors out if any of the patches is not reversed by :py:func:`kernprof.main` @@ -2340,7 +2340,10 @@ def _run_test_module( return proc, prof_result -def check_tagged_line_nhits(output: str, tag: str, nhits: int) -> None: +def check_tagged_line_nhits( + output: str, tag: str, nhits: T, *, + comparator: Callable[[T, int], bool] | None = None, +) -> None: """ Check the output of :py:meth:`LineStats.print` for the number of hits on the line tagged with the comment ``# GREP_MARKER[<...>]``. @@ -2355,10 +2358,15 @@ def check_tagged_line_nhits(output: str, tag: str, nhits: int) -> None: actual_nhits += int(n) except Exception: pass - ResultMismatch.compare( + compare = partial( + ResultMismatch.compare, nhits, actual_nhits, expected=f'{nhits} hit(s) on line(s) tagged with {tag!r}', ) + if comparator is None: + compare() + else: + compare(comparator=comparator) run_module = partial(_run_test_module, _run_as_module) diff --git a/tests/test_child_procs/test_abnormal_execution.py b/tests/test_child_procs/test_abnormal_execution.py new file mode 100644 index 00000000..b371c114 --- /dev/null +++ b/tests/test_child_procs/test_abnormal_execution.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +import os +import re +import site +import sys +import sysconfig +from contextlib import ExitStack +from multiprocessing import get_all_start_methods +from pathlib import Path +from stat import S_IWRITE +from types import ModuleType +from typing import Literal + +import pytest + +from ._test_child_procs_utils import ( + run_subproc, strip, check_tagged_line_nhits, +) + + +@pytest.mark.parametrize(('trigger_timeout', 'label1'), + [(True, 'terminate-proc'), (False, 'happy-path')]) +@pytest.mark.parametrize( + ('suppress_error', 'label2'), + [(True, 'suppress-error'), (False, 'propagate-error')]) +def test_prematurely_terminated_process( + tmp_path_factory: pytest.TempPathFactory, + process_test_module_object: ModuleType, + trigger_timeout: bool, + suppress_error: bool, + label1: str, label2: str, +) -> None: + """ + Check that if a :py:class:`multiprocessing.process.BaseProcess` is + explicitly and prematurely ended (e.g. ``.terminate()``-ed or + ``.kill()``-ed, profiling data therein are lost, but it doesn't + crash the session or cause further loss of profiling data. + """ + nlocal = 123 + nchild = 10 + delay = .03125 + nhits = {'INVOCATION': 1, 'LOOP': nlocal} + if trigger_timeout: + # If the timeout isn't enough, exiting the context in the test + # module would have invoked `BaseProcess.terminate()`, impairing + # the collection of profiling data + timeout: float | None = .125 # Would've needed ~ .3 s + else: + timeout = None + nhits['INVOCATION'] += 1 + nhits['LOOP'] += nchild + + tmp = tmp_path_factory.mktemp('mytemp') + test_module = tmp / 'test.py' + out_file = tmp / 'out.lprof' + test_module.write_text(strip(f''' + from __future__ import annotations + + from time import sleep + + from {process_test_module_object.__name__} import Worker + + + def my_sum(n: list[int], delay: float = 0.) -> int: + result: int = 0 # GREP_MARKER[INVOCATION] + for item in range(1, 1 + n): + sleep(delay) + result += item # GREP_MARKER[LOOP] + return result + + + def main() -> None: + my_sum({nlocal}) + try: + with Worker.new(my_sum, args=[{nchild}, {delay}]) as worker: + worker.get_result({timeout}) + except Exception: + if not {suppress_error}: + raise + + + if __name__ == '__main__': + main() + ''')) + + cmd = [ + sys.executable, '-m', 'kernprof', + '--prof-child-procs', + '--line-by-line', + '--view', + f'--prof-mod={test_module}', + f'--outfile={out_file}', + str(test_module), + ] + proc = run_subproc(cmd, capture_output=True, text=True) + + # Check: process termination per-se shouldn't cause `kernprof` to + # error out + assert bool(proc.returncode) == (trigger_timeout and not suppress_error) + # Check: collection of profiling data is as expected + for tag, num in nhits.items(): + check_tagged_line_nhits(proc.stdout, tag, num) + + +@pytest.mark.parametrize(('corrupt', 'label'), + [(True, 'with-corruption'), (False, 'no-corruption')]) +def test_corrupted_child_stats_file( + tmp_path_factory: pytest.TempPathFactory, + corrupt: bool, + label: str, +) -> None: + """ + Check that if a child's stats file is corrupted, the profiling data + of said child are lost and a warning is issued, but it doesn't crash + the session or cause further loss of profiling data. + """ + def between(range: tuple[int, int], x: int) -> bool: + return min(range) <= x <= max(range) + + nlocal = 123 + nchild = [45, 67, 89, 10] + if corrupt: + # Since we assume that there are 2+ child-profiling-stats files + # and we're corrupting one of them, we should capture SOME but + # not ALL of the child-process data + min_invocations = 2 + max_invocations = len(nchild) + min_loops = nlocal + min(nchild) + max_loops = nlocal + sum(nchild) - min(nchild) + else: + min_invocations = max_invocations = len(nchild) + 1 + min_loops = max_loops = sum(nchild) + nlocal + + tmp = tmp_path_factory.mktemp('mytemp') + test_module = tmp / 'test.py' + out_file = tmp / 'out.lprof' + test_module.write_text(strip(f''' + from __future__ import annotations + + import multiprocessing + + from line_profiler._child_process_profiling.cache import ( + LineProfilingCache, + ) + + + def my_sum(n: list[int]) -> int: + result: int = 0 # GREP_MARKER[INVOCATION] + for item in range(1, 1 + n): + result += item # GREP_MARKER[LOOP] + return result + + + def main() -> None: + # Accrue local data + my_sum({nlocal}) + + # Accrue data in child + with multiprocessing.Pool(2) as pool: + pool.map(my_sum, {nchild[0::2]!r}) + + # Boot up another pool to ensure that we at least have had + # two worker processes while handled tasks + with multiprocessing.Pool(2) as pool: + pool.map(my_sum, {nchild[1::2]!r}) + + # Retrieve one of the non-empty profiling-data files and corrupt + # it + if {corrupt}: + cache = LineProfilingCache.load() + stats, _, *__ = [ + path for path in cache._get_profiling_outfiles() + if path.stat().st_size + ] + stats.write_bytes(b'foo bar baz') + + + if __name__ == '__main__': + main() + ''')) + + cmd = [ + sys.executable, '-m', 'kernprof', + '--prof-child-procs', + '--line-by-line', + '--view', + f'--prof-mod={test_module}', + f'--outfile={out_file}', + str(test_module), + ] + + # Check: data corruption shouldn't cause `kernprof` to error out + proc = run_subproc(cmd, capture_output=True, text=True, check=True) + # Check: collection of profiling data is as expected + for tag, min_value, max_value in [ + ('INVOCATION', min_invocations, max_invocations), + ('LOOP', min_loops, max_loops), + ]: + check_tagged_line_nhits( + proc.stdout, tag, (min_value, max_value), comparator=between, + ) + # Check: warning for the corrupted file + if corrupt: + assert re.search( + r'UserWarning: .*1 file\(s\) .*cannot be loaded', proc.stderr, + ) + + +@pytest.mark.parametrize('start_method', ['spawn', 'fork', 'forkserver']) +@pytest.mark.parametrize( + ('make_unwritable', 'label'), + [(True, 'cannot-write-pth'), (False, 'can-write-pth')]) +@pytest.mark.parametrize(('n', 'nprocs'), [(100, 1)]) +def test_unwritable_purelib_path( + tmp_path_factory: pytest.TempPathFactory, + pool_test_module_object: ModuleType, + make_unwritable: bool, + start_method: Literal['spawn', 'fork', 'forkserver'], + n: int, + nprocs: int, + label: str, +) -> None: + """ + Check that if we can't write a .pth file, the profiling data of + child processes are lost, but it doesn't crash the session or cause + further loss of profiling data. + """ + class revoke_write_access: + def __init__(self, path: os.PathLike[str] | str) -> None: + self.path = Path(path) + + def __enter__(self) -> None: + self._should_restore = False + if not self.path.is_dir(): + return + if self.path.stat().st_uid != get_uid(): + return + self._should_restore = self._make_unwritable(self.path) + + def __exit__(self, *_, **__) -> None: + if self._should_restore: + self._make_writable(self.path) + + @staticmethod + def _make_writable(path: Path) -> bool: + mode = path.stat().st_mode + if mode & S_IWRITE: + return False + os.chmod(path, mode | S_IWRITE) + return True + + @staticmethod + def _make_unwritable(path: Path) -> bool: + mode = path.stat().st_mode + if not mode & S_IWRITE: + return False + os.chmod(path, mode - S_IWRITE) + return True + + def get_uid() -> int: + try: + return os.getuid() + except Exception: + # Not available on Windows, work around that by looking at + # the UID of the tempdir + return os.stat(tmp).st_uid + + # With start methods other than `fork`, we can't extend profiling + # into the worker without writing a .pth + nhits = {'LOCAL-INVOCATION': 1, 'LOCAL-LOOP': nprocs} + if start_method not in get_all_start_methods(): + pytest.skip( + f'start method {start_method!r} not available on ' + f'the platform {sys.platform!r}' + ) + if start_method == 'fork' or not make_unwritable: + nhits['LOCAL-INVOCATION'] += nprocs + nhits['LOCAL-LOOP'] += n + + tmp = tmp_path_factory.mktemp('mytemp') + module_name = pool_test_module_object.__name__ + out_file = tmp / 'out.lprof' + cmd = [ + sys.executable, '-m', 'kernprof', + '--prof-child-procs', + '--line-by-line', + '--view', + f'--prof-mod={module_name}', + f'--outfile={out_file}', + '-m', + module_name, + f'--start-method={start_method}', + '-l', str(n), + '-n', str(nprocs), + '--local', + ] + + # Check: even if we can't write a .pth, it shouldn't cause + # `kernprof` to error out + with ExitStack() as stack: + if make_unwritable: + for path in [ + site.getusersitepackages(), sysconfig.get_path('purelib'), + ]: + stack.enter_context(revoke_write_access(path)) + proc = run_subproc(cmd, capture_output=True, text=True, check=True) + # Check: collection of profiling data is as expected + for tag, num in nhits.items(): + check_tagged_line_nhits(proc.stdout, tag, num) + # Check: warning for unwritable .pth locs + if make_unwritable: + assert re.search( + r'UserWarning: .*cannot write a \.pth file', proc.stderr, + ) From 60c9f72ba3b98d2c4613c7cd352529fa1c17ec28 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Jul 2026 22:36:00 +0200 Subject: [PATCH 164/165] WIP: debug `test_unwritable_purelib_path()` tests/test_child_procs/_test_child_procs_utils.py::concat_command_line() Updated conditional check switching between the POSIX and Windows implementations tests/test_child_procs/test_abnormal_execution.py Now printing the debug logs with `kernprof --debug-log=...` test_unwritable_purelib_path() - Now skipping the test on Windows since permissions are wonky thereon - Now directly using `LineProfilingCache._enumerate_pth_installation_locations()` to avoid missing possible paths - Now also checking the group and other writable bits to ensure that we can't write to the directories TODO: see if this works on Linux... --- .../_test_child_procs_utils.py | 2 +- .../test_abnormal_execution.py | 200 +++++++++++------- 2 files changed, 129 insertions(+), 73 deletions(-) diff --git a/tests/test_child_procs/_test_child_procs_utils.py b/tests/test_child_procs/_test_child_procs_utils.py index 2978b85b..5bfd740a 100644 --- a/tests/test_child_procs/_test_child_procs_utils.py +++ b/tests/test_child_procs/_test_child_procs_utils.py @@ -1902,7 +1902,7 @@ def filter_mp_patch_summary(summary: _PatchSummary) -> dict[str, set[str]]: # it's been around since 2.4, seems stable enough, and does exactly what # is needed concat_command_line: Callable[[Sequence[str]], str] -if sys.platform == 'win32': +if sys.platform.startswith('win32'): concat_command_line = subprocess.list2cmdline else: concat_command_line = shlex.join diff --git a/tests/test_child_procs/test_abnormal_execution.py b/tests/test_child_procs/test_abnormal_execution.py index b371c114..edb940a3 100644 --- a/tests/test_child_procs/test_abnormal_execution.py +++ b/tests/test_child_procs/test_abnormal_execution.py @@ -2,23 +2,47 @@ import os import re -import site import sys -import sysconfig from contextlib import ExitStack from multiprocessing import get_all_start_methods from pathlib import Path -from stat import S_IWRITE +# Note: `S_IWRITE` is said to work on Windows but it seems wonky (see +# GitHub issue python/cpython#101675), and it doesn't seem to work on +# Linux either... +from stat import S_IWUSR, S_IWGRP, S_IWOTH +from textwrap import indent from types import ModuleType from typing import Literal import pytest +from line_profiler._child_process_profiling.cache import LineProfilingCache + from ._test_child_procs_utils import ( run_subproc, strip, check_tagged_line_nhits, ) +DEBUG = True + + +class _write_debug_log: + def __init__(self, logfile: os.PathLike[str] | str | None = None) -> None: + self.file = Path(logfile) if logfile else None + + def __enter__(self) -> None: + pass + + def __exit__(self, *_, **__) -> None: + if not (self.file and self.file.exists() and DEBUG): + return + print('-- Combined debug logs --', file=sys.stderr) + print( + indent(self.file.read_text(), ' '), end='', file=sys.stderr, + ) + print('-- End of debug logs --', file=sys.stderr) + + @pytest.mark.parametrize(('trigger_timeout', 'label1'), [(True, 'terminate-proc'), (False, 'happy-path')]) @pytest.mark.parametrize( @@ -54,6 +78,7 @@ def test_prematurely_terminated_process( tmp = tmp_path_factory.mktemp('mytemp') test_module = tmp / 'test.py' out_file = tmp / 'out.lprof' + debug_log: Path | None = None test_module.write_text(strip(f''' from __future__ import annotations @@ -91,16 +116,23 @@ def main() -> None: '--view', f'--prof-mod={test_module}', f'--outfile={out_file}', - str(test_module), ] - proc = run_subproc(cmd, capture_output=True, text=True) - - # Check: process termination per-se shouldn't cause `kernprof` to - # error out - assert bool(proc.returncode) == (trigger_timeout and not suppress_error) - # Check: collection of profiling data is as expected - for tag, num in nhits.items(): - check_tagged_line_nhits(proc.stdout, tag, num) + if DEBUG: + debug_log = tmp / 'debug.log' + cmd.append(f'--debug-log={debug_log}') + cmd.append(str(test_module)) + + with _write_debug_log(debug_log): + proc = run_subproc(cmd, capture_output=True, text=True) + + # Check: process termination per-se shouldn't cause `kernprof` + # to error out + assert bool(proc.returncode) == ( + trigger_timeout and not suppress_error + ) + # Check: collection of profiling data is as expected + for tag, num in nhits.items(): + check_tagged_line_nhits(proc.stdout, tag, num) @pytest.mark.parametrize(('corrupt', 'label'), @@ -135,6 +167,7 @@ def between(range: tuple[int, int], x: int) -> bool: tmp = tmp_path_factory.mktemp('mytemp') test_module = tmp / 'test.py' out_file = tmp / 'out.lprof' + debug_log: Path | None = None test_module.write_text(strip(f''' from __future__ import annotations @@ -187,26 +220,31 @@ def main() -> None: '--view', f'--prof-mod={test_module}', f'--outfile={out_file}', - str(test_module), ] + if DEBUG: + debug_log = tmp / 'debug.log' + cmd.append(f'--debug-log={debug_log}') + cmd.append(str(test_module)) - # Check: data corruption shouldn't cause `kernprof` to error out - proc = run_subproc(cmd, capture_output=True, text=True, check=True) - # Check: collection of profiling data is as expected - for tag, min_value, max_value in [ - ('INVOCATION', min_invocations, max_invocations), - ('LOOP', min_loops, max_loops), - ]: - check_tagged_line_nhits( - proc.stdout, tag, (min_value, max_value), comparator=between, - ) - # Check: warning for the corrupted file - if corrupt: - assert re.search( - r'UserWarning: .*1 file\(s\) .*cannot be loaded', proc.stderr, - ) - - + with _write_debug_log(debug_log): + # Check: data corruption shouldn't cause `kernprof` to error out + proc = run_subproc(cmd, capture_output=True, text=True, check=True) + # Check: collection of profiling data is as expected + for tag, nhits_range in [ + ('INVOCATION', (min_invocations, max_invocations)), + ('LOOP', (min_loops, max_loops)), + ]: + check_tagged_line_nhits( + proc.stdout, tag, nhits_range, comparator=between, + ) + # Check: warning for the corrupted file + if corrupt: + assert re.search( + r'UserWarning: .*1 file\(s\) .*cannot be loaded', proc.stderr, + ) + + +@pytest.mark.skipif(sys.platform.startswith('win32'), reason='POSIX-only test') @pytest.mark.parametrize('start_method', ['spawn', 'fork', 'forkserver']) @pytest.mark.parametrize( ('make_unwritable', 'label'), @@ -225,46 +263,59 @@ def test_unwritable_purelib_path( Check that if we can't write a .pth file, the profiling data of child processes are lost, but it doesn't crash the session or cause further loss of profiling data. + + Note: + Not being able to write a .pth file is an edge case anyway, so + it's probably alright to skip the test on Windows. """ class revoke_write_access: def __init__(self, path: os.PathLike[str] | str) -> None: self.path = Path(path) def __enter__(self) -> None: - self._should_restore = False + self._perms: int | None = None if not self.path.is_dir(): return - if self.path.stat().st_uid != get_uid(): + if self.path.stat().st_uid != os.getuid(): + return + + mode = self.mode + for bit in S_IWUSR, S_IWGRP, S_IWOTH: + self._make_unwritable(self.path, bit) + if mode == self.mode: return - self._should_restore = self._make_unwritable(self.path) + self._perms = mode + if DEBUG: + print( + f'Updated {str(self.path)!r}:', + f'{oct(mode)} -> {oct(self.mode)}', + '(disabling write)', + ) def __exit__(self, *_, **__) -> None: - if self._should_restore: - self._make_writable(self.path) + if self._perms is None: + return + mode = self.mode + os.chmod(self.path, self._perms) + if DEBUG: + print( + f'Updated {str(self.path)!r}:', + f'{oct(mode)} -> {oct(self.mode)}', + '(enabling write)', + ) @staticmethod - def _make_writable(path: Path) -> bool: + def _make_unwritable(path: Path, writable_bytes: int) -> None: mode = path.stat().st_mode - if mode & S_IWRITE: - return False - os.chmod(path, mode | S_IWRITE) - return True + mask = mode & writable_bytes + if mask: + os.chmod(path, mode - mask) - @staticmethod - def _make_unwritable(path: Path) -> bool: - mode = path.stat().st_mode - if not mode & S_IWRITE: - return False - os.chmod(path, mode - S_IWRITE) - return True + @property + def mode(self) -> int: + return self.path.stat().st_mode - def get_uid() -> int: - try: - return os.getuid() - except Exception: - # Not available on Windows, work around that by looking at - # the UID of the tempdir - return os.stat(tmp).st_uid + get_pth_locs = LineProfilingCache._enumerate_pth_installation_locations # With start methods other than `fork`, we can't extend profiling # into the worker without writing a .pth @@ -281,6 +332,7 @@ def get_uid() -> int: tmp = tmp_path_factory.mktemp('mytemp') module_name = pool_test_module_object.__name__ out_file = tmp / 'out.lprof' + debug_log: Path | None = None cmd = [ sys.executable, '-m', 'kernprof', '--prof-child-procs', @@ -288,28 +340,32 @@ def get_uid() -> int: '--view', f'--prof-mod={module_name}', f'--outfile={out_file}', + ] + if DEBUG: + debug_log = tmp / 'debug.log' + cmd.append(f'--debug-log={debug_log}') + cmd.extend([ '-m', module_name, f'--start-method={start_method}', '-l', str(n), '-n', str(nprocs), '--local', - ] - - # Check: even if we can't write a .pth, it shouldn't cause - # `kernprof` to error out - with ExitStack() as stack: + ]) + + with _write_debug_log(debug_log): + # Check: even if we can't write a .pth, it shouldn't cause + # `kernprof` to error out + with ExitStack() as stack: + if make_unwritable: + for path in get_pth_locs(): + stack.enter_context(revoke_write_access(path)) + proc = run_subproc(cmd, capture_output=True, text=True, check=True) + # Check: collection of profiling data is as expected + for tag, num in nhits.items(): + check_tagged_line_nhits(proc.stdout, tag, num) + # Check: warning for unwritable .pth locs if make_unwritable: - for path in [ - site.getusersitepackages(), sysconfig.get_path('purelib'), - ]: - stack.enter_context(revoke_write_access(path)) - proc = run_subproc(cmd, capture_output=True, text=True, check=True) - # Check: collection of profiling data is as expected - for tag, num in nhits.items(): - check_tagged_line_nhits(proc.stdout, tag, num) - # Check: warning for unwritable .pth locs - if make_unwritable: - assert re.search( - r'UserWarning: .*cannot write a \.pth file', proc.stderr, - ) + assert re.search( + r'UserWarning: .*cannot write a \.pth file', proc.stderr, + ) From a7234769d134d6e3e1a281f03ff8b5250be56f95 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Jul 2026 23:34:57 +0200 Subject: [PATCH 165/165] Fix (?) process-table retrieval on Windows line_profiler/_child_process_profiling/_retrieve_pids.py ::get_active_processes_windows() - Removed erroneous check on the header line of the CSV data (column-header strings can be quoted) - Now wrapping errors raised during parsing in a `RuntimeError` which shows the first lines in the `tasklist` output to help with debugging --- .../_retrieve_pids.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/line_profiler/_child_process_profiling/_retrieve_pids.py b/line_profiler/_child_process_profiling/_retrieve_pids.py index 27c76af2..d4c82274 100644 --- a/line_profiler/_child_process_profiling/_retrieve_pids.py +++ b/line_profiler/_child_process_profiling/_retrieve_pids.py @@ -6,6 +6,7 @@ import sys from collections.abc import Callable, Collection from shutil import which +from textwrap import indent from typing import NoReturn @@ -55,18 +56,19 @@ def get_active_processes_windows() -> set[int]: # CSV formatted output cmd = [_find_executable('tasklist'), '/fo', 'csv'] tasklist = subprocess.run(cmd, capture_output=True, text=True, check=True) - header_line, *rows = tasklist.stdout.splitlines() - headers = {h.strip() for h in header_line.lower().split(',')} + rows = tasklist.stdout.splitlines() # XXX: this is *probably* locale-safe given that even JP has PID as - # a header... but who knows - if 'pid' not in headers: - raise RuntimeError( - 'cannot find PID column in `tasklist` output, ' - f'got CSV header: {header_line!r}' - ) - return { - int(row['pid']) for row in csv.DictReader([header_line.lower(), *rows]) - } + # a header... but who knows. + # Might have to test in an RTL language... + try: + return {int(row['pid']) for row in csv.DictReader(rows)} + except Exception as e: # nocover + n = 10 + head = '\n'.join(rows[:n]) + raise RuntimeError( # Context for debugging + 'Error reading PIDs from `tasklist` output ' + f'(first {n} line(s)):\n{indent(head, " ")}' + ) from e def get_active_processes_not_supported() -> NoReturn: # nocover