From f69b54ca201ac5a79913000a138e7ce3ef2e65bc Mon Sep 17 00:00:00 2001 From: Test User Date: Sat, 4 Jul 2026 18:11:53 -0400 Subject: [PATCH 1/2] Streamline README and move long usage notes into docs --- README.rst | 422 +++------------------------ docs/source/index.rst | 8 +- docs/source/manual/index.rst | 11 + docs/source/manual/legacy_readme.rst | 389 ++++++++++++++++++++++++ 4 files changed, 448 insertions(+), 382 deletions(-) create mode 100644 docs/source/manual/index.rst create mode 100644 docs/source/manual/legacy_readme.rst diff --git a/README.rst b/README.rst index 5f2030b6..b0c41f82 100644 --- a/README.rst +++ b/README.rst @@ -23,8 +23,9 @@ This fork is the official continuation of the project. ``line_profiler`` is a module for doing line-by-line profiling of functions. -kernprof is a convenient script for running either ``line_profiler`` or the Python -standard library's cProfile or profile modules, depending on what is available. +Use it when function-level profiling identifies a slow function, but you need +to see which individual source lines account for the time. ``kernprof`` is the +command-line helper included with the package. They are available under a `BSD license`_. @@ -33,40 +34,6 @@ They are available under a `BSD license`_. .. contents:: -Quick Start (Modern) -==================== - -This guide is for versions of line profiler starting at ``4.1.0``. - -To profile a python script: - -* Install line_profiler: ``pip install line_profiler``. - -* In the relevant file(s), import line profiler and decorate function(s) you - want to profile with ``@line_profiler.profile``. - -* Set the environment variable ``LINE_PROFILE=1`` and run your script as normal. - When the script ends a summary of profile results, files written to disk, and - instructions for inspecting details will be written to stdout. - -For more details and a short tutorial see `Line Profiler Basic Usage `_. - - -Quick Start (Legacy) -==================== - -This section is the original quick-start guide, and may eventually be removed -from the README. This will work with current and older (pre ``4.1.0``) versions -of line profiler. - -To profile a python script: - -* Install line_profiler: ``pip install line_profiler``. - -* Decorate function(s) you want to profile with @profile. The decorator will be made automatically available on run. - -* Run ``kernprof -lv script_to_profile.py``. - Installation ============ @@ -74,381 +41,76 @@ Releases of ``line_profiler`` can be installed using pip:: $ pip install line_profiler -Installation while ensuring a compatible IPython version can also be installed using pip:: - - $ pip install line_profiler[ipython] - -To check out the development sources, you can use Git_:: - - $ git clone https://github.com/pyutils/line_profiler.git - -You may also download source tarballs of any snapshot from that URL. - -Source releases will require a C compiler in order to build `line_profiler`. -In addition, git checkouts will also require Cython. Source releases -on PyPI should contain the pregenerated C sources, so Cython should not be -required in that case. - -``kernprof`` is a single-file pure Python script and does not require -a compiler. If you wish to use it to run cProfile and not line-by-line -profiling, you may copy it to a directory on your ``PATH`` manually and avoid -trying to build any C extensions. - -As of 2021-06-04 Linux (x86_64 and i686), OSX (10_9_x86_64), and Win32 (win32, -and amd64) binaries are available on pypi. - -The last version of line profiler to support Python 2.7 was 3.1.0 and the last -version to support Python 3.5 was 3.3.1. - -.. _git: http://git-scm.com/ -.. _Cython: http://www.cython.org -.. _build and install: http://docs.python.org/install/index.html - - -line_profiler -============= - -The current profiling tools supported in Python only time -function calls. This is a good first step for locating hotspots in one's program -and is frequently all one needs to do to optimize the program. However, -sometimes the cause of the hotspot is actually a single line in the function, -and that line may not be obvious from just reading the source code. These cases -are particularly frequent in scientific computing. Functions tend to be larger -(sometimes because of legitimate algorithmic complexity, sometimes because the -programmer is still trying to write FORTRAN code), and a single statement -without function calls can trigger lots of computation when using libraries like -numpy. cProfile only times explicit function calls, not special methods called -because of syntax. Consequently, a relatively slow numpy operation on large -arrays like this, :: - - a[large_index_array] = some_other_large_array - -is a hotspot that never gets broken out by cProfile because there is no explicit -function call in that statement. - -LineProfiler can be given functions to profile, and it will time the execution -of each individual line inside those functions. In a typical workflow, one only -cares about line timings of a few functions because wading through the results -of timing every single line of code would be overwhelming. However, LineProfiler -does need to be explicitly told what functions to profile. The easiest way to -get started is to use the ``kernprof`` script. :: - - $ kernprof -l script_to_profile.py - -``kernprof`` will create an instance of LineProfiler and insert it into the -``__builtins__`` namespace with the name ``profile``. It has been written to be -used as a decorator, so in your script, you decorate the functions you want -to profile with @profile. :: - - @profile - def slow_function(a, b, c): - ... - -The default behavior of ``kernprof`` is to put the results into a binary file -script_to_profile.py.lprof . You can tell ``kernprof`` to immediately view the -formatted results at the terminal with the [-v/--view] option. Otherwise, you -can view the results later like so:: - - $ python -m line_profiler script_to_profile.py.lprof - -For example, here are the results of profiling a single function from -a decorated version of the pystone.py benchmark (the first two lines are output -from ``pystone.py``, not ``kernprof``):: - - Pystone(1.1) time for 50000 passes = 2.48 - This machine benchmarks at 20161.3 pystones/second - Wrote profile results to pystone.py.lprof - Timer unit: 1e-06 s - - File: pystone.py - Function: Proc2 at line 149 - Total time: 0.606656 s - - Line # Hits Time Per Hit % Time Line Contents - ============================================================== - 149 @profile - 150 def Proc2(IntParIO): - 151 50000 82003 1.6 13.5 IntLoc = IntParIO + 10 - 152 50000 63162 1.3 10.4 while 1: - 153 50000 69065 1.4 11.4 if Char1Glob == 'A': - 154 50000 66354 1.3 10.9 IntLoc = IntLoc - 1 - 155 50000 67263 1.3 11.1 IntParIO = IntLoc - IntGlob - 156 50000 65494 1.3 10.8 EnumLoc = Ident1 - 157 50000 68001 1.4 11.2 if EnumLoc == Ident1: - 158 50000 63739 1.3 10.5 break - 159 50000 61575 1.2 10.1 return IntParIO - - -The source code of the function is printed with the timing information for each -line. There are six columns of information. - - * Line #: The line number in the file. - - * Hits: The number of times that line was executed. - - * Time: The total amount of time spent executing the line in the timer's - units. In the header information before the tables, you will see a line - "Timer unit:" giving the conversion factor to seconds. It may be different - on different systems. - - * Per Hit: The average amount of time spent executing the line once in the - timer's units. - - * % Time: The percentage of time spent on that line relative to the total - amount of recorded time spent in the function. - - * Line Contents: The actual source code. Note that this is always read from - disk when the formatted results are viewed, *not* when the code was - executed. If you have edited the file in the meantime, the lines will not - match up, and the formatter may not even be able to locate the function - for display. - -If you are using IPython, there is an implementation of an %lprun magic command -which will let you specify functions to profile and a statement to execute. It -will also add its LineProfiler instance into the __builtins__, but typically, -you would not use it like that. - -For IPython 0.11+, you can install it by editing the IPython configuration file -``~/.ipython/profile_default/ipython_config.py`` to add the ``'line_profiler'`` -item to the extensions list:: - - c.TerminalIPythonApp.extensions = [ - 'line_profiler', - ] - -Or explicitly call:: - - %load_ext line_profiler - -To get usage help for %lprun, use the standard IPython help mechanism:: - - In [1]: %lprun? - -These two methods are expected to be the most frequent user-level ways of using -LineProfiler and will usually be the easiest. However, if you are building other -tools with LineProfiler, you will need to use the API. There are two ways to -inform LineProfiler of functions to profile: you can pass them as arguments to -the constructor or use the ``add_function(f)`` method after instantiation. :: +Current releases require Python 3.10 or newer. Older Python versions require +older ``line_profiler`` releases. - profile = LineProfiler(f, g) - profile.add_function(h) +Installation while ensuring a compatible IPython version can also be installed +using pip:: -LineProfiler has the same ``run()``, ``runctx()``, and ``runcall()`` methods as -cProfile.Profile as well as ``enable()`` and ``disable()``. It should be noted, -though, that ``enable()`` and ``disable()`` are not entirely safe when nested. -Nesting is common when using LineProfiler as a decorator. In order to support -nesting, use ``enable_by_count()`` and ``disable_by_count()``. These functions will -increment and decrement a counter and only actually enable or disable the -profiler when the count transitions from or to 0. - -After profiling, the ``dump_stats(filename)`` method will pickle the results out -to the given file. ``print_stats([stream])`` will print the formatted results to -sys.stdout or whatever stream you specify. ``get_stats()`` will return LineStats -object, which just holds two attributes: a dictionary containing the results and -the timer unit. - - -kernprof -======== - -``kernprof`` also works with cProfile, its third-party incarnation lsprof, or the -pure-Python profile module depending on what is available. It has a few main -features: - - * Encapsulation of profiling concerns. You do not have to modify your script - in order to initiate profiling and save the results. Unless if you want to - use the advanced __builtins__ features, of course. - - * Robust script execution. Many scripts require things like __name__, - __file__, and sys.path to be set relative to it. A naive approach at - encapsulation would just use execfile(), but many scripts which rely on - that information will fail. kernprof will set those variables correctly - before executing the script. - - * Easy executable location. If you are profiling an application installed on - your PATH, you can just give the name of the executable. If kernprof does - not find the given script in the current directory, it will search your - PATH for it. - - * Inserting the profiler into __builtins__. Sometimes, you just want to - profile a small part of your code. With the [-b/--builtin] argument, the - Profiler will be instantiated and inserted into your __builtins__ with the - name "profile". Like LineProfiler, it may be used as a decorator, or - enabled/disabled with ``enable_by_count()`` and ``disable_by_count()``, or - even as a context manager with the "with profile:" statement. - - * Pre-profiling setup. With the [-s/--setup] option, you can provide - a script which will be executed without profiling before executing the - main script. This is typically useful for cases where imports of large - libraries like wxPython or VTK are interfering with your results. If you - can modify your source code, the __builtins__ approach may be - easier. - -The results of profile script_to_profile.py will be written to -script_to_profile.py.prof by default. It will be a typical marshalled file that -can be read with pstats.Stats(). They may be interactively viewed with the -command:: - - $ python -m pstats script_to_profile.py.prof - - -Such files may also be viewed with graphical tools. A list of 3rd party tools -built on ``cProfile`` or ``line_profiler`` are as follows: - -* `pyprof2calltree `_: converts profiling data to a format - that can be visualized using kcachegrind_ (linux only), wincachegrind_ - (windows only, unmaintained), or qcachegrind_. - -* `Line Profiler GUI `_: Qt GUI for line_profiler. - -* `SnakeViz `_: A web viewer for Python profiling data. - -* `SnakeRunner `_: A fork of RunSnakeRun_, ported to Python 3. - -* `Pycharm plugin `_: A PyCharm plugin for line_profiler. - -* `Spyder plugin `_: A plugin to run line_profiler from within the Spyder IDE. - -* `pprof `_: A render web report for ``line_profiler``. - -.. _qcachegrind: https://sourceforge.net/projects/qcachegrindwin/ -.. _kcachegrind: https://kcachegrind.github.io/html/Home.html -.. _wincachegrind: https://github.com/ceefour/wincachegrind -.. _pyprof2calltree: http://pypi.python.org/pypi/pyprof2calltree/ -.. _SnakeViz: https://github.com/jiffyclub/snakeviz/ -.. _SnakeRunner: https://github.com/venthur/snakerunner -.. _RunSnakeRun: https://pypi.org/project/RunSnakeRun/ -.. _qt_profiler_gui: https://github.com/Nodd/lineprofilergui -.. _pycharm_line_profiler_plugin: https://plugins.jetbrains.com/plugin/16536-line-profiler -.. _spyder_line_profiler_plugin: https://github.com/spyder-ide/spyder-line-profiler -.. _web_profiler_ui: https://github.com/mirecl/pprof - - -Related Work -============ - -Check out these other Python profilers: - -* `Scalene `_: A CPU+GPU+memory sampling based profiler. - -* `PyInstrument `_: A call stack profiler. - -* `Yappi `_: A tracing profiler that is multithreading, asyncio and gevent aware. - -* `profile / cProfile `_: The builtin profile module. - -* `timeit `_: The builtin timeit module for profiling single statements. - -* `timerit `_: A multi-statements alternative to the builtin ``timeit`` module. - -Frequently Asked Questions -========================== + $ pip install line_profiler[ipython] -* Why the name "kernprof"? +Source installs may require a C compiler. Git checkouts also require Cython. +Wheels are published for common platforms. If no wheel is available for your +platform or Python version, installation may build from source. - I didn't manage to come up with a meaningful name, so I named it after - myself. -* The line-by-line timings don't add up when one profiled function calls - another. What's up with that? +Quick Start +=========== - Let's say you have function F() calling function G(), and you are using - LineProfiler on both. The total time reported for G() is less than the time - reported on the line in F() that calls G(). The reason is that I'm being - reasonably clever (and possibly too clever) in recording the times. - Basically, I try to prevent recording the time spent inside LineProfiler - doing all of the bookkeeping for each line. Each time Python's tracing - facility issues a line event (which happens just before a line actually gets - executed), LineProfiler will find two timestamps, one at the beginning - before it does anything (t_begin) and one as close to the end as possible - (t_end). Almost all of the overhead of LineProfiler's data structures - happens in between these two times. +The recommended way to use ``line_profiler`` is to import ``profile`` and enable +profiling with the ``LINE_PROFILE`` environment variable. - When a line event comes in, LineProfiler finds the function it belongs to. - If it's the first line in the function, we record the line number and - *t_end* associated with the function. The next time we see a line event - belonging to that function, we take t_begin of the new event and subtract - the old t_end from it to find the amount of time spent in the old line. Then - we record the new t_end as the active line for this function. This way, we - are removing most of LineProfiler's overhead from the results. Well almost. - When one profiled function F calls another profiled function G, the line in - F that calls G basically records the total time spent executing the line, - which includes the time spent inside the profiler while inside G. +To profile a python script: - The first time this question was asked, the questioner had the G() function - call as part of a larger expression, and he wanted to try to estimate how - much time was being spent in the function as opposed to the rest of the - expression. My response was that, even if I could remove the effect, it - might still be misleading. G() might be called elsewhere, not just from the - relevant line in F(). The workaround would be to modify the code to split it - up into two lines, one which just assigns the result of G() to a temporary - variable and the other with the rest of the expression. +* Install line_profiler: ``pip install line_profiler``. - I am open to suggestions on how to make this more robust. Or simple - admonitions against trying to be clever. +* In the relevant file(s), import ``profile`` and decorate function(s) you want + to profile:: -* Why do my list comprehensions have so many hits when I use the LineProfiler? + from line_profiler import profile - LineProfiler records the line with the list comprehension once for each - iteration of the list comprehension. -* Why is kernprof distributed with line_profiler? It works with just cProfile, - right? + @profile + def slow_function(): + ... - Partly because kernprof.py is essential to using line_profiler effectively, - but mostly because I'm lazy and don't want to maintain the overhead of two - projects for modules as small as these. However, kernprof.py is - a standalone, pure Python script that can be used to do function profiling - with just the Python standard library. You may grab it and install it by - itself without ``line_profiler``. +* Set the environment variable ``LINE_PROFILE=1`` and run your script as normal. + When the script ends a summary of profile results, files written to disk, and + instructions for inspecting details will be written to stdout. -* Do I need a C compiler to build ``line_profiler``? kernprof.py? +For more details and a short tutorial see `Line Profiler Basic Usage `_. - You do need a C compiler for line_profiler. kernprof.py is a pure Python - script and can be installed separately, though. -* Do I need Cython to build ``line_profiler``? +Older ``kernprof`` workflow +=========================== - Wheels for supported versions of Python are available on PyPI and support - linux, osx, and windows for x86-64 architectures. Linux additionally ships - with i686 wheels for manylinux and musllinux. If you have a different CPU - architecture, or an unsupported Python version, then you will need to build - from source. +The older ``kernprof -l`` workflow is still supported, but it is no longer the +main README quick start. See `Usage Notes and FAQ `_ +for the previous README material covering ``kernprof``, IPython, lower-level API +usage, related tools, and FAQ entries. -* What version of Python do I need? +The short version is:: - Both ``line_profiler`` and ``kernprof`` have been tested with Python 3.6-3.11. - Older versions of ``line_profiler`` support older versions of Python. + $ kernprof -lv script_to_profile.py -To Do -===== +Documentation +============= -cProfile uses a neat "rotating trees" data structure to minimize the overhead of -looking up and recording entries. LineProfiler uses Python dictionaries and -extension objects thanks to Cython. This mostly started out as a prototype that -I wanted to play with as quickly as possible, so I passed on stealing the -rotating trees for now. As usual, I got it working, and it seems to have -acceptable performance, so I am much less motivated to use a different strategy -now. Maybe later. Contributions accepted! +* `Full documentation `_ +* `Examples `_ +* `Usage Notes and FAQ `_ +* `Changelog `_ Bugs and Such ============= -Bugs and pull requested can be submitted on GitHub_. +Bugs and pull requests can be submitted on GitHub_. .. _GitHub: https://github.com/pyutils/line_profiler -Changes -======= - -See `CHANGELOG`_. - -.. _CHANGELOG: CHANGELOG.rst - - .. |CircleCI| image:: https://circleci.com/gh/pyutils/line_profiler.svg?style=svg :target: https://circleci.com/gh/pyutils/line_profiler .. |Travis| image:: https://img.shields.io/travis/pyutils/line_profiler/master.svg?label=Travis%20CI diff --git a/docs/source/index.rst b/docs/source/index.rst index 034f16dd..e34fd57d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -2,6 +2,12 @@ .. automodule:: line_profiler.__init__ :show-inheritance: +.. toctree:: + :maxdepth: 2 + :caption: User Guide + + manual/index + .. toctree:: :maxdepth: 8 :caption: Package Layout @@ -11,8 +17,6 @@ auto/line_profiler.explicit_profiler auto/kernprof - manual/examples/index - Indices and tables ================== diff --git a/docs/source/manual/index.rst b/docs/source/manual/index.rst new file mode 100644 index 00000000..a0db0773 --- /dev/null +++ b/docs/source/manual/index.rst @@ -0,0 +1,11 @@ +User Manual +=========== + +Longer usage notes, examples, and FAQ entries live here so the top-level +README can stay short. + +.. toctree:: + :maxdepth: 2 + + examples/index + legacy_readme diff --git a/docs/source/manual/legacy_readme.rst b/docs/source/manual/legacy_readme.rst new file mode 100644 index 00000000..dba5734c --- /dev/null +++ b/docs/source/manual/legacy_readme.rst @@ -0,0 +1,389 @@ +Usage Notes and FAQ +=================== + +This page preserves the longer usage notes that historically lived in the +top-level ``README.rst``. Keeping this material in the manual lets the README +stay short without dropping useful explanations, older workflows, API notes, +related tools, or FAQ entries. + +Command-Line Quick Start +------------------------ + +This workflow uses the ``kernprof`` command-line runner. It works with current +versions and remains useful when you want to run a script under a profiling +wrapper. + +To profile a python script: + +* Install line_profiler: ``pip install line_profiler``. + +* Decorate function(s) you want to profile with @profile. The decorator will be made automatically available on run. + +* Run ``kernprof -lv script_to_profile.py``. + +For more details, see the `line_profiler documentation `_. + +Installation +------------ + +Releases of ``line_profiler`` can be installed using pip:: + + $ pip install line_profiler + +Installation while ensuring a compatible IPython version can also be installed using pip:: + + $ pip install line_profiler[ipython] + +To check out the development sources, you can use Git_:: + + $ git clone https://github.com/pyutils/line_profiler.git + +You may also download source tarballs of any snapshot from that URL. + +Source releases will require a C compiler in order to build `line_profiler`. +In addition, git checkouts will also require Cython. Source releases +on PyPI should contain the pregenerated C sources, so Cython should not be +required in that case. + +``kernprof`` is a single-file pure Python script and does not require +a compiler. If you wish to use it to run cProfile and not line-by-line +profiling, you may copy it to a directory on your ``PATH`` manually and avoid +trying to build any C extensions. + +Wheels are published for common platforms. If no wheel is available for your +platform or Python version, installation may build from source. + +The last version of line profiler to support Python 2.7 was 3.1.0 and the last +version to support Python 3.5 was 3.3.1. + +.. _git: http://git-scm.com/ +.. _Cython: http://www.cython.org +.. _build and install: http://docs.python.org/install/index.html + + +line_profiler +------------- + +The current profiling tools supported in Python only time +function calls. This is a good first step for locating hotspots in one's program +and is frequently all one needs to do to optimize the program. However, +sometimes the cause of the hotspot is actually a single line in the function, +and that line may not be obvious from just reading the source code. These cases +are particularly frequent in scientific computing. Functions tend to be larger +(sometimes because of legitimate algorithmic complexity, sometimes because the +programmer is still trying to write FORTRAN code), and a single statement +without function calls can trigger lots of computation when using libraries like +numpy. cProfile only times explicit function calls, not special methods called +because of syntax. Consequently, a relatively slow numpy operation on large +arrays like this, :: + + a[large_index_array] = some_other_large_array + +is a hotspot that never gets broken out by cProfile because there is no explicit +function call in that statement. + +LineProfiler can be given functions to profile, and it will time the execution +of each individual line inside those functions. In a typical workflow, one only +cares about line timings of a few functions because wading through the results +of timing every single line of code would be overwhelming. However, LineProfiler +does need to be explicitly told what functions to profile. The easiest way to +get started is to use the ``kernprof`` script. :: + + $ kernprof -l script_to_profile.py + +``kernprof`` will create an instance of LineProfiler and insert it into the +``__builtins__`` namespace with the name ``profile``. It has been written to be +used as a decorator, so in your script, you decorate the functions you want +to profile with @profile. :: + + @profile + def slow_function(a, b, c): + ... + +The default behavior of ``kernprof`` is to put the results into a binary file +script_to_profile.py.lprof . You can tell ``kernprof`` to immediately view the +formatted results at the terminal with the [-v/--view] option. Otherwise, you +can view the results later like so:: + + $ python -m line_profiler script_to_profile.py.lprof + +For example, here are the results of profiling a single function from +a decorated version of the pystone.py benchmark (the first two lines are output +from ``pystone.py``, not ``kernprof``):: + + Pystone(1.1) time for 50000 passes = 2.48 + This machine benchmarks at 20161.3 pystones/second + Wrote profile results to pystone.py.lprof + Timer unit: 1e-06 s + + File: pystone.py + Function: Proc2 at line 149 + Total time: 0.606656 s + + Line # Hits Time Per Hit % Time Line Contents + ============================================================== + 149 @profile + 150 def Proc2(IntParIO): + 151 50000 82003 1.6 13.5 IntLoc = IntParIO + 10 + 152 50000 63162 1.3 10.4 while 1: + 153 50000 69065 1.4 11.4 if Char1Glob == 'A': + 154 50000 66354 1.3 10.9 IntLoc = IntLoc - 1 + 155 50000 67263 1.3 11.1 IntParIO = IntLoc - IntGlob + 156 50000 65494 1.3 10.8 EnumLoc = Ident1 + 157 50000 68001 1.4 11.2 if EnumLoc == Ident1: + 158 50000 63739 1.3 10.5 break + 159 50000 61575 1.2 10.1 return IntParIO + + +The source code of the function is printed with the timing information for each +line. There are six columns of information. + + * Line #: The line number in the file. + + * Hits: The number of times that line was executed. + + * Time: The total amount of time spent executing the line in the timer's + units. In the header information before the tables, you will see a line + "Timer unit:" giving the conversion factor to seconds. It may be different + on different systems. + + * Per Hit: The average amount of time spent executing the line once in the + timer's units. + + * % Time: The percentage of time spent on that line relative to the total + amount of recorded time spent in the function. + + * Line Contents: The actual source code. Note that this is always read from + disk when the formatted results are viewed, *not* when the code was + executed. If you have edited the file in the meantime, the lines will not + match up, and the formatter may not even be able to locate the function + for display. + +If you are using IPython, there is an implementation of an %lprun magic command +which will let you specify functions to profile and a statement to execute. It +will also add its LineProfiler instance into the __builtins__, but typically, +you would not use it like that. + +For IPython 0.11+, you can install it by editing the IPython configuration file +``~/.ipython/profile_default/ipython_config.py`` to add the ``'line_profiler'`` +item to the extensions list:: + + c.TerminalIPythonApp.extensions = [ + 'line_profiler', + ] + +Or explicitly call:: + + %load_ext line_profiler + +To get usage help for %lprun, use the standard IPython help mechanism:: + + In [1]: %lprun? + +These two methods are expected to be the most frequent user-level ways of using +LineProfiler and will usually be the easiest. However, if you are building other +tools with LineProfiler, you will need to use the API. There are two ways to +inform LineProfiler of functions to profile: you can pass them as arguments to +the constructor or use the ``add_function(f)`` method after instantiation. :: + + profile = LineProfiler(f, g) + profile.add_function(h) + +LineProfiler has the same ``run()``, ``runctx()``, and ``runcall()`` methods as +cProfile.Profile as well as ``enable()`` and ``disable()``. It should be noted, +though, that ``enable()`` and ``disable()`` are not entirely safe when nested. +Nesting is common when using LineProfiler as a decorator. In order to support +nesting, use ``enable_by_count()`` and ``disable_by_count()``. These functions will +increment and decrement a counter and only actually enable or disable the +profiler when the count transitions from or to 0. + +After profiling, the ``dump_stats(filename)`` method will pickle the results out +to the given file. ``print_stats([stream])`` will print the formatted results to +sys.stdout or whatever stream you specify. ``get_stats()`` will return LineStats +object, which just holds two attributes: a dictionary containing the results and +the timer unit. + + +kernprof +-------- + +``kernprof`` also works with cProfile, its third-party incarnation lsprof, or the +pure-Python profile module depending on what is available. It has a few main +features: + + * Encapsulation of profiling concerns. You do not have to modify your script + in order to initiate profiling and save the results. Unless if you want to + use the advanced __builtins__ features, of course. + + * Robust script execution. Many scripts require things like __name__, + __file__, and sys.path to be set relative to it. A naive approach at + encapsulation would just use execfile(), but many scripts which rely on + that information will fail. kernprof will set those variables correctly + before executing the script. + + * Easy executable location. If you are profiling an application installed on + your PATH, you can just give the name of the executable. If kernprof does + not find the given script in the current directory, it will search your + PATH for it. + + * Inserting the profiler into __builtins__. Sometimes, you just want to + profile a small part of your code. With the [-b/--builtin] argument, the + Profiler will be instantiated and inserted into your __builtins__ with the + name "profile". Like LineProfiler, it may be used as a decorator, or + enabled/disabled with ``enable_by_count()`` and ``disable_by_count()``, or + even as a context manager with the "with profile:" statement. + + * Pre-profiling setup. With the [-s/--setup] option, you can provide + a script which will be executed without profiling before executing the + main script. This is typically useful for cases where imports of large + libraries like wxPython or VTK are interfering with your results. If you + can modify your source code, the __builtins__ approach may be + easier. + +The results of profile script_to_profile.py will be written to +script_to_profile.py.prof by default. It will be a typical marshalled file that +can be read with pstats.Stats(). They may be interactively viewed with the +command:: + + $ python -m pstats script_to_profile.py.prof + + +Such files may also be viewed with graphical tools. A list of 3rd party tools +built on ``cProfile`` or ``line_profiler`` are as follows: + +* `pyprof2calltree `_: converts profiling data to a format + that can be visualized using kcachegrind_ (linux only), wincachegrind_ + (windows only, unmaintained), or qcachegrind_. + +* `Line Profiler GUI `_: Qt GUI for line_profiler. + +* `SnakeViz `_: A web viewer for Python profiling data. + +* `SnakeRunner `_: A fork of RunSnakeRun_, ported to Python 3. + +* `Pycharm plugin `_: A PyCharm plugin for line_profiler. + +* `Spyder plugin `_: A plugin to run line_profiler from within the Spyder IDE. + +* `pprof `_: A render web report for ``line_profiler``. + +.. _qcachegrind: https://sourceforge.net/projects/qcachegrindwin/ +.. _kcachegrind: https://kcachegrind.github.io/html/Home.html +.. _wincachegrind: https://github.com/ceefour/wincachegrind +.. _pyprof2calltree: http://pypi.python.org/pypi/pyprof2calltree/ +.. _SnakeViz: https://github.com/jiffyclub/snakeviz/ +.. _SnakeRunner: https://github.com/venthur/snakerunner +.. _RunSnakeRun: https://pypi.org/project/RunSnakeRun/ +.. _qt_profiler_gui: https://github.com/Nodd/lineprofilergui +.. _pycharm_line_profiler_plugin: https://plugins.jetbrains.com/plugin/16536-line-profiler +.. _spyder_line_profiler_plugin: https://github.com/spyder-ide/spyder-line-profiler +.. _web_profiler_ui: https://github.com/mirecl/pprof + + +Related Work +------------ + +Check out these other Python profilers: + +* `Scalene `_: A CPU+GPU+memory sampling based profiler. + +* `PyInstrument `_: A call stack profiler. + +* `Yappi `_: A tracing profiler that is multithreading, asyncio and gevent aware. + +* `profile / cProfile `_: The builtin profile module. + +* `timeit `_: The builtin timeit module for profiling single statements. + +* `timerit `_: A multi-statements alternative to the builtin ``timeit`` module. + +Frequently Asked Questions +-------------------------- + +* Why the name "kernprof"? + + I didn't manage to come up with a meaningful name, so I named it after + myself. + +* The line-by-line timings don't add up when one profiled function calls + another. What's up with that? + + Let's say you have function F() calling function G(), and you are using + LineProfiler on both. The total time reported for G() is less than the time + reported on the line in F() that calls G(). The reason is that I'm being + reasonably clever (and possibly too clever) in recording the times. + Basically, I try to prevent recording the time spent inside LineProfiler + doing all of the bookkeeping for each line. Each time Python's tracing + facility issues a line event (which happens just before a line actually gets + executed), LineProfiler will find two timestamps, one at the beginning + before it does anything (t_begin) and one as close to the end as possible + (t_end). Almost all of the overhead of LineProfiler's data structures + happens in between these two times. + + When a line event comes in, LineProfiler finds the function it belongs to. + If it's the first line in the function, we record the line number and + *t_end* associated with the function. The next time we see a line event + belonging to that function, we take t_begin of the new event and subtract + the old t_end from it to find the amount of time spent in the old line. Then + we record the new t_end as the active line for this function. This way, we + are removing most of LineProfiler's overhead from the results. Well almost. + When one profiled function F calls another profiled function G, the line in + F that calls G basically records the total time spent executing the line, + which includes the time spent inside the profiler while inside G. + + The first time this question was asked, the questioner had the G() function + call as part of a larger expression, and he wanted to try to estimate how + much time was being spent in the function as opposed to the rest of the + expression. My response was that, even if I could remove the effect, it + might still be misleading. G() might be called elsewhere, not just from the + relevant line in F(). The workaround would be to modify the code to split it + up into two lines, one which just assigns the result of G() to a temporary + variable and the other with the rest of the expression. + + I am open to suggestions on how to make this more robust. Or simple + admonitions against trying to be clever. + +* Why do my list comprehensions have so many hits when I use the LineProfiler? + + LineProfiler records the line with the list comprehension once for each + iteration of the list comprehension. + +* Why is kernprof distributed with line_profiler? It works with just cProfile, + right? + + Partly because kernprof.py is essential to using line_profiler effectively, + but mostly because I'm lazy and don't want to maintain the overhead of two + projects for modules as small as these. However, kernprof.py is + a standalone, pure Python script that can be used to do function profiling + with just the Python standard library. You may grab it and install it by + itself without ``line_profiler``. + +* Do I need a C compiler to build ``line_profiler``? kernprof.py? + + You do need a C compiler for line_profiler. kernprof.py is a pure Python + script and can be installed separately, though. + +* Do I need Cython to build ``line_profiler``? + + Wheels for supported versions of Python are available on PyPI and support + linux, osx, and windows for x86-64 architectures. Linux additionally ships + with i686 wheels for manylinux and musllinux. If you have a different CPU + architecture, or an unsupported Python version, then you will need to build + from source. + +* What version of Python do I need? + + Current versions support Python 3.10 and newer. Older versions of + ``line_profiler`` support older versions of Python. + + +To Do +----- + +cProfile uses a neat "rotating trees" data structure to minimize the overhead of +looking up and recording entries. LineProfiler uses Python dictionaries and +extension objects thanks to Cython. This mostly started out as a prototype that +I wanted to play with as quickly as possible, so I passed on stealing the +rotating trees for now. As usual, I got it working, and it seems to have +acceptable performance, so I am much less motivated to use a different strategy +now. Maybe later. Contributions accepted! From 5d0ee19ebb9dab5dc8502ca7e8be1e32afd3eac5 Mon Sep 17 00:00:00 2001 From: Test User Date: Sat, 4 Jul 2026 18:36:03 -0400 Subject: [PATCH 2/2] Clean up docs warnings --- docs/source/_static/.gitkeep | 0 docs/source/auto/line_profiler.rst | 7 ++- .../source/auto/line_profiler.toml_config.rst | 2 +- docs/source/conf.py | 17 +++++-- .../manual/examples/example_kernprof.rst | 6 ++- docs/source/manual/examples/example_units.rst | 1 + docs/source/manual/examples/index.rst | 15 ++++--- line_profiler/_line_profiler.pyx | 45 +++++-------------- .../autoprofile/ast_profile_transformer.py | 3 +- .../autoprofile/line_profiler_utils.py | 6 +-- line_profiler/cli_utils.py | 5 ++- line_profiler/ipython_extension.py | 10 ++--- line_profiler/line_profiler.py | 5 ++- 13 files changed, 58 insertions(+), 64 deletions(-) create mode 100644 docs/source/_static/.gitkeep diff --git a/docs/source/_static/.gitkeep b/docs/source/_static/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docs/source/auto/line_profiler.rst b/docs/source/auto/line_profiler.rst index 38985a4c..66700f23 100644 --- a/docs/source/auto/line_profiler.rst +++ b/docs/source/auto/line_profiler.rst @@ -28,7 +28,6 @@ Submodules Module contents --------------- -.. automodule:: line_profiler - :members: - :undoc-members: - :show-inheritance: +The package overview and top-level usage documentation are rendered on the +main documentation page. This page keeps the generated subpackage and submodule +navigation in one place. diff --git a/docs/source/auto/line_profiler.toml_config.rst b/docs/source/auto/line_profiler.toml_config.rst index 54a81532..fe1ed172 100644 --- a/docs/source/auto/line_profiler.toml_config.rst +++ b/docs/source/auto/line_profiler.toml_config.rst @@ -1,5 +1,5 @@ line\_profiler.toml\_config module -================================ +====================================== .. automodule:: line_profiler.toml_config :members: diff --git a/docs/source/conf.py b/docs/source/conf.py index 0604cedd..21e4bb9e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -274,7 +274,10 @@ def visit_Assign(self, node): # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -source_suffix = ['.rst', '.md'] +source_suffix = { + '.rst': 'restructuredtext', + '.md': 'markdown', +} # The master toctree document. master_doc = 'index' @@ -289,7 +292,15 @@ def visit_Assign(self, node): # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . -exclude_patterns = [] +exclude_patterns = [ + # Stale sphinx-apidoc entry points. The canonical API navigation is + # rooted at auto/line_profiler.rst and docs/source/index.rst. + 'modules.rst', + 'auto/modules.rst', + # There is no importable Python module named line_profiler.timers; the + # timers implementation lives in C support files. + 'auto/line_profiler.timers.rst', +] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' @@ -301,7 +312,6 @@ def visit_Assign(self, node): # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -309,7 +319,6 @@ def visit_Assign(self, node): # html_theme_options = { 'collapse_navigation': False, - 'display_version': True, 'navigation_depth': -1, # 'logo_only': True, } diff --git a/docs/source/manual/examples/example_kernprof.rst b/docs/source/manual/examples/example_kernprof.rst index 97c157ca..9a80de22 100644 --- a/docs/source/manual/examples/example_kernprof.rst +++ b/docs/source/manual/examples/example_kernprof.rst @@ -262,8 +262,10 @@ like ``python -c``: argument thereafter (the executed code). If there isn't one, an error is raised as :ref:`above ` with ``kernprof -m``. - * .. _kernprof-c-note: - Since the temporary file containing the executed code will not + + .. _kernprof-c-note: + + * Since the temporary file containing the executed code will not exist beyond the ``kernprof`` process, profiling results pertaining to targets (function definitions) local to said code :ref:`will not be accessible later ` by diff --git a/docs/source/manual/examples/example_units.rst b/docs/source/manual/examples/example_units.rst index 51475a3e..2dfb2252 100644 --- a/docs/source/manual/examples/example_units.rst +++ b/docs/source/manual/examples/example_units.rst @@ -48,6 +48,7 @@ reported use the ``--unit`` command line argument. The following example shows 4 variants: .. code:: bash + LINE_PROFILE=1 python script.py # Use different values for the unit report diff --git a/docs/source/manual/examples/index.rst b/docs/source/manual/examples/index.rst index b0aa80c7..b58d2eac 100644 --- a/docs/source/manual/examples/index.rst +++ b/docs/source/manual/examples/index.rst @@ -3,14 +3,17 @@ Examples Examples of line profiler usage: -+ `Basic Usage <../../index.html#line-profiler-basic-usage>`_ +.. toctree:: + :maxdepth: 1 -+ `Kernprof Usage `_ + Kernprof Usage + Timing Units + TOML Config Usage -+ `Auto Profiling <../../auto/line_profiler.autoprofile.html#auto-profiling>`_ +Additional API-focused examples: -+ `Explicit Profiler <../../auto/line_profiler.explicit_profiler.html#module-line_profiler.explicit_profiler>`_ ++ `Basic Usage <../../index.html#line-profiler-basic-usage>`_ -+ `Timing Units `_ ++ `Auto Profiling <../../auto/line_profiler.autoprofile.html#auto-profiling>`_ -+ `TOML Config Usage `_ ++ `Explicit Profiler <../../auto/line_profiler.explicit_profiler.html#module-line_profiler.explicit_profiler>`_ diff --git a/line_profiler/_line_profiler.pyx b/line_profiler/_line_profiler.pyx index ad2df605..042f23c6 100644 --- a/line_profiler/_line_profiler.pyx +++ b/line_profiler/_line_profiler.pyx @@ -678,12 +678,8 @@ line_profiler/blob/main/line_profiler/_line_profiler.pyx @cython.profile(False) cpdef handle_line_event(self, object code, int lineno): """ - Line-event (|LINE|_) callback passed to - :py:func:`sys.monitoring.register_callback`. - - .. |LINE| replace:: :py:attr:`!sys.monitoring.events.LINE` - .. _LINE: https://docs.python.org/3/library/\ -sys.monitoring.html#monitoring-event-LINE + Line-event callback for :py:attr:`!sys.monitoring.events.LINE`, + passed to :py:func:`sys.monitoring.register_callback`. """ self._base_callback( 1, sys.monitoring.events.LINE, code, lineno, (lineno,), ()) @@ -692,13 +688,8 @@ sys.monitoring.html#monitoring-event-LINE cpdef handle_return_event( self, object code, int instruction_offset, object retval): """ - Return-event (|PY_RETURN|_) callback passed to - :py:func:`sys.monitoring.register_callback`. - - .. |PY_RETURN| replace:: \ -:py:attr:`!sys.monitoring.events.PY_RETURN` - .. _PY_RETURN: https://docs.python.org/3/library/\ -sys.monitoring.html#monitoring-event-PY_RETURN + Return-event callback for :py:attr:`!sys.monitoring.events.PY_RETURN`, + passed to :py:func:`sys.monitoring.register_callback`. """ self._handle_exit_event( sys.monitoring.events.PY_RETURN, code, instruction_offset, retval) @@ -707,13 +698,8 @@ sys.monitoring.html#monitoring-event-PY_RETURN cpdef handle_yield_event( self, object code, int instruction_offset, object retval): """ - Yield-event (|PY_YIELD|_) callback passed to - :py:func:`sys.monitoring.register_callback`. - - .. |PY_YIELD| replace:: \ -:py:attr:`!sys.monitoring.events.PY_YIELD` - .. _PY_YIELD: https://docs.python.org/3/library/\ -sys.monitoring.html#monitoring-event-PY_YIELD + Yield-event callback for :py:attr:`!sys.monitoring.events.PY_YIELD`, + passed to :py:func:`sys.monitoring.register_callback`. """ self._handle_exit_event( sys.monitoring.events.PY_YIELD, code, instruction_offset, retval) @@ -722,12 +708,8 @@ sys.monitoring.html#monitoring-event-PY_YIELD cpdef handle_raise_event( self, object code, int instruction_offset, object exception): """ - Raise-event (|RAISE|_) callback passed to - :py:func:`sys.monitoring.register_callback`. - - .. |RAISE| replace:: :py:attr:`!sys.monitoring.events.RAISE` - .. _RAISE: https://docs.python.org/3/library/\ -sys.monitoring.html#monitoring-event-RAISE + Raise-event callback for :py:attr:`!sys.monitoring.events.RAISE`, + passed to :py:func:`sys.monitoring.register_callback`. """ self._handle_exit_event( sys.monitoring.events.RAISE, code, instruction_offset, exception) @@ -736,12 +718,8 @@ sys.monitoring.html#monitoring-event-RAISE cpdef handle_reraise_event( self, object code, int instruction_offset, object exception): """ - Re-raise-event (|RERAISE|_) callback passed to - :py:func:`sys.monitoring.register_callback`. - - .. |RERAISE| replace:: :py:attr:`!sys.monitoring.events.RERAISE` - .. _RERAISE: https://docs.python.org/3/library/\ -sys.monitoring.html#monitoring-event-RERAISE + Re-raise-event callback for :py:attr:`!sys.monitoring.events.RERAISE`, + passed to :py:func:`sys.monitoring.register_callback`. """ self._handle_exit_event( sys.monitoring.events.RERAISE, code, instruction_offset, exception) @@ -1307,8 +1285,7 @@ datamodel.html#user-defined-functions def c_last_time(self): """ Raises: - KeyError - If no profiling data is available on the current thread. + KeyError: If no profiling data is available on the current thread. """ try: return (self._c_last_time)[PyThread_get_thread_ident()] diff --git a/line_profiler/autoprofile/ast_profile_transformer.py b/line_profiler/autoprofile/ast_profile_transformer.py index 96b65c49..d4df3a60 100644 --- a/line_profiler/autoprofile/ast_profile_transformer.py +++ b/line_profiler/autoprofile/ast_profile_transformer.py @@ -15,7 +15,8 @@ def ast_create_profile_node( passes modname to it. At runtime, this adds the object to the profiler so it can be profiled. This node must be added after the first instance of modname in the AST and before it is used. - The node will look like: + The node will look like:: + >>> # xdoctest: +SKIP >>> import foo.bar >>> profile.add_imported_function_or_module(foo.bar) diff --git a/line_profiler/autoprofile/line_profiler_utils.py b/line_profiler/autoprofile/line_profiler_utils.py index 31d95c7e..f11c45cc 100644 --- a/line_profiler/autoprofile/line_profiler_utils.py +++ b/line_profiler/autoprofile/line_profiler_utils.py @@ -91,9 +91,9 @@ def add_imported_function_or_module( See also: :py:data:`~line_profiler.line_profiler\ .DEFAULT_SCOPING_POLICIES`, - :py:meth:`.LineProfiler.add_callable()`, - :py:meth:`.LineProfiler.add_module()`, - :py:meth:`.LineProfiler.add_class()`, + :py:meth:`add_callable() `, + :py:meth:`add_module() `, + :py:meth:`add_class() `, :py:class:`~.ScopingPolicy`, :py:meth:`ScopingPolicy.to_policies() \ ` diff --git a/line_profiler/cli_utils.py b/line_profiler/cli_utils.py index 470a06d3..26947332 100644 --- a/line_profiler/cli_utils.py +++ b/line_profiler/cli_utils.py @@ -40,6 +40,7 @@ def add_argument( * Set the destination value to the corresponding value in the no-arg form, but also allow (for long options) for a single arg which is parsed by :py:func:`.boolean()`. + Also automatically generates complementary boolean options for ``action='store_true'`` options. If ``hide_complementary_options`` is @@ -56,8 +57,8 @@ def add_argument( Whether to hide the auto-generated complementary options to ``action='store_true'`` options from the help text for brevity. - arg, *args, **kwargs - Passed to ``parser_like.add_argument()`` + arg, *args, **kwargs: + Passed to ``parser_like.add_argument()``. Returns: Any: action_like diff --git a/line_profiler/ipython_extension.py b/line_profiler/ipython_extension.py index 4b5b307d..de4ee105 100644 --- a/line_profiler/ipython_extension.py +++ b/line_profiler/ipython_extension.py @@ -4,7 +4,7 @@ If you are using IPython, there is an implementation of an |lprun| magic command which will let you specify functions to profile and a statement to execute. It will also add its -:py:class:`~.LineProfiler` instance into the |builtins|, but typically, +:py:class:`LineProfiler ` instance into the |builtins|, but typically, you would not use it like that. You can also use |lprun_all|, which profiles the whole cell you're @@ -398,14 +398,14 @@ def lprun(self, parameter_s=''): %lprun [] The given statement (which doesn't require quote marks) is run - via the :py:class:`~.LineProfiler`. Profiling is enabled for + via the :py:class:`LineProfiler `. Profiling is enabled for the functions specified by the ``-f`` options. The statistics will be shown side-by-side with the code through the pager once the statement has completed. Options: - ``-f ``: :py:class:`~.LineProfiler` only profiles + ``-f ``: :py:class:`LineProfiler ` only profiles functions and methods it is told to profile. This option tells the profiler about these functions. Multiple ``-f`` options may be used. The argument may be any expression that gives @@ -425,7 +425,7 @@ def lprun(self, parameter_s=''): ``-T ``: dump the text-formatted statistics with the code side-by-side out to a text file. - ``-r``: return the :py:class:`~.LineProfiler` object after it + ``-r``: return the :py:class:`LineProfiler ` object after it has completed profiling. ``-s``: strip out all entries from the print-out that have @@ -504,7 +504,7 @@ def lprun_all(self, parameter_s='', cell=''): ``-T ``: dump the text-formatted statistics with the code side-by-side out to a text file. - ``-r``: return the :py:class:`~.LineProfiler` object after it + ``-r``: return the :py:class:`LineProfiler ` object after it has completed profiling. ``-z``: strip out all entries from the print-out that have diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index bede6f6c..1be5ea8c 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -78,9 +78,10 @@ def get_column_widths( config: bool | str | None = False, ) -> Mapping[ColumnLiterals, int]: """ - Arguments - config (bool | str | None) + Args: + config (bool | str | None): Passed to :py:meth:`.ConfigSource.from_config`. + Note: * Results are cached. * The default value (:py:data:`False`) loads the config from the