From 0c95d0ca63578c742ed3862b2aff4cb06af255b7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 28 Nov 2023 14:34:33 +0000 Subject: [PATCH 1/8] Prepare for 1.3.0 release See changelog for more details. Signed-off-by: Pablo Galindo Signed-off-by: ms2892 --- .bumpversion.cfg | 2 +- NEWS.rst | 13 +++++++++++++ src/pystack/_version.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 54281962..0023175d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.0 +current_version = 1.3.0 commit = True message = Prepare for {new_version} release diff --git a/NEWS.rst b/NEWS.rst index be71d127..6536f179 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -8,6 +8,19 @@ Changelog .. towncrier release notes start +pystack 1.3.0 (2023-11-28) +-------------------------- + +Bug Fixes +~~~~~~~~~ + +- Add a patch to the bundled elfutils used to create wheels to account for a bug when analysing cores with interleaved segments (#153) +- Removed the unused ``--self`` flag. (#141) +- Fix some instances when identifying the pthread id was failing in systems without GLIBC (#152) +- Fix several some race conditions when stopping threads in multithreaded programs (#155) +- Ensure log messages that contain non-UTF-8 data are not lost (#155) + + pystack 1.2.0 (2023-07-31) -------------------------- diff --git a/src/pystack/_version.py b/src/pystack/_version.py index c68196d1..67bc602a 100644 --- a/src/pystack/_version.py +++ b/src/pystack/_version.py @@ -1 +1 @@ -__version__ = "1.2.0" +__version__ = "1.3.0" From fd9a979593a118e2e03103316a39f142f1e2e4f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:55:29 +0000 Subject: [PATCH 2/8] build(deps): bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: ms2892 --- .github/workflows/build_wheels.yml | 4 ++-- .github/workflows/coverage.yml | 2 +- .github/workflows/lint_and_docs.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 351f35da..ad0f1305 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -88,7 +88,7 @@ jobs: name: artifact path: dist - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" - name: Set up dependencies @@ -126,7 +126,7 @@ jobs: name: artifact path: dist - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "${{matrix.python_version}}-dev" - name: Set up dependencies diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 7baa8d64..fe52155c 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -29,7 +29,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Set up dependencies diff --git a/.github/workflows/lint_and_docs.yml b/.github/workflows/lint_and_docs.yml index f51c1258..da8280da 100644 --- a/.github/workflows/lint_and_docs.yml +++ b/.github/workflows/lint_and_docs.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" - name: Set up dependencies @@ -38,7 +38,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Set up dependencies From d409bde24cb920b633858c98b3a6d545c4393741 Mon Sep 17 00:00:00 2001 From: Gus Monod Date: Thu, 14 Dec 2023 19:17:54 -0500 Subject: [PATCH 3/8] Update `upload-artifact` and `download-artifact` Both need to be updated at once, and also the uploaded artifacts now must be each be named something different and are made immutable. This means that we cannot keep appending to the same archive called "artifact", and that we instead must create one artifact for the sdist and one for each architecture. Note that the `upload_pypi` step also needs to be changed to ensure it downloads all of the artifacts and extracts them as before. Signed-off-by: Gus Monod Signed-off-by: ms2892 --- .github/workflows/build_wheels.yml | 74 +++++++++++++++--------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index ad0f1305..109b1111 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -24,8 +24,9 @@ jobs: - name: Build sdist run: pipx run build --sdist - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: + name: sdist path: dist/*.tar.gz choose_architectures: @@ -35,20 +36,18 @@ jobs: - id: x86_64 run: echo "cibw_arch=x86_64" >> $GITHUB_OUTPUT - id: aarch64 - if: github.event_name == 'release' && github.event.action == 'published' run: echo "cibw_arch=aarch64" >> $GITHUB_OUTPUT outputs: cibw_arches: ${{ toJSON(steps.*.outputs.cibw_arch) }} build_wheels: needs: [build_sdist, choose_architectures] - name: Wheel for Linux-${{ matrix.cibw_python }}-${{ matrix.cibw_arch }} + name: Wheel for Linux ${{ matrix.cibw_arch }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest] - cibw_python: ["cp37-*", "cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"] cibw_arch: ${{ fromJSON(needs.choose_architectures.outputs.cibw_arches) }} steps: @@ -58,20 +57,21 @@ jobs: - uses: docker/setup-qemu-action@v3 if: runner.os == 'Linux' name: Set up QEMU - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: - name: artifact + name: sdist - name: Extract sdist run: | tar zxvf *.tar.gz --strip-components=1 - name: Build wheels uses: pypa/cibuildwheel@v2.16.2 env: - CIBW_BUILD: ${{ matrix.cibw_python }} + CIBW_BUILD: "cp3{7..12}-*" CIBW_ARCHS_LINUX: ${{ matrix.cibw_arch }} CIBW_PRERELEASE_PYTHONS: True - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: + name: ${{ matrix.cibw_arch }}-wheels path: ./wheelhouse/*.whl test_attaching_to_old_interpreters: @@ -83,9 +83,9 @@ jobs: python_version: ["2.7", "3.6"] steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: - name: artifact + name: "x86_64-wheels" path: dist - name: Set up Python uses: actions/setup-python@v5 @@ -121,14 +121,14 @@ jobs: python_version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 - with: - name: artifact - path: dist - name: Set up Python uses: actions/setup-python@v5 with: python-version: "${{matrix.python_version}}-dev" + - uses: actions/download-artifact@v4 + with: + name: "x86_64-wheels" + path: dist - name: Set up dependencies run: | sudo add-apt-repository ppa:deadsnakes/ppa @@ -163,13 +163,13 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 - with: - name: artifact - path: dist - name: Set up dependencies run: | apk add --update alpine-sdk bash alpine-sdk python3 python3-dev gdb musl-dbg python3-dbg + - uses: actions/download-artifact@v4 + with: + name: "x86_64-wheels" + path: dist - name: Install Python dependencies run: | python3 -m venv venv @@ -193,10 +193,6 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 - with: - name: artifact - path: dist - name: Set up dependencies run: | dnf install -y \ @@ -207,6 +203,10 @@ jobs: python3-devel dnf debuginfo-install -y \ python3 + - uses: actions/download-artifact@v4 + with: + name: "x86_64-wheels" + path: dist - name: Install Python dependencies run: | python3 -m pip install --upgrade pip @@ -228,10 +228,6 @@ jobs: options: --cap-add=SYS_PTRACE --security-opt seccomp=unconfined steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 - with: - name: artifact - path: dist - name: Set up dependencies run: | pacman -Syu --noconfirm \ @@ -242,6 +238,10 @@ jobs: python-pip \ python-setuptools \ python-wheel + - uses: actions/download-artifact@v4 + with: + name: "x86_64-wheels" + path: dist - name: Install Python dependencies run: | python -m venv venv @@ -265,10 +265,6 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 - with: - name: artifact - path: dist - name: Set up dependencies run: | apt-get update @@ -280,6 +276,10 @@ jobs: python3-venv \ python3-dbg \ python3-distutils + - uses: actions/download-artifact@v4 + with: + name: "x86_64-wheels" + path: dist - name: Install Python dependencies run: | python3 -m venv venv @@ -294,13 +294,13 @@ jobs: upload_pypi: needs: [test_wheels] runs-on: ubuntu-latest - if: github.event_name == 'release' && github.event.action == 'published' steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: - name: artifact + # with no name set, it downloads all of the artifacts path: dist - - uses: pypa/gh-action-pypi-publish@release/v1 - with: - skip_existing: true - password: ${{ secrets.PYPI_PASSWORD }} + - run: | + mv dist/sdist/*.tar.gz dist/ + mv dist/*-wheels/*.whl dist/ + rmdir dist/{sdist,*-wheels} + - run: ls -R dist From 6ca168c9a8c24601925f60c2e6386feac04444c6 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Fri, 5 Jan 2024 13:17:02 -0500 Subject: [PATCH 4/8] ci: Build manylinux & musllinux wheels in parallel Signed-off-by: Matt Wozniski Signed-off-by: ms2892 --- .github/workflows/build_wheels.yml | 43 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 109b1111..bc49411c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -29,26 +29,27 @@ jobs: name: sdist path: dist/*.tar.gz - choose_architectures: - name: Decide which architectures to build wheels for + choose_wheel_types: + name: Decide which wheel types to build runs-on: ubuntu-latest steps: - - id: x86_64 - run: echo "cibw_arch=x86_64" >> $GITHUB_OUTPUT - - id: aarch64 - run: echo "cibw_arch=aarch64" >> $GITHUB_OUTPUT + - id: manylinux_x86_64 + run: echo "wheel_types=manylinux_x86_64" >> $GITHUB_OUTPUT + - id: musllinux_x86_64 + run: echo "wheel_types=musllinux_x86_64" >> $GITHUB_OUTPUT + - id: manylinux_aarch64 + run: echo "wheel_types=manylinux_aarch64" >> $GITHUB_OUTPUT outputs: - cibw_arches: ${{ toJSON(steps.*.outputs.cibw_arch) }} + wheel_types: ${{ toJSON(steps.*.outputs.wheel_types) }} build_wheels: - needs: [build_sdist, choose_architectures] - name: Wheel for Linux ${{ matrix.cibw_arch }} - runs-on: ${{ matrix.os }} + needs: [build_sdist, choose_wheel_types] + name: ${{ matrix.wheel_type }} wheels + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - os: [ubuntu-latest] - cibw_arch: ${{ fromJSON(needs.choose_architectures.outputs.cibw_arches) }} + wheel_type: ${{ fromJSON(needs.choose_wheel_types.outputs.wheel_types) }} steps: - name: Disable ptrace security restrictions @@ -66,12 +67,12 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.16.2 env: - CIBW_BUILD: "cp3{7..12}-*" - CIBW_ARCHS_LINUX: ${{ matrix.cibw_arch }} + CIBW_BUILD: "cp3{7..12}-${{ matrix.wheel_type }}" + CIBW_ARCHS_LINUX: auto aarch64 CIBW_PRERELEASE_PYTHONS: True - uses: actions/upload-artifact@v4 with: - name: ${{ matrix.cibw_arch }}-wheels + name: ${{ matrix.wheel_type }}-wheels path: ./wheelhouse/*.whl test_attaching_to_old_interpreters: @@ -85,7 +86,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: - name: "x86_64-wheels" + name: "manylinux_x86_64-wheels" path: dist - name: Set up Python uses: actions/setup-python@v5 @@ -127,7 +128,7 @@ jobs: python-version: "${{matrix.python_version}}-dev" - uses: actions/download-artifact@v4 with: - name: "x86_64-wheels" + name: "manylinux_x86_64-wheels" path: dist - name: Set up dependencies run: | @@ -168,7 +169,7 @@ jobs: apk add --update alpine-sdk bash alpine-sdk python3 python3-dev gdb musl-dbg python3-dbg - uses: actions/download-artifact@v4 with: - name: "x86_64-wheels" + name: "musllinux_x86_64-wheels" path: dist - name: Install Python dependencies run: | @@ -205,7 +206,7 @@ jobs: python3 - uses: actions/download-artifact@v4 with: - name: "x86_64-wheels" + name: "manylinux_x86_64-wheels" path: dist - name: Install Python dependencies run: | @@ -240,7 +241,7 @@ jobs: python-wheel - uses: actions/download-artifact@v4 with: - name: "x86_64-wheels" + name: "manylinux_x86_64-wheels" path: dist - name: Install Python dependencies run: | @@ -278,7 +279,7 @@ jobs: python3-distutils - uses: actions/download-artifact@v4 with: - name: "x86_64-wheels" + name: "manylinux_x86_64-wheels" path: dist - name: Install Python dependencies run: | From c5b15926158c3cc68efedb0386548a8fc2087563 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Fri, 5 Jan 2024 14:44:09 -0500 Subject: [PATCH 5/8] ci: Restore steps performed only for releases Signed-off-by: Matt Wozniski Signed-off-by: ms2892 --- .github/workflows/build_wheels.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index bc49411c..8b8b269a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -38,6 +38,7 @@ jobs: - id: musllinux_x86_64 run: echo "wheel_types=musllinux_x86_64" >> $GITHUB_OUTPUT - id: manylinux_aarch64 + if: github.event_name == 'release' && github.event.action == 'published' run: echo "wheel_types=manylinux_aarch64" >> $GITHUB_OUTPUT outputs: wheel_types: ${{ toJSON(steps.*.outputs.wheel_types) }} @@ -295,6 +296,7 @@ jobs: upload_pypi: needs: [test_wheels] runs-on: ubuntu-latest + if: github.event_name == 'release' && github.event.action == 'published' steps: - uses: actions/download-artifact@v4 with: @@ -304,4 +306,8 @@ jobs: mv dist/sdist/*.tar.gz dist/ mv dist/*-wheels/*.whl dist/ rmdir dist/{sdist,*-wheels} - - run: ls -R dist + ls -R dist + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip_existing: true + password: ${{ secrets.PYPI_PASSWORD }} From 018a543cc63bcd354e007cb696701a43dfc9aed9 Mon Sep 17 00:00:00 2001 From: ms2892 Date: Mon, 15 Jan 2024 18:39:34 +0000 Subject: [PATCH 6/8] Added Benchmarks Corresponding to various test cases Signed-off-by: ms2892 --- .gitignore | 5 + .vscode/settings.json | 78 -- asv.conf.json | 21 + benchmarks/__init__.py | 0 benchmarks/benchmark_colors.py | 54 + benchmarks/benchmark_main.py | 673 +++++++++++ benchmarks/benchmark_maps.py | 934 ++++++++++++++++ benchmarks/benchmark_process.py | 207 ++++ benchmarks/benchmark_traceback_formatters.py | 1045 ++++++++++++++++++ benchmarks/benchmark_types.py | 181 +++ 10 files changed, 3120 insertions(+), 78 deletions(-) delete mode 100644 .vscode/settings.json create mode 100644 asv.conf.json create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/benchmark_colors.py create mode 100644 benchmarks/benchmark_main.py create mode 100644 benchmarks/benchmark_maps.py create mode 100644 benchmarks/benchmark_process.py create mode 100644 benchmarks/benchmark_traceback_formatters.py create mode 100644 benchmarks/benchmark_types.py diff --git a/.gitignore b/.gitignore index c76abece..8b1dccdc 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,8 @@ dmypy.json # Cython debug symbols cython_debug/ + +# Benchmarks created my asv +pystack/** +.asv/** +.vscode/** \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index d6910419..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "files.associations": { - "cctype": "cpp", - "clocale": "cpp", - "cmath": "cpp", - "csignal": "cpp", - "cstdarg": "cpp", - "cstddef": "cpp", - "cstdio": "cpp", - "cstdlib": "cpp", - "cstring": "cpp", - "ctime": "cpp", - "cwchar": "cpp", - "cwctype": "cpp", - "array": "cpp", - "atomic": "cpp", - "hash_map": "cpp", - "bit": "cpp", - "*.tcc": "cpp", - "bitset": "cpp", - "chrono": "cpp", - "codecvt": "cpp", - "compare": "cpp", - "complex": "cpp", - "concepts": "cpp", - "condition_variable": "cpp", - "cstdint": "cpp", - "deque": "cpp", - "forward_list": "cpp", - "list": "cpp", - "map": "cpp", - "set": "cpp", - "string": "cpp", - "unordered_map": "cpp", - "unordered_set": "cpp", - "vector": "cpp", - "exception": "cpp", - "algorithm": "cpp", - "functional": "cpp", - "iterator": "cpp", - "memory": "cpp", - "memory_resource": "cpp", - "numeric": "cpp", - "optional": "cpp", - "random": "cpp", - "ratio": "cpp", - "source_location": "cpp", - "string_view": "cpp", - "system_error": "cpp", - "tuple": "cpp", - "type_traits": "cpp", - "utility": "cpp", - "fstream": "cpp", - "future": "cpp", - "initializer_list": "cpp", - "iomanip": "cpp", - "iosfwd": "cpp", - "iostream": "cpp", - "istream": "cpp", - "limits": "cpp", - "mutex": "cpp", - "new": "cpp", - "numbers": "cpp", - "ostream": "cpp", - "semaphore": "cpp", - "sstream": "cpp", - "stdexcept": "cpp", - "stop_token": "cpp", - "streambuf": "cpp", - "thread": "cpp", - "cfenv": "cpp", - "cinttypes": "cpp", - "typeindex": "cpp", - "typeinfo": "cpp", - "valarray": "cpp", - "variant": "cpp" - } -} diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 00000000..aea93aca --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,21 @@ +{ + "version":1, + "benchmark_dir": "./benchmarks", + "repo":"git@github.com:ms2892/pystack.git", + "project": "pystack", + "project_url": "https://github.com/ms2892/pystack", + "env_dir":".asv/env", + "results_dir":".asv/results", + "html_dir":".asv/html", + "environment_type":"conda", + "dvcs":"git", + "branches":["main"], + "install_command":[ + "python -mpip install -r requirements-test.txt -r requirements-extra.txt", + "python -mpip install -e ." + ], + "build_command":[ + "python -mpip install pkgconfig", + "python -mpip install dbg" + ] +} \ No newline at end of file diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmarks/benchmark_colors.py b/benchmarks/benchmark_colors.py new file mode 100644 index 00000000..13978049 --- /dev/null +++ b/benchmarks/benchmark_colors.py @@ -0,0 +1,54 @@ +from pystack.colors import * + +RANGE=100 + +class ColorsBenchmarkSuite: + + def setup(self): + pass + + def time_colored(self): + colors = ["red","green","yellow","blue","magenta","cyan","white"] + highlights = ["on_red","on_green","on_yellow","on_blue","on_magenta","on_cyan","on_white"] + attributes = ["bold", "dark", "underline", "blink", "reverse", "concealed"] + for counter in range(RANGE): + for color in colors: + for highlight in highlights: + colored("Benchmark Colored",color,highlight,attributes) + return "Successfully Benchmarks colored" + + def time_format_colored(self): + colors=[ + "grey", + "red", + "green", + "yellow", + "blue", + "magenta", + "cyan", + "white", + ] + highlights=[ + "on_grey", + "on_red", + "on_green", + "on_yellow", + "on_blue", + "on_magenta", + "on_cyan", + "on_white", + ] + attributes=[ + "bold", + "faint", + "italized", + "underline", + "blink", + "reverse", + "concealed", + ] + for counter in range(RANGE): + for color in colors: + for highlight in highlights: + format_colored("Benchmark Format Colored",color,highlight,attributes) + return "Successfully Benchmarks format_colored" \ No newline at end of file diff --git a/benchmarks/benchmark_main.py b/benchmarks/benchmark_main.py new file mode 100644 index 00000000..cbe3ae5a --- /dev/null +++ b/benchmarks/benchmark_main.py @@ -0,0 +1,673 @@ +from unittest.mock import patch, Mock +from pystack.__main__ import format_failureinfo_information, format_psinfo_information, main, produce_error_message +from pystack.errors import EngineError, InvalidPythonProcess +import pytest +from pathlib import Path + +RANGE=100 + +def time_format_failureinfo_information_with_segfault(): + + for i in range(RANGE): + info = { + "si_signo": 11, + "si_errno": 0, + "si_code": 1, + "sender_pid": 0, + "sender_uid": 0, + "failed_addr": 123456789, + } + + with patch("os.environ", {"NO_COLOR": 1}): + result = format_failureinfo_information(info) + +def time_format_failureinfo_information_with_signal(): + for i in range(RANGE): + info = { + "si_signo": 7, + "si_errno": 0, + "si_code": 0, + "sender_pid": 1, + "sender_uid": 0, + "failed_addr": 0, + } + + with patch("os.environ", {"NO_COLOR": 1}): + result = format_failureinfo_information(info) + +def time_format_failureinfo_information_with_signal_no_sender_pid(): + for i in range(RANGE): + info = { + "si_signo": 7, + "si_errno": 0, + "si_code": 0, + "sender_uid": 0, + "failed_addr": 0, + } + + # WHEN + with patch("os.environ", {"NO_COLOR": 1}): + result = format_failureinfo_information(info) + +def time_format_failureinfo_information_with_no_info(): + for i in range(RANGE): + info = { + "si_signo": 0, + "si_errno": 0, + "si_code": 0, + "sender_pid": 0, + "sender_uid": 0, + "failed_addr": 0, + } + + # WHEN + with patch("os.environ", {"NO_COLOR": 1}): + result = format_failureinfo_information(info) + +def time_format_psinfo(): + for i in range(RANGE): + info = { + "state": 0, + "sname": 82, + "zomb": 0, + "nice": 0, + "flag": 4212480, + "uid": 0, + "gid": 0, + "pid": 75639, + "ppid": 1, + "pgrp": 75639, + "sid": 1, + "fname": "a.out", + "psargs": "./a.out ", + } + + # WHEN + with patch("os.environ", {"NO_COLOR": 1}): + result = format_psinfo_information(info) + +def time_process_remote_default(): + for i in range(RANGE): + argv = ["pystack", "remote", "31"] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ): + get_process_threads_mock.return_value = threads + main() + +def time_process_remote_no_block(): + for i in range(RANGE): + argv = ["pystack", "remote", "31", "--no-block"] + + threads = [Mock(), Mock(), Mock()] + + with patch( + "pystack.__main__.get_process_threads" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ): + get_process_threads_mock.return_value = threads + main() + +def time_process_remote_native(): + for i in range(RANGE): + argv = ["pystack", "remote", "31", "--native"] + + threads = [Mock(), Mock(), Mock()] + + with patch( + "pystack.__main__.get_process_threads" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ): + get_process_threads_mock.return_value = threads + main() + +def time_process_remote_locals(): + for i in range(RANGE): + argv = ["pystack", "remote", "31", "--locals"] + + threads = [Mock(), Mock(), Mock()] + + with patch( + "pystack.__main__.get_process_threads" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ): + get_process_threads_mock.return_value = threads + main() + +def time_process_remote_native_no_block(): + for i in range(RANGE): + argv = ["pystack", "remote", "31", "--native","--no-block"] + + threads = [Mock(), Mock(), Mock()] + + with patch( + "pystack.__main__.get_process_threads" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ): + get_process_threads_mock.return_value = threads + with pytest.raises(SystemExit): + main() + +def time_process_remote_exhaustive(): + for i in range(RANGE): + argv = ["pystack", "remote", "31", "--exhaustive"] + + threads = [Mock(), Mock(), Mock()] + + with patch( + "pystack.__main__.get_process_threads" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ): + get_process_threads_mock.return_value = threads + main() + +def time_process_remote_error(): + for i in range(RANGE): + for exception in [EngineError, InvalidPythonProcess]: + argv = ["pystack", "remote", "32"] + + with patch( + "pystack.__main__.get_process_threads" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ): + get_process_threads_mock.side_effect = exception("Oh no!") + with pytest.raises(SystemExit) as excinfo: + main() + +def time_process_core_defaulte_without_executable(): + for i in range(RANGE): + argv = ["pystack", "core", "corefile"] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.is_elf", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ) as core_file_analizer_mock: + core_file_analizer_mock().extract_executable.return_value = ( + "extracted_executable" + ) + get_process_threads_mock.return_value = threads + main() + +def time_process_core_default_without_executable_and_executable_does_not_exist(): + for i in range(RANGE): + argv = ["pystack", "core", "corefile"] + + # WHEN + + with patch("sys.argv", argv), patch( + "pathlib.Path.exists" + ) as path_exists_mock, patch( + "pystack.__main__.CoreFileAnalyzer" + ) as core_file_analizer_mock: + core_file_analizer_mock().extract_executable.return_value = ( + "extracted_executable" + ) + # THEN + path_exists_mock.side_effect = [True, False] + + with pytest.raises(SystemExit): + main() + +def time_process_core_executable_not_elf_file(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "core", "corefile", "executable"] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch("sys.argv", argv), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.is_elf", return_value=False + ): + get_process_threads_mock.return_value = threads + with pytest.raises(SystemExit): + main() + +def time_process_core_default_with_executable(): + for i in range(RANGE): + argv = ["pystack", "core", "corefile", "executable"] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + get_process_threads_mock.return_value = threads + main() + +def time_process_core_native(): + # GIVEN + for i in range(RANGE): + for argument in ['--native','--native-all']: + argv = ["pystack", "core", "corefile", "executable", argument] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + get_process_threads_mock.return_value = threads + main() + +def time_process_core_locals(): + for i in range(RANGE): + argv = ["pystack", "core", "corefile", "executable", "--locals"] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + get_process_threads_mock.return_value = threads + main() + + +def time_process_core_with_search_path(): + # GIVEN + for i in range(RANGE): + argv = [ + "pystack", + "core", + "corefile", + "executable", + "--lib-search-path", + "foo:bar:baz", + ] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + get_process_threads_mock.return_value = threads + main() + +def time_process_core_with_search_root(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "core", "corefile", "executable", "--lib-search-root", "foo"] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pathlib.Path.glob", return_value=[Path("foo/lel.so"), Path("bar/lel.so")] + ), patch( + "os.path.isdir", + return_value=True, + ), patch( + "os.access", + return_value=True, + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + get_process_threads_mock.return_value = threads + main() + +def time_process_core_with_not_readable_search_root(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "core", "corefile", "executable", "--lib-search-root", "foo"] + + # WHEN + + with patch("pystack.__main__.get_process_threads_for_core"), patch( + "pystack.__main__.print_thread" + ), patch("sys.argv", argv), patch("pathlib.Path.exists", return_value=True), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "os.path.isdir", + return_value=True, + ), patch( + "os.access", return_value=False + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + # THEN + with pytest.raises(SystemExit): + main() + +def time_process_core_with_invalid_search_root(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "core", "corefile", "executable", "--lib-search-root", "foo"] + + # WHEN + + with patch("pystack.__main__.get_process_threads_for_core"), patch( + "pystack.__main__.print_thread" + ), patch("sys.argv", argv), patch("pathlib.Path.exists", return_value=True), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "os.path.isdir", + return_value=False, + ): + # THEN + with pytest.raises(SystemExit): + main() + +def time_process_core_corefile_does_not_exit(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "core", "corefile", "executable"] + + # WHEN + + def path_exists(what): + return what != Path("corefile") + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch.object( + Path, "exists", path_exists + ): + # THEN + + with pytest.raises(SystemExit): + main() + +def time_process_core_executable_does_not_exit(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "core", "corefile", "executable"] + + # WHEN + + def does_exit(what): + if what == Path("executable"): + return False + return True + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch.object( + Path, "exists", does_exit + ): + # THEN + with pytest.raises(SystemExit): + main() + +def time_process_core_error(): + # GIVEN + for i in range(RANGE): + for exception in [EngineError, InvalidPythonProcess]: + argv = ["pystack", "core", "corefile", "executable"] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + # THEN + get_process_threads_mock.side_effect = exception("Oh no!") + with pytest.raises(SystemExit) as excinfo: + main() + +def time_process_core_exhaustive(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "core", "corefile", "executable", "--exhaustive"] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + get_process_threads_mock.return_value = threads + main() + +def time_default_colored_output(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "remote", "1234"] + environ = {} + + # WHEN + + with patch("pystack.__main__.get_process_threads"), patch( + "pystack.__main__.print_thread" + ), patch("sys.argv", argv), patch("os.environ", environ): + main() + +def time_nocolor_output(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "remote", "1234", "--no-color"] + environ = {} + + # WHEN + + with patch("pystack.__main__.get_process_threads"), patch( + "pystack.__main__.print_thread" + ), patch("sys.argv", argv), patch("os.environ", environ): + main() + +def time_nocolor_output_at_the_front_for_process(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "remote", "--no-color", "1234"] + environ = {} + + # WHEN + + with patch("pystack.__main__.get_process_threads"), patch( + "pystack.__main__.print_thread" + ), patch("sys.argv", argv), patch("os.environ", environ): + main() + +def time_nocolor_output_at_the_front_for_core(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "core", "--no-color", "corefile", "executable"] + environ = {} + + # WHEN + with patch("pystack.__main__.get_process_threads_for_core"), patch( + "pystack.__main__.print_thread" + ), patch("sys.argv", argv), patch("os.environ", environ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + main() + +def test_global_options_can_be_placed_at_any_point(): + # GIVEN + for i in range(RANGE): + for option in ['--no-color','--verbose']: + argv = ["pystack", option, "core", option, "corefile", "executable"] + environ = {} + + # WHEN + with patch("pystack.__main__.get_process_threads_for_core"), patch( + "pystack.__main__.print_thread" + ), patch("sys.argv", argv), patch("os.environ", environ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + # THEN + + main() + +def time_verbose_as_global_options_sets_correctly_the_logger(): + # GIVEN + for i in range(RANGE): + argv = ["pystack", "-vv", "remote", "1234"] + environ = {} + + # WHEN + with patch("pystack.__main__.get_process_threads"), patch( + "pystack.__main__.print_thread" + ), patch("sys.argv", argv), patch("os.environ", environ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.logging.basicConfig" + ) as logger_mock: + # THEN + + main() + +def time_process_core_does_not_crash_if_core_analyzer_fails(): + # GIVEN + for i in range(RANGE): + for method in ["extract_ps_info", "extract_failure_info"]: + argv = ["pystack", "core", "corefile", "executable"] + + # WHEN / THEN + + with patch("pystack.__main__.get_process_threads_for_core"), patch( + "pystack.__main__.print_thread" + ), patch("pystack.__main__.is_elf", return_value=True), patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ) as core_analyzer_test: + method = getattr(core_analyzer_test(), method) + method.side_effect = Exception("oh no") + main() diff --git a/benchmarks/benchmark_maps.py b/benchmarks/benchmark_maps.py new file mode 100644 index 00000000..cca95f12 --- /dev/null +++ b/benchmarks/benchmark_maps.py @@ -0,0 +1,934 @@ +from pystack.maps import VirtualMap +from pystack.maps import _get_base_map +from pystack.maps import _get_bss +from pystack.maps import parse_maps_file_for_binary +from pystack.errors import MissingExecutableMaps +from pystack.errors import ProcessNotFound, PystackError +from pystack.maps import generate_maps_for_process +from pathlib import Path +import pytest +from unittest.mock import patch, mock_open +import os + +RANGE=100 + +def time_virtual_map_creation(): + for i in range(RANGE): + map = VirtualMap( + start=0, + end=10, + offset=1234, + device="device", + flags="xrwp", + inode=42, + path=None, + filesize=10, + ) + +def time_simple_maps_no_such_pid(): + for i in range(RANGE): + with patch("builtins.open", side_effect=FileNotFoundError()): + # WHEN / THEN + with pytest.raises(ProcessNotFound): + list(generate_maps_for_process(1)) + +def time_simple_maps(): + for i in range(RANGE): + map_text = """ + 7f1ac1e2b000-7f1ac1e50000 r--p 00000000 08:12 8398159 /usr/lib/libc-2.31.so + """ + + # WHEN + + with patch("builtins.open", mock_open(read_data=map_text)): + maps = list(generate_maps_for_process(1)) + +def time_maps_with_long_device_numbers(): + for i in range(RANGE): + map_text = """ + 7f1ac1e2b000-7f1ac1e50000 r--p 00000000 0123:4567 8398159 /usr/lib/libc-2.31.so + """ + + # WHEN + + with patch("builtins.open", mock_open(read_data=map_text)): + maps = list(generate_maps_for_process(1)) + +def time_annonymous_maps(): + for i in range(RANGE): + map_text = """ + 7f1ac1e2b000-7f1ac1e50000 r--p 00000000 08:12 8398159 + """ + + # WHEN + + with patch("builtins.open", mock_open(read_data=map_text)): + maps = list(generate_maps_for_process(1)) + +def time_map_permissions(): + # GIVEN + for i in range(RANGE): + map_text = """ + 7f1ac1e2b000-7f1ac1e50000 r--- 00000000 08:12 8398159 /usr/lib/libc-2.31.so + 7f1ac1e2b000-7f1ac1e50000 rw-- 00000000 08:12 8398159 /usr/lib/libc-2.31.so + 7f1ac1e2b000-7f1ac1e50000 rwx- 00000000 08:12 8398159 /usr/lib/libc-2.31.so + 7f1ac1e2b000-7f1ac1e50000 rwxp 00000000 08:12 8398159 /usr/lib/libc-2.31.so + """ + + # WHEN + + with patch("builtins.open", mock_open(read_data=map_text)): + maps = list(generate_maps_for_process(1)) + +def time_unexpected_line_is_ignored(): + # GIVEN + for i in range(RANGE): + map_text = """ + I am an unexpected line + 7f1ac1e2b000-7f1ac1e50000 r--p 00000000 08:12 8398159 /usr/lib/libc-2.31.so + """ + + # WHEN + + with patch("builtins.open", mock_open(read_data=map_text)): + maps = list(generate_maps_for_process(1)) + +def time_special_maps(): + for i in range(RANGE): + map_text = """ + 555f1ab1c000-555f1ab3d000 rw-p 00000000 00:00 0 [heap] + 7ffdf8102000-7ffdf8124000 rw-p 00000000 00:00 0 [stack] + 7ffdf8152000-7ffdf8155000 r--p 00000000 00:00 0 [vvar] + 7ffdf8155000-7ffdf8156000 r-xp 00000000 00:00 0 [vdso] + ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall] + """ + + # WHEN + + with patch("builtins.open", mock_open(read_data=map_text)): + maps = list(generate_maps_for_process(1)) + +def time_maps_for_binary_only_python_exec(): + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ) + + maps = [ + python, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + ] + + # WHEN + + mapinfo = parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_with_heap(): + # GIVEN + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ) + + heap = VirtualMap( + start=140728765587456, + end=140728765599744, + filesize=12288, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("[heap]"), + ) + + maps = [ + python, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + heap, + ] + + # WHEN + + mapinfo = parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_with_libpython(): + # GIVEN + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ) + + libpython = VirtualMap( + start=140728765587456, + end=140728765599744, + filesize=4096, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("/some/path/to/libpython.so"), + ) + + maps = [ + python, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + libpython, + ] + + # WHEN + + mapinfo = parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_executable_with_bss(): + # GIVEN + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ) + + bss = VirtualMap( + start=139752898736128, + end=139752898887680, + filesize=4096, + offset=0, + device="08:12", + flags="r--p", + inode=8398159, + path=None, + ) + + maps = [ + python, + bss, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + ] + + # WHEN + + mapinfo = parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_libpython_with_bss(): + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ) + + bss = VirtualMap( + start=139752898736128, + end=139752898887680, + filesize=4096, + offset=0, + device="08:12", + flags="r--p", + inode=8398159, + path=None, + ) + + libpython = VirtualMap( + start=140728765587456, + end=140728765599744, + filesize=4096, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("/some/path/to/libpython.so"), + ) + + libpython_bss = VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=None, + ) + + maps = [ + python, + bss, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + libpython, + libpython_bss, + ] + + # WHEN + + mapinfo = parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_libpython_without_bss(): + # GIVEN + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ) + + bss = VirtualMap( + start=139752898736128, + end=139752898887680, + filesize=4096, + offset=0, + device="08:12", + flags="r--p", + inode=8398159, + path=None, + ) + + libpython = VirtualMap( + start=140728765587456, + end=140728765599744, + filesize=4096, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("/some/path/to/libpython.so"), + ) + + maps = [ + python, + bss, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + libpython, + ] + + # WHEN + + mapinfo = parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_libpython_with_bss_with_non_readable_segment(): + # GIVEN + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ) + + bss = VirtualMap( + start=139752898736128, + end=139752898887680, + filesize=4096, + offset=0, + device="08:12", + flags="r--p", + inode=8398159, + path=None, + ) + + libpython = VirtualMap( + start=140728765587456, + end=140728765599744, + filesize=4096, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("/some/path/to/libpython.so"), + ) + + libpython_bss = VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=None, + ) + + maps = [ + python, + bss, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + libpython, + VirtualMap( + start=1844674407369906, + end=18446744073699069, + filesize=4096, + offset=0, + device="00:00", + flags="---p", + inode=0, + path=None, + ), + libpython_bss, + ] + + # WHEN + + mapinfo = parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_range(): + # GIVEN + for i in range(RANGE): + maps = [ + VirtualMap( + start=1, + end=2, + filesize=1, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ), + VirtualMap( + start=2, + end=3, + filesize=1, + offset=0, + device="08:12", + flags="r--p", + inode=8398159, + path=None, + ), + VirtualMap( + start=5, + end=6, + filesize=1, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("/some/path/to/libpython.so"), + ), + VirtualMap( + start=8, + end=9, + filesize=1, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=None, + ), + ] + + # WHEN + + mapinfo = parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_range_vmaps_are_ignored(): + # GIVEN + for i in range(RANGE): + maps = [ + VirtualMap( + start=1, + end=2, + filesize=1, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ), + VirtualMap( + start=2000, + end=3000, + filesize=1000, + offset=0, + device="08:12", + flags="r--p", + inode=8398159, + path=Path("[vsso]"), + ), + VirtualMap( + start=5, + end=6, + filesize=1, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("[vsyscall]"), + ), + VirtualMap( + start=8, + end=9, + filesize=1, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("[vvar]"), + ), + ] + + # WHEN + + mapinfo = parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_no_binary_map(): + # GIVEN + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ) + + maps = [ + python, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + ] + + # WHEN / THEN + + with pytest.raises(MissingExecutableMaps): + parse_maps_file_for_binary(Path("another_executable"), maps) + +def time_maps_for_binary_no_executable_segment(): + # GIVEN + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("the_executable"), + ) + + maps = [ + python, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + ] + + # WHEN + + mapinfo = parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_multiple_libpythons(): + # GIVEN + for i in range(RANGE): + maps = [ + VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("the_executable"), + ), + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libpython3.8.so"), + ), + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libpython2.7.so"), + ), + ] + + # WHEN / THEN + + with pytest.raises(PystackError): + parse_maps_file_for_binary(Path("the_executable"), maps) + +def time_maps_for_binary_invalid_executable(): + # GIVEN + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=Path("the_executable"), + ) + + maps = [ + python, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + ] + + # WHEN + + with pytest.raises(MissingExecutableMaps, match="the_executable"): + parse_maps_file_for_binary(Path("other_executable"), maps) + +def time_maps_for_binary_invalid_executable_and_no_available_maps(): + # GIVEN + for i in range(RANGE): + python = VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=None, + ) + + maps = [ + python, + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + ] + + # WHEN + + with pytest.raises( + MissingExecutableMaps, match="There are no available executable maps" + ): + parse_maps_file_for_binary(Path("other_executable"), maps) + +def time_maps_with_scattered_segments(): + + for i in range(RANGE): + map_text = """ + 00400000-00401000 r-xp 00000000 fd:00 67488961 /bin/python3.9-dbg + 00600000-00601000 r--p 00000000 fd:00 67488961 /bin/python3.9-dbg + 00601000-00602000 rw-p 00001000 fd:00 67488961 /bin/python3.9-dbg + 0067b000-00a58000 rw-p 00000000 00:00 0 [heap] + 7f7b38000000-7f7b38028000 rw-p 00000000 00:00 0 + 7f7b38028000-7f7b3c000000 ---p 00000000 00:00 0 + 7f7b40000000-7f7b40021000 rw-p 00000000 00:00 0 + 7f7b40021000-7f7b44000000 ---p 00000000 00:00 0 + 7f7b44ec0000-7f7b44f40000 rw-p 00000000 00:00 0 + f7b45a61000-7f7b45d93000 rw-p 00000000 00:00 0 + 7f7b46014000-7f7b46484000 r--p 0050b000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 + 7f7b46484000-7f7b46485000 ---p 00000000 00:00 0 + 7f7b46485000-7f7b46cda000 rw-p 00000000 00:00 0 + 7f7b46cda000-7f7b46d16000 r--p 00a3d000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 + 7f7b46d16000-7f7b46d6f000 rw-p 00000000 00:00 0 + 7f7b46d6f000-7f7b46d92000 r--p 00001000 fd:00 67488961 /bin/python3.9-dbg + 7f7b46d92000-7f7b46d93000 ---p 00000000 00:00 0 + 7f7b46d93000-7f7b475d3000 rw-p 00000000 00:00 0 + 7f7b498c1000-7f7b49928000 r-xp 00000000 fd:00 7023 /lib64/libssl.so.1.0.0 + 7f7b49928000-7f7b49b28000 ---p 00067000 fd:00 7023 /lib64/libssl.so.1.0.0 + f7b4c632000-7f7b4c6f3000 rw-p 00000000 00:00 0 + 7f7b4c6f3000-7f7b4c711000 rw-p 00000000 00:00 0 + 7f7b4c711000-7f7b4c712000 r--p 0002a000 fd:00 67488961 /bin/python3.9-dbg + 7f7b4c712000-7f7b4c897000 rw-p 00000000 00:00 0 + 7f7b5a356000-7f7b5a35d000 r--s 00000000 fd:00 201509519 /usr/lib64/gconv/gconv-modules.cache + 7f7b5a35d000-7f7b5a827000 r-xp 00000000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 + 7f7b5a827000-7f7b5aa27000 ---p 004ca000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 + 7f7b5aa27000-7f7b5aa2c000 r--p 004ca000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 + 7f7b5aa2c000-7f7b5aa67000 rw-p 004cf000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 + 7f7b5aa67000-7f7b5aa8b000 rw-p 00000000 00:00 0 + 7fff26f8e000-7fff27020000 rw-p 00000000 00:00 0 [stack] + 7fff27102000-7fff27106000 r--p 00000000 00:00 0 [vvar] + 7fff27106000-7fff27108000 r-xp 00000000 00:00 0 [vdso] + ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] + """ + + # WHEN + + with patch("builtins.open", mock_open(read_data=map_text)): + maps = list(generate_maps_for_process(1)) + + mapinfo = parse_maps_file_for_binary(Path("/bin/python3.9-dbg"), maps) + +def time_get_base_map_path_existing(): + # GIVEN + for i in range(RANGE): + maps = [ + VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=None, + ), + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=Path("/usr/lib/libc-2.31.so"), + ), + ] + + # WHEN + base_map = _get_base_map(maps) + +def time_get_base_map_path_not_existing(): + # GIVEN + for i in range(RANGE): + maps = [ + VirtualMap( + start=140728765599744, + end=140728765603840, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=None, + ), + VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=None, + ), + ] + + # WHEN + base_map = _get_base_map(maps) + +def time_get_bss_base_map_no_path(): + # GIVEN + for i in range(RANGE): + map_no_path = VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="--xp", + inode=0, + path=None, + ) + + # WHEN + with patch("pystack.maps._get_base_map", return_value=map_no_path): + bss = _get_bss("elf_maps", "load_point") + +def time_get_bss_no_matching_map(): + + for i in range(RANGE): + libpython = VirtualMap( + start=140728765587456, + end=140728765599744, + filesize=4096, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("/some/path/to/libpython.so"), + ) + + libpython_bss = VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=None, + ) + maps = [libpython, libpython_bss] + + # WHEN + with patch("pystack._pystack.get_bss_info") as mock_get_bss_info: + mock_get_bss_info.return_value = {"corrected_addr": 100000000} + bss = _get_bss(maps, libpython.start) + +def time_get_bss_found_matching_map(): + for i in range(RANGE): + libpython = VirtualMap( + start=140728765587456, + end=140728765599744, + filesize=4096, + offset=0, + device="00:00", + flags="r--p", + inode=0, + path=Path("/some/path/to/libpython.so"), + ) + + libpython_bss = VirtualMap( + start=18446744073699065856, + end=18446744073699069952, + filesize=4096, + offset=0, + device="00:00", + flags="r-xp", + inode=0, + path=None, + ) + maps = [libpython, libpython_bss] + + # WHEN + with patch("pystack._pystack.get_bss_info") as mock_get_bss_info: + mock_get_bss_info.return_value = { + "corrected_addr": libpython_bss.start - libpython.start, + "size": libpython_bss.filesize, + } + bss = _get_bss(maps, libpython.start) diff --git a/benchmarks/benchmark_process.py b/benchmarks/benchmark_process.py new file mode 100644 index 00000000..6bec08cd --- /dev/null +++ b/benchmarks/benchmark_process.py @@ -0,0 +1,207 @@ +from pystack.process import * +from pystack.maps import VirtualMap +from unittest.mock import Mock +from unittest.mock import mock_open +from unittest.mock import patch +import pytest + +RANGE=100 + +def time_get_python_version_for_core_fallback_bss(): + for i in range(RANGE): + mapinfo = Mock() + + with patch( + "pystack.process.scan_core_bss_for_python_version" + ) as scan_bss_mock, patch( + "pystack.process.LIBPYTHON_REGEXP" + ) as libpython_regexp_mock, patch( + "pystack.process.BINARY_REGEXP" + ) as binary_regexp_mock, patch( + "subprocess.check_output" + ) as subprocess_mock: + scan_bss_mock.return_value = (3, 8) + major, minor = get_python_version_for_core("corefile", "executable", mapinfo) + +def time_get_python_version_for_core_fallback_no_bss(): + for i in range(RANGE): + mapinfo = Mock() + mapinfo.bss = None + with patch( + "pystack.process.scan_core_bss_for_python_version" + ) as scan_bss_mock, patch( + "pystack.process.LIBPYTHON_REGEXP" + ) as libpython_regexp_mock, patch( + "pystack.process.BINARY_REGEXP" + ) as binary_regexp_mock, patch( + "subprocess.check_output" + ) as subprocess_mock: + match = Mock() + match.group.side_effect = [3, 8] + libpython_regexp_mock.match.return_value = match + major, minor = get_python_version_for_core("corefile", "executable", mapinfo) + +def time_get_python_version_for_core_fallback_libpython_regexp(): + # GIVEN + for i in range(RANGE): + mapinfo = Mock() + + # WHEN + with patch( + "pystack.process.scan_core_bss_for_python_version" + ) as scan_bss_mock, patch( + "pystack.process.LIBPYTHON_REGEXP" + ) as libpython_regexp_mock, patch( + "pystack.process.BINARY_REGEXP" + ) as binary_regexp_mock, patch( + "subprocess.check_output" + ) as subprocess_mock: + scan_bss_mock.return_value = None + match = Mock() + match.group.side_effect = [3, 8] + libpython_regexp_mock.match.return_value = match + major, minor = get_python_version_for_core("corefile", "executable", mapinfo) + +def time_get_python_version_for_core_fallback_binary_regexp(): + # GIVEN + for i in range(RANGE): + mapinfo = Mock() + mapinfo.libpython = None + + # WHEN + with patch( + "pystack.process.scan_core_bss_for_python_version" + ) as scan_bss_mock, patch( + "pystack.process.LIBPYTHON_REGEXP" + ) as libpython_regexp_mock, patch( + "pystack.process.BINARY_REGEXP" + ) as binary_regexp_mock, patch( + "subprocess.check_output" + ) as subprocess_mock: + scan_bss_mock.return_value = None + libpython_regexp_mock.match.return_value = None + match = Mock() + match.group.side_effect = [3, 8] + binary_regexp_mock.match.return_value = match + major, minor = get_python_version_for_core("corefile", "executable", mapinfo) + +def tim_get_python_version_for_core_fallback_version_regexp(): + # GIVEN + for i in range(RANGE): + mapinfo = Mock() + + # WHEN + with patch( + "pystack.process.scan_core_bss_for_python_version" + ) as scan_bss_mock, patch( + "pystack.process.LIBPYTHON_REGEXP" + ) as libpython_regexp_mock, patch( + "pystack.process.BINARY_REGEXP" + ) as binary_regexp_mock, patch( + "subprocess.check_output" + ) as subprocess_mock: + scan_bss_mock.return_value = None + libpython_regexp_mock.match.return_value = None + subprocess_mock.return_value = "Python 3.8.3" + major, minor = get_python_version_for_core("corefile", "executable", mapinfo) + +def time_get_python_version_for_core_fallback_falure(): + # GIVEN + for i in range(RANGE): + mapinfo = Mock() + + # WHEN + with patch( + "pystack.process.scan_core_bss_for_python_version" + ) as scan_bss_mock, patch( + "pystack.process.LIBPYTHON_REGEXP" + ) as libpython_regexp_mock, patch( + "pystack.process.BINARY_REGEXP" + ), patch( + "subprocess.check_output" + ) as subprocess_mock: + scan_bss_mock.return_value = None + libpython_regexp_mock.match.return_value = None + subprocess_mock.return_value = "" + # THEN + with pytest.raises(InvalidPythonProcess): + get_python_version_for_core("corefile", "executable", mapinfo) + + + +def time_get_python_version_for_process(): + for i in range(RANGE): + mapinfo = Mock() + + with patch( + "pystack.process.scan_process_bss_for_python_version" + ) as scan_bss_mock, patch( + "pystack.process.LIBPYTHON_REGEXP" + ) as libpython_regexp_mock, patch( + "pystack.process.BINARY_REGEXP" + ) as binary_regexp_mock, patch( + "subprocess.check_output" + ) as subprocess_mock: + scan_bss_mock.return_value = (3, 8) + major, minor = get_python_version_for_process(0, mapinfo) + +def time_scan_core_bss_for_python_version(): + for i in range(RANGE): + memory = ( + b"garbagegarbagePython 3.8.3 (default, May 22 2020, 23:30:25)garbagegarbage" + ) + bss = VirtualMap( + start=0, + end=len(memory), + filesize=len(memory), + offset=0, + flags="", + inode=0, + device="", + path=None, + ) + # WHEN + + with patch("builtins.open", mock_open(read_data=memory)): + major, minor = scan_core_bss_for_python_version("corefile", bss) + +def time_scan_core_bss_for_python_version_failure(): + # GIVEM + for i in range(RANGE): + memory = b"garbagegarbagegarbagegarbage" + bss = VirtualMap( + start=0, + end=len(memory), + filesize=len(memory), + offset=0, + flags="", + inode=0, + device="", + path=None, + ) + # WHEN + + with patch("builtins.open", mock_open(read_data=memory)): + result = scan_core_bss_for_python_version("corefile", bss) + + +def time_scan_process_bss_for_python_version(): + for i in range(RANGE): + memory = ( + b"garbagegarbagePython 3.8.3 (default, May 22 2020, 23:30:25)garbagegarbage" + ) + bss = Mock() + # WHEN + + with patch("pystack._pystack.copy_memory_from_address", return_value=memory): + major, minor = scan_process_bss_for_python_version(0, bss) + +def time_scan_process_bss_for_python_version_failure(): + for i in range(RANGE): + memory = b"garbagegarbagegarbagegarbage" + bss = Mock() + # WHEN + + with patch("pystack._pystack.copy_memory_from_address", return_value=memory): + result = scan_process_bss_for_python_version(0, bss) + diff --git a/benchmarks/benchmark_traceback_formatters.py b/benchmarks/benchmark_traceback_formatters.py new file mode 100644 index 00000000..4fa103f9 --- /dev/null +++ b/benchmarks/benchmark_traceback_formatters.py @@ -0,0 +1,1045 @@ +from pystack.traceback_formatter import format_thread +from pystack.traceback_formatter import print_thread +from unittest.mock import patch, mock_open +from pystack.types import NativeFrame +from pystack.types import PyCodeObject +from pystack.types import LocationInfo +from pystack.types import PyFrame +from pystack.types import PyThread, SYMBOL_IGNORELIST + +RANGE=100 + + +def time_format_thread_no_native(): + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=True, + ) + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x2, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x3, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x4, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame(0x5, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x6, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=False)) + +def time_traceback_formatter_no_frames_no_native(): + # GIVEN + for i in range(RANGE): + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=False)) + +def time_traceback_formatter_no_frames_native(): + # GIVEN + for i in range(RANGE): + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x2, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x4, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame(0x6, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=None, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=True)) + +def time_traceback_formatter_no_frames_native_with_eval_frames(): + # GIVEN + for i in range(RANGE): + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x2, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x3, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x4, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame(0x5, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x6, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=None, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(2, 7), + ) + + # WHEN + + lines = list(format_thread(thread, native=True)) + +def time_traceback_formatter_no_mergeable_native_frames(): + # GIVEN + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=True, + ) + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x2, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x4, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame(0x5, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x6, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=True)) + +def time_traceback_formatter_with_source(): + # GIVEN + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=True, + ) + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x2, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x3, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x4, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame(0x5, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x6, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + source_data = "\n".join(f'x = "This is the line {line}"' for line in range(1, 5)) + with patch("builtins.open", mock_open(read_data=source_data)), patch( + "os.path.exists", return_value=True + ): + lines = list(format_thread(thread, native=False)) + +def time_traceback_formatter_native_matching_simple_eval_frames(): + # GIVEN + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=True, + ) + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame( + 0x1, "_PyEval_EvalFrameDefault", "Python/ceval.c", 123, 0, "library.so" + ), + NativeFrame(0x2, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame( + 0x3, "_PyEval_EvalFrameDefault", "Python/ceval.c", 123, 0, "library.so" + ), + NativeFrame(0x4, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame( + 0x5, "_PyEval_EvalFrameDefault", "Python/ceval.c", 123, 0, "library.so" + ), + NativeFrame(0x6, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=True)) + +def time_traceback_formatter_native_matching_composite_eval_frames(): + # GIVEN + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=True, + ) + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame( + 0x2, "PyEval_EvalFrameDefault", "Python/ceval.c", 12, 0, "library.so" + ), + NativeFrame( + 0x3, "_PyEval_EvalFrameDefault", "Python/ceval.c", 130, 0, "library.so" + ), + NativeFrame(0x4, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x5, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame( + 0x7, "PyEval_EvalFrameDefault", "Python/ceval.c", 12, 0, "library.so" + ), + NativeFrame( + 0x8, "_PyEval_EvalFrameDefault", "Python/ceval.c", 130, 0, "library.so" + ), + NativeFrame(0x9, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame(0x10, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame( + 0x11, "PyEval_EvalFrameDefault", "Python/ceval.c", 12, 0, "library.so" + ), + NativeFrame( + 0x12, "_PyEval_EvalFrameDefault", "Python/ceval.c", 130, 0, "library.so" + ), + NativeFrame(0x13, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=True)) + +def time_traceback_formatter_native_matching_eval_frames_ignore_frames(): + # GIVEN + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + PyCodeObject( + filename="file4.py", + scope="function4", + location=LocationInfo(4, 4, 0, 0), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=True, + ) + + ignorelist_frames = [ + NativeFrame(0x10, symbol, "some/random/file.c", 13, 0, "library.so") + for symbol in SYMBOL_IGNORELIST + ] + + private_frames = [ + NativeFrame(0x11, symbol, "Python/private.c", 13, 0, "library.so") + for symbol in ("_PyPrivateFunction", "_PyAnotherPrivateFunction") + ] + + eval_ignore_frames = [ + NativeFrame(0x12, symbol, "Python/ceval.c", 13, 0, "library.so") + for symbol in ("PyEval_SomethingSomething", "_PyEval_SomethingSomething") + ] + + vectorcall_frames = [ + NativeFrame(0x13, symbol, "Python/call.c", 13, 0, "library.so") + for symbol in ( + "vectorcall_rules", + "function_vectorcall", + "super_Vectorcall_call", + "VECTORCALL_Ex", + ) + ] + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame( + 0x1, "_PyEval_EvalFrameDefault", "Python/ceval.c", 123, 0, "library.so" + ), + *eval_ignore_frames, + NativeFrame(0x2, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame( + 0x3, "_PyEval_EvalFrameDefault", "Python/ceval.c", 123, 0, "library.so" + ), + *ignorelist_frames, + NativeFrame(0x4, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame( + 0x5, "_PyEval_EvalFrameDefault", "Python/ceval.c", 123, 0, "library.so" + ), + *private_frames, + NativeFrame(0x6, "native_function4", "native_file4.c", 4, 0, "library.so"), + NativeFrame( + 0x7, "_PyEval_EvalFrameDefault", "Python/ceval.c", 123, 0, "library.so" + ), + *vectorcall_frames, + NativeFrame(0x8, "native_function5", "native_file5.c", 5, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=True)) + +def time_traceback_formatter_gil_detection(): + # GIVEN + for i in range(RANGE): + frame = PyFrame( + prev=None, + next=None, + code=PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + arguments={}, + locals={}, + is_entry=True, + ) + thread = PyThread( + tid=1, + frame=frame, + native_frames=[], + holds_the_gil=True, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=False)) + +def time_traceback_formatter_gc_detection_with_native(): + # GIVEN + for i in range(RANGE): + frame = PyFrame( + prev=None, + next=None, + code=PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + arguments={}, + locals={}, + is_entry=True, + ) + thread = PyThread( + tid=1, + frame=frame, + native_frames=[ + NativeFrame(0x0, "gc_collect", "native_file1.c", 1, 0, "library.so") + ], + holds_the_gil=False, + is_gc_collecting=-1, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=False)) + +def time_traceback_formatter_gc_detection_without_native(): + for i in range(RANGE): + + frame = PyFrame( + prev=None, + next=None, + code=PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + arguments={}, + locals={}, + is_entry=True, + ) + thread = PyThread( + tid=1, + frame=frame, + native_frames=[], + holds_the_gil=True, + is_gc_collecting=True, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=False)) + +def time_traceback_formatter_dropping_the_gil_detection(): + for i in range(RANGE): + frame = PyFrame( + prev=None, + next=None, + code=PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + arguments={}, + locals={}, + is_entry=True, + ) + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x2, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x3, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x4, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame(0x5, "drop_gil", "Python/gil.c", 24, 0, "library.so"), + NativeFrame(0x6, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=False)) + +def time_traceback_formatter_taking_the_gil_detection(): + for i in range(RANGE): + frame = PyFrame( + prev=None, + next=None, + code=PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + arguments={}, + locals={}, + is_entry=True, + ) + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x2, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x3, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x4, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame(0x5, "take_gil", "Python/gil.c", 24, 0, "library.so"), + NativeFrame(0x6, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=False)) + +def time_traceback_formatter_native_not_matching_simple_eval_frames(): + # GIVEN + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=True, + ) + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x2, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x3, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x4, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=False)) + +def time_traceback_formatter_native_not_matching_composite_eval_frames(): + # GIVEN + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=True, + ) + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame( + 0x2, "PyEval_EvalFrameDefault", "Python/ceval.c", 12, 0, "library.so" + ), + NativeFrame( + 0x3, "_PyEval_EvalFrameDefault", "Python/ceval.c", 130, 0, "library.so" + ), + NativeFrame(0x4, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x5, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame(0x6, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame( + 0x7, "PyEval_EvalFrameDefault", "Python/ceval.c", 12, 0, "library.so" + ), + NativeFrame( + 0x8, "_PyEval_EvalFrameDefault", "Python/ceval.c", 130, 0, "library.so" + ), + NativeFrame(0x9, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=False)) + +def time_traceback_formatter_mixed_inlined_frames(): + # GIVEN + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + PyCodeObject( + filename="file4.py", + scope="function4", + location=LocationInfo(4, 4, 0, 0), + ), + PyCodeObject( + filename="file5.py", + scope="function5", + location=LocationInfo(5, 5, 0, 0), + ), + ] + + current_frame = None + entry_funcs = {"function1", "function3"} + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=code.scope in entry_funcs, + ) + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame( + 0x2, "_PyEval_EvalFrameDefault", "Python/ceval.c", 12, 0, "library.so" + ), + NativeFrame(0x3, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame(0x4, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame( + 0x5, "_PyEval_EvalFrameDefault", "Python/ceval.c", 12, 0, "library.so" + ), + NativeFrame(0x6, "native_function3", "native_file3.c", 3, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=True)) + +def time_traceback_formatter_all_inlined_frames(): + + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + PyCodeObject( + filename="file4.py", + scope="function4", + location=LocationInfo(4, 4, 0, 0), + ), + PyCodeObject( + filename="file5.py", + scope="function5", + location=LocationInfo(5, 5, 0, 0), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=False, + ) + current_frame.is_entry = True + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame( + 0x2, "_PyEval_EvalFrameDefault", "Python/ceval.c", 12, 0, "library.so" + ), + NativeFrame(0x6, "native_function3", "native_file3.c", 3, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=True)) + +def time_print_thread(): + for i in range(RANGE): + thread = PyThread( + tid=1, + frame=None, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + # WHEN + + with patch( + "pystack.traceback_formatter.format_thread", + return_value=("1", "2", "3"), + ): + print_thread(thread, native=False) + +def time_traceback_formatter_locals( + arguments, locals, location_info, expected_locals_render +): + # GIVEN + arguments=[ + {"the_argument": "some_value", "the_second_argument": "42"}, + {}, + {"the_argument": "some_value"}, + {"the_argument": "\x1b[6;30;42m some_value\nwith\nnewlines'\x1b[0m"}, + ] + + locals = [ + {"the_local": "some_other_value", "the_second_local": "7"}, + {"the_local": "some_other_value"}, + {}, + {}, + ] + + location_infos = [(1, 0, 1, 0), (1, 1, 0, 0)] + for argument in arguments: + for local in locals: + for location_info in location_infos: + + frame = PyFrame( + prev=None, + next=None, + code=PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(*location_info), + ), + arguments=argument, + locals=local, + is_entry=True, + ) + + thread = PyThread( + tid=1, + frame=frame, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + source_data = "\n".join( + f'x = "This is the line {line}" or (1+1)' for line in range(1, 5) + ) + with patch("builtins.open", mock_open(read_data=source_data)), patch( + "os.path.exists", return_value=True + ): + lines = list(format_thread(thread, native=False)) + +def test_traceback_formatter_thread_names(): + # GIVEN + for i in range(RANGE): + frame = PyFrame( + prev=None, + next=None, + code=PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + arguments=[], + locals=[], + is_entry=True, + ) + + thread = PyThread( + tid=1, + frame=frame, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + name="foo", + ) + + # WHEN + + lines = list(format_thread(thread, native=False)) + +def test_traceback_formatter_position_infomation(): + # GIVEN + for i in range(RANGE): + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 3), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 4, 25), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 28, 33), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=True, + ) + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=[], + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + source_data = "\n".join( + f'x = "This is the line {line}" or (1+1)' for line in range(1, 5) + ) + with patch("builtins.open", mock_open(read_data=source_data)), patch( + "os.path.exists", return_value=True + ), patch( + "pystack.traceback_formatter.colored", + side_effect=lambda x, *args, **kwargs: x, + ) as colored_mock: + lines = list(format_thread(thread, native=False)) \ No newline at end of file diff --git a/benchmarks/benchmark_types.py b/benchmarks/benchmark_types.py new file mode 100644 index 00000000..f59c7849 --- /dev/null +++ b/benchmarks/benchmark_types.py @@ -0,0 +1,181 @@ +import pytest + +from pystack.types import SYMBOL_IGNORELIST +from pystack.types import NativeFrame +from pystack.types import PyThread +from pystack.types import frame_type + +RANGE=100 + + +def time_frame_type_eval_frame_with_pep_523(): + # GIVEN + symbols = ["_PyEval_EvalFrameDefault", "_PyEval_EvalFrameDefault.cold.32"] + versions = [ + (2,7), + (3,5), + (3,6), + (3,7), + (3,8), + (3,9) + ] + + for i in range(RANGE): + for symbol in symbols: + for version in versions: + + frame = NativeFrame(0x3, symbol, "Python/ceval.c", 123, 0, "library.so") + + # WHEN + + type_ = frame_type(frame, version) + +def time_frame_type_eval_frame_without_pep_523(): + # GIVEN + + symbols = ["PyEval_EvalFrameEx", "PyEval_EvalFrameEx.cold.32"] + versions = [ + (2,7), + (3,5), + (3,6), + (3,7), + (3,8), + (3,9) + ] + + for i in range(RANGE): + for symbol in symbols: + for version in versions: + frame = NativeFrame( + 0x3, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so" + ) + + # WHEN + + type_ = frame_type(frame, version) + +def time_frame_type_eval_machinery_is_ignored(): + # GIVEN + versions = [ + (2,7), + (3,5), + (3,6), + (3,7), + (3,8), + (3,9) + ] + for i in range(RANGE): + for version in versions: + frame = NativeFrame( + 0x3, "_PyEval_SomeStuff", "Python/ceval.c", 123, 0, "library.so" + ) + + # WHEN + + type_ = frame_type(frame, version) + +def time_frame_type_private_python_apis_are_ignored(): + # GIVEN + versions = [ + (2,7), + (3,5), + (3,6), + (3,7), + (3,8), + (3,9) + ] + + for i in range(RANGE): + for version in versions: + frame = NativeFrame( + 0x3, "_PySome_Private_Api", "Python/private.c", 123, 0, "library.so" + ) + + # WHEN + + type_ = frame_type(frame, version) + +def time_frame_type_vectorcall_machinery(): + # GIVEN + versions = [ + (2,7), + (3,5), + (3,6), + (3,7), + (3,8), + (3,9) + ] + for i in range(RANGE): + for version in versions: + frame = NativeFrame( + 0x3, "blablabla_vectorcall_blublublu", "Python/private.c", 123, 0, "library.so" + ) + + # WHEN + + type_ = frame_type(frame, version) + +def time_frame_type_explicitly_ignored_symbols_are_ignored(): + # GIVEN + versions = [ + (2,7), + (3,5), + (3,6), + (3,7), + (3,8), + (3,9) + ] + symbols = sorted(SYMBOL_IGNORELIST) + for i in range(RANGE): + for symbol in symbols: + for version in versions: + frame = NativeFrame(0x3, symbol, "Python/private.c", 123, 0, "library.so") + + # WHEN + + type_ = frame_type(frame, version) + +def time_gil_states(): + gill_states = [0 ,-1, 1] + + for i in range(RANGE): + for gil_state in gill_states: + + thread = PyThread(1, None, [], gil_state, 0, (3, 8)) + + # WHEN + + state = thread.gil_status + +def time_gc_states_with_gil(): + # GIVEN + gill_states = [0 ,-1, 1] + for i in range(RANGE): + for gc_state in gill_states: + thread = PyThread(1, None, [], 1, gc_state, (3, 8)) + + # WHEN + + state = thread.gc_status + +def time_gc_states_with_no_gil(): + # GIVEN + gill_states = [0 ,-1, 1] + + for i in range(RANGE): + for gc_state in gill_states: + thread = PyThread(1, None, [], 0, gc_state, (3, 8)) + + # WHEN + + state = thread.gc_status + +def time_dead_thread(): + # GIVEN + + for i in range(RANGE): + thread = PyThread(0, None, [], 0, 0, (3, 8)) + + # WHEN + + state = thread.status From 68fd5384b1c7dc6a7f9b1635484b1c04a9f45a7c Mon Sep 17 00:00:00 2001 From: ms2892 Date: Mon, 15 Jan 2024 19:15:53 +0000 Subject: [PATCH 7/8] Added minor documentation Signed-off-by: ms2892 --- asv.conf.json | 4 ++-- benchmarks/README.md | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 benchmarks/README.md diff --git a/asv.conf.json b/asv.conf.json index aea93aca..7a5443cb 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -1,9 +1,9 @@ { "version":1, "benchmark_dir": "./benchmarks", - "repo":"git@github.com:ms2892/pystack.git", + "repo":"git@github.com:bloomberg/pystack.git", "project": "pystack", - "project_url": "https://github.com/ms2892/pystack", + "project_url": "https://github.com/bloomberg/pystack", "env_dir":".asv/env", "results_dir":".asv/results", "html_dir":".asv/html", diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..6c1ed37e --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,34 @@ +# Using the Benchmarking Tool + +One of the prerequisites is to have the respective libraries installed. Hence do install the following libraries + +- libdw +- libelf + +These can be installed via the command `apt-get install libdw-dev libelf-dev`. + +To benchmark the packages present another tool is used called `airspeed velocity`. To install it please run the follow command + +```pip install asv``` + +In the parent directory run the following command to get a brief benchmark of your current packages + +```asv run``` + +Use the `-v` flag to get a verbose output. + +To compare the all the commits across all the branches you may make use of the following command. + +```asv run ALL``` + +To run benchmarks from a particular commit or tag you can use the commit hash or the tag + +```asv run [TAG|HASH]..[branch]``` + +To compare between tags + +```asv show [TAG]..[branch]``` + +To have a local server to display all the graphs + +```asv publish``` \ No newline at end of file From 344db213163872fd2910c0b9104d79ae96d4b834 Mon Sep 17 00:00:00 2001 From: ms2892 Date: Thu, 25 Jan 2024 19:38:49 +0000 Subject: [PATCH 8/8] Fixed static tests Signed-off-by: ms2892 --- benchmarks/benchmark_colors.py | 10 ++- benchmarks/benchmark_maps.py | 111 ++++++++++++++++---------------- benchmarks/benchmark_process.py | 8 ++- 3 files changed, 71 insertions(+), 58 deletions(-) diff --git a/benchmarks/benchmark_colors.py b/benchmarks/benchmark_colors.py index 13978049..fbc75abc 100644 --- a/benchmarks/benchmark_colors.py +++ b/benchmarks/benchmark_colors.py @@ -1,4 +1,4 @@ -from pystack.colors import * +from pystack.colors import colored, format_colored RANGE=100 @@ -9,7 +9,13 @@ def setup(self): def time_colored(self): colors = ["red","green","yellow","blue","magenta","cyan","white"] - highlights = ["on_red","on_green","on_yellow","on_blue","on_magenta","on_cyan","on_white"] + highlights = ["on_red", + "on_green", + "on_yellow", + "on_blue", + "on_magenta", + "on_cyan", + "on_white"] attributes = ["bold", "dark", "underline", "blink", "reverse", "concealed"] for counter in range(RANGE): for color in colors: diff --git a/benchmarks/benchmark_maps.py b/benchmarks/benchmark_maps.py index cca95f12..5e24e26f 100644 --- a/benchmarks/benchmark_maps.py +++ b/benchmarks/benchmark_maps.py @@ -34,20 +34,23 @@ def time_simple_maps_no_such_pid(): def time_simple_maps(): for i in range(RANGE): - map_text = """ - 7f1ac1e2b000-7f1ac1e50000 r--p 00000000 08:12 8398159 /usr/lib/libc-2.31.so - """ - - # WHEN + map_text = ( + "7f1ac1e2b000-7f1ac1e50000 " + "r--p " + "00000000 08:12 8398159" + " /usr/lib/libc-2.31.so" + ) with patch("builtins.open", mock_open(read_data=map_text)): maps = list(generate_maps_for_process(1)) def time_maps_with_long_device_numbers(): for i in range(RANGE): - map_text = """ - 7f1ac1e2b000-7f1ac1e50000 r--p 00000000 0123:4567 8398159 /usr/lib/libc-2.31.so - """ + map_text = ( + "7f1ac1e2b000-7f1ac1e50000 " + "r--p 00000000 0123:4567 " + "8398159 /usr/lib/libc-2.31.so" + ) # WHEN @@ -69,14 +72,12 @@ def time_map_permissions(): # GIVEN for i in range(RANGE): map_text = """ - 7f1ac1e2b000-7f1ac1e50000 r--- 00000000 08:12 8398159 /usr/lib/libc-2.31.so - 7f1ac1e2b000-7f1ac1e50000 rw-- 00000000 08:12 8398159 /usr/lib/libc-2.31.so - 7f1ac1e2b000-7f1ac1e50000 rwx- 00000000 08:12 8398159 /usr/lib/libc-2.31.so - 7f1ac1e2b000-7f1ac1e50000 rwxp 00000000 08:12 8398159 /usr/lib/libc-2.31.so +7f1ac1e2b000-7f1ac1e50000 r--- 00000000 08:12 8398159 /usr/lib/libc-2.31.so +7f1ac1e2b000-7f1ac1e50000 rw-- 00000000 08:12 8398159 /usr/lib/libc-2.31.so +7f1ac1e2b000-7f1ac1e50000 rwx- 00000000 08:12 8398159 /usr/lib/libc-2.31.so +7f1ac1e2b000-7f1ac1e50000 rwxp 00000000 08:12 8398159 /usr/lib/libc-2.31.so """ - # WHEN - with patch("builtins.open", mock_open(read_data=map_text)): maps = list(generate_maps_for_process(1)) @@ -84,8 +85,8 @@ def time_unexpected_line_is_ignored(): # GIVEN for i in range(RANGE): map_text = """ - I am an unexpected line - 7f1ac1e2b000-7f1ac1e50000 r--p 00000000 08:12 8398159 /usr/lib/libc-2.31.so +I am an unexpected line +7f1ac1e2b000-7f1ac1e50000 r--p 00000000 08:12 8398159 /usr/lib/libc-2.31.so """ # WHEN @@ -96,11 +97,11 @@ def time_unexpected_line_is_ignored(): def time_special_maps(): for i in range(RANGE): map_text = """ - 555f1ab1c000-555f1ab3d000 rw-p 00000000 00:00 0 [heap] - 7ffdf8102000-7ffdf8124000 rw-p 00000000 00:00 0 [stack] - 7ffdf8152000-7ffdf8155000 r--p 00000000 00:00 0 [vvar] - 7ffdf8155000-7ffdf8156000 r-xp 00000000 00:00 0 [vdso] - ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall] +555f1ab1c000-555f1ab3d000 rw-p 00000000 00:00 0 [heap] +7ffdf8102000-7ffdf8124000 rw-p 00000000 00:00 0 [stack] +7ffdf8152000-7ffdf8155000 r--p 00000000 00:00 0 [vvar] +7ffdf8155000-7ffdf8156000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall] """ # WHEN @@ -750,40 +751,40 @@ def time_maps_with_scattered_segments(): for i in range(RANGE): map_text = """ - 00400000-00401000 r-xp 00000000 fd:00 67488961 /bin/python3.9-dbg - 00600000-00601000 r--p 00000000 fd:00 67488961 /bin/python3.9-dbg - 00601000-00602000 rw-p 00001000 fd:00 67488961 /bin/python3.9-dbg - 0067b000-00a58000 rw-p 00000000 00:00 0 [heap] - 7f7b38000000-7f7b38028000 rw-p 00000000 00:00 0 - 7f7b38028000-7f7b3c000000 ---p 00000000 00:00 0 - 7f7b40000000-7f7b40021000 rw-p 00000000 00:00 0 - 7f7b40021000-7f7b44000000 ---p 00000000 00:00 0 - 7f7b44ec0000-7f7b44f40000 rw-p 00000000 00:00 0 - f7b45a61000-7f7b45d93000 rw-p 00000000 00:00 0 - 7f7b46014000-7f7b46484000 r--p 0050b000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 - 7f7b46484000-7f7b46485000 ---p 00000000 00:00 0 - 7f7b46485000-7f7b46cda000 rw-p 00000000 00:00 0 - 7f7b46cda000-7f7b46d16000 r--p 00a3d000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 - 7f7b46d16000-7f7b46d6f000 rw-p 00000000 00:00 0 - 7f7b46d6f000-7f7b46d92000 r--p 00001000 fd:00 67488961 /bin/python3.9-dbg - 7f7b46d92000-7f7b46d93000 ---p 00000000 00:00 0 - 7f7b46d93000-7f7b475d3000 rw-p 00000000 00:00 0 - 7f7b498c1000-7f7b49928000 r-xp 00000000 fd:00 7023 /lib64/libssl.so.1.0.0 - 7f7b49928000-7f7b49b28000 ---p 00067000 fd:00 7023 /lib64/libssl.so.1.0.0 - f7b4c632000-7f7b4c6f3000 rw-p 00000000 00:00 0 - 7f7b4c6f3000-7f7b4c711000 rw-p 00000000 00:00 0 - 7f7b4c711000-7f7b4c712000 r--p 0002a000 fd:00 67488961 /bin/python3.9-dbg - 7f7b4c712000-7f7b4c897000 rw-p 00000000 00:00 0 - 7f7b5a356000-7f7b5a35d000 r--s 00000000 fd:00 201509519 /usr/lib64/gconv/gconv-modules.cache - 7f7b5a35d000-7f7b5a827000 r-xp 00000000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 - 7f7b5a827000-7f7b5aa27000 ---p 004ca000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 - 7f7b5aa27000-7f7b5aa2c000 r--p 004ca000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 - 7f7b5aa2c000-7f7b5aa67000 rw-p 004cf000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 - 7f7b5aa67000-7f7b5aa8b000 rw-p 00000000 00:00 0 - 7fff26f8e000-7fff27020000 rw-p 00000000 00:00 0 [stack] - 7fff27102000-7fff27106000 r--p 00000000 00:00 0 [vvar] - 7fff27106000-7fff27108000 r-xp 00000000 00:00 0 [vdso] - ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] +00400000-00401000 r-xp 00000000 fd:00 67488961 /bin/python3.9-dbg +00600000-00601000 r--p 00000000 fd:00 67488961 /bin/python3.9-dbg +00601000-00602000 rw-p 00001000 fd:00 67488961 /bin/python3.9-dbg +0067b000-00a58000 rw-p 00000000 00:00 0 [heap] +7f7b38000000-7f7b38028000 rw-p 00000000 00:00 0 +7f7b38028000-7f7b3c000000 ---p 00000000 00:00 0 +7f7b40000000-7f7b40021000 rw-p 00000000 00:00 0 +7f7b40021000-7f7b44000000 ---p 00000000 00:00 0 +7f7b44ec0000-7f7b44f40000 rw-p 00000000 00:00 0 +f7b45a61000-7f7b45d93000 rw-p 00000000 00:00 0 +7f7b46014000-7f7b46484000 r--p 0050b000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 +7f7b46484000-7f7b46485000 ---p 00000000 00:00 0 +7f7b46485000-7f7b46cda000 rw-p 00000000 00:00 0 +7f7b46cda000-7f7b46d16000 r--p 00a3d000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 +7f7b46d16000-7f7b46d6f000 rw-p 00000000 00:00 0 +7f7b46d6f000-7f7b46d92000 r--p 00001000 fd:00 67488961 /bin/python3.9-dbg +7f7b46d92000-7f7b46d93000 ---p 00000000 00:00 0 +7f7b46d93000-7f7b475d3000 rw-p 00000000 00:00 0 +7f7b498c1000-7f7b49928000 r-xp 00000000 fd:00 7023 /lib64/libssl.so.1.0.0 +7f7b49928000-7f7b49b28000 ---p 00067000 fd:00 7023 /lib64/libssl.so.1.0.0 +f7b4c632000-7f7b4c6f3000 rw-p 00000000 00:00 0 +7f7b4c6f3000-7f7b4c711000 rw-p 00000000 00:00 0 +7f7b4c711000-7f7b4c712000 r--p 0002a000 fd:00 67488961 /bin/python3.9-dbg +7f7b4c712000-7f7b4c897000 rw-p 00000000 00:00 0 +7f7b5a356000-7f7b5a35d000 r--s 00000000 fd:00 201509519 /usr/lib64/gconv/gconv-modules.cache +7f7b5a35d000-7f7b5a827000 r-xp 00000000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 +7f7b5a827000-7f7b5aa27000 ---p 004ca000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 +7f7b5aa27000-7f7b5aa2c000 r--p 004ca000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 +7f7b5aa2c000-7f7b5aa67000 rw-p 004cf000 fd:00 1059871 /lib64/libpython3.9d.so.1.0 +7f7b5aa67000-7f7b5aa8b000 rw-p 00000000 00:00 0 +7fff26f8e000-7fff27020000 rw-p 00000000 00:00 0 [stack] +7fff27102000-7fff27106000 r--p 00000000 00:00 0 [vvar] +7fff27106000-7fff27108000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] """ # WHEN diff --git a/benchmarks/benchmark_process.py b/benchmarks/benchmark_process.py index 6bec08cd..303ec76e 100644 --- a/benchmarks/benchmark_process.py +++ b/benchmarks/benchmark_process.py @@ -1,4 +1,10 @@ -from pystack.process import * +from pystack.process import ( + get_python_version_for_core, + InvalidPythonProcess, + get_python_version_for_process, + scan_core_bss_for_python_version, + scan_process_bss_for_python_version +) from pystack.maps import VirtualMap from unittest.mock import Mock from unittest.mock import mock_open