diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 6f8fe005621c88..a4ada1b66bf476 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,11 +2,11 @@ FROM docker.io/library/fedora:40 ENV CC=clang -ENV WASI_SDK_VERSION=21 +ENV WASI_SDK_VERSION=22 ENV WASI_SDK_PATH=/opt/wasi-sdk ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=18.0.3 +ENV WASMTIME_VERSION=22.0.0 ENV WASMTIME_CPU_ARCH=x86_64 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e8f4a4693a814c..9aa5004b0cdb7f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -72,6 +72,7 @@ Include/internal/pycore_freelist.h @ericsnowcurrently Include/internal/pycore_global_objects.h @ericsnowcurrently Include/internal/pycore_obmalloc.h @ericsnowcurrently Include/internal/pycore_pymem.h @ericsnowcurrently +Include/internal/pycore_stackref.h @Fidget-Spinner Modules/main.c @ericsnowcurrently Programs/_bootstrap_python.c @ericsnowcurrently Programs/python.c @ericsnowcurrently @@ -213,6 +214,7 @@ Doc/c-api/stable.rst @encukou **/*idlelib* @terryjreedy /Doc/library/idle.rst @terryjreedy +**/*annotationlib* @JelleZijlstra **/*typing* @JelleZijlstra @AlexWaygood **/*ftplib @giampaolo diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 750aa1ed87bca1..6568b50c3a6a13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,11 +27,32 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: + # Some of the referenced steps set outputs conditionally and there may be + # cases when referencing them evaluates to empty strings. It is nice to + # work with proper booleans so they have to be evaluated through JSON + # conversion in the expressions. However, empty strings used like that + # may trigger all sorts of undefined and hard-to-debug behaviors in + # GitHub Actions CI/CD. To help with this, all of the outputs set here + # that are meant to be used as boolean flags (and not arbitrary strings), + # MUST have fallbacks with default values set. A common pattern would be + # to add ` || false` to all such expressions here, in the output + # definitions. They can then later be safely used through the following + # idiom in job conditionals and other expressions. Here's some examples: + # + # if: fromJSON(needs.check_source.outputs.run-docs) + # + # ${{ + # fromJSON(needs.check_source.outputs.run_tests) + # && 'truthy-branch' + # || 'falsy-branch' + # }} + # run-docs: ${{ steps.docs-changes.outputs.run-docs || false }} - run_tests: ${{ steps.check.outputs.run_tests }} - run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} - run_cifuzz: ${{ steps.check.outputs.run_cifuzz }} - config_hash: ${{ steps.config_hash.outputs.hash }} + run-win-msi: ${{ steps.win-msi-changes.outputs.run-win-msi || false }} + run_tests: ${{ steps.check.outputs.run_tests || false }} + run_hypothesis: ${{ steps.check.outputs.run_hypothesis || false }} + run_cifuzz: ${{ steps.check.outputs.run_cifuzz || false }} + config_hash: ${{ steps.config_hash.outputs.hash }} # str steps: - uses: actions/checkout@v4 - name: Check for source changes @@ -103,6 +124,20 @@ jobs: id: docs-changes run: | echo "run-docs=true" >> "${GITHUB_OUTPUT}" + - name: Get a list of the MSI installer-related files + id: changed-win-msi-files + uses: Ana06/get-changed-files@v2.3.0 + with: + filter: | + Tools/msi/** + .github/workflows/reusable-windows-msi.yml + format: csv # works for paths with spaces + - name: Check for changes in MSI installer-related files + if: >- + steps.changed-win-msi-files.outputs.added_modified_renamed != '' + id: win-msi-changes + run: | + echo "run-win-msi=true" >> "${GITHUB_OUTPUT}" check-docs: name: Docs @@ -179,68 +214,89 @@ jobs: run: make check-c-globals build_windows: - name: 'Windows' - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-windows.yml - - build_windows_free_threading: - name: 'Windows (free-threading)' + name: >- + Windows + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: fromJSON(needs.check_source.outputs.run_tests) + strategy: + matrix: + arch: + - Win32 + - x64 + - arm64 + free-threading: + - false + - true uses: ./.github/workflows/reusable-windows.yml with: - free-threading: true + arch: ${{ matrix.arch }} + free-threading: ${{ matrix.free-threading }} - build_macos: - name: 'macOS' + build_windows_msi: + name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category + Windows MSI${{ '' }} needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-macos.yml + if: fromJSON(needs.check_source.outputs.run-win-msi) + strategy: + matrix: + arch: + - x86 + - x64 + - arm64 + uses: ./.github/workflows/reusable-windows-msi.yml with: - config_hash: ${{ needs.check_source.outputs.config_hash }} - # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. - # Cirrus used for upstream, macos-14 for forks. - os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-14", "macos-13"]' + arch: ${{ matrix.arch }} - build_macos_free_threading: - name: 'macOS (free-threading)' + build_macos: + name: >- + macOS + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source if: needs.check_source.outputs.run_tests == 'true' + strategy: + fail-fast: false + matrix: + # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. + # macOS 13 only runs tests against the GIL-enabled CPython. + # Cirrus used for upstream, macos-14 for forks. + os: + - ghcr.io/cirruslabs/macos-runner:sonoma + - macos-14 + - macos-13 + is-fork: # only used for the exclusion trick + - ${{ github.repository_owner != 'python' }} + free-threading: + - false + - true + exclude: + - os: ghcr.io/cirruslabs/macos-runner:sonoma + is-fork: true + - os: macos-14 + is-fork: false + - os: macos-13 + free-threading: true uses: ./.github/workflows/reusable-macos.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - free-threading: true - # Cirrus and macos-14 are M1. - # Cirrus used for upstream, macos-14 for forks. - os-matrix: '["ghcr.io/cirruslabs/macos-runner:sonoma", "macos-14"]' + free-threading: ${{ matrix.free-threading }} + os: ${{ matrix.os }} build_ubuntu: - name: 'Ubuntu' - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-ubuntu.yml - with: - config_hash: ${{ needs.check_source.outputs.config_hash }} - options: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --with-openssl=$OPENSSL_DIR - - build_ubuntu_free_threading: - name: 'Ubuntu (free-threading)' + name: >- + Ubuntu + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source if: needs.check_source.outputs.run_tests == 'true' + strategy: + matrix: + free-threading: + - false + - true uses: ./.github/workflows/reusable-ubuntu.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} - options: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --with-openssl=$OPENSSL_DIR \ - --disable-gil + free-threading: ${{ matrix.free-threading }} build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' @@ -292,7 +348,7 @@ jobs: with: save: false - name: Configure CPython - run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR + run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl=$OPENSSL_DIR - name: Build CPython run: make -j4 - name: Display build info @@ -365,6 +421,7 @@ jobs: ../cpython-ro-srcdir/configure \ --config-cache \ --with-pydebug \ + --enable-slower-safety \ --with-openssl=$OPENSSL_DIR - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} @@ -393,7 +450,7 @@ jobs: path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} restore-keys: | - - hypothesis-database- + hypothesis-database- - name: "Run tests" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -550,13 +607,11 @@ jobs: - check-docs - check_generated_files - build_macos - - build_macos_free_threading - build_ubuntu - - build_ubuntu_free_threading - build_ubuntu_ssltests - build_wasi - build_windows - - build_windows_free_threading + - build_windows_msi - test_hypothesis - build_asan - build_tsan @@ -571,6 +626,7 @@ jobs: with: allowed-failures: >- build_ubuntu_ssltests, + build_windows_msi, cifuzz, test_hypothesis, allowed-skips: >- @@ -586,13 +642,10 @@ jobs: && ' check_generated_files, build_macos, - build_macos_free_threading, build_ubuntu, - build_ubuntu_free_threading, build_ubuntu_ssltests, build_wasi, build_windows, - build_windows_free_threading, build_asan, build_tsan, build_tsan_free_threading, diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml deleted file mode 100644 index 65d32c734e7745..00000000000000 --- a/.github/workflows/build_msi.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: TestsMSI - -on: - workflow_dispatch: - push: - branches: - - 'main' - - '3.*' - paths: - - 'Tools/msi/**' - - '.github/workflows/build_msi.yml' - pull_request: - branches: - - 'main' - - '3.*' - paths: - - 'Tools/msi/**' - - '.github/workflows/build_msi.yml' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build: - name: Windows Installer - runs-on: windows-latest - timeout-minutes: 60 - strategy: - matrix: - type: [x86, x64, arm64] - env: - IncludeFreethreaded: true - steps: - - uses: actions/checkout@v4 - - name: Build CPython installer - run: .\Tools\msi\build.bat --doc -${{ matrix.type }} diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index f825d1a7b3f69a..64ef2c91329d81 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -8,13 +8,14 @@ on: required: false type: boolean default: false - os-matrix: - required: false + os: + description: OS to run the job + required: true type: string jobs: build_macos: - name: build and test (${{ matrix.os }}) + name: build and test (${{ inputs.os }}) timeout-minutes: 60 env: HOMEBREW_NO_ANALYTICS: 1 @@ -23,18 +24,7 @@ jobs: HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux - strategy: - fail-fast: false - matrix: - os: ${{fromJson(inputs.os-matrix)}} - is-fork: - - ${{ github.repository_owner != 'python' }} - exclude: - - os: "ghcr.io/cirruslabs/macos-runner:sonoma" - is-fork: true - - os: "macos-14" - is-fork: false - runs-on: ${{ matrix.os }} + runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v4 - name: Runner image version @@ -43,7 +33,7 @@ jobs: uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ matrix.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk - name: Configure CPython @@ -53,6 +43,7 @@ jobs: ./configure \ --config-cache \ --with-pydebug \ + --enable-slower-safety \ ${{ inputs.free-threading && '--disable-gil' || '' }} \ --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index fa450ed3376321..c6289a74e9a5f6 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -4,9 +4,11 @@ on: config_hash: required: true type: string - options: - required: true - type: string + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false jobs: build_ubuntu_reusable: @@ -63,13 +65,22 @@ jobs: key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: ${{ inputs.options }} + run: >- + ../cpython-ro-srcdir/configure + CFLAGS="-fdiagnostics-format=json" + --config-cache + --with-pydebug + --enable-slower-safety + --with-openssl=$OPENSSL_DIR + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 + run: make -j4 &> compiler_output.txt - name: Display build info working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo + - name: Check compiler warnings + run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index c389fe9e173b38..ffa143b3457e5a 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -11,8 +11,8 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-22.04 env: - WASMTIME_VERSION: 18.0.3 - WASI_SDK_VERSION: 21 + WASMTIME_VERSION: 22.0.0 + WASI_SDK_VERSION: 22 WASI_SDK_PATH: /opt/wasi-sdk CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasi @@ -20,9 +20,9 @@ jobs: - uses: actions/checkout@v4 # No problem resolver registered as one doesn't currently exist for Clang. - name: "Install wasmtime" - uses: jcbhmr/setup-wasmtime@v2 + uses: bytecodealliance/actions/wasmtime/setup@v1 with: - wasmtime-version: ${{ env.WASMTIME_VERSION }} + version: ${{ env.WASMTIME_VERSION }} - name: "Restore WASI SDK" id: cache-wasi-sdk uses: actions/cache@v4 @@ -50,8 +50,10 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}-${{ env.pythonLocation }} + # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. + # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. + # (Make sure to keep the key in sync with the other config.cache step below.) + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} - name: "Configure build Python" run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" @@ -60,8 +62,8 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CROSS_BUILD_WASI }}/config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }}-${{ env.pythonLocation }} + # Should be kept in sync with the other config.cache step above. + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} - name: "Configure host" # `--with-pydebug` inferred from configure-build-python run: python3 Tools/wasm/wasi.py configure-host -- --config-cache diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml new file mode 100644 index 00000000000000..fc34ab7c3eb1f2 --- /dev/null +++ b/.github/workflows/reusable-windows-msi.yml @@ -0,0 +1,24 @@ +name: TestsMSI + +on: + workflow_call: + inputs: + arch: + description: CPU architecture + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: installer for ${{ inputs.arch }} + runs-on: windows-latest + timeout-minutes: 60 + env: + IncludeFreethreaded: true + steps: + - uses: actions/checkout@v4 + - name: Build CPython installer + run: .\Tools\msi\build.bat --doc -${{ inputs.arch }} diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index c0209e0e1c92e9..e9c3c8e05a801c 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -1,53 +1,45 @@ on: workflow_call: inputs: + arch: + description: CPU architecture + required: true + type: string free-threading: + description: Whether to compile CPython in free-threading mode required: false type: boolean default: false -jobs: - build_win32: - name: 'build and test (x86)' - runs-on: windows-latest - timeout-minutes: 60 - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p Win32 ${{ inputs.free-threading && '--disable-gil' || '' }} - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }} +env: + IncludeUwp: >- + true - build_win_amd64: - name: 'build and test (x64)' +jobs: + build: + name: >- + build${{ inputs.arch != 'arm64' && ' and test' || '' }} + (${{ inputs.arch }}) runs-on: windows-latest timeout-minutes: 60 - env: - IncludeUwp: 'true' steps: - uses: actions/checkout@v4 - name: Register MSVC problem matcher + if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p x64 ${{ inputs.free-threading && '--disable-gil' || '' }} + run: >- + .\PCbuild\build.bat + -e -d -v + -p ${{ inputs.arch }} + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Display build info + if: inputs.arch != 'arm64' run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }} - - build_win_arm64: - name: 'build (arm64)' - runs-on: windows-latest - timeout-minutes: 60 - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Register MSVC problem matcher - run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -v -p arm64 ${{ inputs.free-threading && '--disable-gil' || '' }} + if: inputs.arch != 'arm64' + run: >- + .\PCbuild\rt.bat + -p ${{ inputs.arch }} + -d -q --fast-ci + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fde9d9149bf62b..b10be5b6bd9904 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,13 +3,21 @@ repos: rev: v0.3.4 hooks: - id: ruff - name: Run Ruff on Lib/test/ + name: Run Ruff (lint) on Doc/ + args: [--exit-non-zero-on-fix] + files: ^Doc/ + - id: ruff + name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ - id: ruff - name: Run Ruff on Argument Clinic + name: Run Ruff (lint) on Argument Clinic args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py + - id: ruff-format + name: Run Ruff (format) on Doc/ + args: [--check] + files: ^Doc/ - repo: https://github.com/psf/black-pre-commit-mirror rev: 24.4.2 diff --git a/Doc/.ruff.toml b/Doc/.ruff.toml new file mode 100644 index 00000000000000..111ce03b91df38 --- /dev/null +++ b/Doc/.ruff.toml @@ -0,0 +1,42 @@ +target-version = "py312" # Align with the version in oldest_supported_sphinx +fix = true +output-format = "full" +line-length = 79 +extend-exclude = [ + "includes/*", + # Temporary exclusions: + "tools/extensions/pyspecific.py", +] + +[lint] +preview = true +select = [ + "C4", # flake8-comprehensions + "B", # flake8-bugbear + "E", # pycodestyle + "F", # pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format + "I", # isort + "LOG", # flake8-logging + "N", # pep8-naming + "PERF", # perflint + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "TCH", # flake8-type-checking + "UP", # pyupgrade + "W", # pycodestyle +] +ignore = [ + "E501", # Ignore line length errors (we use auto-formatting) +] + +[format] +preview = true +quote-style = "preserve" +docstring-code-format = true +exclude = [ + "tools/extensions/lexers/*", +] diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 9aff2f0ff5187d..5d0f68ca69675e 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -16,21 +16,16 @@ Documentation bugs ================== If you find a bug in this documentation or would like to propose an improvement, -please submit a bug report on the :ref:`tracker `. If you +please submit a bug report on the :ref:`issue tracker `. If you have a suggestion on how to fix it, include that as well. You can also open a discussion item on our `Documentation Discourse forum `_. If you find a bug in the theme (HTML / CSS / JavaScript) of the -documentation, please submit a bug report on the `python-doc-theme bug +documentation, please submit a bug report on the `python-doc-theme issue tracker `_. -If you're short on time, you can also email documentation bug reports to -docs@python.org (behavioral bugs can be sent to python-list@python.org). -'docs@' is a mailing list run by volunteers; your request will be noticed, -though it may take a while to be processed. - .. seealso:: `Documentation bugs`_ diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 834aae9372fe3b..3201bdc82691f4 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -280,10 +280,10 @@ Numbers length 1, to a C :c:expr:`int`. ``f`` (:class:`float`) [float] - Convert a Python floating point number to a C :c:expr:`float`. + Convert a Python floating-point number to a C :c:expr:`float`. ``d`` (:class:`float`) [double] - Convert a Python floating point number to a C :c:expr:`double`. + Convert a Python floating-point number to a C :c:expr:`double`. ``D`` (:class:`complex`) [Py_complex] Convert a Python complex number to a C :c:type:`Py_complex` structure. @@ -642,10 +642,10 @@ Building values object of length 1. ``d`` (:class:`float`) [double] - Convert a C :c:expr:`double` to a Python floating point number. + Convert a C :c:expr:`double` to a Python floating-point number. ``f`` (:class:`float`) [float] - Convert a C :c:expr:`float` to a Python floating point number. + Convert a C :c:expr:`float` to a Python floating-point number. ``D`` (:class:`complex`) [Py_complex \*] Convert a C :c:type:`Py_complex` structure to a Python complex number. diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index f8cd0344fdd1c0..61eb994c370946 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -39,7 +39,8 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: PyObject* PyCell_Get(PyObject *cell) - Return the contents of the cell *cell*. + Return the contents of the cell *cell*, which can be ``NULL``. + If *cell* is not a cell object, returns ``NULL`` with an exception set. .. c:function:: PyObject* PyCell_GET(PyObject *cell) @@ -52,8 +53,10 @@ Cell objects are not likely to be useful elsewhere. Set the contents of the cell object *cell* to *value*. This releases the reference to any current content of the cell. *value* may be ``NULL``. *cell* - must be non-``NULL``; if it is not a cell object, ``-1`` will be returned. On - success, ``0`` will be returned. + must be non-``NULL``. + + On success, return ``0``. + If *cell* is not a cell object, set an exception and return ``-1``. .. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 5a0474869071d9..67d0c5f144e075 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -25,12 +25,16 @@ pointers. This is consistent throughout the API. The C structure which corresponds to the value portion of a Python complex number object. Most of the functions for dealing with complex number objects - use structures of this type as input or output values, as appropriate. It is - defined as:: + use structures of this type as input or output values, as appropriate. + + .. c:member:: double real + double imag + + The structure is defined as:: typedef struct { - double real; - double imag; + double real; + double imag; } Py_complex; @@ -106,11 +110,13 @@ Complex Numbers as Python Objects .. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) Create a new Python complex number object from a C :c:type:`Py_complex` value. + Return ``NULL`` with an exception set on error. .. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) Return a new :c:type:`PyComplexObject` object from *real* and *imag*. + Return ``NULL`` with an exception set on error. .. c:function:: double PyComplex_RealAsDouble(PyObject *op) @@ -121,7 +127,9 @@ Complex Numbers as Python Objects :meth:`~object.__complex__` method, this method will first be called to convert *op* to a Python complex number object. If :meth:`!__complex__` is not defined then it falls back to call :c:func:`PyFloat_AsDouble` and - returns its result. Upon failure, this method returns ``-1.0``, so one + returns its result. + + Upon failure, this method returns ``-1.0`` with an exception set, so one should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.13 @@ -135,8 +143,10 @@ Complex Numbers as Python Objects :meth:`~object.__complex__` method, this method will first be called to convert *op* to a Python complex number object. If :meth:`!__complex__` is not defined then it falls back to call :c:func:`PyFloat_AsDouble` and - returns ``0.0`` on success. Upon failure, this method returns ``-1.0``, so - one should call :c:func:`PyErr_Occurred` to check for errors. + returns ``0.0`` on success. + + Upon failure, this method returns ``-1.0`` with an exception set, so one + should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.13 Use :meth:`~object.__complex__` if available. @@ -149,8 +159,11 @@ Complex Numbers as Python Objects method, this method will first be called to convert *op* to a Python complex number object. If :meth:`!__complex__` is not defined then it falls back to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back - to :meth:`~object.__index__`. Upon failure, this method returns ``-1.0`` as a real - value. + to :meth:`~object.__index__`. + + Upon failure, this method returns :c:type:`Py_complex` + with :c:member:`~Py_complex.real` set to ``-1.0`` and with an exception set, so one + should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.8 Use :meth:`~object.__index__` if available. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 4f6ac0d8175c6b..1da37a5bcaeef9 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -2,20 +2,20 @@ .. _floatobjects: -Floating Point Objects +Floating-Point Objects ====================== -.. index:: pair: object; floating point +.. index:: pair: object; floating-point .. c:type:: PyFloatObject - This subtype of :c:type:`PyObject` represents a Python floating point object. + This subtype of :c:type:`PyObject` represents a Python floating-point object. .. c:var:: PyTypeObject PyFloat_Type - This instance of :c:type:`PyTypeObject` represents the Python floating point + This instance of :c:type:`PyTypeObject` represents the Python floating-point type. This is the same object as :class:`float` in the Python layer. @@ -45,7 +45,7 @@ Floating Point Objects .. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) Return a C :c:expr:`double` representation of the contents of *pyfloat*. If - *pyfloat* is not a Python floating point object but has a :meth:`~object.__float__` + *pyfloat* is not a Python floating-point object but has a :meth:`~object.__float__` method, this method will first be called to convert *pyfloat* into a float. If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. This method returns ``-1.0`` upon failure, so one should call diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 5195f6cccfe9df..6e2e04fba55a45 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -509,7 +509,7 @@ PyConfig The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are - strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + stripped from :c:member:`PyConfig.argv`, parsing arguments twice would parse the application options as Python options. :ref:`Preinitialize Python ` if needed. @@ -1041,7 +1041,7 @@ PyConfig The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are - strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + stripped from :c:member:`PyConfig.argv`, parsing arguments twice would parse the application options as Python options. Default: ``1`` in Python mode, ``0`` in isolated mode. diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 53eb54d3e1021a..758415a76e5cb4 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -38,9 +38,12 @@ List Objects .. note:: If *len* is greater than zero, the returned list object's items are - set to ``NULL``. Thus you cannot use abstract API functions such as - :c:func:`PySequence_SetItem` or expose the object to Python code before - setting all items to a real object with :c:func:`PyList_SetItem`. + set to ``NULL``. Thus you cannot use abstract API functions such as + :c:func:`PySequence_SetItem` or expose the object to Python code before + setting all items to a real object with :c:func:`PyList_SetItem` or + :c:func:`PyList_SET_ITEM()`. The following APIs are safe APIs before + the list is fully initialized: :c:func:`PyList_SetItem()` and :c:func:`PyList_SET_ITEM()`. + .. c:function:: Py_ssize_t PyList_Size(PyObject *list) diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 489f1580a414b2..b9085ad3ec361d 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -15,7 +15,7 @@ Numeric values are stored with the least significant byte first. The module supports two versions of the data format: version 0 is the historical version, version 1 shares interned strings in the file, and upon -unmarshalling. Version 2 uses a binary format for floating point numbers. +unmarshalling. Version 2 uses a binary format for floating-point numbers. ``Py_MARSHAL_VERSION`` indicates the current file format (currently 2). diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 63e3bed6727987..2b68907d1e0fc8 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -43,6 +43,8 @@ Module Objects to ``None``); the caller is responsible for providing a :attr:`__file__` attribute. + Return ``NULL`` with an exception set on error. + .. versionadded:: 3.3 .. versionchanged:: 3.4 @@ -265,6 +267,8 @@ of the following two module creation functions: API version *module_api_version*. If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. + Return ``NULL`` with an exception set on error. + .. note:: Most uses of this function should be using :c:func:`PyModule_Create` @@ -338,7 +342,8 @@ The available slot types are: The *value* pointer of this slot must point to a function of the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) - :noindex: + :no-index-entry: + :no-contents-entry: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. @@ -373,7 +378,8 @@ The available slot types are: The signature of the function is: .. c:function:: int exec_module(PyObject* module) - :noindex: + :no-index-entry: + :no-contents-entry: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. @@ -461,6 +467,8 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. + Return ``NULL`` with an exception set on error. + .. note:: Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` @@ -511,7 +519,7 @@ state: On success, return ``0``. On error, raise an exception and return ``-1``. - Return ``NULL`` if *value* is ``NULL``. It must be called with an exception + Return ``-1`` if *value* is ``NULL``. It must be called with an exception raised in this case. Example usage:: @@ -543,6 +551,14 @@ state: Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in this case, since *obj* can be ``NULL``. + The number of different *name* strings passed to this function + should be kept small, usually by only using statically allocated strings + as *name*. + For names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_SetAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object. + .. versionadded:: 3.10 @@ -601,15 +617,23 @@ state: .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) Add an integer constant to *module* as *name*. This convenience function can be - used from the module's initialization function. Return ``-1`` on error, ``0`` on - success. + used from the module's initialization function. + Return ``-1`` with an exception set on error, ``0`` on success. + + This is a convenience function that calls :c:func:`PyLong_FromLong` and + :c:func:`PyModule_AddObjectRef`; see their documentation for details. .. c:function:: int PyModule_AddStringConstant(PyObject *module, const char *name, const char *value) Add a string constant to *module* as *name*. This convenience function can be used from the module's initialization function. The string *value* must be - ``NULL``-terminated. Return ``-1`` on error, ``0`` on success. + ``NULL``-terminated. + Return ``-1`` with an exception set on error, ``0`` on success. + + This is a convenience function that calls + :c:func:`PyUnicode_InternFromString` and :c:func:`PyModule_AddObjectRef`; + see their documentation for details. .. c:macro:: PyModule_AddIntMacro(module, macro) @@ -617,7 +641,7 @@ state: Add an int constant to *module*. The name and the value are taken from *macro*. For example ``PyModule_AddIntMacro(module, AF_INET)`` adds the int constant *AF_INET* with the value of *AF_INET* to *module*. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. c:macro:: PyModule_AddStringMacro(module, macro) @@ -630,7 +654,7 @@ state: The type object is finalized by calling internally :c:func:`PyType_Ready`. The name of the type object is taken from the last component of :c:member:`~PyTypeObject.tp_name` after dot. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.9 @@ -643,7 +667,7 @@ state: import machinery assumes the module does not support running without the GIL. This function is only available in Python builds configured with :option:`--disable-gil`. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.13 @@ -682,14 +706,14 @@ since multiple such modules can be created from a single definition. The caller must hold the GIL. - Return 0 on success or -1 on failure. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.3 .. c:function:: int PyState_RemoveModule(PyModuleDef *def) Removes the module object created from *def* from the interpreter state. - Return 0 on success or -1 on failure. + Return ``-1`` with an exception set on error, ``0`` on success. The caller must hold the GIL. diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 13d3c5af956905..ad8b5935258fa7 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -51,8 +51,8 @@ Number Protocol Return a reasonable approximation for the mathematical value of *o1* divided by *o2*, or ``NULL`` on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when + floating-point numbers are approximate; it is not possible to represent all real + numbers in base two. This function can return a floating-point value when passed two integers. This is the equivalent of the Python expression ``o1 / o2``. @@ -177,8 +177,8 @@ Number Protocol Return a reasonable approximation for the mathematical value of *o1* divided by *o2*, or ``NULL`` on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when + floating-point numbers are approximate; it is not possible to represent all real + numbers in base two. This function can return a floating-point value when passed two integers. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 /= o2``. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 2103a64d8ffbb7..1c28f30321bd7a 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -206,6 +206,13 @@ Object Protocol If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. + The number of different attribute names passed to this function + should be kept small, usually by using a statically allocated string + as *attr_name*. + For attribute names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_SetAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object. .. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) @@ -231,6 +238,14 @@ Object Protocol specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. + The number of different attribute names passed to this function + should be kept small, usually by using a statically allocated string + as *attr_name*. + For attribute names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_DelAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object for lookup. + .. c:function:: PyObject* PyObject_GenericGetDict(PyObject *o, void *context) diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 27a1757c745d8b..8adf6a961378a3 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -23,7 +23,9 @@ Slice Objects Return a new slice object with the given values. The *start*, *stop*, and *step* parameters are used as the values of the slice object attributes of the same names. Any of the values may be ``NULL``, in which case the - ``None`` will be used for the corresponding attribute. Return ``NULL`` if + ``None`` will be used for the corresponding attribute. + + Return ``NULL`` with an exception set if the new object could not be allocated. @@ -52,7 +54,7 @@ Slice Objects of bounds indices are clipped in a manner consistent with the handling of normal slices. - Returns ``0`` on success and ``-1`` on error with exception set. + Return ``0`` on success and ``-1`` on error with an exception set. .. note:: This function is considered not safe for resizable sequences. @@ -95,7 +97,7 @@ Slice Objects ``PY_SSIZE_T_MIN`` to ``PY_SSIZE_T_MIN``, and silently boost the step values less than ``-PY_SSIZE_T_MAX`` to ``-PY_SSIZE_T_MAX``. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.6.1 diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 246cf47df62e78..958fafd47ac81b 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1490,18 +1490,43 @@ They all return ``NULL`` or ``-1`` if an exception occurs. existing interned string that is the same as :c:expr:`*p_unicode`, it sets :c:expr:`*p_unicode` to it (releasing the reference to the old string object and creating a new :term:`strong reference` to the interned string object), otherwise it leaves - :c:expr:`*p_unicode` alone and interns it (creating a new :term:`strong reference`). + :c:expr:`*p_unicode` alone and interns it. + (Clarification: even though there is a lot of talk about references, think - of this function as reference-neutral; you own the object after the call - if and only if you owned it before the call.) + of this function as reference-neutral. You must own the object you pass in; + after the call you no longer own the passed-in reference, but you newly own + the result.) + + This function never raises an exception. + On error, it leaves its argument unchanged without interning it. + + Instances of subclasses of :py:class:`str` may not be interned, that is, + :c:expr:`PyUnicode_CheckExact(*p_unicode)` must be true. If it is not, + then -- as with any other error -- the argument is left unchanged. + + Note that interned strings are not “immortal”. + You must keep a reference to the result to benefit from interning. .. c:function:: PyObject* PyUnicode_InternFromString(const char *str) A combination of :c:func:`PyUnicode_FromString` and - :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string - object that has been interned, or a new ("owned") reference to an earlier - interned string object with the same value. + :c:func:`PyUnicode_InternInPlace`, meant for statically allocated strings. + + Return a new ("owned") reference to either a new Unicode string object + that has been interned, or an earlier interned string object with the + same value. + + Python may keep a reference to the result, or make it :term:`immortal`, + preventing it from being garbage-collected promptly. + For interning an unbounded number of different strings, such as ones coming + from user input, prefer calling :c:func:`PyUnicode_FromString` and + :c:func:`PyUnicode_InternInPlace` directly. + + .. impl-detail:: + + Strings interned this way are made :term:`immortal`. + PyUnicodeWriter ^^^^^^^^^^^^^^^ diff --git a/Doc/conf.py b/Doc/conf.py index 8a14646801ebac..4841b69e380085 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -6,9 +6,11 @@ # The contents of this file are pickled, so don't put values in the namespace # that aren't pickleable (module imports are okay, they're removed automatically). +import importlib import os import sys import time + sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) @@ -18,11 +20,9 @@ # --------------------- extensions = [ - 'asdl_highlight', 'c_annotations', - 'escape4chm', 'glossary_search', - 'peg_highlight', + 'lexers', 'pyspecific', 'sphinx.ext.coverage', 'sphinx.ext.doctest', @@ -31,13 +31,13 @@ # Skip if downstream redistributors haven't installed them try: - import notfound.extension + import notfound.extension # noqa: F401 except ImportError: pass else: extensions.append('notfound.extension') try: - import sphinxext.opengraph + import sphinxext.opengraph # noqa: F401 except ImportError: pass else: @@ -64,8 +64,8 @@ # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. -import patchlevel -version, release = patchlevel.get_version_info() +# See Doc/tools/extensions/patchlevel.py +version, release = importlib.import_module('patchlevel').get_version_info() rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` @@ -83,7 +83,7 @@ highlight_language = 'python3' # Minimum version of sphinx required -needs_sphinx = '4.2' +needs_sphinx = '6.2.1' # Create table of contents entries for domain objects (e.g. functions, classes, # attributes, etc.). Default is True. @@ -272,6 +272,9 @@ ('c:data', 'PyExc_UnicodeWarning'), ('c:data', 'PyExc_UserWarning'), ('c:data', 'PyExc_Warning'), + # Undocumented public C macros + ('c:macro', 'Py_BUILD_ASSERT'), + ('c:macro', 'Py_BUILD_ASSERT_EXPR'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. @@ -296,7 +299,8 @@ # Disable Docutils smartquotes for several translations smartquotes_excludes = { - 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'], + 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], + 'builders': ['man', 'text'], } # Avoid a warning with Sphinx >= 4.0 @@ -317,11 +321,13 @@ 'collapsiblesidebar': True, 'issues_url': '/bugs.html', 'license_url': '/license.html', - 'root_include_title': False # We use the version switcher instead. + 'root_include_title': False, # We use the version switcher instead. } if os.getenv("READTHEDOCS"): - html_theme_options["hosted_on"] = 'Read the Docs' + html_theme_options["hosted_on"] = ( + 'Read the Docs' + ) # Override stylesheet fingerprinting for Windows CHM htmlhelp to fix GH-91207 # https://github.com/python/cpython/issues/91207 @@ -335,16 +341,21 @@ # Deployment preview information # (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) -repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL") +is_deployment_preview = os.getenv("READTHEDOCS_VERSION_TYPE") == "external" +repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL", "") +repository_url = repository_url.removesuffix(".git") html_context = { - "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external", - "repository_url": repository_url.removesuffix(".git") if repository_url else None, + "is_deployment_preview": is_deployment_preview, + "repository_url": repository_url or None, "pr_id": os.getenv("READTHEDOCS_VERSION"), "enable_analytics": os.getenv("PYTHON_DOCS_ENABLE_ANALYTICS"), } # This 'Last updated on:' timestamp is inserted at the bottom of every page. -html_last_updated_fmt = time.strftime('%b %d, %Y (%H:%M UTC)', time.gmtime()) +html_time = int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) +html_last_updated_fmt = time.strftime( + '%b %d, %Y (%H:%M UTC)', time.gmtime(html_time) +) # Path to find HTML templates. templates_path = ['tools/templates'] @@ -404,30 +415,70 @@ # (source start file, target name, title, author, document class [howto/manual]). _stdauthor = 'Guido van Rossum and the Python development team' latex_documents = [ - ('c-api/index', 'c-api.tex', - 'The Python/C API', _stdauthor, 'manual'), - ('extending/index', 'extending.tex', - 'Extending and Embedding Python', _stdauthor, 'manual'), - ('installing/index', 'installing.tex', - 'Installing Python Modules', _stdauthor, 'manual'), - ('library/index', 'library.tex', - 'The Python Library Reference', _stdauthor, 'manual'), - ('reference/index', 'reference.tex', - 'The Python Language Reference', _stdauthor, 'manual'), - ('tutorial/index', 'tutorial.tex', - 'Python Tutorial', _stdauthor, 'manual'), - ('using/index', 'using.tex', - 'Python Setup and Usage', _stdauthor, 'manual'), - ('faq/index', 'faq.tex', - 'Python Frequently Asked Questions', _stdauthor, 'manual'), - ('whatsnew/' + version, 'whatsnew.tex', - 'What\'s New in Python', 'A. M. Kuchling', 'howto'), + ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), + ( + 'extending/index', + 'extending.tex', + 'Extending and Embedding Python', + _stdauthor, + 'manual', + ), + ( + 'installing/index', + 'installing.tex', + 'Installing Python Modules', + _stdauthor, + 'manual', + ), + ( + 'library/index', + 'library.tex', + 'The Python Library Reference', + _stdauthor, + 'manual', + ), + ( + 'reference/index', + 'reference.tex', + 'The Python Language Reference', + _stdauthor, + 'manual', + ), + ( + 'tutorial/index', + 'tutorial.tex', + 'Python Tutorial', + _stdauthor, + 'manual', + ), + ( + 'using/index', + 'using.tex', + 'Python Setup and Usage', + _stdauthor, + 'manual', + ), + ( + 'faq/index', + 'faq.tex', + 'Python Frequently Asked Questions', + _stdauthor, + 'manual', + ), + ( + 'whatsnew/' + version, + 'whatsnew.tex', + 'What\'s New in Python', + 'A. M. Kuchling', + 'howto', + ), ] # Collect all HOWTOs individually -latex_documents.extend(('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', - '', _stdauthor, 'howto') - for fn in os.listdir('howto') - if fn.endswith('.rst') and fn != 'index.rst') +latex_documents.extend( + ('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', '', _stdauthor, 'howto') + for fn in os.listdir('howto') + if fn.endswith('.rst') and fn != 'index.rst' +) # Documents to append as an appendix to all manuals. latex_appendices = ['glossary', 'about', 'license', 'copyright'] @@ -455,8 +506,7 @@ 'test($|_)', ] -coverage_ignore_classes = [ -] +coverage_ignore_classes = [] # Glob patterns for C source files for C API coverage, relative to this directory. coverage_c_path = [ @@ -473,7 +523,7 @@ # The coverage checker will ignore all C items whose names match these regexes # (using re.match) -- the keys must be the same as in coverage_c_regexes. coverage_ignore_c_items = { -# 'cfunction': [...] + # 'cfunction': [...] } @@ -538,14 +588,16 @@ } extlinks_detect_hardcoded_links = True -# Options for extensions -# ---------------------- +# Options for c_annotations +# ------------------------- # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' -# sphinxext-opengraph config +# Options for sphinxext-opengraph +# ------------------------------- + ogp_site_url = 'https://docs.python.org/3/' ogp_site_name = 'Python documentation' ogp_image = '_static/og-image.png' diff --git a/Doc/contents.rst b/Doc/contents.rst index 24ceacb0076b5e..b57f4b09a5dcb6 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -14,6 +14,7 @@ installing/index.rst howto/index.rst faq/index.rst + deprecations/index.rst glossary.rst about.rst diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 1f7af436a4150b..90ddb3fd8213ca 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,888 +1,888 @@ role,name,added,ifdef_note,struct_abi_kind macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, -function,PyAIter_Check,3.10,, -function,PyArg_Parse,3.2,, -function,PyArg_ParseTuple,3.2,, -function,PyArg_ParseTupleAndKeywords,3.2,, -function,PyArg_UnpackTuple,3.2,, -function,PyArg_VaParse,3.2,, -function,PyArg_VaParseTupleAndKeywords,3.2,, -function,PyArg_ValidateKeywordArguments,3.2,, -var,PyBaseObject_Type,3.2,, -function,PyBool_FromLong,3.2,, -var,PyBool_Type,3.2,, -function,PyBuffer_FillContiguousStrides,3.11,, -function,PyBuffer_FillInfo,3.11,, -function,PyBuffer_FromContiguous,3.11,, -function,PyBuffer_GetPointer,3.11,, -function,PyBuffer_IsContiguous,3.11,, -function,PyBuffer_Release,3.11,, -function,PyBuffer_SizeFromFormat,3.11,, -function,PyBuffer_ToContiguous,3.11,, -var,PyByteArrayIter_Type,3.2,, -function,PyByteArray_AsString,3.2,, -function,PyByteArray_Concat,3.2,, -function,PyByteArray_FromObject,3.2,, -function,PyByteArray_FromStringAndSize,3.2,, -function,PyByteArray_Resize,3.2,, -function,PyByteArray_Size,3.2,, -var,PyByteArray_Type,3.2,, -var,PyBytesIter_Type,3.2,, -function,PyBytes_AsString,3.2,, -function,PyBytes_AsStringAndSize,3.2,, -function,PyBytes_Concat,3.2,, -function,PyBytes_ConcatAndDel,3.2,, -function,PyBytes_DecodeEscape,3.2,, -function,PyBytes_FromFormat,3.2,, -function,PyBytes_FromFormatV,3.2,, -function,PyBytes_FromObject,3.2,, -function,PyBytes_FromString,3.2,, -function,PyBytes_FromStringAndSize,3.2,, -function,PyBytes_Repr,3.2,, -function,PyBytes_Size,3.2,, -var,PyBytes_Type,3.2,, +func,PyAIter_Check,3.10,, +func,PyArg_Parse,3.2,, +func,PyArg_ParseTuple,3.2,, +func,PyArg_ParseTupleAndKeywords,3.2,, +func,PyArg_UnpackTuple,3.2,, +func,PyArg_VaParse,3.2,, +func,PyArg_VaParseTupleAndKeywords,3.2,, +func,PyArg_ValidateKeywordArguments,3.2,, +data,PyBaseObject_Type,3.2,, +func,PyBool_FromLong,3.2,, +data,PyBool_Type,3.2,, +func,PyBuffer_FillContiguousStrides,3.11,, +func,PyBuffer_FillInfo,3.11,, +func,PyBuffer_FromContiguous,3.11,, +func,PyBuffer_GetPointer,3.11,, +func,PyBuffer_IsContiguous,3.11,, +func,PyBuffer_Release,3.11,, +func,PyBuffer_SizeFromFormat,3.11,, +func,PyBuffer_ToContiguous,3.11,, +data,PyByteArrayIter_Type,3.2,, +func,PyByteArray_AsString,3.2,, +func,PyByteArray_Concat,3.2,, +func,PyByteArray_FromObject,3.2,, +func,PyByteArray_FromStringAndSize,3.2,, +func,PyByteArray_Resize,3.2,, +func,PyByteArray_Size,3.2,, +data,PyByteArray_Type,3.2,, +data,PyBytesIter_Type,3.2,, +func,PyBytes_AsString,3.2,, +func,PyBytes_AsStringAndSize,3.2,, +func,PyBytes_Concat,3.2,, +func,PyBytes_ConcatAndDel,3.2,, +func,PyBytes_DecodeEscape,3.2,, +func,PyBytes_FromFormat,3.2,, +func,PyBytes_FromFormatV,3.2,, +func,PyBytes_FromObject,3.2,, +func,PyBytes_FromString,3.2,, +func,PyBytes_FromStringAndSize,3.2,, +func,PyBytes_Repr,3.2,, +func,PyBytes_Size,3.2,, +data,PyBytes_Type,3.2,, type,PyCFunction,3.2,, type,PyCFunctionFast,3.13,, type,PyCFunctionFastWithKeywords,3.13,, type,PyCFunctionWithKeywords,3.2,, -function,PyCFunction_GetFlags,3.2,, -function,PyCFunction_GetFunction,3.2,, -function,PyCFunction_GetSelf,3.2,, -function,PyCFunction_New,3.4,, -function,PyCFunction_NewEx,3.2,, -var,PyCFunction_Type,3.2,, -function,PyCMethod_New,3.9,, -function,PyCallIter_New,3.2,, -var,PyCallIter_Type,3.2,, -function,PyCallable_Check,3.2,, +func,PyCFunction_GetFlags,3.2,, +func,PyCFunction_GetFunction,3.2,, +func,PyCFunction_GetSelf,3.2,, +func,PyCFunction_New,3.4,, +func,PyCFunction_NewEx,3.2,, +data,PyCFunction_Type,3.2,, +func,PyCMethod_New,3.9,, +func,PyCallIter_New,3.2,, +data,PyCallIter_Type,3.2,, +func,PyCallable_Check,3.2,, type,PyCapsule_Destructor,3.2,, -function,PyCapsule_GetContext,3.2,, -function,PyCapsule_GetDestructor,3.2,, -function,PyCapsule_GetName,3.2,, -function,PyCapsule_GetPointer,3.2,, -function,PyCapsule_Import,3.2,, -function,PyCapsule_IsValid,3.2,, -function,PyCapsule_New,3.2,, -function,PyCapsule_SetContext,3.2,, -function,PyCapsule_SetDestructor,3.2,, -function,PyCapsule_SetName,3.2,, -function,PyCapsule_SetPointer,3.2,, -var,PyCapsule_Type,3.2,, -var,PyClassMethodDescr_Type,3.2,, -function,PyCodec_BackslashReplaceErrors,3.2,, -function,PyCodec_Decode,3.2,, -function,PyCodec_Decoder,3.2,, -function,PyCodec_Encode,3.2,, -function,PyCodec_Encoder,3.2,, -function,PyCodec_IgnoreErrors,3.2,, -function,PyCodec_IncrementalDecoder,3.2,, -function,PyCodec_IncrementalEncoder,3.2,, -function,PyCodec_KnownEncoding,3.2,, -function,PyCodec_LookupError,3.2,, -function,PyCodec_NameReplaceErrors,3.7,, -function,PyCodec_Register,3.2,, -function,PyCodec_RegisterError,3.2,, -function,PyCodec_ReplaceErrors,3.2,, -function,PyCodec_StreamReader,3.2,, -function,PyCodec_StreamWriter,3.2,, -function,PyCodec_StrictErrors,3.2,, -function,PyCodec_Unregister,3.10,, -function,PyCodec_XMLCharRefReplaceErrors,3.2,, -function,PyComplex_FromDoubles,3.2,, -function,PyComplex_ImagAsDouble,3.2,, -function,PyComplex_RealAsDouble,3.2,, -var,PyComplex_Type,3.2,, -function,PyDescr_NewClassMethod,3.2,, -function,PyDescr_NewGetSet,3.2,, -function,PyDescr_NewMember,3.2,, -function,PyDescr_NewMethod,3.2,, -var,PyDictItems_Type,3.2,, -var,PyDictIterItem_Type,3.2,, -var,PyDictIterKey_Type,3.2,, -var,PyDictIterValue_Type,3.2,, -var,PyDictKeys_Type,3.2,, -function,PyDictProxy_New,3.2,, -var,PyDictProxy_Type,3.2,, -var,PyDictRevIterItem_Type,3.8,, -var,PyDictRevIterKey_Type,3.8,, -var,PyDictRevIterValue_Type,3.8,, -var,PyDictValues_Type,3.2,, -function,PyDict_Clear,3.2,, -function,PyDict_Contains,3.2,, -function,PyDict_Copy,3.2,, -function,PyDict_DelItem,3.2,, -function,PyDict_DelItemString,3.2,, -function,PyDict_GetItem,3.2,, -function,PyDict_GetItemRef,3.13,, -function,PyDict_GetItemString,3.2,, -function,PyDict_GetItemStringRef,3.13,, -function,PyDict_GetItemWithError,3.2,, -function,PyDict_Items,3.2,, -function,PyDict_Keys,3.2,, -function,PyDict_Merge,3.2,, -function,PyDict_MergeFromSeq2,3.2,, -function,PyDict_New,3.2,, -function,PyDict_Next,3.2,, -function,PyDict_SetItem,3.2,, -function,PyDict_SetItemString,3.2,, -function,PyDict_Size,3.2,, -var,PyDict_Type,3.2,, -function,PyDict_Update,3.2,, -function,PyDict_Values,3.2,, -var,PyEllipsis_Type,3.2,, -var,PyEnum_Type,3.2,, -function,PyErr_BadArgument,3.2,, -function,PyErr_BadInternalCall,3.2,, -function,PyErr_CheckSignals,3.2,, -function,PyErr_Clear,3.2,, -function,PyErr_Display,3.2,, -function,PyErr_DisplayException,3.12,, -function,PyErr_ExceptionMatches,3.2,, -function,PyErr_Fetch,3.2,, -function,PyErr_Format,3.2,, -function,PyErr_FormatV,3.5,, -function,PyErr_GetExcInfo,3.7,, -function,PyErr_GetHandledException,3.11,, -function,PyErr_GetRaisedException,3.12,, -function,PyErr_GivenExceptionMatches,3.2,, -function,PyErr_NewException,3.2,, -function,PyErr_NewExceptionWithDoc,3.2,, -function,PyErr_NoMemory,3.2,, -function,PyErr_NormalizeException,3.2,, -function,PyErr_Occurred,3.2,, -function,PyErr_Print,3.2,, -function,PyErr_PrintEx,3.2,, -function,PyErr_ProgramText,3.2,, -function,PyErr_ResourceWarning,3.6,, -function,PyErr_Restore,3.2,, -function,PyErr_SetExcFromWindowsErr,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilename,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilenameObject,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilenameObjects,3.7,on Windows, -function,PyErr_SetExcInfo,3.7,, -function,PyErr_SetFromErrno,3.2,, -function,PyErr_SetFromErrnoWithFilename,3.2,, -function,PyErr_SetFromErrnoWithFilenameObject,3.2,, -function,PyErr_SetFromErrnoWithFilenameObjects,3.7,, -function,PyErr_SetFromWindowsErr,3.7,on Windows, -function,PyErr_SetFromWindowsErrWithFilename,3.7,on Windows, -function,PyErr_SetHandledException,3.11,, -function,PyErr_SetImportError,3.7,, -function,PyErr_SetImportErrorSubclass,3.6,, -function,PyErr_SetInterrupt,3.2,, -function,PyErr_SetInterruptEx,3.10,, -function,PyErr_SetNone,3.2,, -function,PyErr_SetObject,3.2,, -function,PyErr_SetRaisedException,3.12,, -function,PyErr_SetString,3.2,, -function,PyErr_SyntaxLocation,3.2,, -function,PyErr_SyntaxLocationEx,3.7,, -function,PyErr_WarnEx,3.2,, -function,PyErr_WarnExplicit,3.2,, -function,PyErr_WarnFormat,3.2,, -function,PyErr_WriteUnraisable,3.2,, -function,PyEval_AcquireThread,3.2,, -function,PyEval_EvalCode,3.2,, -function,PyEval_EvalCodeEx,3.2,, -function,PyEval_EvalFrame,3.2,, -function,PyEval_EvalFrameEx,3.2,, -function,PyEval_GetBuiltins,3.2,, -function,PyEval_GetFrame,3.2,, -function,PyEval_GetFrameBuiltins,3.13,, -function,PyEval_GetFrameGlobals,3.13,, -function,PyEval_GetFrameLocals,3.13,, -function,PyEval_GetFuncDesc,3.2,, -function,PyEval_GetFuncName,3.2,, -function,PyEval_GetGlobals,3.2,, -function,PyEval_GetLocals,3.2,, -function,PyEval_InitThreads,3.2,, -function,PyEval_ReleaseThread,3.2,, -function,PyEval_RestoreThread,3.2,, -function,PyEval_SaveThread,3.2,, -var,PyExc_ArithmeticError,3.2,, -var,PyExc_AssertionError,3.2,, -var,PyExc_AttributeError,3.2,, -var,PyExc_BaseException,3.2,, -var,PyExc_BaseExceptionGroup,3.11,, -var,PyExc_BlockingIOError,3.7,, -var,PyExc_BrokenPipeError,3.7,, -var,PyExc_BufferError,3.2,, -var,PyExc_BytesWarning,3.2,, -var,PyExc_ChildProcessError,3.7,, -var,PyExc_ConnectionAbortedError,3.7,, -var,PyExc_ConnectionError,3.7,, -var,PyExc_ConnectionRefusedError,3.7,, -var,PyExc_ConnectionResetError,3.7,, -var,PyExc_DeprecationWarning,3.2,, -var,PyExc_EOFError,3.2,, -var,PyExc_EncodingWarning,3.10,, -var,PyExc_EnvironmentError,3.2,, -var,PyExc_Exception,3.2,, -var,PyExc_FileExistsError,3.7,, -var,PyExc_FileNotFoundError,3.7,, -var,PyExc_FloatingPointError,3.2,, -var,PyExc_FutureWarning,3.2,, -var,PyExc_GeneratorExit,3.2,, -var,PyExc_IOError,3.2,, -var,PyExc_ImportError,3.2,, -var,PyExc_ImportWarning,3.2,, -var,PyExc_IndentationError,3.2,, -var,PyExc_IndexError,3.2,, -var,PyExc_InterruptedError,3.7,, -var,PyExc_IsADirectoryError,3.7,, -var,PyExc_KeyError,3.2,, -var,PyExc_KeyboardInterrupt,3.2,, -var,PyExc_LookupError,3.2,, -var,PyExc_MemoryError,3.2,, -var,PyExc_ModuleNotFoundError,3.6,, -var,PyExc_NameError,3.2,, -var,PyExc_NotADirectoryError,3.7,, -var,PyExc_NotImplementedError,3.2,, -var,PyExc_OSError,3.2,, -var,PyExc_OverflowError,3.2,, -var,PyExc_PendingDeprecationWarning,3.2,, -var,PyExc_PermissionError,3.7,, -var,PyExc_ProcessLookupError,3.7,, -var,PyExc_RecursionError,3.7,, -var,PyExc_ReferenceError,3.2,, -var,PyExc_ResourceWarning,3.7,, -var,PyExc_RuntimeError,3.2,, -var,PyExc_RuntimeWarning,3.2,, -var,PyExc_StopAsyncIteration,3.7,, -var,PyExc_StopIteration,3.2,, -var,PyExc_SyntaxError,3.2,, -var,PyExc_SyntaxWarning,3.2,, -var,PyExc_SystemError,3.2,, -var,PyExc_SystemExit,3.2,, -var,PyExc_TabError,3.2,, -var,PyExc_TimeoutError,3.7,, -var,PyExc_TypeError,3.2,, -var,PyExc_UnboundLocalError,3.2,, -var,PyExc_UnicodeDecodeError,3.2,, -var,PyExc_UnicodeEncodeError,3.2,, -var,PyExc_UnicodeError,3.2,, -var,PyExc_UnicodeTranslateError,3.2,, -var,PyExc_UnicodeWarning,3.2,, -var,PyExc_UserWarning,3.2,, -var,PyExc_ValueError,3.2,, -var,PyExc_Warning,3.2,, -var,PyExc_WindowsError,3.7,on Windows, -var,PyExc_ZeroDivisionError,3.2,, -function,PyExceptionClass_Name,3.8,, -function,PyException_GetArgs,3.12,, -function,PyException_GetCause,3.2,, -function,PyException_GetContext,3.2,, -function,PyException_GetTraceback,3.2,, -function,PyException_SetArgs,3.12,, -function,PyException_SetCause,3.2,, -function,PyException_SetContext,3.2,, -function,PyException_SetTraceback,3.2,, -function,PyFile_FromFd,3.2,, -function,PyFile_GetLine,3.2,, -function,PyFile_WriteObject,3.2,, -function,PyFile_WriteString,3.2,, -var,PyFilter_Type,3.2,, -function,PyFloat_AsDouble,3.2,, -function,PyFloat_FromDouble,3.2,, -function,PyFloat_FromString,3.2,, -function,PyFloat_GetInfo,3.2,, -function,PyFloat_GetMax,3.2,, -function,PyFloat_GetMin,3.2,, -var,PyFloat_Type,3.2,, +func,PyCapsule_GetContext,3.2,, +func,PyCapsule_GetDestructor,3.2,, +func,PyCapsule_GetName,3.2,, +func,PyCapsule_GetPointer,3.2,, +func,PyCapsule_Import,3.2,, +func,PyCapsule_IsValid,3.2,, +func,PyCapsule_New,3.2,, +func,PyCapsule_SetContext,3.2,, +func,PyCapsule_SetDestructor,3.2,, +func,PyCapsule_SetName,3.2,, +func,PyCapsule_SetPointer,3.2,, +data,PyCapsule_Type,3.2,, +data,PyClassMethodDescr_Type,3.2,, +func,PyCodec_BackslashReplaceErrors,3.2,, +func,PyCodec_Decode,3.2,, +func,PyCodec_Decoder,3.2,, +func,PyCodec_Encode,3.2,, +func,PyCodec_Encoder,3.2,, +func,PyCodec_IgnoreErrors,3.2,, +func,PyCodec_IncrementalDecoder,3.2,, +func,PyCodec_IncrementalEncoder,3.2,, +func,PyCodec_KnownEncoding,3.2,, +func,PyCodec_LookupError,3.2,, +func,PyCodec_NameReplaceErrors,3.7,, +func,PyCodec_Register,3.2,, +func,PyCodec_RegisterError,3.2,, +func,PyCodec_ReplaceErrors,3.2,, +func,PyCodec_StreamReader,3.2,, +func,PyCodec_StreamWriter,3.2,, +func,PyCodec_StrictErrors,3.2,, +func,PyCodec_Unregister,3.10,, +func,PyCodec_XMLCharRefReplaceErrors,3.2,, +func,PyComplex_FromDoubles,3.2,, +func,PyComplex_ImagAsDouble,3.2,, +func,PyComplex_RealAsDouble,3.2,, +data,PyComplex_Type,3.2,, +func,PyDescr_NewClassMethod,3.2,, +func,PyDescr_NewGetSet,3.2,, +func,PyDescr_NewMember,3.2,, +func,PyDescr_NewMethod,3.2,, +data,PyDictItems_Type,3.2,, +data,PyDictIterItem_Type,3.2,, +data,PyDictIterKey_Type,3.2,, +data,PyDictIterValue_Type,3.2,, +data,PyDictKeys_Type,3.2,, +func,PyDictProxy_New,3.2,, +data,PyDictProxy_Type,3.2,, +data,PyDictRevIterItem_Type,3.8,, +data,PyDictRevIterKey_Type,3.8,, +data,PyDictRevIterValue_Type,3.8,, +data,PyDictValues_Type,3.2,, +func,PyDict_Clear,3.2,, +func,PyDict_Contains,3.2,, +func,PyDict_Copy,3.2,, +func,PyDict_DelItem,3.2,, +func,PyDict_DelItemString,3.2,, +func,PyDict_GetItem,3.2,, +func,PyDict_GetItemRef,3.13,, +func,PyDict_GetItemString,3.2,, +func,PyDict_GetItemStringRef,3.13,, +func,PyDict_GetItemWithError,3.2,, +func,PyDict_Items,3.2,, +func,PyDict_Keys,3.2,, +func,PyDict_Merge,3.2,, +func,PyDict_MergeFromSeq2,3.2,, +func,PyDict_New,3.2,, +func,PyDict_Next,3.2,, +func,PyDict_SetItem,3.2,, +func,PyDict_SetItemString,3.2,, +func,PyDict_Size,3.2,, +data,PyDict_Type,3.2,, +func,PyDict_Update,3.2,, +func,PyDict_Values,3.2,, +data,PyEllipsis_Type,3.2,, +data,PyEnum_Type,3.2,, +func,PyErr_BadArgument,3.2,, +func,PyErr_BadInternalCall,3.2,, +func,PyErr_CheckSignals,3.2,, +func,PyErr_Clear,3.2,, +func,PyErr_Display,3.2,, +func,PyErr_DisplayException,3.12,, +func,PyErr_ExceptionMatches,3.2,, +func,PyErr_Fetch,3.2,, +func,PyErr_Format,3.2,, +func,PyErr_FormatV,3.5,, +func,PyErr_GetExcInfo,3.7,, +func,PyErr_GetHandledException,3.11,, +func,PyErr_GetRaisedException,3.12,, +func,PyErr_GivenExceptionMatches,3.2,, +func,PyErr_NewException,3.2,, +func,PyErr_NewExceptionWithDoc,3.2,, +func,PyErr_NoMemory,3.2,, +func,PyErr_NormalizeException,3.2,, +func,PyErr_Occurred,3.2,, +func,PyErr_Print,3.2,, +func,PyErr_PrintEx,3.2,, +func,PyErr_ProgramText,3.2,, +func,PyErr_ResourceWarning,3.6,, +func,PyErr_Restore,3.2,, +func,PyErr_SetExcFromWindowsErr,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilename,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilenameObject,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilenameObjects,3.7,on Windows, +func,PyErr_SetExcInfo,3.7,, +func,PyErr_SetFromErrno,3.2,, +func,PyErr_SetFromErrnoWithFilename,3.2,, +func,PyErr_SetFromErrnoWithFilenameObject,3.2,, +func,PyErr_SetFromErrnoWithFilenameObjects,3.7,, +func,PyErr_SetFromWindowsErr,3.7,on Windows, +func,PyErr_SetFromWindowsErrWithFilename,3.7,on Windows, +func,PyErr_SetHandledException,3.11,, +func,PyErr_SetImportError,3.7,, +func,PyErr_SetImportErrorSubclass,3.6,, +func,PyErr_SetInterrupt,3.2,, +func,PyErr_SetInterruptEx,3.10,, +func,PyErr_SetNone,3.2,, +func,PyErr_SetObject,3.2,, +func,PyErr_SetRaisedException,3.12,, +func,PyErr_SetString,3.2,, +func,PyErr_SyntaxLocation,3.2,, +func,PyErr_SyntaxLocationEx,3.7,, +func,PyErr_WarnEx,3.2,, +func,PyErr_WarnExplicit,3.2,, +func,PyErr_WarnFormat,3.2,, +func,PyErr_WriteUnraisable,3.2,, +func,PyEval_AcquireThread,3.2,, +func,PyEval_EvalCode,3.2,, +func,PyEval_EvalCodeEx,3.2,, +func,PyEval_EvalFrame,3.2,, +func,PyEval_EvalFrameEx,3.2,, +func,PyEval_GetBuiltins,3.2,, +func,PyEval_GetFrame,3.2,, +func,PyEval_GetFrameBuiltins,3.13,, +func,PyEval_GetFrameGlobals,3.13,, +func,PyEval_GetFrameLocals,3.13,, +func,PyEval_GetFuncDesc,3.2,, +func,PyEval_GetFuncName,3.2,, +func,PyEval_GetGlobals,3.2,, +func,PyEval_GetLocals,3.2,, +func,PyEval_InitThreads,3.2,, +func,PyEval_ReleaseThread,3.2,, +func,PyEval_RestoreThread,3.2,, +func,PyEval_SaveThread,3.2,, +data,PyExc_ArithmeticError,3.2,, +data,PyExc_AssertionError,3.2,, +data,PyExc_AttributeError,3.2,, +data,PyExc_BaseException,3.2,, +data,PyExc_BaseExceptionGroup,3.11,, +data,PyExc_BlockingIOError,3.7,, +data,PyExc_BrokenPipeError,3.7,, +data,PyExc_BufferError,3.2,, +data,PyExc_BytesWarning,3.2,, +data,PyExc_ChildProcessError,3.7,, +data,PyExc_ConnectionAbortedError,3.7,, +data,PyExc_ConnectionError,3.7,, +data,PyExc_ConnectionRefusedError,3.7,, +data,PyExc_ConnectionResetError,3.7,, +data,PyExc_DeprecationWarning,3.2,, +data,PyExc_EOFError,3.2,, +data,PyExc_EncodingWarning,3.10,, +data,PyExc_EnvironmentError,3.2,, +data,PyExc_Exception,3.2,, +data,PyExc_FileExistsError,3.7,, +data,PyExc_FileNotFoundError,3.7,, +data,PyExc_FloatingPointError,3.2,, +data,PyExc_FutureWarning,3.2,, +data,PyExc_GeneratorExit,3.2,, +data,PyExc_IOError,3.2,, +data,PyExc_ImportError,3.2,, +data,PyExc_ImportWarning,3.2,, +data,PyExc_IndentationError,3.2,, +data,PyExc_IndexError,3.2,, +data,PyExc_InterruptedError,3.7,, +data,PyExc_IsADirectoryError,3.7,, +data,PyExc_KeyError,3.2,, +data,PyExc_KeyboardInterrupt,3.2,, +data,PyExc_LookupError,3.2,, +data,PyExc_MemoryError,3.2,, +data,PyExc_ModuleNotFoundError,3.6,, +data,PyExc_NameError,3.2,, +data,PyExc_NotADirectoryError,3.7,, +data,PyExc_NotImplementedError,3.2,, +data,PyExc_OSError,3.2,, +data,PyExc_OverflowError,3.2,, +data,PyExc_PendingDeprecationWarning,3.2,, +data,PyExc_PermissionError,3.7,, +data,PyExc_ProcessLookupError,3.7,, +data,PyExc_RecursionError,3.7,, +data,PyExc_ReferenceError,3.2,, +data,PyExc_ResourceWarning,3.7,, +data,PyExc_RuntimeError,3.2,, +data,PyExc_RuntimeWarning,3.2,, +data,PyExc_StopAsyncIteration,3.7,, +data,PyExc_StopIteration,3.2,, +data,PyExc_SyntaxError,3.2,, +data,PyExc_SyntaxWarning,3.2,, +data,PyExc_SystemError,3.2,, +data,PyExc_SystemExit,3.2,, +data,PyExc_TabError,3.2,, +data,PyExc_TimeoutError,3.7,, +data,PyExc_TypeError,3.2,, +data,PyExc_UnboundLocalError,3.2,, +data,PyExc_UnicodeDecodeError,3.2,, +data,PyExc_UnicodeEncodeError,3.2,, +data,PyExc_UnicodeError,3.2,, +data,PyExc_UnicodeTranslateError,3.2,, +data,PyExc_UnicodeWarning,3.2,, +data,PyExc_UserWarning,3.2,, +data,PyExc_ValueError,3.2,, +data,PyExc_Warning,3.2,, +data,PyExc_WindowsError,3.7,on Windows, +data,PyExc_ZeroDivisionError,3.2,, +func,PyExceptionClass_Name,3.8,, +func,PyException_GetArgs,3.12,, +func,PyException_GetCause,3.2,, +func,PyException_GetContext,3.2,, +func,PyException_GetTraceback,3.2,, +func,PyException_SetArgs,3.12,, +func,PyException_SetCause,3.2,, +func,PyException_SetContext,3.2,, +func,PyException_SetTraceback,3.2,, +func,PyFile_FromFd,3.2,, +func,PyFile_GetLine,3.2,, +func,PyFile_WriteObject,3.2,, +func,PyFile_WriteString,3.2,, +data,PyFilter_Type,3.2,, +func,PyFloat_AsDouble,3.2,, +func,PyFloat_FromDouble,3.2,, +func,PyFloat_FromString,3.2,, +func,PyFloat_GetInfo,3.2,, +func,PyFloat_GetMax,3.2,, +func,PyFloat_GetMin,3.2,, +data,PyFloat_Type,3.2,, type,PyFrameObject,3.2,,opaque -function,PyFrame_GetCode,3.10,, -function,PyFrame_GetLineNumber,3.10,, -function,PyFrozenSet_New,3.2,, -var,PyFrozenSet_Type,3.2,, -function,PyGC_Collect,3.2,, -function,PyGC_Disable,3.10,, -function,PyGC_Enable,3.10,, -function,PyGC_IsEnabled,3.10,, -function,PyGILState_Ensure,3.2,, -function,PyGILState_GetThisThreadState,3.2,, -function,PyGILState_Release,3.2,, +func,PyFrame_GetCode,3.10,, +func,PyFrame_GetLineNumber,3.10,, +func,PyFrozenSet_New,3.2,, +data,PyFrozenSet_Type,3.2,, +func,PyGC_Collect,3.2,, +func,PyGC_Disable,3.10,, +func,PyGC_Enable,3.10,, +func,PyGC_IsEnabled,3.10,, +func,PyGILState_Ensure,3.2,, +func,PyGILState_GetThisThreadState,3.2,, +func,PyGILState_Release,3.2,, type,PyGILState_STATE,3.2,, type,PyGetSetDef,3.2,,full-abi -var,PyGetSetDescr_Type,3.2,, -function,PyImport_AddModule,3.2,, -function,PyImport_AddModuleObject,3.7,, -function,PyImport_AddModuleRef,3.13,, -function,PyImport_AppendInittab,3.2,, -function,PyImport_ExecCodeModule,3.2,, -function,PyImport_ExecCodeModuleEx,3.2,, -function,PyImport_ExecCodeModuleObject,3.7,, -function,PyImport_ExecCodeModuleWithPathnames,3.2,, -function,PyImport_GetImporter,3.2,, -function,PyImport_GetMagicNumber,3.2,, -function,PyImport_GetMagicTag,3.2,, -function,PyImport_GetModule,3.8,, -function,PyImport_GetModuleDict,3.2,, -function,PyImport_Import,3.2,, -function,PyImport_ImportFrozenModule,3.2,, -function,PyImport_ImportFrozenModuleObject,3.7,, -function,PyImport_ImportModule,3.2,, -function,PyImport_ImportModuleLevel,3.2,, -function,PyImport_ImportModuleLevelObject,3.7,, -function,PyImport_ImportModuleNoBlock,3.2,, -function,PyImport_ReloadModule,3.2,, -function,PyIndex_Check,3.8,, +data,PyGetSetDescr_Type,3.2,, +func,PyImport_AddModule,3.2,, +func,PyImport_AddModuleObject,3.7,, +func,PyImport_AddModuleRef,3.13,, +func,PyImport_AppendInittab,3.2,, +func,PyImport_ExecCodeModule,3.2,, +func,PyImport_ExecCodeModuleEx,3.2,, +func,PyImport_ExecCodeModuleObject,3.7,, +func,PyImport_ExecCodeModuleWithPathnames,3.2,, +func,PyImport_GetImporter,3.2,, +func,PyImport_GetMagicNumber,3.2,, +func,PyImport_GetMagicTag,3.2,, +func,PyImport_GetModule,3.8,, +func,PyImport_GetModuleDict,3.2,, +func,PyImport_Import,3.2,, +func,PyImport_ImportFrozenModule,3.2,, +func,PyImport_ImportFrozenModuleObject,3.7,, +func,PyImport_ImportModule,3.2,, +func,PyImport_ImportModuleLevel,3.2,, +func,PyImport_ImportModuleLevelObject,3.7,, +func,PyImport_ImportModuleNoBlock,3.2,, +func,PyImport_ReloadModule,3.2,, +func,PyIndex_Check,3.8,, type,PyInterpreterState,3.2,,opaque -function,PyInterpreterState_Clear,3.2,, -function,PyInterpreterState_Delete,3.2,, -function,PyInterpreterState_Get,3.9,, -function,PyInterpreterState_GetDict,3.8,, -function,PyInterpreterState_GetID,3.7,, -function,PyInterpreterState_New,3.2,, -function,PyIter_Check,3.8,, -function,PyIter_Next,3.2,, -function,PyIter_Send,3.10,, -var,PyListIter_Type,3.2,, -var,PyListRevIter_Type,3.2,, -function,PyList_Append,3.2,, -function,PyList_AsTuple,3.2,, -function,PyList_GetItem,3.2,, -function,PyList_GetItemRef,3.13,, -function,PyList_GetSlice,3.2,, -function,PyList_Insert,3.2,, -function,PyList_New,3.2,, -function,PyList_Reverse,3.2,, -function,PyList_SetItem,3.2,, -function,PyList_SetSlice,3.2,, -function,PyList_Size,3.2,, -function,PyList_Sort,3.2,, -var,PyList_Type,3.2,, +func,PyInterpreterState_Clear,3.2,, +func,PyInterpreterState_Delete,3.2,, +func,PyInterpreterState_Get,3.9,, +func,PyInterpreterState_GetDict,3.8,, +func,PyInterpreterState_GetID,3.7,, +func,PyInterpreterState_New,3.2,, +func,PyIter_Check,3.8,, +func,PyIter_Next,3.2,, +func,PyIter_Send,3.10,, +data,PyListIter_Type,3.2,, +data,PyListRevIter_Type,3.2,, +func,PyList_Append,3.2,, +func,PyList_AsTuple,3.2,, +func,PyList_GetItem,3.2,, +func,PyList_GetItemRef,3.13,, +func,PyList_GetSlice,3.2,, +func,PyList_Insert,3.2,, +func,PyList_New,3.2,, +func,PyList_Reverse,3.2,, +func,PyList_SetItem,3.2,, +func,PyList_SetSlice,3.2,, +func,PyList_Size,3.2,, +func,PyList_Sort,3.2,, +data,PyList_Type,3.2,, type,PyLongObject,3.2,,opaque -var,PyLongRangeIter_Type,3.2,, -function,PyLong_AsDouble,3.2,, -function,PyLong_AsInt,3.13,, -function,PyLong_AsLong,3.2,, -function,PyLong_AsLongAndOverflow,3.2,, -function,PyLong_AsLongLong,3.2,, -function,PyLong_AsLongLongAndOverflow,3.2,, -function,PyLong_AsSize_t,3.2,, -function,PyLong_AsSsize_t,3.2,, -function,PyLong_AsUnsignedLong,3.2,, -function,PyLong_AsUnsignedLongLong,3.2,, -function,PyLong_AsUnsignedLongLongMask,3.2,, -function,PyLong_AsUnsignedLongMask,3.2,, -function,PyLong_AsVoidPtr,3.2,, -function,PyLong_FromDouble,3.2,, -function,PyLong_FromLong,3.2,, -function,PyLong_FromLongLong,3.2,, -function,PyLong_FromSize_t,3.2,, -function,PyLong_FromSsize_t,3.2,, -function,PyLong_FromString,3.2,, -function,PyLong_FromUnsignedLong,3.2,, -function,PyLong_FromUnsignedLongLong,3.2,, -function,PyLong_FromVoidPtr,3.2,, -function,PyLong_GetInfo,3.2,, -var,PyLong_Type,3.2,, -var,PyMap_Type,3.2,, -function,PyMapping_Check,3.2,, -function,PyMapping_GetItemString,3.2,, -function,PyMapping_GetOptionalItem,3.13,, -function,PyMapping_GetOptionalItemString,3.13,, -function,PyMapping_HasKey,3.2,, -function,PyMapping_HasKeyString,3.2,, -function,PyMapping_HasKeyStringWithError,3.13,, -function,PyMapping_HasKeyWithError,3.13,, -function,PyMapping_Items,3.2,, -function,PyMapping_Keys,3.2,, -function,PyMapping_Length,3.2,, -function,PyMapping_SetItemString,3.2,, -function,PyMapping_Size,3.2,, -function,PyMapping_Values,3.2,, -function,PyMem_Calloc,3.7,, -function,PyMem_Free,3.2,, -function,PyMem_Malloc,3.2,, -function,PyMem_RawCalloc,3.13,, -function,PyMem_RawFree,3.13,, -function,PyMem_RawMalloc,3.13,, -function,PyMem_RawRealloc,3.13,, -function,PyMem_Realloc,3.2,, +data,PyLongRangeIter_Type,3.2,, +func,PyLong_AsDouble,3.2,, +func,PyLong_AsInt,3.13,, +func,PyLong_AsLong,3.2,, +func,PyLong_AsLongAndOverflow,3.2,, +func,PyLong_AsLongLong,3.2,, +func,PyLong_AsLongLongAndOverflow,3.2,, +func,PyLong_AsSize_t,3.2,, +func,PyLong_AsSsize_t,3.2,, +func,PyLong_AsUnsignedLong,3.2,, +func,PyLong_AsUnsignedLongLong,3.2,, +func,PyLong_AsUnsignedLongLongMask,3.2,, +func,PyLong_AsUnsignedLongMask,3.2,, +func,PyLong_AsVoidPtr,3.2,, +func,PyLong_FromDouble,3.2,, +func,PyLong_FromLong,3.2,, +func,PyLong_FromLongLong,3.2,, +func,PyLong_FromSize_t,3.2,, +func,PyLong_FromSsize_t,3.2,, +func,PyLong_FromString,3.2,, +func,PyLong_FromUnsignedLong,3.2,, +func,PyLong_FromUnsignedLongLong,3.2,, +func,PyLong_FromVoidPtr,3.2,, +func,PyLong_GetInfo,3.2,, +data,PyLong_Type,3.2,, +data,PyMap_Type,3.2,, +func,PyMapping_Check,3.2,, +func,PyMapping_GetItemString,3.2,, +func,PyMapping_GetOptionalItem,3.13,, +func,PyMapping_GetOptionalItemString,3.13,, +func,PyMapping_HasKey,3.2,, +func,PyMapping_HasKeyString,3.2,, +func,PyMapping_HasKeyStringWithError,3.13,, +func,PyMapping_HasKeyWithError,3.13,, +func,PyMapping_Items,3.2,, +func,PyMapping_Keys,3.2,, +func,PyMapping_Length,3.2,, +func,PyMapping_SetItemString,3.2,, +func,PyMapping_Size,3.2,, +func,PyMapping_Values,3.2,, +func,PyMem_Calloc,3.7,, +func,PyMem_Free,3.2,, +func,PyMem_Malloc,3.2,, +func,PyMem_RawCalloc,3.13,, +func,PyMem_RawFree,3.13,, +func,PyMem_RawMalloc,3.13,, +func,PyMem_RawRealloc,3.13,, +func,PyMem_Realloc,3.2,, type,PyMemberDef,3.2,,full-abi -var,PyMemberDescr_Type,3.2,, -function,PyMember_GetOne,3.2,, -function,PyMember_SetOne,3.2,, -function,PyMemoryView_FromBuffer,3.11,, -function,PyMemoryView_FromMemory,3.7,, -function,PyMemoryView_FromObject,3.2,, -function,PyMemoryView_GetContiguous,3.2,, -var,PyMemoryView_Type,3.2,, +data,PyMemberDescr_Type,3.2,, +func,PyMember_GetOne,3.2,, +func,PyMember_SetOne,3.2,, +func,PyMemoryView_FromBuffer,3.11,, +func,PyMemoryView_FromMemory,3.7,, +func,PyMemoryView_FromObject,3.2,, +func,PyMemoryView_GetContiguous,3.2,, +data,PyMemoryView_Type,3.2,, type,PyMethodDef,3.2,,full-abi -var,PyMethodDescr_Type,3.2,, +data,PyMethodDescr_Type,3.2,, type,PyModuleDef,3.2,,full-abi type,PyModuleDef_Base,3.2,,full-abi -function,PyModuleDef_Init,3.5,, -var,PyModuleDef_Type,3.5,, -function,PyModule_Add,3.13,, -function,PyModule_AddFunctions,3.7,, -function,PyModule_AddIntConstant,3.2,, -function,PyModule_AddObject,3.2,, -function,PyModule_AddObjectRef,3.10,, -function,PyModule_AddStringConstant,3.2,, -function,PyModule_AddType,3.10,, -function,PyModule_Create2,3.2,, -function,PyModule_ExecDef,3.7,, -function,PyModule_FromDefAndSpec2,3.7,, -function,PyModule_GetDef,3.2,, -function,PyModule_GetDict,3.2,, -function,PyModule_GetFilename,3.2,, -function,PyModule_GetFilenameObject,3.2,, -function,PyModule_GetName,3.2,, -function,PyModule_GetNameObject,3.7,, -function,PyModule_GetState,3.2,, -function,PyModule_New,3.2,, -function,PyModule_NewObject,3.7,, -function,PyModule_SetDocString,3.7,, -var,PyModule_Type,3.2,, -function,PyNumber_Absolute,3.2,, -function,PyNumber_Add,3.2,, -function,PyNumber_And,3.2,, -function,PyNumber_AsSsize_t,3.2,, -function,PyNumber_Check,3.2,, -function,PyNumber_Divmod,3.2,, -function,PyNumber_Float,3.2,, -function,PyNumber_FloorDivide,3.2,, -function,PyNumber_InPlaceAdd,3.2,, -function,PyNumber_InPlaceAnd,3.2,, -function,PyNumber_InPlaceFloorDivide,3.2,, -function,PyNumber_InPlaceLshift,3.2,, -function,PyNumber_InPlaceMatrixMultiply,3.7,, -function,PyNumber_InPlaceMultiply,3.2,, -function,PyNumber_InPlaceOr,3.2,, -function,PyNumber_InPlacePower,3.2,, -function,PyNumber_InPlaceRemainder,3.2,, -function,PyNumber_InPlaceRshift,3.2,, -function,PyNumber_InPlaceSubtract,3.2,, -function,PyNumber_InPlaceTrueDivide,3.2,, -function,PyNumber_InPlaceXor,3.2,, -function,PyNumber_Index,3.2,, -function,PyNumber_Invert,3.2,, -function,PyNumber_Long,3.2,, -function,PyNumber_Lshift,3.2,, -function,PyNumber_MatrixMultiply,3.7,, -function,PyNumber_Multiply,3.2,, -function,PyNumber_Negative,3.2,, -function,PyNumber_Or,3.2,, -function,PyNumber_Positive,3.2,, -function,PyNumber_Power,3.2,, -function,PyNumber_Remainder,3.2,, -function,PyNumber_Rshift,3.2,, -function,PyNumber_Subtract,3.2,, -function,PyNumber_ToBase,3.2,, -function,PyNumber_TrueDivide,3.2,, -function,PyNumber_Xor,3.2,, -function,PyOS_AfterFork,3.2,on platforms with fork(), -function,PyOS_AfterFork_Child,3.7,on platforms with fork(), -function,PyOS_AfterFork_Parent,3.7,on platforms with fork(), -function,PyOS_BeforeFork,3.7,on platforms with fork(), -function,PyOS_CheckStack,3.7,on platforms with USE_STACKCHECK, -function,PyOS_FSPath,3.6,, -var,PyOS_InputHook,3.2,, -function,PyOS_InterruptOccurred,3.2,, -function,PyOS_double_to_string,3.2,, -function,PyOS_getsig,3.2,, -function,PyOS_mystricmp,3.2,, -function,PyOS_mystrnicmp,3.2,, -function,PyOS_setsig,3.2,, +func,PyModuleDef_Init,3.5,, +data,PyModuleDef_Type,3.5,, +func,PyModule_Add,3.13,, +func,PyModule_AddFunctions,3.7,, +func,PyModule_AddIntConstant,3.2,, +func,PyModule_AddObject,3.2,, +func,PyModule_AddObjectRef,3.10,, +func,PyModule_AddStringConstant,3.2,, +func,PyModule_AddType,3.10,, +func,PyModule_Create2,3.2,, +func,PyModule_ExecDef,3.7,, +func,PyModule_FromDefAndSpec2,3.7,, +func,PyModule_GetDef,3.2,, +func,PyModule_GetDict,3.2,, +func,PyModule_GetFilename,3.2,, +func,PyModule_GetFilenameObject,3.2,, +func,PyModule_GetName,3.2,, +func,PyModule_GetNameObject,3.7,, +func,PyModule_GetState,3.2,, +func,PyModule_New,3.2,, +func,PyModule_NewObject,3.7,, +func,PyModule_SetDocString,3.7,, +data,PyModule_Type,3.2,, +func,PyNumber_Absolute,3.2,, +func,PyNumber_Add,3.2,, +func,PyNumber_And,3.2,, +func,PyNumber_AsSsize_t,3.2,, +func,PyNumber_Check,3.2,, +func,PyNumber_Divmod,3.2,, +func,PyNumber_Float,3.2,, +func,PyNumber_FloorDivide,3.2,, +func,PyNumber_InPlaceAdd,3.2,, +func,PyNumber_InPlaceAnd,3.2,, +func,PyNumber_InPlaceFloorDivide,3.2,, +func,PyNumber_InPlaceLshift,3.2,, +func,PyNumber_InPlaceMatrixMultiply,3.7,, +func,PyNumber_InPlaceMultiply,3.2,, +func,PyNumber_InPlaceOr,3.2,, +func,PyNumber_InPlacePower,3.2,, +func,PyNumber_InPlaceRemainder,3.2,, +func,PyNumber_InPlaceRshift,3.2,, +func,PyNumber_InPlaceSubtract,3.2,, +func,PyNumber_InPlaceTrueDivide,3.2,, +func,PyNumber_InPlaceXor,3.2,, +func,PyNumber_Index,3.2,, +func,PyNumber_Invert,3.2,, +func,PyNumber_Long,3.2,, +func,PyNumber_Lshift,3.2,, +func,PyNumber_MatrixMultiply,3.7,, +func,PyNumber_Multiply,3.2,, +func,PyNumber_Negative,3.2,, +func,PyNumber_Or,3.2,, +func,PyNumber_Positive,3.2,, +func,PyNumber_Power,3.2,, +func,PyNumber_Remainder,3.2,, +func,PyNumber_Rshift,3.2,, +func,PyNumber_Subtract,3.2,, +func,PyNumber_ToBase,3.2,, +func,PyNumber_TrueDivide,3.2,, +func,PyNumber_Xor,3.2,, +func,PyOS_AfterFork,3.2,on platforms with fork(), +func,PyOS_AfterFork_Child,3.7,on platforms with fork(), +func,PyOS_AfterFork_Parent,3.7,on platforms with fork(), +func,PyOS_BeforeFork,3.7,on platforms with fork(), +func,PyOS_CheckStack,3.7,on platforms with USE_STACKCHECK, +func,PyOS_FSPath,3.6,, +data,PyOS_InputHook,3.2,, +func,PyOS_InterruptOccurred,3.2,, +func,PyOS_double_to_string,3.2,, +func,PyOS_getsig,3.2,, +func,PyOS_mystricmp,3.2,, +func,PyOS_mystrnicmp,3.2,, +func,PyOS_setsig,3.2,, type,PyOS_sighandler_t,3.2,, -function,PyOS_snprintf,3.2,, -function,PyOS_string_to_double,3.2,, -function,PyOS_strtol,3.2,, -function,PyOS_strtoul,3.2,, -function,PyOS_vsnprintf,3.2,, +func,PyOS_snprintf,3.2,, +func,PyOS_string_to_double,3.2,, +func,PyOS_strtol,3.2,, +func,PyOS_strtoul,3.2,, +func,PyOS_vsnprintf,3.2,, type,PyObject,3.2,,members member,PyObject.ob_refcnt,3.2,, member,PyObject.ob_type,3.2,, -function,PyObject_ASCII,3.2,, -function,PyObject_AsFileDescriptor,3.2,, -function,PyObject_Bytes,3.2,, -function,PyObject_Call,3.2,, -function,PyObject_CallFunction,3.2,, -function,PyObject_CallFunctionObjArgs,3.2,, -function,PyObject_CallMethod,3.2,, -function,PyObject_CallMethodObjArgs,3.2,, -function,PyObject_CallNoArgs,3.10,, -function,PyObject_CallObject,3.2,, -function,PyObject_Calloc,3.7,, -function,PyObject_CheckBuffer,3.11,, -function,PyObject_ClearWeakRefs,3.2,, -function,PyObject_CopyData,3.11,, -function,PyObject_DelAttr,3.13,, -function,PyObject_DelAttrString,3.13,, -function,PyObject_DelItem,3.2,, -function,PyObject_DelItemString,3.2,, -function,PyObject_Dir,3.2,, -function,PyObject_Format,3.2,, -function,PyObject_Free,3.2,, -function,PyObject_GC_Del,3.2,, -function,PyObject_GC_IsFinalized,3.9,, -function,PyObject_GC_IsTracked,3.9,, -function,PyObject_GC_Track,3.2,, -function,PyObject_GC_UnTrack,3.2,, -function,PyObject_GenericGetAttr,3.2,, -function,PyObject_GenericGetDict,3.10,, -function,PyObject_GenericSetAttr,3.2,, -function,PyObject_GenericSetDict,3.7,, -function,PyObject_GetAIter,3.10,, -function,PyObject_GetAttr,3.2,, -function,PyObject_GetAttrString,3.2,, -function,PyObject_GetBuffer,3.11,, -function,PyObject_GetItem,3.2,, -function,PyObject_GetIter,3.2,, -function,PyObject_GetOptionalAttr,3.13,, -function,PyObject_GetOptionalAttrString,3.13,, -function,PyObject_GetTypeData,3.12,, -function,PyObject_HasAttr,3.2,, -function,PyObject_HasAttrString,3.2,, -function,PyObject_HasAttrStringWithError,3.13,, -function,PyObject_HasAttrWithError,3.13,, -function,PyObject_Hash,3.2,, -function,PyObject_HashNotImplemented,3.2,, -function,PyObject_Init,3.2,, -function,PyObject_InitVar,3.2,, -function,PyObject_IsInstance,3.2,, -function,PyObject_IsSubclass,3.2,, -function,PyObject_IsTrue,3.2,, -function,PyObject_Length,3.2,, -function,PyObject_Malloc,3.2,, -function,PyObject_Not,3.2,, -function,PyObject_Realloc,3.2,, -function,PyObject_Repr,3.2,, -function,PyObject_RichCompare,3.2,, -function,PyObject_RichCompareBool,3.2,, -function,PyObject_SelfIter,3.2,, -function,PyObject_SetAttr,3.2,, -function,PyObject_SetAttrString,3.2,, -function,PyObject_SetItem,3.2,, -function,PyObject_Size,3.2,, -function,PyObject_Str,3.2,, -function,PyObject_Type,3.2,, -function,PyObject_Vectorcall,3.12,, -function,PyObject_VectorcallMethod,3.12,, -var,PyProperty_Type,3.2,, -var,PyRangeIter_Type,3.2,, -var,PyRange_Type,3.2,, -var,PyReversed_Type,3.2,, -function,PySeqIter_New,3.2,, -var,PySeqIter_Type,3.2,, -function,PySequence_Check,3.2,, -function,PySequence_Concat,3.2,, -function,PySequence_Contains,3.2,, -function,PySequence_Count,3.2,, -function,PySequence_DelItem,3.2,, -function,PySequence_DelSlice,3.2,, -function,PySequence_Fast,3.2,, -function,PySequence_GetItem,3.2,, -function,PySequence_GetSlice,3.2,, -function,PySequence_In,3.2,, -function,PySequence_InPlaceConcat,3.2,, -function,PySequence_InPlaceRepeat,3.2,, -function,PySequence_Index,3.2,, -function,PySequence_Length,3.2,, -function,PySequence_List,3.2,, -function,PySequence_Repeat,3.2,, -function,PySequence_SetItem,3.2,, -function,PySequence_SetSlice,3.2,, -function,PySequence_Size,3.2,, -function,PySequence_Tuple,3.2,, -var,PySetIter_Type,3.2,, -function,PySet_Add,3.2,, -function,PySet_Clear,3.2,, -function,PySet_Contains,3.2,, -function,PySet_Discard,3.2,, -function,PySet_New,3.2,, -function,PySet_Pop,3.2,, -function,PySet_Size,3.2,, -var,PySet_Type,3.2,, -function,PySlice_AdjustIndices,3.7,, -function,PySlice_GetIndices,3.2,, -function,PySlice_GetIndicesEx,3.2,, -function,PySlice_New,3.2,, -var,PySlice_Type,3.2,, -function,PySlice_Unpack,3.7,, -function,PyState_AddModule,3.3,, -function,PyState_FindModule,3.2,, -function,PyState_RemoveModule,3.3,, +func,PyObject_ASCII,3.2,, +func,PyObject_AsFileDescriptor,3.2,, +func,PyObject_Bytes,3.2,, +func,PyObject_Call,3.2,, +func,PyObject_CallFunction,3.2,, +func,PyObject_CallFunctionObjArgs,3.2,, +func,PyObject_CallMethod,3.2,, +func,PyObject_CallMethodObjArgs,3.2,, +func,PyObject_CallNoArgs,3.10,, +func,PyObject_CallObject,3.2,, +func,PyObject_Calloc,3.7,, +func,PyObject_CheckBuffer,3.11,, +func,PyObject_ClearWeakRefs,3.2,, +func,PyObject_CopyData,3.11,, +func,PyObject_DelAttr,3.13,, +func,PyObject_DelAttrString,3.13,, +func,PyObject_DelItem,3.2,, +func,PyObject_DelItemString,3.2,, +func,PyObject_Dir,3.2,, +func,PyObject_Format,3.2,, +func,PyObject_Free,3.2,, +func,PyObject_GC_Del,3.2,, +func,PyObject_GC_IsFinalized,3.9,, +func,PyObject_GC_IsTracked,3.9,, +func,PyObject_GC_Track,3.2,, +func,PyObject_GC_UnTrack,3.2,, +func,PyObject_GenericGetAttr,3.2,, +func,PyObject_GenericGetDict,3.10,, +func,PyObject_GenericSetAttr,3.2,, +func,PyObject_GenericSetDict,3.7,, +func,PyObject_GetAIter,3.10,, +func,PyObject_GetAttr,3.2,, +func,PyObject_GetAttrString,3.2,, +func,PyObject_GetBuffer,3.11,, +func,PyObject_GetItem,3.2,, +func,PyObject_GetIter,3.2,, +func,PyObject_GetOptionalAttr,3.13,, +func,PyObject_GetOptionalAttrString,3.13,, +func,PyObject_GetTypeData,3.12,, +func,PyObject_HasAttr,3.2,, +func,PyObject_HasAttrString,3.2,, +func,PyObject_HasAttrStringWithError,3.13,, +func,PyObject_HasAttrWithError,3.13,, +func,PyObject_Hash,3.2,, +func,PyObject_HashNotImplemented,3.2,, +func,PyObject_Init,3.2,, +func,PyObject_InitVar,3.2,, +func,PyObject_IsInstance,3.2,, +func,PyObject_IsSubclass,3.2,, +func,PyObject_IsTrue,3.2,, +func,PyObject_Length,3.2,, +func,PyObject_Malloc,3.2,, +func,PyObject_Not,3.2,, +func,PyObject_Realloc,3.2,, +func,PyObject_Repr,3.2,, +func,PyObject_RichCompare,3.2,, +func,PyObject_RichCompareBool,3.2,, +func,PyObject_SelfIter,3.2,, +func,PyObject_SetAttr,3.2,, +func,PyObject_SetAttrString,3.2,, +func,PyObject_SetItem,3.2,, +func,PyObject_Size,3.2,, +func,PyObject_Str,3.2,, +func,PyObject_Type,3.2,, +func,PyObject_Vectorcall,3.12,, +func,PyObject_VectorcallMethod,3.12,, +data,PyProperty_Type,3.2,, +data,PyRangeIter_Type,3.2,, +data,PyRange_Type,3.2,, +data,PyReversed_Type,3.2,, +func,PySeqIter_New,3.2,, +data,PySeqIter_Type,3.2,, +func,PySequence_Check,3.2,, +func,PySequence_Concat,3.2,, +func,PySequence_Contains,3.2,, +func,PySequence_Count,3.2,, +func,PySequence_DelItem,3.2,, +func,PySequence_DelSlice,3.2,, +func,PySequence_Fast,3.2,, +func,PySequence_GetItem,3.2,, +func,PySequence_GetSlice,3.2,, +func,PySequence_In,3.2,, +func,PySequence_InPlaceConcat,3.2,, +func,PySequence_InPlaceRepeat,3.2,, +func,PySequence_Index,3.2,, +func,PySequence_Length,3.2,, +func,PySequence_List,3.2,, +func,PySequence_Repeat,3.2,, +func,PySequence_SetItem,3.2,, +func,PySequence_SetSlice,3.2,, +func,PySequence_Size,3.2,, +func,PySequence_Tuple,3.2,, +data,PySetIter_Type,3.2,, +func,PySet_Add,3.2,, +func,PySet_Clear,3.2,, +func,PySet_Contains,3.2,, +func,PySet_Discard,3.2,, +func,PySet_New,3.2,, +func,PySet_Pop,3.2,, +func,PySet_Size,3.2,, +data,PySet_Type,3.2,, +func,PySlice_AdjustIndices,3.7,, +func,PySlice_GetIndices,3.2,, +func,PySlice_GetIndicesEx,3.2,, +func,PySlice_New,3.2,, +data,PySlice_Type,3.2,, +func,PySlice_Unpack,3.7,, +func,PyState_AddModule,3.3,, +func,PyState_FindModule,3.2,, +func,PyState_RemoveModule,3.3,, type,PyStructSequence_Desc,3.2,,full-abi type,PyStructSequence_Field,3.2,,full-abi -function,PyStructSequence_GetItem,3.2,, -function,PyStructSequence_New,3.2,, -function,PyStructSequence_NewType,3.2,, -function,PyStructSequence_SetItem,3.2,, -var,PyStructSequence_UnnamedField,3.11,, -var,PySuper_Type,3.2,, -function,PySys_Audit,3.13,, -function,PySys_AuditTuple,3.13,, -function,PySys_FormatStderr,3.2,, -function,PySys_FormatStdout,3.2,, -function,PySys_GetObject,3.2,, -function,PySys_GetXOptions,3.7,, -function,PySys_ResetWarnOptions,3.2,, -function,PySys_SetArgv,3.2,, -function,PySys_SetArgvEx,3.2,, -function,PySys_SetObject,3.2,, -function,PySys_WriteStderr,3.2,, -function,PySys_WriteStdout,3.2,, +func,PyStructSequence_GetItem,3.2,, +func,PyStructSequence_New,3.2,, +func,PyStructSequence_NewType,3.2,, +func,PyStructSequence_SetItem,3.2,, +data,PyStructSequence_UnnamedField,3.11,, +data,PySuper_Type,3.2,, +func,PySys_Audit,3.13,, +func,PySys_AuditTuple,3.13,, +func,PySys_FormatStderr,3.2,, +func,PySys_FormatStdout,3.2,, +func,PySys_GetObject,3.2,, +func,PySys_GetXOptions,3.7,, +func,PySys_ResetWarnOptions,3.2,, +func,PySys_SetArgv,3.2,, +func,PySys_SetArgvEx,3.2,, +func,PySys_SetObject,3.2,, +func,PySys_WriteStderr,3.2,, +func,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque -function,PyThreadState_Clear,3.2,, -function,PyThreadState_Delete,3.2,, -function,PyThreadState_Get,3.2,, -function,PyThreadState_GetDict,3.2,, -function,PyThreadState_GetFrame,3.10,, -function,PyThreadState_GetID,3.10,, -function,PyThreadState_GetInterpreter,3.10,, -function,PyThreadState_New,3.2,, -function,PyThreadState_SetAsyncExc,3.2,, -function,PyThreadState_Swap,3.2,, -function,PyThread_GetInfo,3.3,, -function,PyThread_ReInitTLS,3.2,, -function,PyThread_acquire_lock,3.2,, -function,PyThread_acquire_lock_timed,3.2,, -function,PyThread_allocate_lock,3.2,, -function,PyThread_create_key,3.2,, -function,PyThread_delete_key,3.2,, -function,PyThread_delete_key_value,3.2,, -function,PyThread_exit_thread,3.2,, -function,PyThread_free_lock,3.2,, -function,PyThread_get_key_value,3.2,, -function,PyThread_get_stacksize,3.2,, -function,PyThread_get_thread_ident,3.2,, -function,PyThread_get_thread_native_id,3.2,on platforms with native thread IDs, -function,PyThread_init_thread,3.2,, -function,PyThread_release_lock,3.2,, -function,PyThread_set_key_value,3.2,, -function,PyThread_set_stacksize,3.2,, -function,PyThread_start_new_thread,3.2,, -function,PyThread_tss_alloc,3.7,, -function,PyThread_tss_create,3.7,, -function,PyThread_tss_delete,3.7,, -function,PyThread_tss_free,3.7,, -function,PyThread_tss_get,3.7,, -function,PyThread_tss_is_created,3.7,, -function,PyThread_tss_set,3.7,, -function,PyTraceBack_Here,3.2,, -function,PyTraceBack_Print,3.2,, -var,PyTraceBack_Type,3.2,, -var,PyTupleIter_Type,3.2,, -function,PyTuple_GetItem,3.2,, -function,PyTuple_GetSlice,3.2,, -function,PyTuple_New,3.2,, -function,PyTuple_Pack,3.2,, -function,PyTuple_SetItem,3.2,, -function,PyTuple_Size,3.2,, -var,PyTuple_Type,3.2,, +func,PyThreadState_Clear,3.2,, +func,PyThreadState_Delete,3.2,, +func,PyThreadState_Get,3.2,, +func,PyThreadState_GetDict,3.2,, +func,PyThreadState_GetFrame,3.10,, +func,PyThreadState_GetID,3.10,, +func,PyThreadState_GetInterpreter,3.10,, +func,PyThreadState_New,3.2,, +func,PyThreadState_SetAsyncExc,3.2,, +func,PyThreadState_Swap,3.2,, +func,PyThread_GetInfo,3.3,, +func,PyThread_ReInitTLS,3.2,, +func,PyThread_acquire_lock,3.2,, +func,PyThread_acquire_lock_timed,3.2,, +func,PyThread_allocate_lock,3.2,, +func,PyThread_create_key,3.2,, +func,PyThread_delete_key,3.2,, +func,PyThread_delete_key_value,3.2,, +func,PyThread_exit_thread,3.2,, +func,PyThread_free_lock,3.2,, +func,PyThread_get_key_value,3.2,, +func,PyThread_get_stacksize,3.2,, +func,PyThread_get_thread_ident,3.2,, +func,PyThread_get_thread_native_id,3.2,on platforms with native thread IDs, +func,PyThread_init_thread,3.2,, +func,PyThread_release_lock,3.2,, +func,PyThread_set_key_value,3.2,, +func,PyThread_set_stacksize,3.2,, +func,PyThread_start_new_thread,3.2,, +func,PyThread_tss_alloc,3.7,, +func,PyThread_tss_create,3.7,, +func,PyThread_tss_delete,3.7,, +func,PyThread_tss_free,3.7,, +func,PyThread_tss_get,3.7,, +func,PyThread_tss_is_created,3.7,, +func,PyThread_tss_set,3.7,, +func,PyTraceBack_Here,3.2,, +func,PyTraceBack_Print,3.2,, +data,PyTraceBack_Type,3.2,, +data,PyTupleIter_Type,3.2,, +func,PyTuple_GetItem,3.2,, +func,PyTuple_GetSlice,3.2,, +func,PyTuple_New,3.2,, +func,PyTuple_Pack,3.2,, +func,PyTuple_SetItem,3.2,, +func,PyTuple_Size,3.2,, +data,PyTuple_Type,3.2,, type,PyTypeObject,3.2,,opaque -function,PyType_ClearCache,3.2,, -function,PyType_FromMetaclass,3.12,, -function,PyType_FromModuleAndSpec,3.10,, -function,PyType_FromSpec,3.2,, -function,PyType_FromSpecWithBases,3.3,, -function,PyType_GenericAlloc,3.2,, -function,PyType_GenericNew,3.2,, -function,PyType_GetFlags,3.2,, -function,PyType_GetFullyQualifiedName,3.13,, -function,PyType_GetModule,3.10,, -function,PyType_GetModuleByDef,3.13,, -function,PyType_GetModuleName,3.13,, -function,PyType_GetModuleState,3.10,, -function,PyType_GetName,3.11,, -function,PyType_GetQualName,3.11,, -function,PyType_GetSlot,3.4,, -function,PyType_GetTypeDataSize,3.12,, -function,PyType_IsSubtype,3.2,, -function,PyType_Modified,3.2,, -function,PyType_Ready,3.2,, +func,PyType_ClearCache,3.2,, +func,PyType_FromMetaclass,3.12,, +func,PyType_FromModuleAndSpec,3.10,, +func,PyType_FromSpec,3.2,, +func,PyType_FromSpecWithBases,3.3,, +func,PyType_GenericAlloc,3.2,, +func,PyType_GenericNew,3.2,, +func,PyType_GetFlags,3.2,, +func,PyType_GetFullyQualifiedName,3.13,, +func,PyType_GetModule,3.10,, +func,PyType_GetModuleByDef,3.13,, +func,PyType_GetModuleName,3.13,, +func,PyType_GetModuleState,3.10,, +func,PyType_GetName,3.11,, +func,PyType_GetQualName,3.11,, +func,PyType_GetSlot,3.4,, +func,PyType_GetTypeDataSize,3.12,, +func,PyType_IsSubtype,3.2,, +func,PyType_Modified,3.2,, +func,PyType_Ready,3.2,, type,PyType_Slot,3.2,,full-abi type,PyType_Spec,3.2,,full-abi -var,PyType_Type,3.2,, -function,PyUnicodeDecodeError_Create,3.2,, -function,PyUnicodeDecodeError_GetEncoding,3.2,, -function,PyUnicodeDecodeError_GetEnd,3.2,, -function,PyUnicodeDecodeError_GetObject,3.2,, -function,PyUnicodeDecodeError_GetReason,3.2,, -function,PyUnicodeDecodeError_GetStart,3.2,, -function,PyUnicodeDecodeError_SetEnd,3.2,, -function,PyUnicodeDecodeError_SetReason,3.2,, -function,PyUnicodeDecodeError_SetStart,3.2,, -function,PyUnicodeEncodeError_GetEncoding,3.2,, -function,PyUnicodeEncodeError_GetEnd,3.2,, -function,PyUnicodeEncodeError_GetObject,3.2,, -function,PyUnicodeEncodeError_GetReason,3.2,, -function,PyUnicodeEncodeError_GetStart,3.2,, -function,PyUnicodeEncodeError_SetEnd,3.2,, -function,PyUnicodeEncodeError_SetReason,3.2,, -function,PyUnicodeEncodeError_SetStart,3.2,, -var,PyUnicodeIter_Type,3.2,, -function,PyUnicodeTranslateError_GetEnd,3.2,, -function,PyUnicodeTranslateError_GetObject,3.2,, -function,PyUnicodeTranslateError_GetReason,3.2,, -function,PyUnicodeTranslateError_GetStart,3.2,, -function,PyUnicodeTranslateError_SetEnd,3.2,, -function,PyUnicodeTranslateError_SetReason,3.2,, -function,PyUnicodeTranslateError_SetStart,3.2,, -function,PyUnicode_Append,3.2,, -function,PyUnicode_AppendAndDel,3.2,, -function,PyUnicode_AsASCIIString,3.2,, -function,PyUnicode_AsCharmapString,3.2,, -function,PyUnicode_AsDecodedObject,3.2,, -function,PyUnicode_AsDecodedUnicode,3.2,, -function,PyUnicode_AsEncodedObject,3.2,, -function,PyUnicode_AsEncodedString,3.2,, -function,PyUnicode_AsEncodedUnicode,3.2,, -function,PyUnicode_AsLatin1String,3.2,, -function,PyUnicode_AsMBCSString,3.7,on Windows, -function,PyUnicode_AsRawUnicodeEscapeString,3.2,, -function,PyUnicode_AsUCS4,3.7,, -function,PyUnicode_AsUCS4Copy,3.7,, -function,PyUnicode_AsUTF16String,3.2,, -function,PyUnicode_AsUTF32String,3.2,, -function,PyUnicode_AsUTF8AndSize,3.10,, -function,PyUnicode_AsUTF8String,3.2,, -function,PyUnicode_AsUnicodeEscapeString,3.2,, -function,PyUnicode_AsWideChar,3.2,, -function,PyUnicode_AsWideCharString,3.7,, -function,PyUnicode_BuildEncodingMap,3.2,, -function,PyUnicode_Compare,3.2,, -function,PyUnicode_CompareWithASCIIString,3.2,, -function,PyUnicode_Concat,3.2,, -function,PyUnicode_Contains,3.2,, -function,PyUnicode_Count,3.2,, -function,PyUnicode_Decode,3.2,, -function,PyUnicode_DecodeASCII,3.2,, -function,PyUnicode_DecodeCharmap,3.2,, -function,PyUnicode_DecodeCodePageStateful,3.7,on Windows, -function,PyUnicode_DecodeFSDefault,3.2,, -function,PyUnicode_DecodeFSDefaultAndSize,3.2,, -function,PyUnicode_DecodeLatin1,3.2,, -function,PyUnicode_DecodeLocale,3.7,, -function,PyUnicode_DecodeLocaleAndSize,3.7,, -function,PyUnicode_DecodeMBCS,3.7,on Windows, -function,PyUnicode_DecodeMBCSStateful,3.7,on Windows, -function,PyUnicode_DecodeRawUnicodeEscape,3.2,, -function,PyUnicode_DecodeUTF16,3.2,, -function,PyUnicode_DecodeUTF16Stateful,3.2,, -function,PyUnicode_DecodeUTF32,3.2,, -function,PyUnicode_DecodeUTF32Stateful,3.2,, -function,PyUnicode_DecodeUTF7,3.2,, -function,PyUnicode_DecodeUTF7Stateful,3.2,, -function,PyUnicode_DecodeUTF8,3.2,, -function,PyUnicode_DecodeUTF8Stateful,3.2,, -function,PyUnicode_DecodeUnicodeEscape,3.2,, -function,PyUnicode_EncodeCodePage,3.7,on Windows, -function,PyUnicode_EncodeFSDefault,3.2,, -function,PyUnicode_EncodeLocale,3.7,, -function,PyUnicode_EqualToUTF8,3.13,, -function,PyUnicode_EqualToUTF8AndSize,3.13,, -function,PyUnicode_FSConverter,3.2,, -function,PyUnicode_FSDecoder,3.2,, -function,PyUnicode_Find,3.2,, -function,PyUnicode_FindChar,3.7,, -function,PyUnicode_Format,3.2,, -function,PyUnicode_FromEncodedObject,3.2,, -function,PyUnicode_FromFormat,3.2,, -function,PyUnicode_FromFormatV,3.2,, -function,PyUnicode_FromObject,3.2,, -function,PyUnicode_FromOrdinal,3.2,, -function,PyUnicode_FromString,3.2,, -function,PyUnicode_FromStringAndSize,3.2,, -function,PyUnicode_FromWideChar,3.2,, -function,PyUnicode_GetDefaultEncoding,3.2,, -function,PyUnicode_GetLength,3.7,, -function,PyUnicode_InternFromString,3.2,, -function,PyUnicode_InternInPlace,3.2,, -function,PyUnicode_IsIdentifier,3.2,, -function,PyUnicode_Join,3.2,, -function,PyUnicode_Partition,3.2,, -function,PyUnicode_RPartition,3.2,, -function,PyUnicode_RSplit,3.2,, -function,PyUnicode_ReadChar,3.7,, -function,PyUnicode_Replace,3.2,, -function,PyUnicode_Resize,3.2,, -function,PyUnicode_RichCompare,3.2,, -function,PyUnicode_Split,3.2,, -function,PyUnicode_Splitlines,3.2,, -function,PyUnicode_Substring,3.7,, -function,PyUnicode_Tailmatch,3.2,, -function,PyUnicode_Translate,3.2,, -var,PyUnicode_Type,3.2,, -function,PyUnicode_WriteChar,3.7,, +data,PyType_Type,3.2,, +func,PyUnicodeDecodeError_Create,3.2,, +func,PyUnicodeDecodeError_GetEncoding,3.2,, +func,PyUnicodeDecodeError_GetEnd,3.2,, +func,PyUnicodeDecodeError_GetObject,3.2,, +func,PyUnicodeDecodeError_GetReason,3.2,, +func,PyUnicodeDecodeError_GetStart,3.2,, +func,PyUnicodeDecodeError_SetEnd,3.2,, +func,PyUnicodeDecodeError_SetReason,3.2,, +func,PyUnicodeDecodeError_SetStart,3.2,, +func,PyUnicodeEncodeError_GetEncoding,3.2,, +func,PyUnicodeEncodeError_GetEnd,3.2,, +func,PyUnicodeEncodeError_GetObject,3.2,, +func,PyUnicodeEncodeError_GetReason,3.2,, +func,PyUnicodeEncodeError_GetStart,3.2,, +func,PyUnicodeEncodeError_SetEnd,3.2,, +func,PyUnicodeEncodeError_SetReason,3.2,, +func,PyUnicodeEncodeError_SetStart,3.2,, +data,PyUnicodeIter_Type,3.2,, +func,PyUnicodeTranslateError_GetEnd,3.2,, +func,PyUnicodeTranslateError_GetObject,3.2,, +func,PyUnicodeTranslateError_GetReason,3.2,, +func,PyUnicodeTranslateError_GetStart,3.2,, +func,PyUnicodeTranslateError_SetEnd,3.2,, +func,PyUnicodeTranslateError_SetReason,3.2,, +func,PyUnicodeTranslateError_SetStart,3.2,, +func,PyUnicode_Append,3.2,, +func,PyUnicode_AppendAndDel,3.2,, +func,PyUnicode_AsASCIIString,3.2,, +func,PyUnicode_AsCharmapString,3.2,, +func,PyUnicode_AsDecodedObject,3.2,, +func,PyUnicode_AsDecodedUnicode,3.2,, +func,PyUnicode_AsEncodedObject,3.2,, +func,PyUnicode_AsEncodedString,3.2,, +func,PyUnicode_AsEncodedUnicode,3.2,, +func,PyUnicode_AsLatin1String,3.2,, +func,PyUnicode_AsMBCSString,3.7,on Windows, +func,PyUnicode_AsRawUnicodeEscapeString,3.2,, +func,PyUnicode_AsUCS4,3.7,, +func,PyUnicode_AsUCS4Copy,3.7,, +func,PyUnicode_AsUTF16String,3.2,, +func,PyUnicode_AsUTF32String,3.2,, +func,PyUnicode_AsUTF8AndSize,3.10,, +func,PyUnicode_AsUTF8String,3.2,, +func,PyUnicode_AsUnicodeEscapeString,3.2,, +func,PyUnicode_AsWideChar,3.2,, +func,PyUnicode_AsWideCharString,3.7,, +func,PyUnicode_BuildEncodingMap,3.2,, +func,PyUnicode_Compare,3.2,, +func,PyUnicode_CompareWithASCIIString,3.2,, +func,PyUnicode_Concat,3.2,, +func,PyUnicode_Contains,3.2,, +func,PyUnicode_Count,3.2,, +func,PyUnicode_Decode,3.2,, +func,PyUnicode_DecodeASCII,3.2,, +func,PyUnicode_DecodeCharmap,3.2,, +func,PyUnicode_DecodeCodePageStateful,3.7,on Windows, +func,PyUnicode_DecodeFSDefault,3.2,, +func,PyUnicode_DecodeFSDefaultAndSize,3.2,, +func,PyUnicode_DecodeLatin1,3.2,, +func,PyUnicode_DecodeLocale,3.7,, +func,PyUnicode_DecodeLocaleAndSize,3.7,, +func,PyUnicode_DecodeMBCS,3.7,on Windows, +func,PyUnicode_DecodeMBCSStateful,3.7,on Windows, +func,PyUnicode_DecodeRawUnicodeEscape,3.2,, +func,PyUnicode_DecodeUTF16,3.2,, +func,PyUnicode_DecodeUTF16Stateful,3.2,, +func,PyUnicode_DecodeUTF32,3.2,, +func,PyUnicode_DecodeUTF32Stateful,3.2,, +func,PyUnicode_DecodeUTF7,3.2,, +func,PyUnicode_DecodeUTF7Stateful,3.2,, +func,PyUnicode_DecodeUTF8,3.2,, +func,PyUnicode_DecodeUTF8Stateful,3.2,, +func,PyUnicode_DecodeUnicodeEscape,3.2,, +func,PyUnicode_EncodeCodePage,3.7,on Windows, +func,PyUnicode_EncodeFSDefault,3.2,, +func,PyUnicode_EncodeLocale,3.7,, +func,PyUnicode_EqualToUTF8,3.13,, +func,PyUnicode_EqualToUTF8AndSize,3.13,, +func,PyUnicode_FSConverter,3.2,, +func,PyUnicode_FSDecoder,3.2,, +func,PyUnicode_Find,3.2,, +func,PyUnicode_FindChar,3.7,, +func,PyUnicode_Format,3.2,, +func,PyUnicode_FromEncodedObject,3.2,, +func,PyUnicode_FromFormat,3.2,, +func,PyUnicode_FromFormatV,3.2,, +func,PyUnicode_FromObject,3.2,, +func,PyUnicode_FromOrdinal,3.2,, +func,PyUnicode_FromString,3.2,, +func,PyUnicode_FromStringAndSize,3.2,, +func,PyUnicode_FromWideChar,3.2,, +func,PyUnicode_GetDefaultEncoding,3.2,, +func,PyUnicode_GetLength,3.7,, +func,PyUnicode_InternFromString,3.2,, +func,PyUnicode_InternInPlace,3.2,, +func,PyUnicode_IsIdentifier,3.2,, +func,PyUnicode_Join,3.2,, +func,PyUnicode_Partition,3.2,, +func,PyUnicode_RPartition,3.2,, +func,PyUnicode_RSplit,3.2,, +func,PyUnicode_ReadChar,3.7,, +func,PyUnicode_Replace,3.2,, +func,PyUnicode_Resize,3.2,, +func,PyUnicode_RichCompare,3.2,, +func,PyUnicode_Split,3.2,, +func,PyUnicode_Splitlines,3.2,, +func,PyUnicode_Substring,3.7,, +func,PyUnicode_Tailmatch,3.2,, +func,PyUnicode_Translate,3.2,, +data,PyUnicode_Type,3.2,, +func,PyUnicode_WriteChar,3.7,, type,PyVarObject,3.2,,members member,PyVarObject.ob_base,3.2,, member,PyVarObject.ob_size,3.2,, -function,PyVectorcall_Call,3.12,, -function,PyVectorcall_NARGS,3.12,, +func,PyVectorcall_Call,3.12,, +func,PyVectorcall_NARGS,3.12,, type,PyWeakReference,3.2,,opaque -function,PyWeakref_GetObject,3.2,, -function,PyWeakref_GetRef,3.13,, -function,PyWeakref_NewProxy,3.2,, -function,PyWeakref_NewRef,3.2,, -var,PyWrapperDescr_Type,3.2,, -function,PyWrapper_New,3.2,, -var,PyZip_Type,3.2,, -function,Py_AddPendingCall,3.2,, -function,Py_AtExit,3.2,, +func,PyWeakref_GetObject,3.2,, +func,PyWeakref_GetRef,3.13,, +func,PyWeakref_NewProxy,3.2,, +func,PyWeakref_NewRef,3.2,, +data,PyWrapperDescr_Type,3.2,, +func,PyWrapper_New,3.2,, +data,PyZip_Type,3.2,, +func,Py_AddPendingCall,3.2,, +func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, macro,Py_BLOCK_THREADS,3.2,, -function,Py_BuildValue,3.2,, -function,Py_BytesMain,3.8,, -function,Py_CompileString,3.2,, -function,Py_DecRef,3.2,, -function,Py_DecodeLocale,3.7,, +func,Py_BuildValue,3.2,, +func,Py_BytesMain,3.8,, +func,Py_CompileString,3.2,, +func,Py_DecRef,3.2,, +func,Py_DecodeLocale,3.7,, macro,Py_END_ALLOW_THREADS,3.2,, -function,Py_EncodeLocale,3.7,, -function,Py_EndInterpreter,3.2,, -function,Py_EnterRecursiveCall,3.9,, -function,Py_Exit,3.2,, -function,Py_FatalError,3.2,, -var,Py_FileSystemDefaultEncodeErrors,3.10,, -var,Py_FileSystemDefaultEncoding,3.2,, -function,Py_Finalize,3.2,, -function,Py_FinalizeEx,3.6,, -function,Py_GenericAlias,3.9,, -var,Py_GenericAliasType,3.9,, -function,Py_GetBuildInfo,3.2,, -function,Py_GetCompiler,3.2,, -function,Py_GetConstant,3.13,, -function,Py_GetConstantBorrowed,3.13,, -function,Py_GetCopyright,3.2,, -function,Py_GetExecPrefix,3.2,, -function,Py_GetPath,3.2,, -function,Py_GetPlatform,3.2,, -function,Py_GetPrefix,3.2,, -function,Py_GetProgramFullPath,3.2,, -function,Py_GetProgramName,3.2,, -function,Py_GetPythonHome,3.2,, -function,Py_GetRecursionLimit,3.2,, -function,Py_GetVersion,3.2,, -var,Py_HasFileSystemDefaultEncoding,3.2,, -function,Py_IncRef,3.2,, -function,Py_Initialize,3.2,, -function,Py_InitializeEx,3.2,, -function,Py_Is,3.10,, -function,Py_IsFalse,3.10,, -function,Py_IsFinalizing,3.13,, -function,Py_IsInitialized,3.2,, -function,Py_IsNone,3.10,, -function,Py_IsTrue,3.10,, -function,Py_LeaveRecursiveCall,3.9,, -function,Py_Main,3.2,, -function,Py_MakePendingCalls,3.2,, -function,Py_NewInterpreter,3.2,, -function,Py_NewRef,3.10,, -function,Py_ReprEnter,3.2,, -function,Py_ReprLeave,3.2,, -function,Py_SetProgramName,3.2,, -function,Py_SetPythonHome,3.2,, -function,Py_SetRecursionLimit,3.2,, -function,Py_TYPE,3.14,, +func,Py_EncodeLocale,3.7,, +func,Py_EndInterpreter,3.2,, +func,Py_EnterRecursiveCall,3.9,, +func,Py_Exit,3.2,, +func,Py_FatalError,3.2,, +data,Py_FileSystemDefaultEncodeErrors,3.10,, +data,Py_FileSystemDefaultEncoding,3.2,, +func,Py_Finalize,3.2,, +func,Py_FinalizeEx,3.6,, +func,Py_GenericAlias,3.9,, +data,Py_GenericAliasType,3.9,, +func,Py_GetBuildInfo,3.2,, +func,Py_GetCompiler,3.2,, +func,Py_GetConstant,3.13,, +func,Py_GetConstantBorrowed,3.13,, +func,Py_GetCopyright,3.2,, +func,Py_GetExecPrefix,3.2,, +func,Py_GetPath,3.2,, +func,Py_GetPlatform,3.2,, +func,Py_GetPrefix,3.2,, +func,Py_GetProgramFullPath,3.2,, +func,Py_GetProgramName,3.2,, +func,Py_GetPythonHome,3.2,, +func,Py_GetRecursionLimit,3.2,, +func,Py_GetVersion,3.2,, +data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_IncRef,3.2,, +func,Py_Initialize,3.2,, +func,Py_InitializeEx,3.2,, +func,Py_Is,3.10,, +func,Py_IsFalse,3.10,, +func,Py_IsFinalizing,3.13,, +func,Py_IsInitialized,3.2,, +func,Py_IsNone,3.10,, +func,Py_IsTrue,3.10,, +func,Py_LeaveRecursiveCall,3.9,, +func,Py_Main,3.2,, +func,Py_MakePendingCalls,3.2,, +func,Py_NewInterpreter,3.2,, +func,Py_NewRef,3.10,, +func,Py_ReprEnter,3.2,, +func,Py_ReprLeave,3.2,, +func,Py_SetProgramName,3.2,, +func,Py_SetPythonHome,3.2,, +func,Py_SetRecursionLimit,3.2,, +func,Py_TYPE,3.14,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, -var,Py_UTF8Mode,3.8,, -function,Py_VaBuildValue,3.2,, -var,Py_Version,3.11,, -function,Py_XNewRef,3.10,, +data,Py_UTF8Mode,3.8,, +func,Py_VaBuildValue,3.2,, +data,Py_Version,3.11,, +func,Py_XNewRef,3.10,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, type,Py_ssize_t,3.2,, diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst new file mode 100644 index 00000000000000..cfb30dd09aef6f --- /dev/null +++ b/Doc/deprecations/index.rst @@ -0,0 +1,10 @@ +Deprecations +============ + +.. include:: pending-removal-in-3.14.rst + +.. include:: pending-removal-in-3.15.rst + +.. include:: pending-removal-in-3.16.rst + +.. include:: pending-removal-in-future.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst new file mode 100644 index 00000000000000..03c785bb4b69ac --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -0,0 +1,52 @@ +Pending Removal in Python 3.13 +------------------------------ + +Modules (see :pep:`594`): + +* :mod:`!aifc` +* :mod:`!audioop` +* :mod:`!cgi` +* :mod:`!cgitb` +* :mod:`!chunk` +* :mod:`!crypt` +* :mod:`!imghdr` +* :mod:`!mailcap` +* :mod:`!msilib` +* :mod:`!nis` +* :mod:`!nntplib` +* :mod:`!ossaudiodev` +* :mod:`!pipes` +* :mod:`!sndhdr` +* :mod:`!spwd` +* :mod:`!sunau` +* :mod:`!telnetlib` +* :mod:`!uu` +* :mod:`!xdrlib` + +Other modules: + +* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) + +APIs: + +* :class:`!configparser.LegacyInterpolation` (:gh:`90765`) +* ``locale.resetlocale()`` (:gh:`90817`) +* :meth:`!turtle.RawTurtle.settiltangle` (:gh:`50096`) +* :func:`!unittest.findTestCases` (:gh:`50096`) +* :func:`!unittest.getTestCaseNames` (:gh:`50096`) +* :func:`!unittest.makeSuite` (:gh:`50096`) +* :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) +* :class:`!webbrowser.MacOSX` (:gh:`86421`) +* :class:`classmethod` descriptor chaining (:gh:`89519`) +* :mod:`importlib.resources` deprecated methods: + + * ``contents()`` + * ``is_resource()`` + * ``open_binary()`` + * ``open_text()`` + * ``path()`` + * ``read_binary()`` + * ``read_text()`` + + Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy + `_ (:gh:`106531`) diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst new file mode 100644 index 00000000000000..6c831ae366ced5 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -0,0 +1,118 @@ +Pending Removal in Python 3.14 +------------------------------ + +* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction` are deprecated + and will be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`92248`.) + +* :mod:`ast`: The following features have been deprecated in documentation + since Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at + runtime when they are accessed or used, and will be removed in Python 3.14: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) + +* :mod:`asyncio`: + + * The child watcher classes :class:`!asyncio.MultiLoopChildWatcher`, + :class:`!asyncio.FastChildWatcher`, :class:`!asyncio.AbstractChildWatcher` + and :class:`!asyncio.SafeChildWatcher` are deprecated and + will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + + * :func:`!asyncio.set_child_watcher`, :func:`!asyncio.get_child_watcher`, + :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` and + :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated + and will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + + * The :meth:`~asyncio.get_event_loop` method of the + default event loop policy now emits a :exc:`DeprecationWarning` if there + is no current event loop set and it decides to create one. + (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) + +* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. + Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. + For use in typing, prefer a union, like ``bytes | bytearray``, + or :class:`collections.abc.Buffer`. + (Contributed by Shantanu Jain in :gh:`91896`.) + +* :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. + (Contributed by Alan Williams in :gh:`72346`.) + +* :mod:`importlib`: ``__package__`` and ``__cached__`` will cease to be set or + taken into consideration by the import system (:gh:`97879`). + +* :mod:`importlib.abc` deprecated classes: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + +* :mod:`itertools` had undocumented, inefficient, historically buggy, + and inconsistent support for copy, deepcopy, and pickle operations. + This will be removed in 3.14 for a significant reduction in code + volume and maintenance burden. + (Contributed by Raymond Hettinger in :gh:`101588`.) + +* :mod:`multiprocessing`: The default start method will change to a safer one on + Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently + the default (:gh:`84559`). Adding a runtime warning about this was deemed too + disruptive as the majority of code is not expected to care. Use the + :func:`~multiprocessing.get_context` or + :func:`~multiprocessing.set_start_method` APIs to explicitly specify when + your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. + +* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and + :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is + deprecated. + +* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` + now raise :exc:`DeprecationWarning`; + use :func:`importlib.util.find_spec` instead. + (Contributed by Nikita Sobolev in :gh:`97850`.) + +* :mod:`pty`: + + * ``master_open()``: use :func:`pty.openpty`. + * ``slave_open()``: use :func:`pty.openpty`. + +* :mod:`sqlite3`: + + * :data:`!version` and :data:`!version_info`. + + * :meth:`~sqlite3.Cursor.execute` and :meth:`~sqlite3.Cursor.executemany` + if :ref:`named placeholders ` are used and + *parameters* is a sequence instead of a :class:`dict`. + + * date and datetime adapter, date and timestamp converter: + see the :mod:`sqlite3` documentation for suggested replacement recipes. + +* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + deprecated in :pep:`626` + since 3.10 and was planned to be removed in 3.12, + but it only got a proper :exc:`DeprecationWarning` in 3.12. + May be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`101866`.) + +* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, + now causes a :exc:`DeprecationWarning` to be emitted when it is used. + +* :mod:`urllib`: + :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a + public API. + (Contributed by Gregory P. Smith in :gh:`88168`.) diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst new file mode 100644 index 00000000000000..85eb634a7c5860 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -0,0 +1,57 @@ +Pending Removal in Python 3.15 +------------------------------ + +* :class:`http.server.CGIHTTPRequestHandler` will be removed along with its + related ``--cgi`` flag to ``python -m http.server``. It was obsolete and + rarely used. No direct replacement exists. *Anything* is better than CGI + to interface a web server with a request handler. + +* :class:`locale`: :func:`locale.getdefaultlocale` was deprecated in Python 3.11 + and originally planned for removal in Python 3.13 (:gh:`90817`), + but removal has been postponed to Python 3.15. + Use :func:`locale.setlocale()`, :func:`locale.getencoding()` and + :func:`locale.getlocale()` instead. + (Contributed by Hugo van Kemenade in :gh:`111187`.) + +* :mod:`pathlib`: + :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for + removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved + paths on Windows. + +* :mod:`platform`: + :func:`~platform.java_ver` is deprecated and will be removed in 3.15. + It was largely untested, had a confusing API, + and was only useful for Jython support. + (Contributed by Nikita Sobolev in :gh:`116349`.) + +* :mod:`threading`: + Passing any arguments to :func:`threading.RLock` is now deprecated. + C version allows any numbers of args and kwargs, + but they are just ignored. Python version does not allow any arguments. + All arguments will be removed from :func:`threading.RLock` in Python 3.15. + (Contributed by Nikita Sobolev in :gh:`102029`.) + +* :class:`typing.NamedTuple`: + + * The undocumented keyword argument syntax for creating :class:`!NamedTuple` classes + (``NT = NamedTuple("NT", x=int)``) is deprecated, and will be disallowed in + 3.15. Use the class-based syntax or the functional syntax instead. + + * When using the functional syntax to create a :class:`!NamedTuple` class, failing to + pass a value to the *fields* parameter (``NT = NamedTuple("NT")``) is + deprecated. Passing ``None`` to the *fields* parameter + (``NT = NamedTuple("NT", None)``) is also deprecated. Both will be + disallowed in Python 3.15. To create a :class:`!NamedTuple` class with 0 fields, use + ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", [])``. + +* :class:`typing.TypedDict`: When using the functional syntax to create a + :class:`!TypedDict` class, failing to pass a value to the *fields* parameter (``TD = + TypedDict("TD")``) is deprecated. Passing ``None`` to the *fields* parameter + (``TD = TypedDict("TD", None)``) is also deprecated. Both will be disallowed + in Python 3.15. To create a :class:`!TypedDict` class with 0 fields, use ``class + TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. + +* :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` + methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. + They will be removed in Python 3.15. + (Contributed by Victor Stinner in :gh:`105096`.) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst new file mode 100644 index 00000000000000..10cb5e424a623b --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -0,0 +1,10 @@ +Pending Removal in Python 3.16 +------------------------------ + +* :mod:`array`: + :class:`array.array` ``'u'`` type (:c:type:`wchar_t`): + use the ``'w'`` type instead (``Py_UCS4``). + +* :mod:`symtable`: + Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest. + (Contributed by Bénédikt Tran in :gh:`119698`.) diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst new file mode 100644 index 00000000000000..7f10d9a98257f9 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -0,0 +1,153 @@ +Pending Removal in Future Versions +---------------------------------- + +The following APIs will be removed in the future, +although there is currently no date scheduled for their removal. + +* :mod:`argparse`: Nesting argument groups and nesting mutually exclusive + groups are deprecated. + +* :mod:`array`'s ``'u'`` format code (:gh:`57281`) + +* :mod:`builtins`: + + * ``~bool``, bitwise inversion on bool. + * ``bool(NotImplemented)``. + * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` + signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, + the single argument signature. + * Currently Python accepts numeric literals immediately followed by keywords, + for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing and + ambiguous expressions like ``[0x1for x in y]`` (which can be interpreted as + ``[0x1 for x in y]`` or ``[0x1f or x in y]``). A syntax warning is raised + if the numeric literal is immediately followed by one of keywords + :keyword:`and`, :keyword:`else`, :keyword:`for`, :keyword:`if`, + :keyword:`in`, :keyword:`is` and :keyword:`or`. In a future release it + will be changed to a syntax error. (:gh:`87999`) + * Support for ``__index__()`` and ``__int__()`` method returning non-int type: + these methods will be required to return an instance of a strict subclass of + :class:`int`. + * Support for ``__float__()`` method returning a strict subclass of + :class:`float`: these methods will be required to return an instance of + :class:`float`. + * Support for ``__complex__()`` method returning a strict subclass of + :class:`complex`: these methods will be required to return an instance of + :class:`complex`. + * Delegation of ``int()`` to ``__trunc__()`` method. + * Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; it should only be passed + as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) + +* :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are + deprecated and replaced by :data:`calendar.JANUARY` and + :data:`calendar.FEBRUARY`. + (Contributed by Prince Roshan in :gh:`103636`.) + +* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method + instead. + +* :mod:`datetime`: + + * :meth:`~datetime.datetime.utcnow`: + use ``datetime.datetime.now(tz=datetime.UTC)``. + * :meth:`~datetime.datetime.utcfromtimestamp`: + use ``datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)``. + +* :mod:`gettext`: Plural value must be an integer. + +* :mod:`importlib`: + + * ``load_module()`` method: use ``exec_module()`` instead. + * :func:`~importlib.util.cache_from_source` *debug_override* parameter is + deprecated: use the *optimization* parameter instead. + +* :mod:`importlib.metadata`: + + * ``EntryPoints`` tuple interface. + * Implicit ``None`` on return values. + +* :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use + BytesIO and binary mode instead. + +* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. + +* :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is + deprecated, use an exception instance. + +* :mod:`re`: More strict rules are now applied for numerical group references + and group names in regular expressions. Only sequence of ASCII digits is now + accepted as a numerical reference. The group name in bytes patterns and + replacement strings can now only contain ASCII letters and digits and + underscore. + (Contributed by Serhiy Storchaka in :gh:`91760`.) + +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + +* :mod:`shutil`: :func:`~shutil.rmtree`'s *onerror* parameter is deprecated in + Python 3.12; use the *onexc* parameter instead. + +* :mod:`ssl` options and protocols: + + * :class:`ssl.SSLContext` without protocol argument is deprecated. + * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and + :meth:`!selected_npn_protocol` are deprecated: use ALPN + instead. + * ``ssl.OP_NO_SSL*`` options + * ``ssl.OP_NO_TLS*`` options + * ``ssl.PROTOCOL_SSLv3`` + * ``ssl.PROTOCOL_TLS`` + * ``ssl.PROTOCOL_TLSv1`` + * ``ssl.PROTOCOL_TLSv1_1`` + * ``ssl.PROTOCOL_TLSv1_2`` + * ``ssl.TLSVersion.SSLv3`` + * ``ssl.TLSVersion.TLSv1`` + * ``ssl.TLSVersion.TLSv1_1`` + +* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and + ignored. + +* :mod:`threading` methods: + + * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. + * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. + * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: + use :attr:`threading.Thread.daemon` attribute. + * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: + use :attr:`threading.Thread.name` attribute. + * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. + * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. + +* :class:`typing.Text` (:gh:`92332`). + +* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value + that is not ``None`` from a test case. + +* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead + + * ``splitattr()`` + * ``splithost()`` + * ``splitnport()`` + * ``splitpasswd()`` + * ``splitport()`` + * ``splitquery()`` + * ``splittag()`` + * ``splittype()`` + * ``splituser()`` + * ``splitvalue()`` + * ``to_bytes()`` + +* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and + :class:`~urllib.request.FancyURLopener` style of invoking requests is + deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. + +* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial + writes. + +* :mod:`xml.etree.ElementTree`: Testing the truth value of an + :class:`~xml.etree.ElementTree.Element` is deprecated. In a future release it + will always return ``True``. Prefer explicit ``len(elem)`` or + ``elem is not None`` tests instead. + +* :meth:`zipimport.zipimporter.load_module` is deprecated: + use :meth:`~zipimport.zipimporter.exec_module` instead. diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index c8beb64e39bc1a..ebb6d5ed1288c6 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -70,7 +70,7 @@ operations. This means that as far as floating-point operations are concerned, Python behaves like many popular languages including C and Java. Many numbers that can be written easily in decimal notation cannot be expressed -exactly in binary floating-point. For example, after:: +exactly in binary floating point. For example, after:: >>> x = 1.2 @@ -87,7 +87,7 @@ which is exactly:: The typical precision of 53 bits provides Python floats with 15--16 decimal digits of accuracy. -For a fuller explanation, please see the :ref:`floating point arithmetic +For a fuller explanation, please see the :ref:`floating-point arithmetic ` chapter in the Python tutorial. diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index a2900952d7bef6..d8d75ca6f2ec96 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -718,12 +718,12 @@ is simple:: import random random.random() -This returns a random floating point number in the range [0, 1). +This returns a random floating-point number in the range [0, 1). There are also many other specialized generators in this module, such as: * ``randrange(a, b)`` chooses an integer in the range [a, b). -* ``uniform(a, b)`` chooses a floating point number in the range [a, b). +* ``uniform(a, b)`` chooses a floating-point number in the range [a, b). * ``normalvariate(mean, sdev)`` samples the normal (Gaussian) distribution. Some higher-level functions operate on sequences directly, such as: diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 0a88c5f6384f2b..3ac8cc1e281694 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -869,7 +869,7 @@ How do I convert a string to a number? -------------------------------------- For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') -== 144``. Similarly, :func:`float` converts to floating-point, +== 144``. Similarly, :func:`float` converts to a floating-point number, e.g. ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == @@ -1741,11 +1741,31 @@ but effective way to define class private variables. Any identifier of the form is textually replaced with ``_classname__spam``, where ``classname`` is the current class name with any leading underscores stripped. -This doesn't guarantee privacy: an outside user can still deliberately access -the "_classname__spam" attribute, and private values are visible in the object's -``__dict__``. Many Python programmers never bother to use private variable -names at all. +The identifier can be used unchanged within the class, but to access it outside +the class, the mangled name must be used: +.. code-block:: python + + class A: + def __one(self): + return 1 + def two(self): + return 2 * self.__one() + + class B(A): + def three(self): + return 3 * self._A__one() + + four = 4 * A()._A__one() + +In particular, this does not guarantee privacy since an outside user can still +deliberately access the private attribute; many Python programmers never bother +to use private variable names at all. + +.. seealso:: + + The :ref:`private name mangling specifications ` + for details and special cases. My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index b29488be39a0a3..67e981f9c57abe 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1366,11 +1366,15 @@ Using the non-data descriptor protocol, a pure Python version of def __call__(self, *args, **kwds): return self.f(*args, **kwds) + @property + def __annotations__(self): + return self.f.__annotations__ + The :func:`functools.update_wrapper` call adds a ``__wrapped__`` attribute that refers to the underlying function. Also it carries forward the attributes necessary to make the wrapper look like the wrapped -function: :attr:`~function.__name__`, :attr:`~function.__qualname__`, -:attr:`~function.__doc__`, and :attr:`~function.__annotations__`. +function, including :attr:`~function.__name__`, :attr:`~function.__qualname__`, +and :attr:`~function.__doc__`. .. testcode:: :hide: diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 1ba91b09516f9c..521810e2887f2f 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -188,7 +188,7 @@ Python's memory management C API provides functions in three different :ref:`allocation domains `: "raw", "mem", and "object". For thread-safety, the free-threaded build requires that only Python objects are allocated using the object domain, and that all Python object are -allocated using that domain. This differes from the prior Python versions, +allocated using that domain. This differs from the prior Python versions, where this was only a best practice and not a hard requirement. .. note:: diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 3ed2dd6251afe9..321ec0c0f73871 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -4022,7 +4022,7 @@ As you can see, this output isn't ideal. That's because the underlying code which writes to ``sys.stderr`` makes multiple writes, each of which results in a separate logged line (for example, the last three lines above). To get around this problem, you need to buffer things and only output log lines when newlines -are seen. Let's use a slghtly better implementation of ``LoggerWriter``: +are seen. Let's use a slightly better implementation of ``LoggerWriter``: .. code-block:: python diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 0367c83d9369d3..aa1341c8d4d4a8 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1466,7 +1466,7 @@ printed to standard error when the argument is used:: snake.py: warning: option '--legs' is deprecated Namespace(legs=4) -.. versionchanged:: 3.13 +.. versionadded:: 3.13 Action classes diff --git a/Doc/library/array.rst b/Doc/library/array.rst index d34a1888342e27..e0b1eb89cf6c05 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -9,7 +9,7 @@ -------------- This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating point numbers. Arrays are sequence +basic values: characters, integers, floating-point numbers. Arrays are sequence types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a :dfn:`type code`, which is a single character. The following type codes are @@ -263,7 +263,7 @@ The string representation is guaranteed to be able to be converted back to an array with the same type and value using :func:`eval`, so long as the :class:`~array.array` class has been imported using ``from array import array``. Variables ``inf`` and ``nan`` must also be defined if it contains -corresponding floating point values. +corresponding floating-point values. Examples:: array('l') diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index f7e8afa7000392..dd5dd5ca4e9e32 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -889,7 +889,7 @@ Statements .. class:: AnnAssign(target, annotation, value, simple) An assignment with a type annotation. ``target`` is a single node and can - be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. + be a :class:`Name`, an :class:`Attribute` or a :class:`Subscript`. ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` node. ``value`` is a single optional node. @@ -1981,7 +1981,7 @@ Function and class definitions YieldFrom(value) A ``yield`` or ``yield from`` expression. Because these are expressions, they - must be wrapped in a :class:`Expr` node if the value sent back is not used. + must be wrapped in an :class:`Expr` node if the value sent back is not used. .. doctest:: diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 1d79f78e8e1b67..70bdd154d6c406 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1262,6 +1262,9 @@ Executing code in thread or process pools The *executor* argument should be an :class:`concurrent.futures.Executor` instance. The default executor is used if *executor* is ``None``. + The default executor can be set by :meth:`loop.set_default_executor`, + otherwise, a :class:`concurrent.futures.ThreadPoolExecutor` will be + lazy-initialized and used by :func:`run_in_executor` if needed. Example:: diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 893ae5518f757d..9dce0731411940 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -120,20 +120,20 @@ Future Object a :exc:`CancelledError` exception. If the Future's result isn't yet available, this method raises - a :exc:`InvalidStateError` exception. + an :exc:`InvalidStateError` exception. .. method:: set_result(result) Mark the Future as *done* and set its result. - Raises a :exc:`InvalidStateError` error if the Future is + Raises an :exc:`InvalidStateError` error if the Future is already *done*. .. method:: set_exception(exception) Mark the Future as *done* and set an exception. - Raises a :exc:`InvalidStateError` error if the Future is + Raises an :exc:`InvalidStateError` error if the Future is already *done*. .. method:: done() diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index c5deac7e2748ae..abf1726b34f539 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1170,7 +1170,7 @@ Task Object a :exc:`CancelledError` exception. If the Task's result isn't yet available, this method raises - a :exc:`InvalidStateError` exception. + an :exc:`InvalidStateError` exception. .. method:: exception() diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 184f981c1021aa..5f83b3a2658da4 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -56,9 +56,13 @@ Additionally, there are **low-level** APIs for * :ref:`bridge ` callback-based libraries and code with async/await syntax. +.. include:: ../includes/wasm-notavail.rst + .. _asyncio-cli: -You can experiment with an ``asyncio`` concurrent context in the REPL: +.. rubric:: asyncio REPL + +You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: .. code-block:: pycon @@ -70,7 +74,14 @@ You can experiment with an ``asyncio`` concurrent context in the REPL: >>> await asyncio.sleep(10, result='hello') 'hello' -.. include:: ../includes/wasm-notavail.rst +.. audit-event:: cpython.run_stdin "" "" + +.. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) + Emits audit events. + +.. versionchanged:: 3.13 + Uses PyREPL if possible, in which case :envvar:`PYTHONSTARTUP` is + also executed. Emits audit events. .. We use the "rubric" directive here to avoid creating the "Reference" subsection in the TOC. diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index 125d62b174088a..ffebf4e40dd609 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -14,7 +14,7 @@ The :mod:`colorsys` module defines bidirectional conversions of color values between colors expressed in the RGB (Red Green Blue) color space used in computer monitors and three other coordinate systems: YIQ, HLS (Hue Lightness Saturation) and HSV (Hue Saturation Value). Coordinates in all of these color -spaces are floating point values. In the YIQ space, the Y coordinate is between +spaces are floating-point values. In the YIQ space, the Y coordinate is between 0 and 1, but the I and Q coordinates can be positive or negative. In all other spaces, the coordinates are all between 0 and 1. diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index e84fb513e45267..7aaad932c0104a 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -147,23 +147,28 @@ case-insensitive and stored in lowercase [1]_. It is possible to read several configurations into a single :class:`ConfigParser`, where the most recently added configuration has the highest priority. Any conflicting keys are taken from the more recent -configuration while the previously existing keys are retained. +configuration while the previously existing keys are retained. The example +below reads in an ``override.ini`` file, which will override any conflicting +keys from the ``example.ini`` file. + +.. code-block:: ini + + [DEFAULT] + ServerAliveInterval = -1 .. doctest:: - >>> another_config = configparser.ConfigParser() - >>> another_config.read('example.ini') - ['example.ini'] - >>> another_config['topsecret.server.example']['Port'] - '50022' - >>> another_config.read_string("[topsecret.server.example]\nPort=48484") - >>> another_config['topsecret.server.example']['Port'] - '48484' - >>> another_config.read_dict({"topsecret.server.example": {"Port": 21212}}) - >>> another_config['topsecret.server.example']['Port'] - '21212' - >>> another_config['topsecret.server.example']['ForwardX11'] - 'no' + >>> config_override = configparser.ConfigParser() + >>> config_override['DEFAULT'] = {'ServerAliveInterval': '-1'} + >>> with open('override.ini', 'w') as configfile: + ... config_override.write(configfile) + ... + >>> config_override = configparser.ConfigParser() + >>> config_override.read(['example.ini', 'override.ini']) + ['example.ini', 'override.ini'] + >>> print(config_override.get('DEFAULT', 'ServerAliveInterval')) + -1 + This behaviour is equivalent to a :meth:`ConfigParser.read` call with several files passed to the *filenames* parameter. @@ -984,6 +989,31 @@ ConfigParser Objects converter gets its own corresponding :meth:`!get*()` method on the parser object and section proxies. + It is possible to read several configurations into a single + :class:`ConfigParser`, where the most recently added configuration has the + highest priority. Any conflicting keys are taken from the more recent + configuration while the previously existing keys are retained. The example + below reads in an ``override.ini`` file, which will override any conflicting + keys from the ``example.ini`` file. + + .. code-block:: ini + + [DEFAULT] + ServerAliveInterval = -1 + + .. doctest:: + + >>> config_override = configparser.ConfigParser() + >>> config_override['DEFAULT'] = {'ServerAliveInterval': '-1'} + >>> with open('override.ini', 'w') as configfile: + ... config_override.write(configfile) + ... + >>> config_override = configparser.ConfigParser() + >>> config_override.read(['example.ini', 'override.ini']) + ['example.ini', 'override.ini'] + >>> print(config_override.get('DEFAULT', 'ServerAliveInterval')) + -1 + .. versionchanged:: 3.1 The default *dict_type* is :class:`collections.OrderedDict`. @@ -1153,7 +1183,7 @@ ConfigParser Objects .. method:: getfloat(section, option, *, raw=False, vars=None[, fallback]) A convenience method which coerces the *option* in the specified *section* - to a floating point number. See :meth:`get` for explanation of *raw*, + to a floating-point number. See :meth:`get` for explanation of *raw*, *vars* and *fallback*. diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 27cf99446e5980..f5b349441bcfee 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -322,7 +322,7 @@ Functions and classes provided: .. versionchanged:: 3.12 ``suppress`` now supports suppressing exceptions raised as - part of an :exc:`BaseExceptionGroup`. + part of a :exc:`BaseExceptionGroup`. .. function:: redirect_stdout(new_target) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index e3d74d7dc0d91c..9e69b3dc51a1ac 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -107,7 +107,7 @@ Functions are accessed as attributes of dll objects:: Note that win32 system dlls like ``kernel32`` and ``user32`` often export ANSI as well as UNICODE versions of a function. The UNICODE version is exported with -an ``W`` appended to the name, while the ANSI version is exported with an ``A`` +a ``W`` appended to the name, while the ANSI version is exported with an ``A`` appended to the name. The win32 ``GetModuleHandle`` function, which returns a *module handle* for a given module name, has the following C prototype, and a macro is used to expose one of them as ``GetModuleHandle`` depending on whether diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index b6d8e6e6df07fa..558900dd3b9a4d 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -48,7 +48,7 @@ Aware and Naive Objects ----------------------- Date and time objects may be categorized as "aware" or "naive" depending on -whether or not they include timezone information. +whether or not they include time zone information. With sufficient knowledge of applicable algorithmic and political time adjustments, such as time zone and daylight saving time information, @@ -58,7 +58,7 @@ interpretation. [#]_ A **naive** object does not contain enough information to unambiguously locate itself relative to other date/time objects. Whether a naive object represents -Coordinated Universal Time (UTC), local time, or time in some other timezone is +Coordinated Universal Time (UTC), local time, or time in some other time zone is purely up to the program, just like it is up to the program whether a particular number represents metres, miles, or mass. Naive objects are easy to understand and to work with, at the cost of ignoring some aspects of reality. @@ -70,9 +70,9 @@ These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`!datetime` module. The :class:`timezone` class can -represent simple timezones with fixed offsets from UTC, such as UTC itself or -North American EST and EDT timezones. Supporting timezones at deeper levels of +supplied by the :mod:`!datetime` module. The :class:`!timezone` class can +represent simple time zones with fixed offsets from UTC, such as UTC itself or +North American EST and EDT time zones. Supporting time zones at deeper levels of detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. @@ -95,7 +95,7 @@ The :mod:`!datetime` module exports the following constants: .. attribute:: UTC - Alias for the UTC timezone singleton :attr:`datetime.timezone.utc`. + Alias for the UTC time zone singleton :attr:`datetime.timezone.utc`. .. versionadded:: 3.11 @@ -869,7 +869,7 @@ Other constructors, all class methods: .. classmethod:: datetime.today() - Return the current local datetime, with :attr:`.tzinfo` ``None``. + Return the current local date and time, with :attr:`.tzinfo` ``None``. Equivalent to:: @@ -1070,7 +1070,7 @@ Other constructors, all class methods: Return a :class:`.datetime` corresponding to *date_string*, parsed according to *format*. - If *format* does not contain microseconds or timezone information, this is equivalent to:: + If *format* does not contain microseconds or time zone information, this is equivalent to:: datetime(*(time.strptime(date_string, format)[0:6])) @@ -1311,22 +1311,22 @@ Instance methods: If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self* - is naive, it is presumed to represent time in the system timezone. + is naive, it is presumed to represent time in the system time zone. If called without arguments (or with ``tz=None``) the system local - timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted + time zone is assumed for the target time zone. The ``.tzinfo`` attribute of the converted datetime instance will be set to an instance of :class:`timezone` with the zone name and offset obtained from the OS. If ``self.tzinfo`` is *tz*, ``self.astimezone(tz)`` is equal to *self*: no adjustment of date or time data is performed. Else the result is local - time in the timezone *tz*, representing the same UTC time as *self*: after + time in the time zone *tz*, representing the same UTC time as *self*: after ``astz = dt.astimezone(tz)``, ``astz - astz.utcoffset()`` will have the same date and time data as ``dt - dt.utcoffset()``. - If you merely want to attach a time zone object *tz* to a datetime *dt* without + If you merely want to attach a :class:`timezone` object *tz* to a datetime *dt* without adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you - merely want to remove the time zone object from an aware datetime *dt* without + merely want to remove the :class:`!timezone` object from an aware datetime *dt* without conversion of date and time data, use ``dt.replace(tzinfo=None)``. Note that the default :meth:`tzinfo.fromutc` method can be overridden in a @@ -1336,7 +1336,7 @@ Instance methods: def astimezone(self, tz): if self.tzinfo is tz: return self - # Convert self to UTC, and attach the new time zone object. + # Convert self to UTC, and attach the new timezone object. utc = (self - self.utcoffset()).replace(tzinfo=tz) # Convert from UTC to tz's local time. return tz.fromutc(utc) @@ -1450,7 +1450,7 @@ Instance methods: There is no method to obtain the POSIX timestamp directly from a naive :class:`.datetime` instance representing UTC time. If your - application uses this convention and your system timezone is not + application uses this convention and your system time zone is not set to UTC, you can obtain the POSIX timestamp by supplying ``tzinfo=timezone.utc``:: @@ -2021,7 +2021,7 @@ Examples of working with a :class:`.time` object:: supply implementations of the standard :class:`tzinfo` methods needed by the :class:`.datetime` methods you use. The :mod:`!datetime` module provides :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can - represent timezones with fixed offset from UTC such as UTC itself or North + represent time zones with fixed offset from UTC such as UTC itself or North American EST and EDT. Special requirement for pickling: A :class:`tzinfo` subclass must have an @@ -2146,7 +2146,7 @@ When a :class:`.datetime` object is passed in response to a :class:`.datetime` method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can rely on this, unless user code calls :class:`tzinfo` methods directly. The intent is that the :class:`tzinfo` methods interpret *dt* as being in local -time, and not need worry about objects in other timezones. +time, and not need worry about objects in other time zones. There is one more :class:`tzinfo` method that a subclass may wish to override: @@ -2263,12 +2263,12 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). :mod:`zoneinfo` The :mod:`!datetime` module has a basic :class:`timezone` class (for handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` - attribute (a UTC timezone instance). + attribute (a UTC :class:`!timezone` instance). - ``zoneinfo`` brings the *IANA timezone database* (also known as the Olson + ``zoneinfo`` brings the *IANA time zone database* (also known as the Olson database) to Python, and its usage is recommended. - `IANA timezone database `_ + `IANA time zone database `_ The Time Zone Database (often called tz, tzdata or zoneinfo) contains code and data that represent the history of local time for many representative locations around the globe. It is updated periodically to reflect changes @@ -2282,10 +2282,10 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). ------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each -instance of which represents a timezone defined by a fixed offset from +instance of which represents a time zone defined by a fixed offset from UTC. -Objects of this class cannot be used to represent timezone information in the +Objects of this class cannot be used to represent time zone information in the locations where different offsets are used in different days of the year or where historical changes have been made to civil time. @@ -2346,7 +2346,7 @@ Class attributes: .. attribute:: timezone.utc - The UTC timezone, ``timezone(timedelta(0))``. + The UTC time zone, ``timezone(timedelta(0))``. .. index:: @@ -2555,7 +2555,7 @@ Using ``datetime.strptime(date_string, format)`` is equivalent to:: datetime(*(time.strptime(date_string, format)[0:6])) -except when the format includes sub-second components or timezone offset +except when the format includes sub-second components or time zone offset information, which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index db323802a6f68c..916f17cadfaa7e 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1,4 +1,4 @@ -:mod:`!decimal` --- Decimal fixed point and floating point arithmetic +:mod:`!decimal` --- Decimal fixed-point and floating-point arithmetic ===================================================================== .. module:: decimal @@ -31,7 +31,7 @@ -------------- The :mod:`decimal` module provides support for fast correctly rounded -decimal floating point arithmetic. It offers several advantages over the +decimal floating-point arithmetic. It offers several advantages over the :class:`float` datatype: * Decimal "is based on a floating-point model which was designed with people @@ -207,7 +207,7 @@ a decimal raises :class:`InvalidOperation`:: .. versionchanged:: 3.3 Decimals interact well with much of the rest of Python. Here is a small decimal -floating point flying circus: +floating-point flying circus: .. doctest:: :options: +NORMALIZE_WHITESPACE @@ -373,7 +373,7 @@ Decimal objects digits, and an integer exponent. For example, ``Decimal((0, (1, 4, 1, 4), -3))`` returns ``Decimal('1.414')``. - If *value* is a :class:`float`, the binary floating point value is losslessly + If *value* is a :class:`float`, the binary floating-point value is losslessly converted to its exact decimal equivalent. This conversion can often require 53 or more digits of precision. For example, ``Decimal(float('1.1'))`` converts to @@ -403,7 +403,7 @@ Decimal objects Underscores are allowed for grouping, as with integral and floating-point literals in code. - Decimal floating point objects share many properties with the other built-in + Decimal floating-point objects share many properties with the other built-in numeric types such as :class:`float` and :class:`int`. All of the usual math operations and special methods apply. Likewise, decimal objects can be copied, pickled, printed, used as dictionary keys, used as set elements, @@ -445,7 +445,7 @@ Decimal objects Mixed-type comparisons between :class:`Decimal` instances and other numeric types are now fully supported. - In addition to the standard numeric properties, decimal floating point + In addition to the standard numeric properties, decimal floating-point objects also have a number of specialized methods: @@ -1741,7 +1741,7 @@ The following table summarizes the hierarchy of signals:: .. _decimal-notes: -Floating Point Notes +Floating-Point Notes -------------------- @@ -1754,7 +1754,7 @@ can still incur round-off error when non-zero digits exceed the fixed precision. The effects of round-off error can be amplified by the addition or subtraction of nearly offsetting quantities resulting in loss of significance. Knuth -provides two instructive examples where rounded floating point arithmetic with +provides two instructive examples where rounded floating-point arithmetic with insufficient precision causes the breakdown of the associative and distributive properties of addition: @@ -1844,7 +1844,7 @@ treated as equal and their sign is informational. In addition to the two signed zeros which are distinct yet equal, there are various representations of zero with differing precisions yet equivalent in value. This takes a bit of getting used to. For an eye accustomed to -normalized floating point representations, it is not immediately obvious that +normalized floating-point representations, it is not immediately obvious that the following calculation returns a value equal to zero: >>> 1 / Decimal('Infinity') @@ -2171,7 +2171,7 @@ value unchanged: Q. Is there a way to convert a regular float to a :class:`Decimal`? -A. Yes, any binary floating point number can be exactly expressed as a +A. Yes, any binary floating-point number can be exactly expressed as a Decimal though an exact conversion may take more precision than intuition would suggest: @@ -2225,7 +2225,7 @@ Q. Is the CPython implementation fast for large numbers? A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for -arbitrary precision correctly rounded decimal floating point arithmetic [#]_. +arbitrary precision correctly rounded decimal floating-point arithmetic [#]_. ``libmpdec`` uses `Karatsuba multiplication `_ for medium-sized numbers and the `Number Theoretic Transform diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index e932b865a825a0..26b13c87181000 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1109,15 +1109,6 @@ iterations of the loop. empty dictionary pre-sized to hold *count* items. -.. opcode:: BUILD_CONST_KEY_MAP (count) - - The version of :opcode:`BUILD_MAP` specialized for constant keys. Pops the - top element on the stack which contains a tuple of keys, then starting from - ``STACK[-2]``, pops *count* values to form values in the built dictionary. - - .. versionadded:: 3.6 - - .. opcode:: BUILD_STRING (count) Concatenates *count* strings from the stack and pushes the resulting string @@ -1727,7 +1718,7 @@ iterations of the loop. | ``INTRINSIC_STOPITERATION_ERROR`` | Extracts the return value from a | | | ``StopIteration`` exception. | +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an aync generator value | + | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an async generator value | +-----------------------------------+-----------------------------------+ | ``INTRINSIC_UNARY_POSITIVE`` | Performs the unary ``+`` | | | operation | diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index c4c322a82e1f44..6e27a6e224a733 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -7,6 +7,7 @@ :synopsis: The base class representing email messages in a fashion backward compatible with Python 3.2 :noindex: + :no-index: The :class:`Message` class is very similar to the diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 43e5b25df01f79..611549604fda15 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -158,7 +158,7 @@ of the new API. Fri, 09 Nov 2001 01:08:47 -0000 - Optional *timeval* if given is a floating point time value as accepted by + Optional *timeval* if given is a floating-point time value as accepted by :func:`time.gmtime` and :func:`time.localtime`, otherwise the current time is used. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 7910b306f143d7..b5ba86f1b19223 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -412,8 +412,8 @@ The following exceptions are the exceptions that are usually raised. represented. This cannot occur for integers (which would rather raise :exc:`MemoryError` than give up). However, for historical reasons, OverflowError is sometimes raised for integers that are outside a required - range. Because of the lack of standardization of floating point exception - handling in C, most floating point operations are not checked. + range. Because of the lack of standardization of floating-point exception + handling in C, most floating-point operations are not checked. .. exception:: PythonFinalizationError diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index 2a0670ffcc2cbc..282d0e0d8db5cf 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -70,7 +70,7 @@ The :mod:`filecmp` module defines the following functions: The :class:`dircmp` class ------------------------- -.. class:: dircmp(a, b, ignore=None, hide=None, shallow=True) +.. class:: dircmp(a, b, ignore=None, hide=None, *, shallow=True) Construct a new directory comparison object, to compare the directories *a* and *b*. *ignore* is a list of names to ignore, and defaults to diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 552d6030b1ceda..2ee154952549ac 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -17,25 +17,30 @@ The :mod:`fractions` module provides support for rational number arithmetic. A Fraction instance can be constructed from a pair of integers, from another rational number, or from a string. +.. index:: single: as_integer_ratio() + .. class:: Fraction(numerator=0, denominator=1) - Fraction(other_fraction) - Fraction(float) - Fraction(decimal) + Fraction(number) Fraction(string) The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance with value ``numerator/denominator``. If *denominator* is ``0``, it - raises a :exc:`ZeroDivisionError`. The second version requires that - *other_fraction* is an instance of :class:`numbers.Rational` and returns a - :class:`Fraction` instance with the same value. The next two versions accept - either a :class:`float` or a :class:`decimal.Decimal` instance, and return a - :class:`Fraction` instance with exactly the same value. Note that due to the - usual issues with binary floating-point (see :ref:`tut-fp-issues`), the + raises a :exc:`ZeroDivisionError`. + + The second version requires that *number* is an instance of + :class:`numbers.Rational` or has the :meth:`!as_integer_ratio` method + (this includes :class:`float` and :class:`decimal.Decimal`). + It returns a :class:`Fraction` instance with exactly the same value. + Assumed, that the :meth:`!as_integer_ratio` method returns a pair + of coprime integers and last one is positive. + Note that due to the + usual issues with binary point (see :ref:`tut-fp-issues`), the argument to ``Fraction(1.1)`` is not exactly equal to 11/10, and so ``Fraction(1.1)`` does *not* return ``Fraction(11, 10)`` as one might expect. (But see the documentation for the :meth:`limit_denominator` method below.) - The last version of the constructor expects a string or unicode instance. + + The last version of the constructor expects a string. The usual form for this instance is:: [sign] numerator ['/' denominator] @@ -87,7 +92,7 @@ another rational number, or from a string. .. versionchanged:: 3.9 The :func:`math.gcd` function is now used to normalize the *numerator* - and *denominator*. :func:`math.gcd` always return a :class:`int` type. + and *denominator*. :func:`math.gcd` always returns an :class:`int` type. Previously, the GCD type depended on *numerator* and *denominator*. .. versionchanged:: 3.11 @@ -110,6 +115,10 @@ another rational number, or from a string. Formatting of :class:`Fraction` instances without a presentation type now supports fill, alignment, sign handling, minimum width and grouping. + .. versionchanged:: 3.14 + The :class:`Fraction` constructor now accepts any objects with the + :meth:`!as_integer_ratio` method. + .. attribute:: numerator Numerator of the Fraction in lowest term. diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 8c39dc00f5db02..bb15322067245e 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -243,7 +243,7 @@ FTP objects Retrieve a file in binary transfer mode. :param str cmd: - An appropriate ``STOR`` command: :samp:`"STOR {filename}"`. + An appropriate ``RETR`` command: :samp:`"RETR {filename}"`. :param callback: A single parameter callable that is called diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 17348dd907bf67..2c649376efee70 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -57,7 +57,7 @@ are always available. They are listed here in alphabetical order. .. function:: abs(x) Return the absolute value of a number. The argument may be an - integer, a floating point number, or an object implementing + integer, a floating-point number, or an object implementing :meth:`~object.__abs__`. If the argument is a complex number, its magnitude is returned. @@ -438,6 +438,8 @@ are always available. They are listed here in alphabetical order. If one of arguments is a real number, only its real component is used in the above expressions. + See also :meth:`complex.from_number` which only accepts a single numeric argument. + If all arguments are omitted, returns ``0j``. The complex type is described in :ref:`typesnumeric`. @@ -542,7 +544,7 @@ are always available. They are listed here in alphabetical order. Take two (non-complex) numbers as arguments and return a pair of numbers consisting of their quotient and remainder when using integer division. With mixed operand types, the rules for binary arithmetic operators apply. For - integers, the result is the same as ``(a // b, a % b)``. For floating point + integers, the result is the same as ``(a // b, a % b)``. For floating-point numbers the result is ``(q, a % b)``, where *q* is usually ``math.floor(a / b)`` but may be 1 less than that. In any case ``q * b + a % b`` is very close to *a*, if ``a % b`` is non-zero it has the same sign as *b*, and ``0 @@ -738,7 +740,7 @@ are always available. They are listed here in alphabetical order. single: NaN single: Infinity - Return a floating point number constructed from a number or a string. + Return a floating-point number constructed from a number or a string. Examples: @@ -779,8 +781,8 @@ are always available. They are listed here in alphabetical order. Case is not significant, so, for example, "inf", "Inf", "INFINITY", and "iNfINity" are all acceptable spellings for positive infinity. - Otherwise, if the argument is an integer or a floating point number, a - floating point number with the same value (within Python's floating point + Otherwise, if the argument is an integer or a floating-point number, a + floating-point number with the same value (within Python's floating-point precision) is returned. If the argument is outside the range of a Python float, an :exc:`OverflowError` will be raised. @@ -788,6 +790,8 @@ are always available. They are listed here in alphabetical order. ``x.__float__()``. If :meth:`~object.__float__` is not defined then it falls back to :meth:`~object.__index__`. + See also :meth:`float.from_number` which only accepts a numeric argument. + If no argument is given, ``0.0`` is returned. The float type is described in :ref:`typesnumeric`. @@ -1006,7 +1010,7 @@ are always available. They are listed here in alphabetical order. If the argument defines :meth:`~object.__int__`, ``int(x)`` returns ``x.__int__()``. If the argument defines :meth:`~object.__index__`, it returns ``x.__index__()``. - For floating point numbers, this truncates towards zero. + For floating-point numbers, this truncates towards zero. If the argument is not a number or if *base* is given, then it must be a string, :class:`bytes`, or :class:`bytearray` instance representing an integer @@ -1505,7 +1509,7 @@ are always available. They are listed here in alphabetical order. (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, and :mod:`shutil`. - .. audit-event:: open file,mode,flags open + .. audit-event:: open path,mode,flags open The ``mode`` and ``flags`` arguments may have been modified or inferred from the original call. @@ -1924,7 +1928,7 @@ are always available. They are listed here in alphabetical order. For some use cases, there are good alternatives to :func:`sum`. The preferred, fast way to concatenate a sequence of strings is by calling - ``''.join(sequence)``. To add floating point values with extended precision, + ``''.join(sequence)``. To add floating-point values with extended precision, see :func:`math.fsum`\. To concatenate a series of iterables, consider using :func:`itertools.chain`. diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 3c80fa747d5f1f..9ed92dc6957370 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -378,7 +378,7 @@ provides three different variants: If the request was mapped to a file, it is opened. Any :exc:`OSError` exception in opening the requested file is mapped to a ``404``, - ``'File not found'`` error. If there was a ``'If-Modified-Since'`` + ``'File not found'`` error. If there was an ``'If-Modified-Since'`` header in the request, and the file was not modified after this time, a ``304``, ``'Not Modified'`` response is sent. Otherwise, the content type is guessed by calling the :meth:`guess_type` method, which in turn diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 5ea8044e1ec6ca..54995ddbfbca12 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -22,7 +22,7 @@ something like a data file that lives next to the ``__init__.py`` file of the package. The purpose of this class is to help abstract out the accessing of such data files so that it does not matter if - the package and its data file(s) are stored in a e.g. zip file + the package and its data file(s) are stored e.g. in a zip file versus on the file system. For any of methods of this class, a *resource* argument is diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 2ec15dd171c18a..1206a2d94d22a3 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -657,7 +657,7 @@ ABC hierarchy:: something like a data file that lives next to the ``__init__.py`` file of the package. The purpose of this class is to help abstract out the accessing of such data files so that it does not matter if - the package and its data file(s) are stored in a e.g. zip file + the package and its data file(s) are stored e.g. in a zip file versus on the file system. For any of methods of this class, a *resource* argument is diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 7838eeed2843c4..361f4054856d89 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -457,7 +457,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the - wrapped function is a :term:`asynchronous generator` function. + wrapped function is an :term:`asynchronous generator` function. .. versionchanged:: 3.13 Functions wrapped in :func:`functools.partialmethod` now return ``True`` @@ -938,7 +938,7 @@ function. .. attribute:: Parameter.kind.description - Describes a enum value of :attr:`Parameter.kind`. + Describes an enum value of :attr:`Parameter.kind`. .. versionadded:: 3.8 diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 1fdd00a69da1f3..553abf788b223a 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -337,7 +337,7 @@ loops that truncate the stream. yield n n += step - When counting with floating point numbers, better accuracy can sometimes be + When counting with floating-point numbers, better accuracy can sometimes be achieved by substituting multiplicative code such as: ``(start + step * i for i in count())``. diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 0a8cbd4f95f473..0246f99157024a 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -424,7 +424,7 @@ The :mod:`locale` module defines the following exception and functions: .. function:: format_string(format, val, grouping=False, monetary=False) Formats a number *val* according to the current :const:`LC_NUMERIC` setting. - The format follows the conventions of the ``%`` operator. For floating point + The format follows the conventions of the ``%`` operator. For floating-point values, the decimal point is modified if appropriate. If *grouping* is ``True``, also takes the grouping into account. @@ -455,7 +455,7 @@ The :mod:`locale` module defines the following exception and functions: .. function:: str(float) - Formats a floating point number using the same format as the built-in function + Formats a floating-point number using the same format as the built-in function ``str(float)``, but takes the decimal point into account. diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 40ea71cd342b47..abb32f9bf3457f 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -1387,7 +1387,7 @@ When an :class:`!MHMessage` instance is created based upon a .. method:: get_visible() - Return an :class:`Message` instance whose headers are the message's + Return a :class:`Message` instance whose headers are the message's visible headers and whose body is empty. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index f9ba4d554b0c22..9e4606df0f774e 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -42,8 +42,8 @@ supports a substantially wider range of objects than marshal. Not all Python object types are supported; in general, only objects whose value is independent from a particular invocation of Python can be written and read by -this module. The following types are supported: booleans, integers, floating -point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, +this module. The following types are supported: booleans, integers, floating-point +numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, frozensets, dictionaries, and code objects (if *allow_code* is true), where it should be understood that tuples, lists, sets, frozensets and dictionaries are only supported as long as @@ -142,7 +142,7 @@ In addition, the following constants are defined: Indicates the format that the module uses. Version 0 is the historical format, version 1 shares interned strings and version 2 uses a binary format - for floating point numbers. + for floating-point numbers. Version 3 adds support for object instancing and recursion. The current version is 4. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 316144992d6832..dd2ba419b5bd12 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -123,7 +123,7 @@ Number-theoretic and representation functions .. function:: fsum(iterable) - Return an accurate floating point sum of values in the iterable. Avoids + Return an accurate floating-point sum of values in the iterable. Avoids loss of precision by tracking multiple intermediate partial sums. The algorithm's accuracy depends on IEEE-754 arithmetic guarantees and the @@ -133,7 +133,7 @@ Number-theoretic and representation functions least significant bit. For further discussion and two alternative approaches, see the `ASPN cookbook - recipes for accurate floating point summation + recipes for accurate floating-point summation `_\. @@ -304,7 +304,7 @@ Number-theoretic and representation functions If the result of the remainder operation is zero, that zero will have the same sign as *x*. - On platforms using IEEE 754 binary floating-point, the result of this + On platforms using IEEE 754 binary floating point, the result of this operation is always exactly representable: no rounding error is introduced. .. versionadded:: 3.7 diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 91e8c30f8607b3..8ad4850584a7e1 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -295,3 +295,13 @@ than one MIME-type database; it provides an interface similar to the one of the types, else to the list of non-standard types. .. versionadded:: 3.2 + + + .. method:: MimeTypes.add_type(type, ext, strict=True) + + Add a mapping from the MIME type *type* to the extension *ext*. When the + extension is already known, the new type will replace the old one. When the type + is already known the extension will be added to the list of known extensions. + + When *strict* is ``True`` (the default), the mapping will be added to the + official MIME types, otherwise to the non-standard ones. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 426291c5f0743d..f1f9d087edf7f9 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -254,6 +254,7 @@ processes: p.join() Queues are thread and process safe. + Any object put into a :mod:`~multiprocessing` queue will be serialized. **Pipes** @@ -281,6 +282,8 @@ processes: of corruption from processes using different ends of the pipe at the same time. + The :meth:`~Connection.send` method serializes the the object and + :meth:`~Connection.recv` re-creates the object. Synchronization between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -745,6 +748,11 @@ If you use :class:`JoinableQueue` then you **must** call semaphore used to count the number of unfinished tasks may eventually overflow, raising an exception. +One difference from other Python queue implementations, is that :mod:`multiprocessing` +queues serializes all objects that are put into them using :mod:`pickle`. +The object return by the get method is a re-created object that does not share memory +with the original object. + Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. @@ -811,6 +819,8 @@ For an example of the usage of queues for interprocess communication see used for receiving messages and ``conn2`` can only be used for sending messages. + The :meth:`~multiprocessing.Connection.send` method serializes the the object using + :mod:`pickle` and the :meth:`~multiprocessing.Connection.recv` re-creates the object. .. class:: Queue([maxsize]) @@ -1463,17 +1473,6 @@ object -- see :ref:`multiprocessing-managers`. On macOS, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with a timeout will emulate that function's behavior using a sleeping loop. -.. note:: - - If the SIGINT signal generated by :kbd:`Ctrl-C` arrives while the main thread is - blocked by a call to :meth:`BoundedSemaphore.acquire`, :meth:`Lock.acquire`, - :meth:`RLock.acquire`, :meth:`Semaphore.acquire`, :meth:`Condition.acquire` - or :meth:`Condition.wait` then the call will be immediately interrupted and - :exc:`KeyboardInterrupt` will be raised. - - This differs from the behaviour of :mod:`threading` where SIGINT will be - ignored while the equivalent blocking calls are in progress. - .. note:: Some of this package's functionality requires a functioning shared semaphore diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 3e96259f94d47b..74a49a8fb33666 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -1352,7 +1352,7 @@ The whole point of creating and populating an OptionParser is to call its the list of arguments to process (default: ``sys.argv[1:]``) ``values`` - an :class:`Values` object to store option arguments in (default: a + a :class:`Values` object to store option arguments in (default: a new instance of :class:`Values`) -- if you give an existing object, the option defaults will not be initialized on it diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index b582321515db56..ac24bf05c289b6 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -201,14 +201,14 @@ the :mod:`glob` module.) .. function:: getatime(path) - Return the time of last access of *path*. The return value is a floating point number giving + Return the time of last access of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or is inaccessible. .. function:: getmtime(path) - Return the time of last modification of *path*. The return value is a floating point number + Return the time of last modification of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or is inaccessible. @@ -389,7 +389,7 @@ the :mod:`glob` module.) that contains symbolic links. On Windows, it converts forward slashes to backward slashes. To normalize case, use :func:`normcase`. - .. note:: + .. note:: On POSIX systems, in accordance with `IEEE Std 1003.1 2013 Edition; 4.13 Pathname Resolution `_, if a pathname begins with exactly two slashes, the first component diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 2878d425310d75..b0ff81141db77d 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1562,7 +1562,7 @@ or `the MSDN `_ on Windo .. function:: pwritev(fd, buffers, offset, flags=0, /) - Write the *buffers* contents to file descriptor *fd* at a offset *offset*, + Write the *buffers* contents to file descriptor *fd* at an offset *offset*, leaving the file offset unchanged. *buffers* must be a sequence of :term:`bytes-like objects `. Buffers are processed in array order. Entire contents of the first buffer is written before @@ -3805,7 +3805,7 @@ features: new file descriptor is :ref:`non-inheritable `. *initval* is the initial value of the event counter. The initial value - must be an 32 bit unsigned integer. Please note that the initial value is + must be a 32 bit unsigned integer. Please note that the initial value is limited to a 32 bit unsigned int although the event counter is an unsigned 64 bit integer with a maximum value of 2\ :sup:`64`\ -\ 2. @@ -3884,7 +3884,7 @@ features: .. data:: EFD_SEMAPHORE - Provide semaphore-like semantics for reads from a :func:`eventfd` file + Provide semaphore-like semantics for reads from an :func:`eventfd` file descriptor. On read the internal counter is decremented by one. .. availability:: Linux >= 2.6.30 diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index f139abd2454d69..60c099af928123 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1016,7 +1016,7 @@ Querying file type and status .. method:: Path.stat(*, follow_symlinks=True) - Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. + Return an :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. The result is looked up at each call to this method. This method normally follows symlinks; to stat a symlink add the argument @@ -1557,7 +1557,8 @@ Copying, renaming and deleting .. versionadded:: 3.14 -.. method:: Path.copytree(target, *, follow_symlinks=True, dirs_exist_ok=False, \ +.. method:: Path.copytree(target, *, follow_symlinks=True, \ + preserve_metadata=False, dirs_exist_ok=False, \ ignore=None, on_error=None) Recursively copy this directory tree to the given destination. @@ -1566,6 +1567,13 @@ Copying, renaming and deleting true (the default), the symlink's target is copied. Otherwise, the symlink is recreated in the destination tree. + If *preserve_metadata* is false (the default), only the directory structure + and file data are guaranteed to be copied. Set *preserve_metadata* to true + to ensure that file and directory permissions, flags, last access and + modification times, and extended attributes are copied where supported. + This argument has no effect on Windows, where metadata is always preserved + when copying. + If the destination is an existing directory and *dirs_exist_ok* is false (the default), a :exc:`FileExistsError` is raised. Otherwise, the copying operation will continue if it encounters existing directories, and files @@ -1645,6 +1653,34 @@ Copying, renaming and deleting Remove this directory. The directory must be empty. +.. method:: Path.rmtree(ignore_errors=False, on_error=None) + + Recursively delete this entire directory tree. The path must not refer to a symlink. + + If *ignore_errors* is true, errors resulting from failed removals will be + ignored. If *ignore_errors* is false or omitted, and a function is given to + *on_error*, it will be called each time an exception is raised. If neither + *ignore_errors* nor *on_error* are supplied, exceptions are propagated to + the caller. + + .. note:: + + On platforms that support the necessary fd-based functions, a symlink + attack-resistant version of :meth:`~Path.rmtree` is used by default. On + other platforms, the :func:`~Path.rmtree` implementation is susceptible + to a symlink attack: given proper timing and circumstances, attackers + can manipulate symlinks on the filesystem to delete files they would not + be able to access otherwise. + + If the optional argument *on_error* is specified, it should be a callable; + it will be called with one argument of type :exc:`OSError`. The + callable can handle the error to continue the deletion process or re-raise + it to stop. Note that the filename is available as the :attr:`~OSError.filename` + attribute of the exception object. + + .. versionadded:: 3.14 + + Permissions and ownership ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1828,39 +1864,54 @@ Corresponding tools Below is a table mapping various :mod:`os` functions to their corresponding :class:`PurePath`/:class:`Path` equivalent. -==================================== ============================== -:mod:`os` and :mod:`os.path` :mod:`pathlib` -==================================== ============================== -:func:`os.path.abspath` :meth:`Path.absolute` -:func:`os.path.realpath` :meth:`Path.resolve` -:func:`os.chmod` :meth:`Path.chmod` -:func:`os.mkdir` :meth:`Path.mkdir` -:func:`os.makedirs` :meth:`Path.mkdir` -:func:`os.rename` :meth:`Path.rename` -:func:`os.replace` :meth:`Path.replace` -:func:`os.rmdir` :meth:`Path.rmdir` -:func:`os.remove`, :func:`os.unlink` :meth:`Path.unlink` -:func:`os.getcwd` :func:`Path.cwd` -:func:`os.path.exists` :meth:`Path.exists` -:func:`os.path.expanduser` :meth:`Path.expanduser` and - :meth:`Path.home` -:func:`os.listdir` :meth:`Path.iterdir` -:func:`os.walk` :meth:`Path.walk` -:func:`os.path.isdir` :meth:`Path.is_dir` -:func:`os.path.isfile` :meth:`Path.is_file` -:func:`os.path.islink` :meth:`Path.is_symlink` -:func:`os.link` :meth:`Path.hardlink_to` -:func:`os.symlink` :meth:`Path.symlink_to` -:func:`os.readlink` :meth:`Path.readlink` -:func:`os.path.relpath` :meth:`PurePath.relative_to` -:func:`os.stat` :meth:`Path.stat`, - :meth:`Path.owner`, - :meth:`Path.group` -:func:`os.path.isabs` :meth:`PurePath.is_absolute` -:func:`os.path.join` :func:`PurePath.joinpath` -:func:`os.path.basename` :attr:`PurePath.name` -:func:`os.path.dirname` :attr:`PurePath.parent` -:func:`os.path.samefile` :meth:`Path.samefile` -:func:`os.path.splitext` :attr:`PurePath.stem` and - :attr:`PurePath.suffix` -==================================== ============================== +===================================== ============================================== +:mod:`os` and :mod:`os.path` :mod:`pathlib` +===================================== ============================================== +:func:`os.path.dirname` :attr:`PurePath.parent` +:func:`os.path.basename` :attr:`PurePath.name` +:func:`os.path.splitext` :attr:`PurePath.stem`, :attr:`PurePath.suffix` +:func:`os.path.join` :meth:`PurePath.joinpath` +:func:`os.path.isabs` :meth:`PurePath.is_absolute` +:func:`os.path.relpath` :meth:`PurePath.relative_to` [1]_ +:func:`os.path.expanduser` :meth:`Path.expanduser` [2]_ +:func:`os.path.realpath` :meth:`Path.resolve` +:func:`os.path.abspath` :meth:`Path.absolute` [3]_ +:func:`os.path.exists` :meth:`Path.exists` +:func:`os.path.isfile` :meth:`Path.is_file` +:func:`os.path.isdir` :meth:`Path.is_dir` +:func:`os.path.islink` :meth:`Path.is_symlink` +:func:`os.path.isjunction` :meth:`Path.is_junction` +:func:`os.path.ismount` :meth:`Path.is_mount` +:func:`os.path.samefile` :meth:`Path.samefile` +:func:`os.getcwd` :meth:`Path.cwd` +:func:`os.stat` :meth:`Path.stat` +:func:`os.lstat` :meth:`Path.lstat` +:func:`os.listdir` :meth:`Path.iterdir` +:func:`os.walk` :meth:`Path.walk` [4]_ +:func:`os.mkdir`, :func:`os.makedirs` :meth:`Path.mkdir` +:func:`os.link` :meth:`Path.hardlink_to` +:func:`os.symlink` :meth:`Path.symlink_to` +:func:`os.readlink` :meth:`Path.readlink` +:func:`os.rename` :meth:`Path.rename` +:func:`os.replace` :meth:`Path.replace` +:func:`os.remove`, :func:`os.unlink` :meth:`Path.unlink` +:func:`os.rmdir` :meth:`Path.rmdir` +:func:`os.chmod` :meth:`Path.chmod` +:func:`os.lchmod` :meth:`Path.lchmod` +===================================== ============================================== + +.. rubric:: Footnotes + +.. [1] :func:`os.path.relpath` calls :func:`~os.path.abspath` to make paths + absolute and remove "``..``" parts, whereas :meth:`PurePath.relative_to` + is a lexical operation that raises :exc:`ValueError` when its inputs' + anchors differ (e.g. if one path is absolute and the other relative.) +.. [2] :func:`os.path.expanduser` returns the path unchanged if the home + directory can't be resolved, whereas :meth:`Path.expanduser` raises + :exc:`RuntimeError`. +.. [3] :func:`os.path.abspath` removes "``..``" components without resolving + symlinks, which may change the meaning of the path, whereas + :meth:`Path.absolute` leaves any "``..``" components in the path. +.. [4] :func:`os.walk` always follows symlinks when categorizing paths into + *dirnames* and *filenames*, whereas :meth:`Path.walk` categorizes all + symlinks into *filenames* when *follow_symlinks* is false (the default.) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 57fbe5b6ece6b6..71fe3743c5968d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -156,13 +156,14 @@ to read the pickle produced. * Protocol version 4 was added in Python 3.4. It adds support for very large objects, pickling more kinds of objects, and some data format - optimizations. It is the default protocol starting with Python 3.8. + optimizations. This was the default protocol in Python 3.8--3.13. Refer to :pep:`3154` for information about improvements brought by protocol 4. * Protocol version 5 was added in Python 3.8. It adds support for out-of-band - data and speedup for in-band data. Refer to :pep:`574` for information about - improvements brought by protocol 5. + data and speedup for in-band data. It is the default protocol starting with + Python 3.14. Refer to :pep:`574` for information about improvements brought + by protocol 5. .. note:: Serialization is a more primitive notion than persistence; although @@ -199,8 +200,10 @@ The :mod:`pickle` module provides the following constants: An integer, the default :ref:`protocol version ` used for pickling. May be less than :data:`HIGHEST_PROTOCOL`. Currently the - default protocol is 4, first introduced in Python 3.4 and incompatible - with previous versions. + default protocol is 5, introduced in Python 3.8 and incompatible + with previous versions. This version introduces support for out-of-band + buffers, where :pep:`3118`-compatible data can be transmitted separately + from the main pickle stream. .. versionchanged:: 3.0 @@ -210,6 +213,10 @@ The :mod:`pickle` module provides the following constants: The default protocol is 4. + .. versionchanged:: 3.14 + + The default protocol is 5. + The :mod:`pickle` module provides the following functions to make the pickling process more convenient: diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 5d4ff34ba029a0..f095cc84173737 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -34,9 +34,9 @@ support. *name* argument. This feature is similar to :file:`\*.pth` files (see the :mod:`site` module for more information), except that it doesn't special-case lines starting with ``import``. A :file:`\*.pkg` file is trusted at face - value: apart from checking for duplicates, all entries found in a - :file:`\*.pkg` file are added to the path, regardless of whether they exist - on the filesystem. (This is a feature.) + value: apart from skipping blank lines and ignoring comments, all entries + found in a :file:`\*.pkg` file are added to the path, regardless of whether + they exist on the filesystem (this is a feature). If the input path is not a list (as is the case for frozen packages) it is returned unchanged. The input path is not modified; an extended copy is diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 78b3c2697bd696..2906ebe7822f52 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -117,7 +117,7 @@ This module defines the following functions: when a key of a dictionary is not a string, otherwise such keys are skipped. When *aware_datetime* is true and any field with type ``datetime.datetime`` - is set as a :ref:`aware object `, it will convert to + is set as an :ref:`aware object `, it will convert to UTC timezone before writing it. A :exc:`TypeError` will be raised if the object is of an unsupported type or diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 9721da7220d54d..3334833eba6b8c 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -682,7 +682,7 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`, that you choose (see :ref:`profile-calibration`). For most machines, a timer that returns a lone integer value will provide the best results in terms of low overhead during profiling. (:func:`os.times` is *pretty* bad, as it - returns a tuple of floating point values). If you want to substitute a + returns a tuple of floating-point values). If you want to substitute a better timer in the cleanest fashion, derive a class and hardwire a replacement dispatch method that best handles your timer call, along with the appropriate calibration constant. @@ -699,7 +699,7 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`, As the :class:`cProfile.Profile` class cannot be calibrated, custom timer functions should be used with care and should be as fast as possible. For the best results with a custom timer, it might be necessary to hard-code it - in the C source of the internal :mod:`_lsprof` module. + in the C source of the internal :mod:`!_lsprof` module. Python 3.3 adds several new functions in :mod:`time` that can be used to make precise measurements of process or wall-clock time. For example, see diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 755d1c8908c966..c7f6b0bdd5b822 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -200,8 +200,8 @@ Functions for sequences For a given seed, the :func:`choices` function with equal weighting typically produces a different sequence than repeated calls to - :func:`choice`. The algorithm used by :func:`choices` uses floating - point arithmetic for internal consistency and speed. The algorithm used + :func:`choice`. The algorithm used by :func:`choices` uses floating-point + arithmetic for internal consistency and speed. The algorithm used by :func:`choice` defaults to integer arithmetic with repeated selections to avoid small biases from round-off error. @@ -298,12 +298,12 @@ be found in any statistics text. .. function:: random() - Return the next random floating point number in the range ``0.0 <= X < 1.0`` + Return the next random floating-point number in the range ``0.0 <= X < 1.0`` .. function:: uniform(a, b) - Return a random floating point number *N* such that ``a <= N <= b`` for + Return a random floating-point number *N* such that ``a <= N <= b`` for ``a <= b`` and ``b <= N <= a`` for ``b < a``. The end-point value ``b`` may or may not be included in the range @@ -313,7 +313,7 @@ be found in any statistics text. .. function:: triangular(low, high, mode) - Return a random floating point number *N* such that ``low <= N <= high`` and + Return a random floating-point number *N* such that ``low <= N <= high`` and with the specified *mode* between those bounds. The *low* and *high* bounds default to zero and one. The *mode* argument defaults to the midpoint between the bounds, giving a symmetric distribution. @@ -741,7 +741,7 @@ The following options are accepted: .. option:: -f --float - Print a random floating point number between 1 and N inclusive, + Print a random floating-point number between 1 and N inclusive, using :meth:`uniform`. If no options are given, the output depends on the input: diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 5658b93c81dc99..d03b71597130cf 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -45,6 +45,10 @@ Readline library in general. python:bind -v python:bind ^I rl_complete + Also note that different libraries may use different history file formats. + When switching the underlying library, existing history files may become + unusable. + .. data:: backend The name of the underlying Readline library being used, either diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index dd80b1e6670d92..0515d205bbca0b 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -305,7 +305,7 @@ These functions are used to retrieve resource usage information: elements. The fields :attr:`ru_utime` and :attr:`ru_stime` of the return value are - floating point values representing the amount of time spent executing in user + floating-point values representing the amount of time spent executing in user mode and the amount of time spent executing in system mode, respectively. The remaining values are integers. Consult the :manpage:`getrusage(2)` man page for detailed information about these values. A brief summary is presented here: diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 06ebaf0201e0e7..f23a249f44b485 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -129,7 +129,7 @@ The module defines the following: Empty iterables are allowed, but acceptance of three empty iterables is platform-dependent. (It is known to work on Unix but not on Windows.) The - optional *timeout* argument specifies a time-out as a floating point number + optional *timeout* argument specifies a time-out as a floating-point number in seconds. When the *timeout* argument is omitted the function blocks until at least one file descriptor is ready. A time-out value of zero specifies a poll and never blocks. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index fd32479195eca8..220207e5f3cbbf 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -706,11 +706,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. The keyword-only *filter* argument is passed to the underlying unpacking function. For zip files, *filter* is not accepted. - For tar files, it is recommended to set it to ``'data'``, - unless using features specific to tar and UNIX-like filesystems. + For tar files, it is recommended to use ``'data'`` (default since Python + 3.14), unless using features specific to tar and UNIX-like filesystems. (See :ref:`tarfile-extraction-filter` for details.) - The ``'data'`` filter will become the default for tar files - in Python 3.14. .. audit-event:: shutil.unpack_archive filename,extract_dir,format shutil.unpack_archive @@ -721,6 +719,12 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. the *extract_dir* argument, e.g. members that have absolute filenames starting with "/" or filenames with two dots "..". + Since Python 3.14, the defaults for both built-in formats (zip and tar + files) will prevent the most dangerous of such security issues, + but will not prevent *all* unintended behavior. + Read the :ref:`tarfile-further-verification` + section for tar-specific details. + .. versionchanged:: 3.7 Accepts a :term:`path-like object` for *filename* and *extract_dir*. diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 782fb9b27ae1ba..b1e35e68b132e2 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1926,7 +1926,7 @@ to sockets. .. method:: socket.settimeout(value) Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative floating point number expressing seconds, or ``None``. + nonnegative floating-point number expressing seconds, or ``None``. If a non-zero value is given, subsequent socket operations will raise a :exc:`timeout` exception if the timeout period *value* has elapsed before the operation has completed. If zero is given, the socket is put in diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index dc72f67c6361e2..7d4c1f0f2de347 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1742,7 +1742,7 @@ to speed up repeated connections from the same clients. IDN-encoded internationalized domain name, the *server_name_callback* receives a decoded U-label (``"pythön.org"``). - If there is an decoding error on the server name, the TLS connection will + If there is a decoding error on the server name, the TLS connection will terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS alert message to the client. diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 8453135d2e164d..614f5b905a4a2e 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -73,7 +73,7 @@ or sample. ======================= =============================================================== :func:`mean` Arithmetic mean ("average") of data. -:func:`fmean` Fast, floating point arithmetic mean, with optional weighting. +:func:`fmean` Fast, floating-point arithmetic mean, with optional weighting. :func:`geometric_mean` Geometric mean of data. :func:`harmonic_mean` Harmonic mean of data. :func:`kde` Estimate the probability density distribution of the data. @@ -485,6 +485,12 @@ However, for reading convenience, most of the examples show sorted sequences. >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) 'red' + Only hashable inputs are supported. To handle type :class:`set`, + consider casting to :class:`frozenset`. To handle type :class:`list`, + consider casting to :class:`tuple`. For mixed or nested inputs, consider + using this slower quadratic algorithm that only depends on equality tests: + ``max(data, key=data.count)``. + .. versionchanged:: 3.8 Now handles multimodal datasets by returning the first mode encountered. Formerly, it raised :exc:`StatisticsError` when more than one mode was diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index d3f7cfb01d3c21..b44ac69606706c 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -209,18 +209,18 @@ Numeric Types --- :class:`int`, :class:`float`, :class:`complex` pair: object; numeric pair: object; Boolean pair: object; integer - pair: object; floating point + pair: object; floating-point pair: object; complex number pair: C; language -There are three distinct numeric types: :dfn:`integers`, :dfn:`floating -point numbers`, and :dfn:`complex numbers`. In addition, Booleans are a -subtype of integers. Integers have unlimited precision. Floating point +There are three distinct numeric types: :dfn:`integers`, :dfn:`floating-point +numbers`, and :dfn:`complex numbers`. In addition, Booleans are a +subtype of integers. Integers have unlimited precision. Floating-point numbers are usually implemented using :c:expr:`double` in C; information -about the precision and internal representation of floating point +about the precision and internal representation of floating-point numbers for the machine on which your program is running is available in :data:`sys.float_info`. Complex numbers have a real and imaginary -part, which are each a floating point number. To extract these parts +part, which are each a floating-point number. To extract these parts from a complex number *z*, use ``z.real`` and ``z.imag``. (The standard library includes the additional numeric types :mod:`fractions.Fraction`, for rationals, and :mod:`decimal.Decimal`, for floating-point numbers with @@ -229,7 +229,7 @@ user-definable precision.) .. index:: pair: numeric; literals pair: integer; literals - pair: floating point; literals + pair: floating-point; literals pair: complex number; literals pair: hexadecimal; literals pair: octal; literals @@ -238,7 +238,7 @@ user-definable precision.) Numbers are created by numeric literals or as the result of built-in functions and operators. Unadorned integer literals (including hex, octal and binary numbers) yield integers. Numeric literals containing a decimal point or an -exponent sign yield floating point numbers. Appending ``'j'`` or ``'J'`` to a +exponent sign yield floating-point numbers. Appending ``'j'`` or ``'J'`` to a numeric literal yields an imaginary number (a complex number with a zero real part) which you can add to an integer or float to get a complex number with real and imaginary parts. @@ -625,6 +625,23 @@ Additional Methods on Float The float type implements the :class:`numbers.Real` :term:`abstract base class`. float also has the following additional methods. +.. classmethod:: float.from_number(x) + + Class method to return a floating-point number constructed from a number *x*. + + If the argument is an integer or a floating-point number, a + floating-point number with the same value (within Python's floating-point + precision) is returned. If the argument is outside the range of a Python + float, an :exc:`OverflowError` will be raised. + + For a general Python object ``x``, ``float.from_number(x)`` delegates to + ``x.__float__()``. + If :meth:`~object.__float__` is not defined then it falls back + to :meth:`~object.__index__`. + + .. versionadded:: 3.14 + + .. method:: float.as_integer_ratio() Return a pair of integers whose ratio is exactly equal to the @@ -703,6 +720,25 @@ hexadecimal string representing the same number:: '0x1.d380000000000p+11' +Additional Methods on Complex +----------------------------- + +The :class:`!complex` type implements the :class:`numbers.Complex` +:term:`abstract base class`. +:class:`!complex` also has the following additional methods. + +.. classmethod:: complex.from_number(x) + + Class method to convert a number to a complex number. + + For a general Python object ``x``, ``complex.from_number(x)`` delegates to + ``x.__complex__()``. If :meth:`~object.__complex__` is not defined then it falls back + to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back + to :meth:`~object.__index__`. + + .. versionadded:: 3.14 + + .. _numeric-hash: Hashing of numeric types @@ -1497,8 +1533,8 @@ objects that compare equal might have different :attr:`~range.start`, .. seealso:: * The `linspace recipe `_ - shows how to implement a lazy version of range suitable for floating - point applications. + shows how to implement a lazy version of range suitable for floating-point + applications. .. index:: single: string; text sequence type @@ -2442,19 +2478,19 @@ The conversion types are: +------------+-----------------------------------------------------+-------+ | ``'X'`` | Signed hexadecimal (uppercase). | \(2) | +------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | +| ``'e'`` | Floating-point exponential format (lowercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | +| ``'E'`` | Floating-point exponential format (uppercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | +| ``'f'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | +| ``'F'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | +| ``'g'`` | Floating-point format. Uses lowercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | +| ``'G'`` | Floating-point format. Uses uppercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ @@ -3661,19 +3697,19 @@ The conversion types are: +------------+-----------------------------------------------------+-------+ | ``'X'`` | Signed hexadecimal (uppercase). | \(2) | +------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | +| ``'e'`` | Floating-point exponential format (lowercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | +| ``'E'`` | Floating-point exponential format (uppercase). | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | +| ``'f'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | +| ``'F'`` | Floating-point decimal format. | \(3) | +------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | +| ``'g'`` | Floating-point format. Uses lowercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | +| ``'G'`` | Floating-point format. Uses uppercase exponential | \(4) | | | format if exponent is less than -4 or not less than | | | | precision, decimal format otherwise. | | +------------+-----------------------------------------------------+-------+ @@ -3895,7 +3931,7 @@ copying. >>> a == b False - Note that, as with floating point numbers, ``v is w`` does *not* imply + Note that, as with floating-point numbers, ``v is w`` does *not* imply ``v == w`` for memoryview objects. .. versionchanged:: 3.3 diff --git a/Doc/library/string.rst b/Doc/library/string.rst index c3c0d732cf18d4..1f316307965c11 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -418,7 +418,7 @@ instead. .. index:: single: _ (underscore); in string formatting The ``'_'`` option signals the use of an underscore for a thousands -separator for floating point presentation types and for integer +separator for floating-point presentation types and for integer presentation type ``'d'``. For integer presentation types ``'b'``, ``'o'``, ``'x'``, and ``'X'``, underscores will be inserted every 4 digits. For other presentation types, specifying this option is an @@ -491,9 +491,9 @@ The available integer presentation types are: +---------+----------------------------------------------------------+ In addition to the above presentation types, integers can be formatted -with the floating point presentation types listed below (except +with the floating-point presentation types listed below (except ``'n'`` and ``None``). When doing so, :func:`float` is used to convert the -integer to a floating point number before formatting. +integer to a floating-point number before formatting. The available presentation types for :class:`float` and :class:`~decimal.Decimal` values are: diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index f520d989e0c70d..a0ba97f429bec9 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1126,7 +1126,7 @@ The :mod:`subprocess` module exposes the following constants. .. data:: NORMAL_PRIORITY_CLASS A :class:`Popen` ``creationflags`` parameter to specify that a new process - will have an normal priority. (default) + will have a normal priority. (default) .. versionadded:: 3.7 diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index aa5f8d95925ada..8ebcb3bcf1b7b4 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -191,6 +191,19 @@ Examining Symbol Tables For example: + .. testsetup:: symtable.Class.get_methods + + import warnings + context = warnings.catch_warnings() + context.__enter__() + warnings.simplefilter("ignore", category=DeprecationWarning) + + .. testcleanup:: symtable.Class.get_methods + + context.__exit__() + + .. doctest:: symtable.Class.get_methods + >>> import symtable >>> st = symtable.symtable(''' ... def outer(): pass @@ -214,6 +227,9 @@ Examining Symbol Tables Although ``A().f()`` raises :exc:`TypeError` at runtime, ``A.f`` is still considered as a method-like function. + .. deprecated-removed:: 3.14 3.16 + + .. class:: Symbol An entry in a :class:`SymbolTable` corresponding to an identifier in the diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 0fa06da522049f..3ead20815fa30e 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -226,6 +226,10 @@ To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. :monitoring-event:`STOP_ITERATION` can be locally disabled, unlike :monitoring-event:`RAISE`. +Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` +event for a :exc:`StopIteration` exception are equivalent, and are treated as interchangeable +when generating events. Implementations will favor :monitoring-event:`STOP_ITERATION` for +performance reasons, but may generate a :monitoring-event:`RAISE` event with a :exc:`StopIteration`. Turning events on and off ------------------------- diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 9556da808f8c63..6834f66b386e4b 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -376,7 +376,7 @@ Other functions This is used mainly to distinguish platform-specific build directories and platform-specific built distributions. Typically includes the OS name and - version and the architecture (as supplied by 'os.uname()'), although the + version and the architecture (as supplied by :func:`os.uname`), although the exact information included depends on the OS; e.g., on Linux, the kernel version isn't particularly important. diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 5b624f3533136f..631d869e42d09d 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -40,9 +40,12 @@ Some facts and figures: Archives are extracted using a :ref:`filter `, which makes it possible to either limit surprising/dangerous features, or to acknowledge that they are expected and the archive is fully trusted. - By default, archives are fully trusted, but this default is deprecated - and slated to change in Python 3.14. +.. versionchanged:: 3.14 + Set the default extraction filter to :func:`data `, + which disallows some dangerous features such as links to absolute paths + or paths outside of the destination. Previously, the filter strategy + was equivalent to :func:`fully_trusted `. .. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, **kwargs) @@ -495,18 +498,18 @@ be finalized; only the internally used file object will be closed. See the The *filter* argument specifies how ``members`` are modified or rejected before extraction. See :ref:`tarfile-extraction-filter` for details. - It is recommended to set this explicitly depending on which *tar* features - you need to support. + It is recommended to set this explicitly only if specific *tar* features + are required, or as ``filter='data'`` to support Python versions with a less + secure default (3.13 and lower). .. warning:: Never extract archives from untrusted sources without prior inspection. - It is possible that files are created outside of *path*, e.g. members - that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. - Set ``filter='data'`` to prevent the most dangerous security issues, - and read the :ref:`tarfile-extraction-filter` section for details. + Since Python 3.14, the default (:func:`data `) will prevent + the most dangerous security issues. + However, it will not prevent *all* unintended or insecure behavior. + Read the :ref:`tarfile-extraction-filter` section for details. .. versionchanged:: 3.5 Added the *numeric_owner* parameter. @@ -517,6 +520,9 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.12 Added the *filter* parameter. + .. versionchanged:: 3.14 + The *filter* parameter now defaults to ``'data'``. + .. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False, filter=None) @@ -536,10 +542,8 @@ be finalized; only the internally used file object will be closed. See the .. warning:: - See the warning for :meth:`extractall`. - - Set ``filter='data'`` to prevent the most dangerous security issues, - and read the :ref:`tarfile-extraction-filter` section for details. + Never extract archives from untrusted sources without prior inspection. + See the warning for :meth:`extractall` for details. .. versionchanged:: 3.2 Added the *set_attrs* parameter. @@ -602,14 +606,8 @@ be finalized; only the internally used file object will be closed. See the String names are not allowed for this attribute, unlike the *filter* argument to :meth:`~TarFile.extract`. - If ``extraction_filter`` is ``None`` (the default), - calling an extraction method without a *filter* argument will raise a - ``DeprecationWarning``, - and fall back to the :func:`fully_trusted ` filter, - whose dangerous behavior matches previous versions of Python. - - In Python 3.14+, leaving ``extraction_filter=None`` will cause - extraction methods to use the :func:`data ` filter by default. + If ``extraction_filter`` is ``None`` (the default), extraction methods + will use the :func:`data ` filter by default. The attribute may be set on instances or overridden in subclasses. It also is possible to set it on the ``TarFile`` class itself to set a @@ -619,6 +617,14 @@ be finalized; only the internally used file object will be closed. See the To set a global default this way, a filter function needs to be wrapped in :func:`staticmethod()` to prevent injection of a ``self`` argument. + .. versionchanged:: 3.14 + + The default filter is set to :func:`data `, + which disallows some dangerous features such as links to absolute paths + or paths outside of the destination. + Previously, the default was equivalent to + :func:`fully_trusted `. + .. method:: TarFile.add(name, arcname=None, recursive=True, *, filter=None) Add the file *name* to the archive. *name* may be any type of file @@ -969,6 +975,12 @@ In most cases, the full functionality is not needed. Therefore, *tarfile* supports extraction filters: a mechanism to limit functionality, and thus mitigate some of the security issues. +.. warning:: + + None of the available filters blocks *all* dangerous archive features. + Never extract archives from untrusted sources without prior inspection. + See also :ref:`tarfile-further-verification`. + .. seealso:: :pep:`706` @@ -992,12 +1004,13 @@ can be: * ``None`` (default): Use :attr:`TarFile.extraction_filter`. - If that is also ``None`` (the default), raise a ``DeprecationWarning``, - and fall back to the ``'fully_trusted'`` filter, whose dangerous behavior - matches previous versions of Python. + If that is also ``None`` (the default), the ``'data'`` filter will be used. + + .. versionchanged:: 3.14 - In Python 3.14, the ``'data'`` filter will become the default instead. - It's possible to switch earlier; see :attr:`TarFile.extraction_filter`. + The default filter is set to :func:`data `. + Previously, the default was equivalent to + :func:`fully_trusted `. * A callable which will be called for each extracted member with a :ref:`TarInfo ` describing the member and the destination @@ -1080,6 +1093,9 @@ reused in custom filters: Return the modified ``TarInfo`` member. + Note that this filter does not block *all* dangerous archive features. + See :ref:`tarfile-further-verification` for details. + .. _tarfile-extraction-refuse: @@ -1093,6 +1109,8 @@ With ``errorlevel=0`` the error will be logged and the member will be skipped, but extraction will continue. +.. _tarfile-further-verification: + Hints for further verification ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1110,9 +1128,10 @@ Here is an incomplete list of things to consider: disk, memory and CPU usage. * Check filenames against an allow-list of characters (to filter out control characters, confusables, foreign path separators, - etc.). + and so on). * Check that filenames have expected extensions (discouraging files that - execute when you “click on them”, or extension-less files like Windows special device names). + execute when you “click on them”, or extension-less files like Windows + special device names). * Limit the number of extracted files, total size of extracted data, filename length (including symlink length), and size of individual files. * Check for files that would be shadowed on case-insensitive filesystems. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 7b259e22dc7124..49c2b9b3ccd4fd 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -412,7 +412,7 @@ since it is impossible to detect the termination of alien threads. timeout occurs. When the *timeout* argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). As :meth:`~Thread.join` always returns ``None``, you must call :meth:`~Thread.is_alive` after :meth:`~Thread.join` to decide whether a timeout happened -- if the thread is still alive, the @@ -794,7 +794,7 @@ item to the buffer only needs to wake up one consumer thread. occurs. Once awakened or timed out, it re-acquires the lock and returns. When the *timeout* argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an :class:`RLock`, it is not released using @@ -1021,7 +1021,7 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. the the internal flag did not become true within the given wait time. When the timeout argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds, + floating-point number specifying a timeout for the operation in seconds, or fractions thereof. .. versionchanged:: 3.1 diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 4d7661715aa0af..900e78dbd22b8a 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -69,7 +69,7 @@ An explanation of some terminology and conventions is in order. systems, the clock "ticks" only 50 or 100 times a second. * On the other hand, the precision of :func:`.time` and :func:`sleep` is better - than their Unix equivalents: times are expressed as floating point numbers, + than their Unix equivalents: times are expressed as floating-point numbers, :func:`.time` returns the most accurate time available (using Unix :c:func:`!gettimeofday` where available), and :func:`sleep` will accept a time with a nonzero fraction (Unix :c:func:`!select` is used to implement this, where @@ -273,7 +273,7 @@ Functions This is the inverse function of :func:`localtime`. Its argument is the :class:`struct_time` or full 9-tuple (since the dst flag is needed; use ``-1`` as the dst flag if it is unknown) which expresses the time in *local* time, not - UTC. It returns a floating point number, for compatibility with :func:`.time`. + UTC. It returns a floating-point number, for compatibility with :func:`.time`. If the input value cannot be represented as a valid time, either :exc:`OverflowError` or :exc:`ValueError` will be raised (which depends on whether the invalid value is caught by Python or the underlying C libraries). @@ -376,7 +376,7 @@ Functions .. function:: sleep(secs) Suspend execution of the calling thread for the given number of seconds. - The argument may be a floating point number to indicate a more precise sleep + The argument may be a floating-point number to indicate a more precise sleep time. If the sleep is interrupted by a signal and no exception is raised by the @@ -665,13 +665,13 @@ Functions .. function:: time() -> float - Return the time in seconds since the epoch_ as a floating point + Return the time in seconds since the epoch_ as a floating-point number. The handling of `leap seconds`_ is platform dependent. On Windows and most Unix systems, the leap seconds are not counted towards the time in seconds since the epoch_. This is commonly referred to as `Unix time `_. - Note that even though the time is always returned as a floating point + Note that even though the time is always returned as a floating-point number, not all systems provide time with a better precision than 1 second. While this function normally returns non-decreasing values, it can return a lower value than a previous call if the system clock has been set back diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index d8ba24c3146cf2..757277e102d80e 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -860,6 +860,20 @@ object:: 3 >>> p.assert_called_once_with() +.. caution:: + + If an :exc:`AttributeError` is raised by :class:`PropertyMock`, + it will be interpreted as a missing descriptor and + :meth:`~object.__getattr__` will be called on the parent mock:: + + >>> m = MagicMock() + >>> no_attribute = PropertyMock(side_effect=AttributeError) + >>> type(m).my_property = no_attribute + >>> m.my_property + + + See :meth:`~object.__getattr__` for details. + .. class:: AsyncMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs) @@ -1624,7 +1638,8 @@ patch.dict .. function:: patch.dict(in_dict, values=(), clear=False, **kwargs) Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. + to its original state after the test, where the restored dictionary is a + copy of the dictionary as it was before the test. *in_dict* can be a dictionary or a mapping like container. If it is a mapping then it must at least support getting, setting and deleting items diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index eb42210e096ecb..dc76374d5181eb 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -2316,8 +2316,8 @@ Loading and running tests (see :ref:`Warning control `), otherwise it will be set to ``'default'``. - Calling ``main`` actually returns an instance of the ``TestProgram`` class. - This stores the result of the tests run as the ``result`` attribute. + Calling ``main`` returns an object with the ``result`` attribute that contains + the result of the tests run as a :class:`unittest.TestResult`. .. versionchanged:: 3.1 The *exit* parameter was added. diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 754405e0fbe5b2..88c1f4bf85f662 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -1092,7 +1092,7 @@ FileHandler Objects .. versionchanged:: 3.2 This method is applicable only for local hostnames. When a remote - hostname is given, an :exc:`~urllib.error.URLError` is raised. + hostname is given, a :exc:`~urllib.error.URLError` is raised. .. _data-handler-objects: @@ -1107,7 +1107,7 @@ DataHandler Objects ignores white spaces in base64 encoded data URLs so the URL may be wrapped in whatever source file it comes from. But even though some browsers don't mind about a missing padding at the end of a base64 encoded data URL, this - implementation will raise an :exc:`ValueError` in that case. + implementation will raise a :exc:`ValueError` in that case. .. _ftp-handler-objects: diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 4c1e7bd7e6734a..51bf88313e5b7a 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -874,6 +874,7 @@ Element Objects .. module:: xml.etree.ElementTree :noindex: + :no-index: .. class:: Element(tag, attrib={}, **extra) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 34d8bbee4c6806..144c6f78ccd443 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -218,7 +218,7 @@ properties: * A sign is shown only when the number is negative. -Python distinguishes between integers, floating point numbers, and complex +Python distinguishes between integers, floating-point numbers, and complex numbers: @@ -262,18 +262,18 @@ Booleans (:class:`bool`) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. index:: - pair: object; floating point - pair: floating point; number + pair: object; floating-point + pair: floating-point; number pair: C; language pair: Java; language -These represent machine-level double precision floating point numbers. You are +These represent machine-level double precision floating-point numbers. You are at the mercy of the underlying machine architecture (and C or Java implementation) for the accepted range and handling of overflow. Python does not -support single-precision floating point numbers; the savings in processor and +support single-precision floating-point numbers; the savings in processor and memory usage that are usually the reason for using these are dwarfed by the overhead of using objects in Python, so there is no reason to complicate the -language with two kinds of floating point numbers. +language with two kinds of floating-point numbers. :class:`numbers.Complex` (:class:`complex`) @@ -284,7 +284,7 @@ language with two kinds of floating point numbers. pair: complex; number These represent complex numbers as a pair of machine-level double precision -floating point numbers. The same caveats apply as for floating point numbers. +floating-point numbers. The same caveats apply as for floating-point numbers. The real and imaginary parts of a complex number ``z`` can be retrieved through the read-only attributes ``z.real`` and ``z.imag``. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 95ece0e1608dcc..dc1cd20abe5ba3 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -33,7 +33,7 @@ implementation for built-in types works as follows: * If either argument is a complex number, the other is converted to complex; -* otherwise, if either argument is a floating point number, the other is +* otherwise, if either argument is a floating-point number, the other is converted to floating point; * otherwise, both must be integers and no conversion is necessary. @@ -83,18 +83,47 @@ exception. pair: name; mangling pair: private; names -**Private name mangling:** When an identifier that textually occurs in a class -definition begins with two or more underscore characters and does not end in two -or more underscores, it is considered a :dfn:`private name` of that class. -Private names are transformed to a longer form before code is generated for -them. The transformation inserts the class name, with leading underscores -removed and a single underscore inserted, in front of the name. For example, -the identifier ``__spam`` occurring in a class named ``Ham`` will be transformed -to ``_Ham__spam``. This transformation is independent of the syntactical -context in which the identifier is used. If the transformed name is extremely -long (longer than 255 characters), implementation defined truncation may happen. -If the class name consists only of underscores, no transformation is done. +Private name mangling +^^^^^^^^^^^^^^^^^^^^^ +When an identifier that textually occurs in a class definition begins with two +or more underscore characters and does not end in two or more underscores, it +is considered a :dfn:`private name` of that class. + +.. seealso:: + + The :ref:`class specifications `. + +More precisely, private names are transformed to a longer form before code is +generated for them. If the transformed name is longer than 255 characters, +implementation-defined truncation may happen. + +The transformation is independent of the syntactical context in which the +identifier is used but only the following private identifiers are mangled: + +- Any name used as the name of a variable that is assigned or read or any + name of an attribute being accessed. + + The ``__name__`` attribute of nested functions, classes, and type aliases + is however not mangled. + +- The name of imported modules, e.g., ``__spam`` in ``import __spam``. + If the module is part of a package (i.e., its name contains a dot), + the name is *not* mangled, e.g., the ``__foo`` in ``import __foo.bar`` + is not mangled. + +- The name of an imported member, e.g., ``__f`` in ``from spam import __f``. + +The transformation rule is defined as follows: + +- The class name, with leading underscores removed and a single leading + underscore inserted, is inserted in front of the identifier, e.g., the + identifier ``__spam`` occurring in a class named ``Foo``, ``_Foo`` or + ``__Foo`` is transformed to ``_Foo__spam``. + +- If the class name consists only of underscores, the transformation is the + identity, e.g., the identifier ``__spam`` occurring in a class named ``_`` + or ``__`` is left as is. .. _atom-literals: @@ -110,8 +139,8 @@ Python supports string and bytes literals and various numeric literals: : | `integer` | `floatnumber` | `imagnumber` Evaluation of a literal yields an object of the given type (string, bytes, -integer, floating point number, complex number) with the given value. The value -may be approximated in the case of floating point and imaginary (complex) +integer, floating-point number, complex number) with the given value. The value +may be approximated in the case of floating-point and imaginary (complex) literals. See section :ref:`literals` for details. .. index:: @@ -1332,7 +1361,7 @@ The floor division operation can be customized using the special The ``%`` (modulo) operator yields the remainder from the division of the first argument by the second. The numeric arguments are first converted to a common type. A zero right argument raises the :exc:`ZeroDivisionError` exception. The -arguments may be floating point numbers, e.g., ``3.14%0.7`` equals ``0.34`` +arguments may be floating-point numbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals ``4*0.7 + 0.34``.) The modulo operator always yields a result with the same sign as its second operand (or zero); the absolute value of the result is strictly smaller than the absolute value of the second operand @@ -1352,8 +1381,8 @@ The *modulo* operation can be customized using the special :meth:`~object.__mod_ and :meth:`~object.__rmod__` methods. The floor division operator, the modulo operator, and the :func:`divmod` -function are not defined for complex numbers. Instead, convert to a floating -point number using the :func:`abs` function if appropriate. +function are not defined for complex numbers. Instead, convert to a +floating-point number using the :func:`abs` function if appropriate. .. index:: single: addition diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 41ea89fd234122..594fc713aa8edd 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -879,10 +879,10 @@ Numeric literals ---------------- .. index:: number, numeric literal, integer literal - floating point literal, hexadecimal literal + floating-point literal, hexadecimal literal octal literal, binary literal, decimal literal, imaginary literal, complex literal -There are three types of numeric literals: integers, floating point numbers, and +There are three types of numeric literals: integers, floating-point numbers, and imaginary numbers. There are no complex literals (complex numbers can be formed by adding a real number and an imaginary number). @@ -943,10 +943,10 @@ Some examples of integer literals:: single: _ (underscore); in numeric literal .. _floating: -Floating point literals +Floating-point literals ----------------------- -Floating point literals are described by the following lexical definitions: +Floating-point literals are described by the following lexical definitions: .. productionlist:: python-grammar floatnumber: `pointfloat` | `exponentfloat` @@ -958,10 +958,10 @@ Floating point literals are described by the following lexical definitions: Note that the integer and exponent parts are always interpreted using radix 10. For example, ``077e010`` is legal, and denotes the same number as ``77e10``. The -allowed range of floating point literals is implementation-dependent. As in +allowed range of floating-point literals is implementation-dependent. As in integer literals, underscores are supported for digit grouping. -Some examples of floating point literals:: +Some examples of floating-point literals:: 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 @@ -982,9 +982,9 @@ Imaginary literals are described by the following lexical definitions: imagnumber: (`floatnumber` | `digitpart`) ("j" | "J") An imaginary literal yields a complex number with a real part of 0.0. Complex -numbers are represented as a pair of floating point numbers and have the same +numbers are represented as a pair of floating-point numbers and have the same restrictions on their range. To create a complex number with a nonzero real -part, add a floating point number to it, e.g., ``(3+4j)``. Some examples of +part, add a floating-point number to it, e.g., ``(3+4j)``. Some examples of imaginary literals:: 3.14j 10.j 10j .001j 1e100j 3.14e-10j 3.14_15_93j diff --git a/Doc/requirements.txt b/Doc/requirements.txt index b47a9d8a8635ab..98ad52e17538a4 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -6,12 +6,12 @@ # Sphinx version is pinned so that new versions that introduce new warnings # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. -sphinx~=7.3.0 +sphinx~=7.4.0 blurb -sphinxext-opengraph==0.7.5 -sphinx-notfound-page==1.0.0 +sphinxext-opengraph~=0.9.0 +sphinx-notfound-page~=1.0.0 # The theme used by the documentation is stored separately, so we need # to install that as well. diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py index c50b00636c36ce..c686eecf8d9271 100644 --- a/Doc/tools/check-warnings.py +++ b/Doc/tools/check-warnings.py @@ -2,6 +2,7 @@ """ Check the output of running Sphinx in nit-picky mode (missing references). """ + from __future__ import annotations import argparse @@ -14,7 +15,7 @@ from typing import TextIO # Fail if NEWS nit found before this line number -NEWS_NIT_THRESHOLD = 200 +NEWS_NIT_THRESHOLD = 1700 # Exclude these whether they're dirty or clean, # because they trigger a rebuild of dirty files. @@ -206,7 +207,9 @@ def annotate_diff( def fail_if_regression( - warnings: list[str], files_with_expected_nits: set[str], files_with_nits: set[str] + warnings: list[str], + files_with_expected_nits: set[str], + files_with_nits: set[str], ) -> int: """ Ensure some files always pass Sphinx nit-picky mode (no missing references). @@ -252,17 +255,11 @@ def fail_if_new_news_nit(warnings: list[str], threshold: int) -> int: """ Ensure no warnings are found in the NEWS file before a given line number. """ - news_nits = ( - warning - for warning in warnings - if "/build/NEWS:" in warning - ) + news_nits = (warning for warning in warnings if "/build/NEWS:" in warning) # Nits found before the threshold line new_news_nits = [ - nit - for nit in news_nits - if int(nit.split(":")[1]) <= threshold + nit for nit in news_nits if int(nit.split(":")[1]) <= threshold ] if new_news_nits: @@ -311,7 +308,8 @@ def main(argv: list[str] | None = None) -> int: exit_code = 0 wrong_directory_msg = "Must run this script from the repo root" - assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg + if not Path("Doc").exists() or not Path("Doc").is_dir(): + raise RuntimeError(wrong_directory_msg) with Path("Doc/sphinx-warnings.txt").open(encoding="UTF-8") as f: warnings = f.read().splitlines() @@ -339,7 +337,9 @@ def main(argv: list[str] | None = None) -> int: ) if args.fail_if_improved: - exit_code += fail_if_improved(files_with_expected_nits, files_with_nits) + exit_code += fail_if_improved( + files_with_expected_nits, files_with_nits + ) if args.fail_if_new_news_nit: exit_code += fail_if_new_news_nit(warnings, args.fail_if_new_news_nit) diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 7916b178f1c0f1..a65cf71e4affe3 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -1,226 +1,305 @@ -""" - c_annotations.py - ~~~~~~~~~~~~~~~~ - - Supports annotations for C API elements: +"""Support annotations for C API elements. - * reference count annotations for C API functions. Based on - refcount.py and anno-api.py in the old Python documentation tools. +* Reference count annotations for C API functions. +* Stable ABI annotations +* Limited API annotations - * stable API annotations +Configuration: +* Set ``refcount_file`` to the path to the reference count data file. +* Set ``stable_abi_file`` to the path to stable ABI list. +""" - Usage: - * Set the `refcount_file` config value to the path to the reference - count data file. - * Set the `stable_abi_file` config value to the path to stable ABI list. +from __future__ import annotations - :copyright: Copyright 2007-2014 by Georg Brandl. - :license: Python license. -""" +import csv +import dataclasses +from pathlib import Path +from typing import TYPE_CHECKING -from os import path +import sphinx from docutils import nodes -from docutils.parsers.rst import directives -from docutils.parsers.rst import Directive from docutils.statemachine import StringList -from sphinx.locale import _ as sphinx_gettext -import csv - from sphinx import addnodes -from sphinx.domains.c import CObject +from sphinx.locale import _ as sphinx_gettext +from sphinx.util.docutils import SphinxDirective +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata -REST_ROLE_MAP = { - 'function': 'func', - 'macro': 'macro', - 'member': 'member', - 'type': 'type', - 'var': 'data', +ROLE_TO_OBJECT_TYPE = { + "func": "function", + "macro": "macro", + "member": "member", + "type": "type", + "data": "var", } -class RCEntry: - def __init__(self, name): - self.name = name - self.args = [] - self.result_type = '' - self.result_refs = None - - -class Annotations: - def __init__(self, refcount_filename, stable_abi_file): - self.refcount_data = {} - with open(refcount_filename, encoding='utf8') as fp: - for line in fp: - line = line.strip() - if line[:1] in ("", "#"): - # blank lines and comments - continue - parts = line.split(":", 4) - if len(parts) != 5: - raise ValueError(f"Wrong field count in {line!r}") - function, type, arg, refcount, comment = parts - # Get the entry, creating it if needed: - try: - entry = self.refcount_data[function] - except KeyError: - entry = self.refcount_data[function] = RCEntry(function) - if not refcount or refcount == "null": - refcount = None - else: - refcount = int(refcount) - # Update the entry with the new parameter or the result - # information. - if arg: - entry.args.append((arg, type, refcount)) - else: - entry.result_type = type - entry.result_refs = refcount - - self.stable_abi_data = {} - with open(stable_abi_file, encoding='utf8') as fp: - for record in csv.DictReader(fp): - name = record['name'] - self.stable_abi_data[name] = record - - def add_annotations(self, app, doctree): - for node in doctree.findall(addnodes.desc_content): - par = node.parent - if par['domain'] != 'c': - continue - if not par[0].has_key('ids') or not par[0]['ids']: - continue - name = par[0]['ids'][0] - if name.startswith("c."): - name = name[2:] - - objtype = par['objtype'] - - # Stable ABI annotation. These have two forms: - # Part of the [Stable ABI](link). - # Part of the [Stable ABI](link) since version X.Y. - # For structs, there's some more info in the message: - # Part of the [Limited API](link) (as an opaque struct). - # Part of the [Stable ABI](link) (including all members). - # Part of the [Limited API](link) (Only some members are part - # of the stable ABI.). - # ... all of which can have "since version X.Y" appended. - record = self.stable_abi_data.get(name) - if record: - if record['role'] != objtype: - raise ValueError( - f"Object type mismatch in limited API annotation " - f"for {name}: {record['role']!r} != {objtype!r}") - stable_added = record['added'] - message = sphinx_gettext('Part of the') - message = message.center(len(message) + 2) - emph_node = nodes.emphasis(message, message, - classes=['stableabi']) - ref_node = addnodes.pending_xref( - 'Stable ABI', refdomain="std", reftarget='stable', - reftype='ref', refexplicit="False") - struct_abi_kind = record['struct_abi_kind'] - if struct_abi_kind in {'opaque', 'members'}: - ref_node += nodes.Text(sphinx_gettext('Limited API')) - else: - ref_node += nodes.Text(sphinx_gettext('Stable ABI')) - emph_node += ref_node - if struct_abi_kind == 'opaque': - emph_node += nodes.Text(' ' + sphinx_gettext('(as an opaque struct)')) - elif struct_abi_kind == 'full-abi': - emph_node += nodes.Text(' ' + sphinx_gettext('(including all members)')) - if record['ifdef_note']: - emph_node += nodes.Text(' ' + record['ifdef_note']) - if stable_added == '3.2': - # Stable ABI was introduced in 3.2. - pass - else: - emph_node += nodes.Text(' ' + sphinx_gettext('since version %s') % stable_added) - emph_node += nodes.Text('.') - if struct_abi_kind == 'members': - emph_node += nodes.Text( - ' ' + sphinx_gettext('(Only some members are part of the stable ABI.)')) - node.insert(0, emph_node) - - # Unstable API annotation. - if name.startswith('PyUnstable'): - warn_node = nodes.admonition( - classes=['unstable-c-api', 'warning']) - message = sphinx_gettext('This is') + ' ' - emph_node = nodes.emphasis(message, message) - ref_node = addnodes.pending_xref( - 'Unstable API', refdomain="std", - reftarget='unstable-c-api', - reftype='ref', refexplicit="False") - ref_node += nodes.Text(sphinx_gettext('Unstable API')) - emph_node += ref_node - emph_node += nodes.Text(sphinx_gettext('. It may change without warning in minor releases.')) - warn_node += emph_node - node.insert(0, warn_node) - - # Return value annotation - if objtype != 'function': - continue - entry = self.refcount_data.get(name) - if not entry: - continue - elif not entry.result_type.endswith("Object*"): - continue - classes = ['refcount'] - if entry.result_refs is None: - rc = sphinx_gettext('Return value: Always NULL.') - classes.append('return_null') - elif entry.result_refs: - rc = sphinx_gettext('Return value: New reference.') - classes.append('return_new_ref') - else: - rc = sphinx_gettext('Return value: Borrowed reference.') - classes.append('return_borrowed_ref') - node.insert(0, nodes.emphasis(rc, rc, classes=classes)) - - -def init_annotations(app): - annotations = Annotations( - path.join(app.srcdir, app.config.refcount_file), - path.join(app.srcdir, app.config.stable_abi_file), +@dataclasses.dataclass(slots=True) +class RefCountEntry: + # Name of the function. + name: str + # List of (argument name, type, refcount effect) tuples. + # (Currently not used. If it was, a dataclass might work better.) + args: list = dataclasses.field(default_factory=list) + # Return type of the function. + result_type: str = "" + # Reference count effect for the return value. + result_refs: int | None = None + + +@dataclasses.dataclass(frozen=True, slots=True) +class StableABIEntry: + # Role of the object. + # Source: Each [item_kind] in stable_abi.toml is mapped to a C Domain role. + role: str + # Name of the object. + # Source: [.*] in stable_abi.toml. + name: str + # Version when the object was added to the stable ABI. + # (Source: [.*.added] in stable_abi.toml. + added: str + # An explananatory blurb for the ifdef. + # Source: ``feature_macro.*.doc`` in stable_abi.toml. + ifdef_note: str + # Defines how much of the struct is exposed. Only relevant for structs. + # Source: [.*.struct_abi_kind] in stable_abi.toml. + struct_abi_kind: str + + +def read_refcount_data(refcount_filename: Path) -> dict[str, RefCountEntry]: + refcount_data = {} + refcounts = refcount_filename.read_text(encoding="utf8") + for line in refcounts.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + # blank lines and comments + continue + + # Each line is of the form + # function ':' type ':' [param name] ':' [refcount effect] ':' [comment] + parts = line.split(":", 4) + if len(parts) != 5: + raise ValueError(f"Wrong field count in {line!r}") + function, type, arg, refcount, _comment = parts + + # Get the entry, creating it if needed: + try: + entry = refcount_data[function] + except KeyError: + entry = refcount_data[function] = RefCountEntry(function) + if not refcount or refcount == "null": + refcount = None + else: + refcount = int(refcount) + # Update the entry with the new parameter + # or the result information. + if arg: + entry.args.append((arg, type, refcount)) + else: + entry.result_type = type + entry.result_refs = refcount + + return refcount_data + + +def read_stable_abi_data(stable_abi_file: Path) -> dict[str, StableABIEntry]: + stable_abi_data = {} + with open(stable_abi_file, encoding="utf8") as fp: + for record in csv.DictReader(fp): + name = record["name"] + stable_abi_data[name] = StableABIEntry(**record) + + return stable_abi_data + + +def add_annotations(app: Sphinx, doctree: nodes.document) -> None: + state = app.env.domaindata["c_annotations"] + refcount_data = state["refcount_data"] + stable_abi_data = state["stable_abi_data"] + for node in doctree.findall(addnodes.desc_content): + par = node.parent + if par["domain"] != "c": + continue + if not par[0].get("ids", None): + continue + name = par[0]["ids"][0] + if name.startswith("c."): + name = name[2:] + + objtype = par["objtype"] + + # Stable ABI annotation. + if record := stable_abi_data.get(name): + if ROLE_TO_OBJECT_TYPE[record.role] != objtype: + msg = ( + f"Object type mismatch in limited API annotation for {name}: " + f"{ROLE_TO_OBJECT_TYPE[record.role]!r} != {objtype!r}" + ) + raise ValueError(msg) + annotation = _stable_abi_annotation(record) + node.insert(0, annotation) + + # Unstable API annotation. + if name.startswith("PyUnstable"): + annotation = _unstable_api_annotation() + node.insert(0, annotation) + + # Return value annotation + if objtype != "function": + continue + if name not in refcount_data: + continue + entry = refcount_data[name] + if not entry.result_type.endswith("Object*"): + continue + annotation = _return_value_annotation(entry.result_refs) + node.insert(0, annotation) + + +def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: + """Create the Stable ABI annotation. + + These have two forms: + Part of the `Stable ABI `_. + Part of the `Stable ABI `_ since version X.Y. + For structs, there's some more info in the message: + Part of the `Limited API `_ (as an opaque struct). + Part of the `Stable ABI `_ (including all members). + Part of the `Limited API `_ (Only some members are part + of the stable ABI.). + ... all of which can have "since version X.Y" appended. + """ + stable_added = record.added + message = sphinx_gettext("Part of the") + message = message.center(len(message) + 2) + emph_node = nodes.emphasis(message, message, classes=["stableabi"]) + ref_node = addnodes.pending_xref( + "Stable ABI", + refdomain="std", + reftarget="stable", + reftype="ref", + refexplicit="False", + ) + struct_abi_kind = record.struct_abi_kind + if struct_abi_kind in {"opaque", "members"}: + ref_node += nodes.Text(sphinx_gettext("Limited API")) + else: + ref_node += nodes.Text(sphinx_gettext("Stable ABI")) + emph_node += ref_node + if struct_abi_kind == "opaque": + emph_node += nodes.Text(" " + sphinx_gettext("(as an opaque struct)")) + elif struct_abi_kind == "full-abi": + emph_node += nodes.Text( + " " + sphinx_gettext("(including all members)") + ) + if record.ifdef_note: + emph_node += nodes.Text(f" {record.ifdef_note}") + if stable_added == "3.2": + # Stable ABI was introduced in 3.2. + pass + else: + emph_node += nodes.Text( + " " + sphinx_gettext("since version %s") % stable_added + ) + emph_node += nodes.Text(".") + if struct_abi_kind == "members": + msg = " " + sphinx_gettext( + "(Only some members are part of the stable ABI.)" + ) + emph_node += nodes.Text(msg) + return emph_node + + +def _unstable_api_annotation() -> nodes.admonition: + ref_node = addnodes.pending_xref( + "Unstable API", + nodes.Text(sphinx_gettext("Unstable API")), + refdomain="std", + reftarget="unstable-c-api", + reftype="ref", + refexplicit="False", + ) + emph_node = nodes.emphasis( + "This is ", + sphinx_gettext("This is") + " ", + ref_node, + nodes.Text( + sphinx_gettext( + ". It may change without warning in minor releases." + ) + ), + ) + return nodes.admonition( + "", + emph_node, + classes=["unstable-c-api", "warning"], ) - app.connect('doctree-read', annotations.add_annotations) - class LimitedAPIList(Directive): - has_content = False - required_arguments = 0 - optional_arguments = 0 - final_argument_whitespace = True +def _return_value_annotation(result_refs: int | None) -> nodes.emphasis: + classes = ["refcount"] + if result_refs is None: + rc = sphinx_gettext("Return value: Always NULL.") + classes.append("return_null") + elif result_refs: + rc = sphinx_gettext("Return value: New reference.") + classes.append("return_new_ref") + else: + rc = sphinx_gettext("Return value: Borrowed reference.") + classes.append("return_borrowed_ref") + return nodes.emphasis(rc, rc, classes=classes) + + +class LimitedAPIList(SphinxDirective): + has_content = False + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = True - def run(self): - content = [] - for record in annotations.stable_abi_data.values(): - role = REST_ROLE_MAP[record['role']] - name = record['name'] - content.append(f'* :c:{role}:`{name}`') + def run(self) -> list[nodes.Node]: + state = self.env.domaindata["c_annotations"] + content = [ + f"* :c:{record.role}:`{record.name}`" + for record in state["stable_abi_data"].values() + ] + node = nodes.paragraph() + self.state.nested_parse(StringList(content), 0, node) + return [node] - pnode = nodes.paragraph() - self.state.nested_parse(StringList(content), 0, pnode) - return [pnode] - app.add_directive('limited-api-list', LimitedAPIList) +def init_annotations(app: Sphinx) -> None: + # Using domaindata is a bit hack-ish, + # but allows storing state without a global variable or closure. + app.env.domaindata["c_annotations"] = state = {} + state["refcount_data"] = read_refcount_data( + Path(app.srcdir, app.config.refcount_file) + ) + state["stable_abi_data"] = read_stable_abi_data( + Path(app.srcdir, app.config.stable_abi_file) + ) -def setup(app): - app.add_config_value('refcount_file', '', True) - app.add_config_value('stable_abi_file', '', True) - app.connect('builder-inited', init_annotations) +def setup(app: Sphinx) -> ExtensionMetadata: + app.add_config_value("refcount_file", "", "env", types={str}) + app.add_config_value("stable_abi_file", "", "env", types={str}) + app.add_directive("limited-api-list", LimitedAPIList) + app.connect("builder-inited", init_annotations) + app.connect("doctree-read", add_annotations) - # monkey-patch C object... - CObject.option_spec = { - 'noindex': directives.flag, - 'stableabi': directives.flag, - } - old_handle_signature = CObject.handle_signature + if sphinx.version_info[:2] < (7, 2): + from docutils.parsers.rst import directives + from sphinx.domains.c import CObject - def new_handle_signature(self, sig, signode): - signode.parent['stableabi'] = 'stableabi' in self.options - return old_handle_signature(self, sig, signode) - CObject.handle_signature = new_handle_signature - return {'version': '1.0', 'parallel_read_safe': True} + # monkey-patch C object... + CObject.option_spec |= { + "no-index-entry": directives.flag, + "no-contents-entry": directives.flag, + } + + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/escape4chm.py b/Doc/tools/extensions/escape4chm.py deleted file mode 100644 index 89970975b9032b..00000000000000 --- a/Doc/tools/extensions/escape4chm.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Escape the `body` part of .chm source file to 7-bit ASCII, to fix visual -effect on some MBCS Windows systems. - -https://bugs.python.org/issue32174 -""" - -import pathlib -import re -from html.entities import codepoint2name - -from sphinx.util.logging import getLogger - -# escape the characters which codepoint > 0x7F -def _process(string): - def escape(matchobj): - codepoint = ord(matchobj.group(0)) - - name = codepoint2name.get(codepoint) - if name is None: - return '&#%d;' % codepoint - else: - return '&%s;' % name - - return re.sub(r'[^\x00-\x7F]', escape, string) - -def escape_for_chm(app, pagename, templatename, context, doctree): - # only works for .chm output - if getattr(app.builder, 'name', '') != 'htmlhelp': - return - - # escape the `body` part to 7-bit ASCII - body = context.get('body') - if body is not None: - context['body'] = _process(body) - -def fixup_keywords(app, exception): - # only works for .chm output - if getattr(app.builder, 'name', '') != 'htmlhelp' or exception: - return - - getLogger(__name__).info('fixing HTML escapes in keywords file...') - outdir = pathlib.Path(app.builder.outdir) - outname = app.builder.config.htmlhelp_basename - with open(outdir / (outname + '.hhk'), 'rb') as f: - index = f.read() - with open(outdir / (outname + '.hhk'), 'wb') as f: - f.write(index.replace(b''', b''')) - -def setup(app): - # `html-page-context` event emitted when the HTML builder has - # created a context dictionary to render a template with. - app.connect('html-page-context', escape_for_chm) - # `build-finished` event emitted when all the files have been - # output. - app.connect('build-finished', fixup_keywords) - - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/glossary_search.py b/Doc/tools/extensions/glossary_search.py index 7c93b1e4990603..502b6cd95bcb94 100644 --- a/Doc/tools/extensions/glossary_search.py +++ b/Doc/tools/extensions/glossary_search.py @@ -1,63 +1,63 @@ -# -*- coding: utf-8 -*- -""" - glossary_search.py - ~~~~~~~~~~~~~~~~ +"""Feature search results for glossary items prominently.""" - Feature search results for glossary items prominently. +from __future__ import annotations - :license: Python license. -""" import json -import os.path -from docutils.nodes import definition_list_item +from pathlib import Path +from typing import TYPE_CHECKING + +from docutils import nodes from sphinx.addnodes import glossary from sphinx.util import logging +if TYPE_CHECKING: + from sphinx.application import Sphinx + from sphinx.util.typing import ExtensionMetadata logger = logging.getLogger(__name__) -STATIC_DIR = '_static' -JSON = 'glossary.json' -def process_glossary_nodes(app, doctree, fromdocname): +def process_glossary_nodes( + app: Sphinx, + doctree: nodes.document, + _docname: str, +) -> None: if app.builder.format != 'html' or app.builder.embedded: return - terms = {} + if hasattr(app.env, 'glossary_terms'): + terms = app.env.glossary_terms + else: + terms = app.env.glossary_terms = {} for node in doctree.findall(glossary): - for glossary_item in node.findall(definition_list_item): - term = glossary_item[0].astext().lower() - definition = glossary_item[1] + for glossary_item in node.findall(nodes.definition_list_item): + term = glossary_item[0].astext() + definition = glossary_item[-1] rendered = app.builder.render_partial(definition) - terms[term] = { - 'title': glossary_item[0].astext(), - 'body': rendered['html_body'] + terms[term.lower()] = { + 'title': term, + 'body': rendered['html_body'], } - if hasattr(app.env, 'glossary_terms'): - app.env.glossary_terms.update(terms) - else: - app.env.glossary_terms = terms -def on_build_finish(app, exc): - if not hasattr(app.env, 'glossary_terms'): - return - if not app.env.glossary_terms: +def write_glossary_json(app: Sphinx, _exc: Exception) -> None: + if not getattr(app.env, 'glossary_terms', None): return - logger.info(f'Writing {JSON}', color='green') - - dest_dir = os.path.join(app.outdir, STATIC_DIR) - os.makedirs(dest_dir, exist_ok=True) - - with open(os.path.join(dest_dir, JSON), 'w') as f: - json.dump(app.env.glossary_terms, f) + logger.info('Writing glossary.json', color='green') + dest = Path(app.outdir, '_static', 'glossary.json') + dest.parent.mkdir(exist_ok=True) + dest.write_text(json.dumps(app.env.glossary_terms), encoding='utf-8') -def setup(app): +def setup(app: Sphinx) -> ExtensionMetadata: app.connect('doctree-resolved', process_glossary_nodes) - app.connect('build-finished', on_build_finish) + app.connect('build-finished', write_glossary_json) - return {'version': '0.1', 'parallel_read_safe': True} + return { + 'version': '1.0', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/Doc/tools/extensions/lexers/__init__.py b/Doc/tools/extensions/lexers/__init__.py new file mode 100644 index 00000000000000..e12ac5be8139cc --- /dev/null +++ b/Doc/tools/extensions/lexers/__init__.py @@ -0,0 +1,15 @@ +from .asdl_lexer import ASDLLexer +from .peg_lexer import PEGLexer + + +def setup(app): + # Used for highlighting Parser/Python.asdl in library/ast.rst + app.add_lexer("asdl", ASDLLexer) + # Used for highlighting Grammar/python.gram in reference/grammar.rst + app.add_lexer("peg", PEGLexer) + + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/Doc/tools/extensions/asdl_highlight.py b/Doc/tools/extensions/lexers/asdl_lexer.py similarity index 62% rename from Doc/tools/extensions/asdl_highlight.py rename to Doc/tools/extensions/lexers/asdl_lexer.py index 42863a4b3bcd6a..3a74174a1f7dfb 100644 --- a/Doc/tools/extensions/asdl_highlight.py +++ b/Doc/tools/extensions/lexers/asdl_lexer.py @@ -1,15 +1,6 @@ -import sys -from pathlib import Path +from pygments.lexer import RegexLexer, bygroups, include +from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text -CPYTHON_ROOT = Path(__file__).resolve().parent.parent.parent.parent -sys.path.append(str(CPYTHON_ROOT / "Parser")) - -from pygments.lexer import RegexLexer, bygroups, include, words -from pygments.token import (Comment, Keyword, Name, Operator, - Punctuation, Text) - -from asdl import builtin_types -from sphinx.highlighting import lexers class ASDLLexer(RegexLexer): name = "ASDL" @@ -34,7 +25,10 @@ class ASDLLexer(RegexLexer): r"(\w+)(\*\s|\?\s|\s)(\w+)", bygroups(Name.Builtin.Pseudo, Operator, Name), ), - (words(builtin_types), Name.Builtin), + # Keep in line with ``builtin_types`` from Parser/asdl.py. + # ASDL's 4 builtin types are + # constant, identifier, int, string + ("constant|identifier|int|string", Name.Builtin), (r"attributes", Name.Builtin), ( _name + _text_ws + "(=)", @@ -46,8 +40,3 @@ class ASDLLexer(RegexLexer): (r".", Text), ], } - - -def setup(app): - lexers["asdl"] = ASDLLexer() - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/peg_highlight.py b/Doc/tools/extensions/lexers/peg_lexer.py similarity index 94% rename from Doc/tools/extensions/peg_highlight.py rename to Doc/tools/extensions/lexers/peg_lexer.py index 4bdc2ee1861334..827af205583f61 100644 --- a/Doc/tools/extensions/peg_highlight.py +++ b/Doc/tools/extensions/lexers/peg_lexer.py @@ -1,8 +1,6 @@ from pygments.lexer import RegexLexer, bygroups, include from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text -from sphinx.highlighting import lexers - class PEGLexer(RegexLexer): """Pygments Lexer for PEG grammar (.gram) files @@ -79,8 +77,3 @@ class PEGLexer(RegexLexer): (r".", Text), ], } - - -def setup(app): - lexers["peg"] = PEGLexer() - return {"version": "1.0", "parallel_read_safe": True} diff --git a/Doc/tools/extensions/patchlevel.py b/Doc/tools/extensions/patchlevel.py index 617f28c2527ddf..f2df6db47a2227 100644 --- a/Doc/tools/extensions/patchlevel.py +++ b/Doc/tools/extensions/patchlevel.py @@ -1,68 +1,77 @@ -# -*- coding: utf-8 -*- -""" - patchlevel.py - ~~~~~~~~~~~~~ +"""Extract version information from Include/patchlevel.h.""" - Extract version info from Include/patchlevel.h. - Adapted from Doc/tools/getversioninfo. +import re +import sys +from pathlib import Path +from typing import Literal, NamedTuple - :copyright: 2007-2008 by Georg Brandl. - :license: Python license. -""" +CPYTHON_ROOT = Path( + __file__, # cpython/Doc/tools/extensions/patchlevel.py + "..", # cpython/Doc/tools/extensions + "..", # cpython/Doc/tools + "..", # cpython/Doc + "..", # cpython +).resolve() +PATCHLEVEL_H = CPYTHON_ROOT / "Include" / "patchlevel.h" -from __future__ import print_function +RELEASE_LEVELS = { + "PY_RELEASE_LEVEL_ALPHA": "alpha", + "PY_RELEASE_LEVEL_BETA": "beta", + "PY_RELEASE_LEVEL_GAMMA": "candidate", + "PY_RELEASE_LEVEL_FINAL": "final", +} -import os -import re -import sys -def get_header_version_info(srcdir): - patchlevel_h = os.path.join(srcdir, '..', 'Include', 'patchlevel.h') +class version_info(NamedTuple): # noqa: N801 + major: int #: Major release number + minor: int #: Minor release number + micro: int #: Patch release number + releaselevel: Literal["alpha", "beta", "candidate", "final"] + serial: int #: Serial release number - # This won't pick out all #defines, but it will pick up the ones we - # care about. - rx = re.compile(r'\s*#define\s+([a-zA-Z][a-zA-Z_0-9]*)\s+([a-zA-Z_0-9]+)') - d = {} - with open(patchlevel_h) as f: - for line in f: - m = rx.match(line) - if m is not None: - name, value = m.group(1, 2) - d[name] = value +def get_header_version_info() -> version_info: + # Capture PY_ prefixed #defines. + pat = re.compile(r"\s*#define\s+(PY_\w*)\s+(\w+)", re.ASCII) - release = version = '%s.%s' % (d['PY_MAJOR_VERSION'], d['PY_MINOR_VERSION']) - micro = int(d['PY_MICRO_VERSION']) - release += '.' + str(micro) + defines = {} + patchlevel_h = PATCHLEVEL_H.read_text(encoding="utf-8") + for line in patchlevel_h.splitlines(): + if (m := pat.match(line)) is not None: + name, value = m.groups() + defines[name] = value - level = d['PY_RELEASE_LEVEL'] - suffixes = { - 'PY_RELEASE_LEVEL_ALPHA': 'a', - 'PY_RELEASE_LEVEL_BETA': 'b', - 'PY_RELEASE_LEVEL_GAMMA': 'rc', - } - if level != 'PY_RELEASE_LEVEL_FINAL': - release += suffixes[level] + str(int(d['PY_RELEASE_SERIAL'])) - return version, release + return version_info( + major=int(defines["PY_MAJOR_VERSION"]), + minor=int(defines["PY_MINOR_VERSION"]), + micro=int(defines["PY_MICRO_VERSION"]), + releaselevel=RELEASE_LEVELS[defines["PY_RELEASE_LEVEL"]], + serial=int(defines["PY_RELEASE_SERIAL"]), + ) -def get_sys_version_info(): - major, minor, micro, level, serial = sys.version_info - release = version = '%s.%s' % (major, minor) - release += '.%s' % micro - if level != 'final': - release += '%s%s' % (level[0], serial) +def format_version_info(info: version_info) -> tuple[str, str]: + version = f"{info.major}.{info.minor}" + release = f"{info.major}.{info.minor}.{info.micro}" + if info.releaselevel != "final": + suffix = {"alpha": "a", "beta": "b", "candidate": "rc"} + release += f"{suffix[info.releaselevel]}{info.serial}" return version, release def get_version_info(): try: - return get_header_version_info('.') - except (IOError, OSError): - version, release = get_sys_version_info() - print('Can\'t get version info from Include/patchlevel.h, ' \ - 'using version of this interpreter (%s).' % release, file=sys.stderr) + info = get_header_version_info() + return format_version_info(info) + except OSError: + version, release = format_version_info(sys.version_info) + print( + f"Failed to get version info from Include/patchlevel.h, " + f"using version of this interpreter ({release}).", + file=sys.stderr, + ) return version, release -if __name__ == '__main__': - print(get_header_version_info('.')[1]) + +if __name__ == "__main__": + print(format_version_info(get_header_version_info())[1]) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8b592d4b4adcea..f5be19a8d49cc9 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -15,13 +15,15 @@ from time import asctime from pprint import pformat -from docutils import nodes, utils +import sphinx +from docutils import nodes from docutils.io import StringOutput -from docutils.parsers.rst import Directive -from docutils.utils import new_document +from docutils.parsers.rst import directives +from docutils.utils import new_document, unescape from sphinx import addnodes from sphinx.builders import Builder -from sphinx.domains.python import PyFunction, PyMethod +from sphinx.domains.changeset import VersionChange, versionlabels, versionlabel_classes +from sphinx.domains.python import PyFunction, PyMethod, PyModule from sphinx.errors import NoUri from sphinx.locale import _ as sphinx_gettext from sphinx.util import logging @@ -48,11 +50,14 @@ std.token_re = re.compile(r'`((~?[\w-]*:)?\w+)`') +# backport :no-index: +PyModule.option_spec['no-index'] = directives.flag + # Support for marking up and linking to bugs.python.org issues def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - issue = utils.unescape(text) + issue = unescape(text) # sanity check: there are no bpo issues within these two values if 47261 < int(issue) < 400000: msg = inliner.reporter.error(f'The BPO ID {text!r} seems too high -- ' @@ -67,7 +72,7 @@ def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): # Support for marking up and linking to GitHub issues def gh_issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - issue = utils.unescape(text) + issue = unescape(text) # sanity check: all GitHub issues have ID >= 32426 # even though some of them are also valid BPO IDs if int(issue) < 32426: @@ -82,7 +87,7 @@ def gh_issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): # Support for marking up implementation details -class ImplementationDetail(Directive): +class ImplementationDetail(SphinxDirective): has_content = True final_argument_whitespace = True @@ -214,7 +219,7 @@ def audit_events_merge(app, env, docnames, other): env.all_audit_events[name] = value -class AuditEvent(Directive): +class AuditEvent(SphinxDirective): has_content = True required_arguments = 1 @@ -244,15 +249,14 @@ def run(self): text = label.format(name="``{}``".format(name), args=", ".join("``{}``".format(a) for a in args if a)) - env = self.state.document.settings.env - if not hasattr(env, 'all_audit_events'): - env.all_audit_events = {} + if not hasattr(self.env, 'all_audit_events'): + self.env.all_audit_events = {} new_info = { 'source': [], 'args': args } - info = env.all_audit_events.setdefault(name, new_info) + info = self.env.all_audit_events.setdefault(name, new_info) if info is not new_info: if not self._do_args_match(info['args'], new_info['args']): self.logger.warning( @@ -272,7 +276,7 @@ def run(self): ) ids.append(target) - info['source'].append((env.docname, target)) + info['source'].append((self.env.docname, target)) pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) pnode.line = self.lineno @@ -310,7 +314,7 @@ class audit_event_list(nodes.General, nodes.Element): pass -class AuditEventListDirective(Directive): +class AuditEventListDirective(SphinxDirective): def run(self): return [audit_event_list('')] @@ -395,58 +399,34 @@ def run(self): # Support for documenting version of removal in deprecations -class DeprecatedRemoved(Directive): - has_content = True +class DeprecatedRemoved(VersionChange): required_arguments = 2 - optional_arguments = 1 - final_argument_whitespace = True - option_spec = {} - _deprecated_label = sphinx_gettext('Deprecated since version {deprecated}, will be removed in version {removed}') - _removed_label = sphinx_gettext('Deprecated since version {deprecated}, removed in version {removed}') + _deprecated_label = sphinx_gettext('Deprecated since version %s, will be removed in version %s') + _removed_label = sphinx_gettext('Deprecated since version %s, removed in version %s') def run(self): - node = addnodes.versionmodified() - node.document = self.state.document - node['type'] = 'deprecated-removed' - version = (self.arguments[0], self.arguments[1]) - node['version'] = version - env = self.state.document.settings.env - current_version = tuple(int(e) for e in env.config.version.split('.')) - removed_version = tuple(int(e) for e in self.arguments[1].split('.')) + # Replace the first two arguments (deprecated version and removed version) + # with a single tuple of both versions. + version_deprecated = self.arguments[0] + version_removed = self.arguments.pop(1) + self.arguments[0] = version_deprecated, version_removed + + # Set the label based on if we have reached the removal version + current_version = tuple(map(int, self.config.version.split('.'))) + removed_version = tuple(map(int, version_removed.split('.'))) if current_version < removed_version: - label = self._deprecated_label - else: - label = self._removed_label - - text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) - if len(self.arguments) == 3: - inodes, messages = self.state.inline_text(self.arguments[2], - self.lineno+1) - para = nodes.paragraph(self.arguments[2], '', *inodes, translatable=False) - node.append(para) + versionlabels[self.name] = self._deprecated_label + versionlabel_classes[self.name] = 'deprecated' else: - messages = [] - if self.content: - self.state.nested_parse(self.content, self.content_offset, node) - if len(node): - if isinstance(node[0], nodes.paragraph) and node[0].rawsource: - content = nodes.inline(node[0].rawsource, translatable=True) - content.source = node[0].source - content.line = node[0].line - content += node[0].children - node[0].replace_self(nodes.paragraph('', '', content, translatable=False)) - node[0].insert(0, nodes.inline('', '%s: ' % text, - classes=['versionmodified'])) - else: - para = nodes.paragraph('', '', - nodes.inline('', '%s.' % text, - classes=['versionmodified']), - translatable=False) - node.append(para) - env = self.state.document.settings.env - env.get_domain('changeset').note_changeset(node) - return [node] + messages + versionlabels[self.name] = self._removed_label + versionlabel_classes[self.name] = 'removed' + try: + return super().run() + finally: + # reset versionlabels and versionlabel_classes + versionlabels[self.name] = '' + versionlabel_classes[self.name] = '' # Support for including Misc/NEWS @@ -456,7 +436,7 @@ def run(self): whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") -class MiscNews(Directive): +class MiscNews(SphinxDirective): has_content = False required_arguments = 1 optional_arguments = 0 @@ -471,7 +451,7 @@ def run(self): if not source_dir: source_dir = path.dirname(path.abspath(source)) fpath = path.join(source_dir, fname) - self.state.document.settings.record_dependencies.add(fpath) + self.env.note_dependency(path.abspath(fpath)) try: with io.open(fpath, encoding='utf-8') as fp: content = fp.read() diff --git a/Doc/tools/static/glossary_search.js b/Doc/tools/static/glossary_search.js new file mode 100644 index 00000000000000..13d728dc027f1d --- /dev/null +++ b/Doc/tools/static/glossary_search.js @@ -0,0 +1,47 @@ +"use strict"; + +const GLOSSARY_PAGE = "glossary.html"; + +const glossary_search = async () => { + const response = await fetch("_static/glossary.json"); + if (!response.ok) { + throw new Error("Failed to fetch glossary.json"); + } + const glossary = await response.json(); + + const params = new URLSearchParams(document.location.search).get("q"); + if (!params) { + return; + } + + const searchParam = params.toLowerCase(); + const glossaryItem = glossary[searchParam]; + if (!glossaryItem) { + return; + } + + // set up the title text with a link to the glossary page + const glossaryTitle = document.getElementById("glossary-title"); + glossaryTitle.textContent = "Glossary: " + glossaryItem.title; + const linkTarget = searchParam.replace(/ /g, "-"); + glossaryTitle.href = GLOSSARY_PAGE + "#term-" + linkTarget; + + // rewrite any anchor links (to other glossary terms) + // to have a full reference to the glossary page + const glossaryBody = document.getElementById("glossary-body"); + glossaryBody.innerHTML = glossaryItem.body; + const anchorLinks = glossaryBody.querySelectorAll('a[href^="#"]'); + anchorLinks.forEach(function (link) { + const currentUrl = link.getAttribute("href"); + link.href = GLOSSARY_PAGE + currentUrl; + }); + + const glossaryResult = document.getElementById("glossary-result"); + glossaryResult.style.display = ""; +}; + +if (document.readyState !== "loading") { + glossary_search().catch(console.error); +} else { + document.addEventListener("DOMContentLoaded", glossary_search); +} diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index b5353d6fb77ab4..c3114e584fa942 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -21,23 +21,27 @@

Download Python {{ release }} Documentation

- - + + - - + + - - + + - - - + + + + + + + - +
FormatPacked as .zipPacked as .tar.bz2
PDF (US-Letter paper size)Download (ca. 13 MiB)Download (ca. 13 MiB)Download (ca. 17 MiB)Download (ca. 17 MiB)
PDF (A4 paper size)Download (ca. 13 MiB)Download (ca. 13 MiB)Download (ca. 17 MiB)Download (ca. 17 MiB)
HTMLDownload (ca. 9 MiB)Download (ca. 6 MiB)Download (ca. 13 MiB)Download (ca. 8 MiB)
Plain TextDownload (ca. 3 MiB)Download (ca. 2 MiB)
Plain textDownload (ca. 4 MiB)Download (ca. 3 MiB)
TexinfoDownload (ca. 9 MiB)Download (ca. 7 MiB)
EPUBDownload (ca. 5 MiB)Download (ca. 6 MiB)
diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 6f854e86ab8ef1..f2e9fbb0106452 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -33,6 +33,8 @@

{{ docstitle|e }}

{% trans %}C API reference{% endtrans %}

{% trans %}FAQs{% endtrans %}
{% trans %}Frequently asked questions (with answers!){% endtrans %}

+

{% trans %}Deprecations{% endtrans %}
+ {% trans %}Deprecated functionality{% endtrans %}

diff --git a/Doc/tools/templates/search.html b/Doc/tools/templates/search.html index 852974461380f2..6ddac5f828bab1 100644 --- a/Doc/tools/templates/search.html +++ b/Doc/tools/templates/search.html @@ -2,61 +2,16 @@ {% block extrahead %} {{ super() }} - + +{% endblock %} +{% block searchresults %} +
+ {# For glossary_search.js #} + +
{% endblock %} diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 1b64741c349ee9..675faa8c52477d 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -688,6 +688,11 @@ current class name with leading underscore(s) stripped. This mangling is done without regard to the syntactic position of the identifier, as long as it occurs within the definition of a class. +.. seealso:: + + The :ref:`private name mangling specifications ` + for details and special cases. + Name mangling is helpful for letting subclasses override methods without breaking intraclass method calls. For example:: diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index 0795e2fef98830..6093028f8307a3 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -6,7 +6,7 @@ .. _tut-fp-issues: ************************************************** -Floating Point Arithmetic: Issues and Limitations +Floating-Point Arithmetic: Issues and Limitations ************************************************** .. sectionauthor:: Tim Peters @@ -88,7 +88,7 @@ the one with 17 significant digits, ``0.10000000000000001``. Starting with Python 3.1, Python (on most systems) is now able to choose the shortest of these and simply display ``0.1``. -Note that this is in the very nature of binary floating-point: this is not a bug +Note that this is in the very nature of binary floating point: this is not a bug in Python, and it is not a bug in your code either. You'll see the same kind of thing in all languages that support your hardware's floating-point arithmetic (although some languages may not *display* the difference by default, or in all @@ -148,13 +148,13 @@ Binary floating-point arithmetic holds many surprises like this. The problem with "0.1" is explained in precise detail below, in the "Representation Error" section. See `Examples of Floating Point Problems `_ for -a pleasant summary of how binary floating-point works and the kinds of +a pleasant summary of how binary floating point works and the kinds of problems commonly encountered in practice. Also see `The Perils of Floating Point `_ for a more complete account of other common surprises. As that says near the end, "there are no easy answers." Still, don't be unduly -wary of floating-point! The errors in Python float operations are inherited +wary of floating point! The errors in Python float operations are inherited from the floating-point hardware, and on most machines are on the order of no more than 1 part in 2\*\*53 per operation. That's more than adequate for most tasks, but you do need to keep in mind that it's not decimal arithmetic and diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 0f16dae8b1418f..3ead346c9bfcec 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -62,7 +62,7 @@ For example:: 20 >>> (50 - 5*6) / 4 5.0 - >>> 8 / 5 # division always returns a floating point number + >>> 8 / 5 # division always returns a floating-point number 1.6 The integer numbers (e.g. ``2``, ``4``, ``20``) have type :class:`int`, @@ -544,7 +544,7 @@ This example introduces several new features. * The :func:`print` function writes the value of the argument(s) it is given. It differs from just writing the expression you want to write (as we did earlier in the calculator examples) in the way it handles multiple arguments, - floating point quantities, and strings. Strings are printed without quotes, + floating-point quantities, and strings. Strings are printed without quotes, and a space is inserted between items, so you can format things nicely, like this:: diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 86c94429339dfc..4b3eef313e76d7 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -138,7 +138,7 @@ Mathematics =========== The :mod:`math` module gives access to the underlying C library functions for -floating point math:: +floating-point math:: >>> import math >>> math.cos(math.pi / 4) diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 719f772e687008..a2f96b34b2dead 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -352,11 +352,11 @@ not want to run a full list sort:: .. _tut-decimal-fp: -Decimal Floating Point Arithmetic +Decimal Floating-Point Arithmetic ================================= The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for -decimal floating point arithmetic. Compared to the built-in :class:`float` +decimal floating-point arithmetic. Compared to the built-in :class:`float` implementation of binary floating point, the class is especially helpful for * financial applications and other uses which require exact decimal diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index a575760c963327..c175c4f8b5b1eb 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -793,6 +793,15 @@ conflict. This variable can also be modified by Python code using :data:`os.environ` to force inspect mode on program termination. + .. audit-event:: cpython.run_stdin "" "" + + .. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) + Emits audit events. + + .. versionchanged:: 3.13 + Uses PyREPL if possible, in which case :envvar:`PYTHONSTARTUP` is + also executed. Emits audit events. + .. envvar:: PYTHONUNBUFFERED diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 2a1f06e2d286ff..32adfb0ba6e5fc 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -16,8 +16,8 @@ Features and minimum versions required to build CPython: * On Windows, Microsoft Visual Studio 2017 or later is required. -* Support for `IEEE 754 `_ floating - point numbers and `floating point Not-a-Number (NaN) +* Support for `IEEE 754 `_ + floating-point numbers and `floating-point Not-a-Number (NaN) `_. * Support for threads. @@ -427,7 +427,7 @@ Options for third-party dependencies .. option:: PANEL_CFLAGS .. option:: PANEL_LIBS - C compiler and Linker flags for PANEL, overriding ``pkg-config``. + C compiler and linker flags for PANEL, overriding ``pkg-config``. C compiler and linker flags for ``libpanel`` or ``libpanelw``, used by :mod:`curses.panel` module, overriding ``pkg-config``. @@ -615,7 +615,7 @@ also be used to improve performance. .. option:: --without-mimalloc - Disable the fast mimalloc allocator :ref:`mimalloc ` + Disable the fast :ref:`mimalloc ` allocator (enabled by default). See also :envvar:`PYTHONMALLOC` environment variable. @@ -907,6 +907,25 @@ Security Options The settings ``python`` and *STRING* also set TLS 1.2 as minimum protocol version. +.. option:: --disable-safety + + Disable compiler options that are recommended by `OpenSSF`_ for security reasons with no performance overhead. + If this option is not enabled, CPython will be built based on safety compiler options with no slow down. + + .. _OpenSSF: https://openssf.org/ + + .. versionadded:: 3.14 + +.. option:: --enable-slower-safety + + Enable compiler options that are recommended by `OpenSSF`_ for security reasons which require overhead. + If this option is not enabled, CPython will not be built based on safety compiler options which performance impact. + + .. _OpenSSF: https://openssf.org/ + + .. versionadded:: 3.14 + + macOS Options ------------- diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index 774856e8aec2ac..ad959064a0af48 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -33,6 +33,17 @@ running, so you only need to deal with the Python code itself. Python at runtime on iOS ======================== +iOS version compatibility +------------------------- + +The minimum supported iOS version is specified at compile time, using the +:option:`--host` option to ``configure``. By default, when compiled for iOS, +Python will be compiled with a minimum supported iOS version of 13.0. To use a +different minimum iOS version, provide the version number as part of the +:option:`!--host` argument - for example, +``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 simulator build +with a deployment target of 15.4. + Platform identification ----------------------- diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index b4002f06e92adc..8eafb48461a67c 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -644,9 +644,9 @@ New and Improved Modules lists the function arguments and the local variables for each frame. * Various functions in the :mod:`time` module, such as :func:`~time.asctime` and - :func:`~time.localtime`, require a floating point argument containing the time in + :func:`~time.localtime`, require a floating-point argument containing the time in seconds since the epoch. The most common use of these functions is to work with - the current time, so the floating point argument has been made optional; when a + the current time, so the floating-point argument has been made optional; when a value isn't provided, the current time will be used. For example, log file entries usually need a string containing the current time; in Python 2.1, ``time.asctime()`` can be used, instead of the lengthier diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index d4dbe0570fbda5..5db34fa08c634a 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1249,7 +1249,7 @@ Some of the more notable changes are: * The :func:`pow` built-in function no longer supports 3 arguments when floating-point numbers are supplied. ``pow(x, y, z)`` returns ``(x**y) % z``, - but this is never useful for floating point numbers, and the final result varies + but this is never useful for floating-point numbers, and the final result varies unpredictably depending on the platform. A call such as ``pow(2.0, 8.0, 7.0)`` will now raise a :exc:`TypeError` exception. diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 8adf36e316c6fb..80849ab9a1a3db 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1382,7 +1382,7 @@ complete list of changes, or look through the CVS logs for all the details. In Python 2.4, the default will change to always returning floats. Application developers should enable this feature only if all their libraries - work properly when confronted with floating point time stamps, or if they use + work properly when confronted with floating-point time stamps, or if they use the tuple API. If used, the feature should be activated on an application level instead of trying to enable it on a per-use basis. diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index fc2de7124859a8..3650edbf5426b3 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -3051,7 +3051,7 @@ Changes to Python's build process and to the C API include: * Several functions return information about the platform's floating-point support. :c:func:`PyFloat_GetMax` returns - the maximum representable floating point value, + the maximum representable floating-point value, and :c:func:`PyFloat_GetMin` returns the minimum positive value. :c:func:`PyFloat_GetInfo` returns an object containing more information from the :file:`float.h` file, such as diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index c45f0887b41f4f..857b6bb33a6e10 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1198,7 +1198,7 @@ changes, or look through the Subversion logs for all the details. of the operands. Previously such comparisons would fall back to Python's default rules for comparing objects, which produced arbitrary results based on their type. Note that you still cannot combine - :class:`!Decimal` and floating-point in other operations such as addition, + :class:`!Decimal` and floating point in other operations such as addition, since you should be explicitly choosing how to convert between float and :class:`!Decimal`. (Fixed by Mark Dickinson; :issue:`2531`.) diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index 69b273e58385d2..b9606beb5f9ef9 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -205,9 +205,9 @@ Some smaller changes made to the core Python language are: (Contributed by Mark Dickinson; :issue:`4707`.) -* Python now uses David Gay's algorithm for finding the shortest floating - point representation that doesn't change its value. This should help - mitigate some of the confusion surrounding binary floating point +* Python now uses David Gay's algorithm for finding the shortest floating-point + representation that doesn't change its value. This should help + mitigate some of the confusion surrounding binary floating-point numbers. The significance is easily seen with a number like ``1.1`` which does not @@ -215,7 +215,7 @@ Some smaller changes made to the core Python language are: equivalent, an expression like ``float('1.1')`` evaluates to the nearest representable value which is ``0x1.199999999999ap+0`` in hex or ``1.100000000000000088817841970012523233890533447265625`` in decimal. That - nearest value was and still is used in subsequent floating point + nearest value was and still is used in subsequent floating-point calculations. What is new is how the number gets displayed. Formerly, Python used a @@ -224,7 +224,7 @@ Some smaller changes made to the core Python language are: using 17 digits was that it relied on IEEE-754 guarantees to assure that ``eval(repr(1.1))`` would round-trip exactly to its original value. The disadvantage is that many people found the output to be confusing (mistaking - intrinsic limitations of binary floating point representation as being a + intrinsic limitations of binary floating-point representation as being a problem with Python itself). The new algorithm for ``repr(1.1)`` is smarter and returns ``'1.1'``. @@ -236,8 +236,8 @@ Some smaller changes made to the core Python language are: it does not change the underlying values. So, it is still the case that ``1.1 + 2.2 != 3.3`` even though the representations may suggest otherwise. - The new algorithm depends on certain features in the underlying floating - point implementation. If the required features are not found, the old + The new algorithm depends on certain features in the underlying floating-point + implementation. If the required features are not found, the old algorithm will continue to be used. Also, the text pickle protocols assure cross-platform portability by using the old algorithm. @@ -550,7 +550,7 @@ Porting to Python 3.1 This section lists previously described changes and other bugfixes that may require changes to your code: -* The new floating point string representations can break existing doctests. +* The new floating-point string representations can break existing doctests. For example:: def e(): diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index b601bd453f5be5..20a060ddf4b6d3 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2138,7 +2138,7 @@ Build Changes :issue:`45440` and :issue:`46640`.) * Support for `IEEE 754 `_ - floating point numbers. + floating-point numbers. (Contributed by Victor Stinner in :issue:`46917`.) * The :c:macro:`!Py_NO_NAN` macro has been removed. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index cfc87cb9089c66..fc2b6519fb1307 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1330,155 +1330,15 @@ Deprecated therefore it will be removed in 3.14. (Contributed by Nikita Sobolev in :gh:`101866`.) -Pending Removal in Python 3.13 ------------------------------- - -The following modules and APIs have been deprecated in earlier Python releases, -and will be removed in Python 3.13. - -Modules (see :pep:`594`): - -* :mod:`!aifc` -* :mod:`!audioop` -* :mod:`!cgi` -* :mod:`!cgitb` -* :mod:`!chunk` -* :mod:`!crypt` -* :mod:`!imghdr` -* :mod:`!mailcap` -* :mod:`!msilib` -* :mod:`!nis` -* :mod:`!nntplib` -* :mod:`!ossaudiodev` -* :mod:`!pipes` -* :mod:`!sndhdr` -* :mod:`!spwd` -* :mod:`!sunau` -* :mod:`!telnetlib` -* :mod:`!uu` -* :mod:`!xdrlib` - -Other modules: - -* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) - -APIs: - -* :class:`!configparser.LegacyInterpolation` (:gh:`90765`) -* ``locale.resetlocale()`` (:gh:`90817`) -* :meth:`!turtle.RawTurtle.settiltangle` (:gh:`50096`) -* :func:`!unittest.findTestCases` (:gh:`50096`) -* :func:`!unittest.getTestCaseNames` (:gh:`50096`) -* :func:`!unittest.makeSuite` (:gh:`50096`) -* :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) -* :class:`!webbrowser.MacOSX` (:gh:`86421`) -* :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files()` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) - -Pending Removal in Python 3.14 ------------------------------- - -The following APIs have been deprecated -and will be removed in Python 3.14. - -* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction` - -* :mod:`ast`: - - * :class:`!ast.Num` - * :class:`!ast.Str` - * :class:`!ast.Bytes` - * :class:`!ast.NameConstant` - * :class:`!ast.Ellipsis` - -* :mod:`asyncio`: - - * :class:`!asyncio.MultiLoopChildWatcher` - * :class:`!asyncio.FastChildWatcher` - * :class:`!asyncio.AbstractChildWatcher` - * :class:`!asyncio.SafeChildWatcher` - * :func:`!asyncio.set_child_watcher` - * :func:`!asyncio.get_child_watcher`, - * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` - -* :mod:`collections.abc`: :class:`!collections.abc.ByteString`. - -* :mod:`email`: the *isdst* parameter in :func:`email.utils.localtime`. - -* :mod:`importlib.abc`: - - * :class:`!importlib.abc.ResourceReader` - * :class:`!importlib.abc.Traversable` - * :class:`!importlib.abc.TraversableResources` - -* :mod:`itertools`: Support for copy, deepcopy, and pickle operations. - -* :mod:`pkgutil`: - - * :func:`!pkgutil.find_loader` - * :func:`!pkgutil.get_loader`. - -* :mod:`pty`: - - * :func:`!pty.master_open` - * :func:`!pty.slave_open` - -* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` - -* :mod:`typing`: :class:`!typing.ByteString` - -* The ``__package__`` and ``__cached__`` attributes on module objects. - -* The :attr:`~codeobject.co_lnotab` attribute of code objects. - -Pending Removal in Python 3.15 ------------------------------- - -The following APIs have been deprecated -and will be removed in Python 3.15. - -APIs: - -* :func:`locale.getdefaultlocale` (:gh:`90817`) - - -Pending Removal in Future Versions ----------------------------------- - -The following APIs were deprecated in earlier Python versions and will be removed, -although there is currently no date scheduled for their removal. - -* :mod:`array`'s ``'u'`` format code (:gh:`57281`) +.. include:: ../deprecations/pending-removal-in-3.13.rst -* :class:`typing.Text` (:gh:`92332`) +.. include:: ../deprecations/pending-removal-in-3.14.rst -* :mod:`xml.etree.ElementTree`: Testing the truth value of an - :class:`xml.etree.ElementTree.Element` is deprecated. In a future release it - will always return True. Prefer explicit ``len(elem)`` or - ``elem is not None`` tests instead. +.. include:: ../deprecations/pending-removal-in-3.15.rst -* Currently Python accepts numeric literals immediately followed by keywords, - for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing - and ambiguous expressions like ``[0x1for x in y]`` (which can be - interpreted as ``[0x1 for x in y]`` or ``[0x1f or x in y]``). - A syntax warning is raised if the numeric literal is - immediately followed by one of keywords :keyword:`and`, :keyword:`else`, - :keyword:`for`, :keyword:`if`, :keyword:`in`, :keyword:`is` and :keyword:`or`. - In a future release it will be changed to a syntax error. (:gh:`87999`) +.. include:: ../deprecations/pending-removal-in-3.16.rst +.. include:: ../deprecations/pending-removal-in-future.rst Removed ======= diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 4a9a0b77d068b3..e89abfdd292f48 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -422,6 +422,10 @@ free-threaded build. Other Language Changes ====================== +* The :func:`exec` and :func:`eval` built-ins now accept their ``globals`` + and ``locals`` namespace arguments as keywords. + (Contributed by Raphael Gaschignard in :gh:`105879`) + * Allow the *count* argument of :meth:`str.replace` to be a keyword. (Contributed by Hugo van Kemenade in :gh:`106487`.) @@ -1631,318 +1635,13 @@ New Deprecations .. Add deprecations above alphabetically, not here at the end. -Pending Removal in Python 3.14 ------------------------------- - -* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction` are deprecated - and will be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`92248`.) - -* :mod:`ast`: The following features have been deprecated in documentation - since Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at - runtime when they are accessed or used, and will be removed in Python 3.14: - - * :class:`!ast.Num` - * :class:`!ast.Str` - * :class:`!ast.Bytes` - * :class:`!ast.NameConstant` - * :class:`!ast.Ellipsis` - - Use :class:`ast.Constant` instead. - (Contributed by Serhiy Storchaka in :gh:`90953`.) - -* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. - Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, - or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) - -* :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. - (Contributed by Alan Williams in :gh:`72346`.) - -* :mod:`importlib`: ``__package__`` and ``__cached__`` will cease to be set or - taken into consideration by the import system (:gh:`97879`). - -* :mod:`importlib.abc` deprecated classes: - - * :class:`!importlib.abc.ResourceReader` - * :class:`!importlib.abc.Traversable` - * :class:`!importlib.abc.TraversableResources` - - Use :mod:`importlib.resources.abc` classes instead: - - * :class:`importlib.resources.abc.Traversable` - * :class:`importlib.resources.abc.TraversableResources` - - (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) - -* :mod:`itertools` had undocumented, inefficient, historically buggy, - and inconsistent support for copy, deepcopy, and pickle operations. - This will be removed in 3.14 for a significant reduction in code - volume and maintenance burden. - (Contributed by Raymond Hettinger in :gh:`101588`.) - -* :mod:`multiprocessing`: The default start method will change to a safer one on - Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently - the default (:gh:`84559`). Adding a runtime warning about this was deemed too - disruptive as the majority of code is not expected to care. Use the - :func:`~multiprocessing.get_context` or - :func:`~multiprocessing.set_start_method` APIs to explicitly specify when - your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. - -* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and - :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is - deprecated. - -* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` - now raise :exc:`DeprecationWarning`; - use :func:`importlib.util.find_spec` instead. - (Contributed by Nikita Sobolev in :gh:`97850`.) - -* :mod:`pty`: - - * ``master_open()``: use :func:`pty.openpty`. - * ``slave_open()``: use :func:`pty.openpty`. - -* :mod:`sqlite3`: - - * :data:`!version` and :data:`!version_info`. - - * :meth:`~sqlite3.Cursor.execute` and :meth:`~sqlite3.Cursor.executemany` - if :ref:`named placeholders ` are used and - *parameters* is a sequence instead of a :class:`dict`. - - * date and datetime adapter, date and timestamp converter: - see the :mod:`sqlite3` documentation for suggested replacement recipes. - -* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was - deprecated in :pep:`626` - since 3.10 and was planned to be removed in 3.12, - but it only got a proper :exc:`DeprecationWarning` in 3.12. - May be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`101866`.) - -* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, - now causes a :exc:`DeprecationWarning` to be emitted when it is used. - -* :mod:`urllib`: - :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a - public API. - (Contributed by Gregory P. Smith in :gh:`88168`.) - -Pending Removal in Python 3.15 ------------------------------- - -* :class:`http.server.CGIHTTPRequestHandler` will be removed along with its - related ``--cgi`` flag to ``python -m http.server``. It was obsolete and - rarely used. No direct replacement exists. *Anything* is better than CGI - to interface a web server with a request handler. - -* :class:`locale`: :func:`locale.getdefaultlocale` was deprecated in Python 3.11 - and originally planned for removal in Python 3.13 (:gh:`90817`), - but removal has been postponed to Python 3.15. - Use :func:`locale.setlocale()`, :func:`locale.getencoding()` and - :func:`locale.getlocale()` instead. - (Contributed by Hugo van Kemenade in :gh:`111187`.) - -* :mod:`pathlib`: - :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for - removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved - paths on Windows. - -* :mod:`platform`: - :func:`~platform.java_ver` is deprecated and will be removed in 3.15. - It was largely untested, had a confusing API, - and was only useful for Jython support. - (Contributed by Nikita Sobolev in :gh:`116349`.) - -* :mod:`threading`: - Passing any arguments to :func:`threading.RLock` is now deprecated. - C version allows any numbers of args and kwargs, - but they are just ignored. Python version does not allow any arguments. - All arguments will be removed from :func:`threading.RLock` in Python 3.15. - (Contributed by Nikita Sobolev in :gh:`102029`.) - -* :class:`typing.NamedTuple`: - - * The undocumented keyword argument syntax for creating :class:`!NamedTuple` classes - (``NT = NamedTuple("NT", x=int)``) is deprecated, and will be disallowed in - 3.15. Use the class-based syntax or the functional syntax instead. - - * When using the functional syntax to create a :class:`!NamedTuple` class, failing to - pass a value to the *fields* parameter (``NT = NamedTuple("NT")``) is - deprecated. Passing ``None`` to the *fields* parameter - (``NT = NamedTuple("NT", None)``) is also deprecated. Both will be - disallowed in Python 3.15. To create a :class:`!NamedTuple` class with 0 fields, use - ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", [])``. - -* :class:`typing.TypedDict`: When using the functional syntax to create a - :class:`!TypedDict` class, failing to pass a value to the *fields* parameter (``TD = - TypedDict("TD")``) is deprecated. Passing ``None`` to the *fields* parameter - (``TD = TypedDict("TD", None)``) is also deprecated. Both will be disallowed - in Python 3.15. To create a :class:`!TypedDict` class with 0 fields, use ``class - TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. - -* :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` - methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. - They will be removed in Python 3.15. - (Contributed by Victor Stinner in :gh:`105096`.) - -Pending Removal in Python 3.16 ------------------------------- - -* :class:`array.array` ``'u'`` type (:c:type:`wchar_t`): - use the ``'w'`` type instead (``Py_UCS4``). - -Pending Removal in Future Versions ----------------------------------- - -The following APIs were deprecated in earlier Python versions and will be removed, -although there is currently no date scheduled for their removal. - -* :mod:`argparse`: Nesting argument groups and nesting mutually exclusive - groups are deprecated. - -* :mod:`builtins`: - - * ``~bool``, bitwise inversion on bool. - * ``bool(NotImplemented)``. - * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` - signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, - the single argument signature. - * Currently Python accepts numeric literals immediately followed by keywords, - for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing and - ambiguous expressions like ``[0x1for x in y]`` (which can be interpreted as - ``[0x1 for x in y]`` or ``[0x1f or x in y]``). A syntax warning is raised - if the numeric literal is immediately followed by one of keywords - :keyword:`and`, :keyword:`else`, :keyword:`for`, :keyword:`if`, - :keyword:`in`, :keyword:`is` and :keyword:`or`. In a future release it - will be changed to a syntax error. (:gh:`87999`) - * Support for ``__index__()`` and ``__int__()`` method returning non-int type: - these methods will be required to return an instance of a strict subclass of - :class:`int`. - * Support for ``__float__()`` method returning a strict subclass of - :class:`float`: these methods will be required to return an instance of - :class:`float`. - * Support for ``__complex__()`` method returning a strict subclass of - :class:`complex`: these methods will be required to return an instance of - :class:`complex`. - * Delegation of ``int()`` to ``__trunc__()`` method. - -* :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are - deprecated and replaced by :data:`calendar.JANUARY` and - :data:`calendar.FEBRUARY`. - (Contributed by Prince Roshan in :gh:`103636`.) - -* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method - instead. - -* :mod:`datetime`: - - * :meth:`~datetime.datetime.utcnow`: - use ``datetime.datetime.now(tz=datetime.UTC)``. - * :meth:`~datetime.datetime.utcfromtimestamp`: - use ``datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)``. - -* :mod:`gettext`: Plural value must be an integer. - -* :mod:`importlib`: - - * ``load_module()`` method: use ``exec_module()`` instead. - * :func:`~importlib.util.cache_from_source` *debug_override* parameter is - deprecated: use the *optimization* parameter instead. - -* :mod:`importlib.metadata`: - - * ``EntryPoints`` tuple interface. - * Implicit ``None`` on return values. - -* :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use - BytesIO and binary mode instead. - -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. - -* :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is - deprecated, use an exception instance. - -* :mod:`re`: More strict rules are now applied for numerical group references - and group names in regular expressions. Only sequence of ASCII digits is now - accepted as a numerical reference. The group name in bytes patterns and - replacement strings can now only contain ASCII letters and digits and - underscore. - (Contributed by Serhiy Storchaka in :gh:`91760`.) - -* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. - -* :mod:`shutil`: :func:`~shutil.rmtree`'s *onerror* parameter is deprecated in - Python 3.12; use the *onexc* parameter instead. - -* :mod:`ssl` options and protocols: - - * :class:`ssl.SSLContext` without protocol argument is deprecated. - * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and - :meth:`!selected_npn_protocol` are deprecated: use ALPN - instead. - * ``ssl.OP_NO_SSL*`` options - * ``ssl.OP_NO_TLS*`` options - * ``ssl.PROTOCOL_SSLv3`` - * ``ssl.PROTOCOL_TLS`` - * ``ssl.PROTOCOL_TLSv1`` - * ``ssl.PROTOCOL_TLSv1_1`` - * ``ssl.PROTOCOL_TLSv1_2`` - * ``ssl.TLSVersion.SSLv3`` - * ``ssl.TLSVersion.TLSv1`` - * ``ssl.TLSVersion.TLSv1_1`` - -* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and - ignored. - -* :mod:`threading` methods: - - * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. - * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. - * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: - use :attr:`threading.Thread.daemon` attribute. - * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: - use :attr:`threading.Thread.name` attribute. - * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. - * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. - -* :class:`typing.Text` (:gh:`92332`). - -* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value - that is not ``None`` from a test case. - -* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead - - * ``splitattr()`` - * ``splithost()`` - * ``splitnport()`` - * ``splitpasswd()`` - * ``splitport()`` - * ``splitquery()`` - * ``splittag()`` - * ``splittype()`` - * ``splituser()`` - * ``splitvalue()`` - * ``to_bytes()`` - -* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and - :class:`~urllib.request.FancyURLopener` style of invoking requests is - deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. - -* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial - writes. +.. include:: ../deprecations/pending-removal-in-3.14.rst -* :mod:`xml.etree.ElementTree`: Testing the truth value of an - :class:`~xml.etree.ElementTree.Element` is deprecated. In a future release it - it will always return ``True``. Prefer explicit ``len(elem)`` or - ``elem is not None`` tests instead. +.. include:: ../deprecations/pending-removal-in-3.15.rst -* :meth:`zipimport.zipimporter.load_module` is deprecated: - use :meth:`~zipimport.zipimporter.exec_module` instead. +.. include:: ../deprecations/pending-removal-in-3.16.rst +.. include:: ../deprecations/pending-removal-in-future.rst CPython Bytecode Changes ======================== diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index d02c10ec9cf3f3..7450597e8597ad 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -75,6 +75,20 @@ New Features Other Language Changes ====================== +* Incorrect usage of :keyword:`await` and asynchronous comprehensions + is now detected even if the code is optimized away by the :option:`-O` + command line option. For example, ``python -O -c 'assert await 1'`` + now produces a :exc:`SyntaxError`. (Contributed by Jelle Zijlstra in :gh:`121637`.) + +* Writes to ``__debug__`` are now detected even if the code is optimized + away by the :option:`-O` command line option. For example, + ``python -O -c 'assert (__debug__ := 1)'`` now produces a + :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) + +* Added class methods :meth:`float.from_number` and :meth:`complex.from_number` + to convert a number to :class:`float` or :class:`complex` type correspondingly. + They raise an error if the argument is a string. + (Contributed by Serhiy Storchaka in :gh:`84978`.) New Modules @@ -96,6 +110,20 @@ ast (Contributed by Bénédikt Tran in :gh:`121141`.) +fractions +--------- + +Added support for converting any objects that have the +:meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. +(Contributed by Serhiy Storchaka in :gh:`82017`.) + +json +---- + +Add notes for JSON serialization errors that allow to identify the source +of the error. +(Contributed by Serhiy Storchaka in :gh:`122163`.) + os -- @@ -107,13 +135,33 @@ os pathlib ------- -* Add :meth:`pathlib.Path.copy`, which copies the content of one file to - another, like :func:`shutil.copyfile`. - (Contributed by Barney Gale in :gh:`73991`.) -* Add :meth:`pathlib.Path.copytree`, which copies one directory tree to - another. +* Add methods to :class:`pathlib.Path` to recursively copy or remove files: + + * :meth:`~pathlib.Path.copy` copies the content of one file to another, like + :func:`shutil.copyfile`. + * :meth:`~pathlib.Path.copytree` copies one directory tree to another, like + :func:`shutil.copytree`. + * :meth:`~pathlib.Path.rmtree` recursively removes a directory tree, like + :func:`shutil.rmtree`. + (Contributed by Barney Gale in :gh:`73991`.) +pdb +--- + +* Hard-coded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace()`) now + reuse the most recent :class:`~pdb.Pdb` instance that calls + :meth:`~pdb.Pdb.set_trace()`, instead of creating a new one each time. + As a result, all the instance specific data like :pdbcmd:`display` and + :pdbcmd:`commands` are preserved across hard-coded breakpoints. + (Contributed by Tian Gao in :gh:`121450`.) + +pickle +------ + +* Set the default protocol version on the :mod:`pickle` module to 5. + For more details, please see :ref:`pickle protocols `. + symtable -------- @@ -125,6 +173,7 @@ symtable (Contributed by Bénédikt Tran in :gh:`120029`.) +.. Add improved modules above alphabetically, not here at the end. Optimizations ============= @@ -137,22 +186,32 @@ asyncio reduces memory usage. (Contributed by Kumar Aditya in :gh:`107803`.) - - - Deprecated ========== -* Passing a complex number as the *real* or *imag* argument in the +* :mod:`builtins`: + Passing a complex number as the *real* or *imag* argument in the :func:`complex` constructor is now deprecated; it should only be passed as a single positional argument. (Contributed by Serhiy Storchaka in :gh:`109218`.) -* :term:`Soft deprecate ` :func:`os.popen` and +* :mod:`os`: + :term:`Soft deprecate ` :func:`os.popen` and :func:`os.spawn* ` functions. They should no longer be used to write new code. The :mod:`subprocess` module is recommended instead. (Contributed by Victor Stinner in :gh:`120743`.) +* :mod:`symtable`: + Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest. + (Contributed by Bénédikt Tran in :gh:`119698`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.15.rst + +.. include:: ../deprecations/pending-removal-in-3.16.rst + +.. include:: ../deprecations/pending-removal-in-future.rst Removed ======= @@ -212,7 +271,6 @@ asyncio (Contributed by Kumar Aditya in :gh:`120804`.) - collections.abc --------------- @@ -259,7 +317,7 @@ pathlib arguments are joined onto *other*. pty -___ +--- * Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. They had previously raised a :exc:`DeprecationWarning` since Python 3.12. diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index a6b38207b70c93..ac05b591b62c6b 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -1312,7 +1312,7 @@ An early decision to limit the interoperability of various numeric types has been relaxed. It is still unsupported (and ill-advised) to have implicit mixing in arithmetic expressions such as ``Decimal('1.1') + float('1.1')`` because the latter loses information in the process of constructing the binary -float. However, since existing floating point value can be converted losslessly +float. However, since existing floating-point value can be converted losslessly to either a decimal or rational representation, it makes sense to add them to the constructor and to support mixed-type comparisons. diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 29b4034e32821c..5df976da3c4b46 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1097,12 +1097,12 @@ decimal C-module and libmpdec written by Stefan Krah. The new C version of the decimal module integrates the high speed libmpdec -library for arbitrary precision correctly rounded decimal floating point +library for arbitrary precision correctly rounded decimal floating-point arithmetic. libmpdec conforms to IBM's General Decimal Arithmetic Specification. Performance gains range from 10x for database applications to 100x for numerically intensive applications. These numbers are expected gains -for standard precisions used in decimal floating point arithmetic. Since +for standard precisions used in decimal floating-point arithmetic. Since the precision is user configurable, the exact figures may vary. For example, in integer bignum arithmetic the differences can be significantly higher. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 8aef0f5ac26728..938dd273e7e102 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -1495,7 +1495,7 @@ The dictionary returned by :meth:`.SSLSocket.getpeercert` contains additional stat ---- -The :mod:`stat` module is now backed by a C implementation in :mod:`_stat`. A C +The :mod:`stat` module is now backed by a C implementation in :mod:`!_stat`. A C implementation is required as most of the values aren't standardized and are platform-dependent. (Contributed by Christian Heimes in :issue:`11016`.) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index cd8a903327cc2f..75654f3e78eb16 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -1935,8 +1935,8 @@ specifying the namespace in which the code will be running. tkinter ------- -The :mod:`tkinter._fix` module used for setting up the Tcl/Tk environment -on Windows has been replaced by a private function in the :mod:`_tkinter` +The :mod:`!tkinter._fix` module used for setting up the Tcl/Tk environment +on Windows has been replaced by a private function in the :mod:`!_tkinter` module which makes no permanent changes to environment variables. (Contributed by Zachary Ware in :issue:`20035`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 68ab43462b743a..35352c7014d773 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -261,7 +261,7 @@ allowed. The :ref:`string formatting ` language also now has support for the ``'_'`` option to signal the use of an underscore for a thousands -separator for floating point presentation types and for integer +separator for floating-point presentation types and for integer presentation type ``'d'``. For integer presentation types ``'b'``, ``'o'``, ``'x'``, and ``'X'``, underscores will be inserted every 4 digits:: diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 69d043bcf7efd5..ab460ed85f8243 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -339,7 +339,7 @@ PEP 564: New Time Functions With Nanosecond Resolution ------------------------------------------------------ The resolution of clocks in modern systems can exceed the limited precision -of a floating point number returned by the :func:`time.time` function +of a floating-point number returned by the :func:`time.time` function and its variants. To avoid loss of precision, :pep:`564` adds six new "nanosecond" variants of the existing timer functions to the :mod:`time` module: @@ -2048,7 +2048,7 @@ The :mod:`macpath` is now deprecated and will be removed in Python 3.8. threading --------- -:mod:`dummy_threading` and :mod:`_dummy_thread` have been deprecated. It is +:mod:`!dummy_threading` and :mod:`!_dummy_thread` have been deprecated. It is no longer possible to build Python with threading disabled. Use :mod:`threading` instead. (Contributed by Antoine Pitrou in :issue:`31370`.) @@ -2184,7 +2184,7 @@ The following features and APIs have been removed from Python 3.7: ``socket.socketpair`` on Python 3.5 and newer. * :mod:`asyncio` no longer exports the :mod:`selectors` and - :mod:`_overlapped` modules as ``asyncio.selectors`` and + :mod:`!_overlapped` modules as ``asyncio.selectors`` and ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with ``import selectors``. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 1356f24547b424..bf5e2770559c69 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1192,7 +1192,7 @@ post-handshake authentication. statistics ---------- -Added :func:`statistics.fmean` as a faster, floating point variant of +Added :func:`statistics.fmean` as a faster, floating-point variant of :func:`statistics.mean()`. (Contributed by Raymond Hettinger and Steven D'Aprano in :issue:`35904`.) diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index 816823716e9a6f..41537210b748a1 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -31,3 +31,7 @@ static inline Py_ssize_t PyBytes_GET_SIZE(PyObject *op) { return Py_SIZE(self); } #define PyBytes_GET_SIZE(self) PyBytes_GET_SIZE(_PyObject_CAST(self)) + +/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, + x must be an iterable object. */ +PyAPI_FUNC(PyObject*) _PyBytes_Join(PyObject *sep, PyObject *x); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 07ed5200a59d61..58d93fcfc1066b 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -194,9 +194,6 @@ Py_DEPRECATED(3.13) static inline int PyCode_GetFirstFree(PyCodeObject *op) { return PyUnstable_Code_GetFirstFree(op); } -#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) -#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) - /* Unstable public interface */ PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_New( int, int, int, int, int, PyObject *, PyObject *, diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index d841c043f37fc4..c60ccc463653f9 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -119,18 +119,18 @@ PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( static inline int -_PyLong_IsCompact(PyLongObject* op) { - assert(PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS)); +_PyLong_IsCompact(const PyLongObject* op) { + assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); return op->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS); } #define PyUnstable_Long_IsCompact _PyLong_IsCompact static inline Py_ssize_t -_PyLong_CompactValue(PyLongObject *op) +_PyLong_CompactValue(const PyLongObject *op) { Py_ssize_t sign; - assert(PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS)); + assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); return sign * (Py_ssize_t)op->long_value.ob_digit[0]; diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 55a139bb9158db..4ecef4f56edf42 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -510,6 +510,9 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); // See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence static inline void _Py_atomic_fence_seq_cst(void); +// Acquire fence +static inline void _Py_atomic_fence_acquire(void); + // Release fence static inline void _Py_atomic_fence_release(void); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index f2ebdeeb5524e4..ef09954d53ac1d 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -542,6 +542,10 @@ static inline void _Py_atomic_fence_seq_cst(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); } + static inline void +_Py_atomic_fence_acquire(void) +{ __atomic_thread_fence(__ATOMIC_ACQUIRE); } + static inline void _Py_atomic_fence_release(void) { __atomic_thread_fence(__ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index f32995c1f578ac..84da21bdcbff4f 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1066,6 +1066,18 @@ _Py_atomic_fence_seq_cst(void) #else # error "no implementation of _Py_atomic_fence_seq_cst" #endif +} + + static inline void +_Py_atomic_fence_acquire(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISHLD); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadBarrier(); +#else +# error "no implementation of _Py_atomic_fence_acquire" +#endif } static inline void diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 0cdce4e6dd39f0..7c71e94c68f8e6 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -961,6 +961,13 @@ _Py_atomic_fence_seq_cst(void) atomic_thread_fence(memory_order_seq_cst); } + static inline void +_Py_atomic_fence_acquire(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_acquire); +} + static inline void _Py_atomic_fence_release(void) { diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 42b4b03b10ca20..b36b4681f5dddb 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -100,7 +100,7 @@ PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( /* In signalmodule.c */ -int PySignal_SetWakeupFd(int fd); +PyAPI_FUNC(int) PySignal_SetWakeupFd(int fd); /* Support for adding program text to SyntaxErrors */ diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index bb2af78a376d75..f005729fff11b6 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -192,6 +192,14 @@ struct _ts { PyObject *previous_executor; uint64_t dict_global_version; + + /* Used to store/retrieve `threading.local` keys/values for this thread */ + PyObject *threading_local_key; + + /* Used by `threading.local`s to be remove keys/values for dying threads. + The PyThreadObject must hold the only reference to this value. + */ + PyObject *threading_local_sentinel; }; #ifdef Py_DEBUG diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 38480a4f6cd78f..c4480758f48514 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -19,12 +19,12 @@ // Define _PY_INTERPRETER macro to increment interpreter_increfs and // interpreter_decrefs. Otherwise, increment increfs and decrefs. -#include "pycore_uop_ids.h" - #ifndef Py_CPYTHON_PYSTATS_H # error "this header file must not be included directly" #endif +#define PYSTATS_MAX_UOP_ID 512 + #define SPECIALIZATION_FAILURE_KINDS 36 /* Stats for determining who is calling PyEval_EvalFrame */ @@ -100,7 +100,7 @@ typedef struct _gc_stats { typedef struct _uop_stats { uint64_t execution_count; uint64_t miss; - uint64_t pair_count[MAX_UOP_ID + 1]; + uint64_t pair_count[PYSTATS_MAX_UOP_ID + 1]; } UOpStats; #define _Py_UOP_HIST_SIZE 32 @@ -118,7 +118,7 @@ typedef struct _optimization_stats { uint64_t recursive_call; uint64_t low_confidence; uint64_t executors_invalidated; - UOpStats opcode[MAX_UOP_ID+1]; + UOpStats opcode[PYSTATS_MAX_UOP_ID + 1]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE]; @@ -128,7 +128,7 @@ typedef struct _optimization_stats { uint64_t optimizer_failure_reason_no_memory; uint64_t remove_globals_builtins_changed; uint64_t remove_globals_incorrect_keys; - uint64_t error_in_opcode[MAX_UOP_ID+1]; + uint64_t error_in_opcode[PYSTATS_MAX_UOP_ID + 1]; } OptimizationStats; typedef struct _rare_event_stats { diff --git a/Include/floatobject.h b/Include/floatobject.h index 999441ac536e1d..8963c16832a4bc 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -2,7 +2,7 @@ /* Float object interface */ /* -PyFloatObject represents a (double precision) floating point number. +PyFloatObject represents a (double precision) floating-point number. */ #ifndef Py_FLOATOBJECT_H diff --git a/Include/internal/mimalloc/mimalloc/atomic.h b/Include/internal/mimalloc/mimalloc/atomic.h index 52f82487685cdb..cdd9c372beafd5 100644 --- a/Include/internal/mimalloc/mimalloc/atomic.h +++ b/Include/internal/mimalloc/mimalloc/atomic.h @@ -23,7 +23,9 @@ terms of the MIT license. A copy of the license can be found in the file #define _Atomic(tp) std::atomic #define mi_atomic(name) std::atomic_##name #define mi_memory_order(name) std::memory_order_##name -#if !defined(ATOMIC_VAR_INIT) || (__cplusplus >= 202002L) // c++20, see issue #571 +#if (__cplusplus >= 202002L) // c++20, see issue #571 + #define MI_ATOMIC_VAR_INIT(x) x +#elif !defined(ATOMIC_VAR_INIT) #define MI_ATOMIC_VAR_INIT(x) x #else #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) @@ -39,7 +41,9 @@ terms of the MIT license. A copy of the license can be found in the file #include #define mi_atomic(name) atomic_##name #define mi_memory_order(name) memory_order_##name -#if !defined(ATOMIC_VAR_INIT) || (__STDC_VERSION__ >= 201710L) // c17, see issue #735 +#if (__STDC_VERSION__ >= 201710L) // c17, see issue #735 + #define MI_ATOMIC_VAR_INIT(x) x +#elif !defined(ATOMIC_VAR_INIT) #define MI_ATOMIC_VAR_INIT(x) x #else #define MI_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 94d421a9eb742a..300e7f4896a39e 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -23,10 +23,6 @@ extern PyObject* _PyBytes_FromHex( PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape(const char *, Py_ssize_t, const char *, const char **); -/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, - x must be an iterable object. */ -extern PyObject* _PyBytes_Join(PyObject *sep, PyObject *x); - // Substring Search. // diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index b472d5d446b246..fac4a4d228053e 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -259,6 +259,8 @@ PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObjec PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v, int argcnt, int argcntafter, _PyStackRef *sp); diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index cf2cc7f5b4ac92..cd77fe408fc5d9 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -31,6 +31,9 @@ typedef union { _Py_BackoffCounter counter; // First cache entry of specializable op } _Py_CODEUNIT; +#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) +#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) + /* These macros only remain defined for compatibility. */ #define _Py_OPCODE(word) ((word).op.code) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index a1ac034e3e44af..325243e6a64e1f 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -76,15 +76,6 @@ int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj); // Export for '_opcode' extension module -PyAPI_FUNC(int) _PyCompile_OpcodeIsValid(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasArg(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasConst(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasName(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasJump(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasFree(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasLocal(int opcode); -PyAPI_FUNC(int) _PyCompile_OpcodeHasExc(int opcode); - PyAPI_FUNC(PyObject*) _PyCompile_GetUnaryIntrinsicName(int index); PyAPI_FUNC(PyObject*) _PyCompile_GetBinaryIntrinsicName(int index); diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index ae5c47f195eb7f..2ecb40ba584f7f 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -5,7 +5,6 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_hamt.h" // PyHamtObject @@ -35,9 +34,11 @@ struct _pycontextvarobject { PyObject_HEAD PyObject *var_name; PyObject *var_default; +#ifndef Py_GIL_DISABLED PyObject *var_cached; uint64_t var_cached_tsid; uint64_t var_cached_tsver; +#endif Py_hash_t var_hash; }; diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 9e0e1237915e82..fc304aca7fea10 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -8,15 +8,18 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_object.h" // PyManagedDictPointer #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); -extern int _PyDict_DelItemIf(PyObject *mp, PyObject *key, - int (*predicate)(PyObject *value)); +// Delete an item from a dict if a predicate is true +// Returns -1 on error, 1 if the item was deleted, 0 otherwise +// Export for '_asyncio' shared extension +PyAPI_FUNC(int) _PyDict_DelItemIf(PyObject *mp, PyObject *key, + int (*predicate)(PyObject *value, void *arg), + void *arg); // "KnownHash" variants // Export for '_asyncio' shared extension @@ -322,6 +325,8 @@ _PyInlineValuesSize(PyTypeObject *tp) int _PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj); +PyDictObject *_PyObject_MaterializeManagedDict_LockHeld(PyObject *); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index f984df695696c3..be1c6cc97720d2 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -8,7 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_unicodeobject.h" // _PyUnicodeWriter /* runtime lifecycle */ diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 1e0368faa5b510..d3a5be000fbce7 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -13,7 +13,7 @@ extern "C" { #include "pycore_code.h" // STATS #include "pycore_stackref.h" // _PyStackRef -/* See Objects/frame_layout.md for an explanation of the frame stack +/* See InternalDocs/frames.md for an explanation of the frame stack * including explanation of the PyFrameObject and _PyInterpreterFrame * structs. */ @@ -27,6 +27,10 @@ struct _frame { char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ PyObject *f_extra_locals; /* Dict for locals set by users using f_locals, could be NULL */ + /* This is purely for backwards compatibility for PyEval_GetLocals. + PyEval_GetLocals requires a borrowed reference so the actual reference + is stored here */ + PyObject *f_locals_cache; /* The frame data, if this frame object owns the frame */ PyObject *_f_frame_data[1]; }; @@ -64,7 +68,7 @@ typedef struct _PyInterpreterFrame { PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ - int stacktop; /* Offset of TOS from localsplus */ + _PyStackRef *stackpointer; uint16_t return_offset; /* Only relevant during a function call */ char owner; /* Locals and stack */ @@ -84,20 +88,20 @@ static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { } static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { - assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - assert(!PyStackRef_IsNull(f->localsplus[f->stacktop-1])); - return f->localsplus[f->stacktop-1]; + assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + assert(!PyStackRef_IsNull(f->stackpointer[-1])); + return f->stackpointer[-1]; } static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) { - assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - f->stacktop--; - return f->localsplus[f->stacktop]; + assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + f->stackpointer--; + return *f->stackpointer; } static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, _PyStackRef value) { - f->localsplus[f->stacktop] = value; - f->stacktop++; + *f->stackpointer = value; + f->stackpointer++; } #define FRAME_SPECIALS_SIZE ((int)((sizeof(_PyInterpreterFrame)-1)/sizeof(PyObject *))) @@ -113,9 +117,12 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code) static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) { - assert(src->stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); *dest = *src; - for (int i = 1; i < src->stacktop; i++) { + assert(src->stackpointer != NULL); + int stacktop = (int)(src->stackpointer - src->localsplus); + assert(stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); + dest->stackpointer = dest->localsplus + stacktop; + for (int i = 1; i < stacktop; i++) { dest->localsplus[i] = src->localsplus[i]; } // Don't leave a dangling pointer to the old frame when creating generators @@ -137,7 +144,7 @@ _PyFrame_Initialize( frame->f_builtins = func->func_builtins; frame->f_globals = func->func_globals; frame->f_locals = locals; - frame->stacktop = code->co_nlocalsplus; + frame->stackpointer = frame->localsplus + code->co_nlocalsplus; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); frame->return_offset = 0; @@ -157,22 +164,23 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) return frame->localsplus; } -/* Fetches the stack pointer, and sets stacktop to -1. - Having stacktop <= 0 ensures that invalid - values are not visible to the cycle GC. - We choose -1 rather than 0 to assist debugging. */ +/* Fetches the stack pointer, and sets stackpointer to NULL. + Having stackpointer == NULL ensures that invalid + values are not visible to the cycle GC. */ static inline _PyStackRef* _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - _PyStackRef *sp = frame->localsplus + frame->stacktop; - frame->stacktop = -1; + assert(frame->stackpointer != NULL); + _PyStackRef *sp = frame->stackpointer; + frame->stackpointer = NULL; return sp; } static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { - frame->stacktop = (int)(stack_pointer - frame->localsplus); + assert(frame->stackpointer == NULL); + frame->stackpointer = stack_pointer; } /* Determine whether a frame is incomplete. @@ -300,7 +308,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->f_globals = NULL; #endif frame->f_locals = NULL; - frame->stacktop = code->co_nlocalsplus + stackdepth; + assert(stackdepth <= code->co_stacksize); + frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); frame->owner = FRAME_OWNED_BY_THREAD; diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index e684e084b8bef8..1ac0fed584ebd9 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -8,144 +8,109 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -// PyTuple_MAXSAVESIZE - largest tuple to save on free list -// PyTuple_MAXFREELIST - maximum number of tuples of each size to save - -#ifdef WITH_FREELISTS -// with freelists -# define PyTuple_MAXSAVESIZE 20 -# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE -# define PyTuple_MAXFREELIST 2000 -# define PyList_MAXFREELIST 80 -# define PyDict_MAXFREELIST 80 -# define PyFloat_MAXFREELIST 100 -# define PyContext_MAXFREELIST 255 -# define _PyAsyncGen_MAXFREELIST 80 -# define _PyObjectStackChunk_MAXFREELIST 4 -#else -# define PyTuple_NFREELISTS 0 -# define PyTuple_MAXFREELIST 0 -# define PyList_MAXFREELIST 0 -# define PyDict_MAXFREELIST 0 -# define PyFloat_MAXFREELIST 0 -# define PyContext_MAXFREELIST 0 -# define _PyAsyncGen_MAXFREELIST 0 -# define _PyObjectStackChunk_MAXFREELIST 0 -#endif - -struct _Py_list_freelist { -#ifdef WITH_FREELISTS - PyListObject *items[PyList_MAXFREELIST]; - int numfree; +#include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_object.h" // _PyObject_IS_GC +#include "pycore_pystate.h" // _PyThreadState_GET +#include "pycore_code.h" // OBJECT_STAT_INC + +static inline struct _Py_freelists * +_Py_freelists_GET(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + _Py_EnsureTstateNotNULL(tstate); #endif -}; - -struct _Py_tuple_freelist { -#if WITH_FREELISTS - /* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE. - The empty tuple is handled separately. - Each tuple stored in the array is the head of the linked list - (and the next available tuple) for that size. The actual tuple - object is used as the linked list node, with its first item - (ob_item[0]) pointing to the next node (i.e. the previous head). - Each linked list is initially NULL. */ - PyTupleObject *items[PyTuple_NFREELISTS]; - int numfree[PyTuple_NFREELISTS]; +#ifdef Py_GIL_DISABLED + return &((_PyThreadStateImpl*)tstate)->freelists; #else - char _unused; // Empty structs are not allowed. -#endif -}; - -struct _Py_float_freelist { -#ifdef WITH_FREELISTS - /* Special free list - free_list is a singly-linked list of available PyFloatObjects, - linked via abuse of their ob_type members. */ - int numfree; - PyFloatObject *items; -#endif -}; - -struct _Py_dict_freelist { -#ifdef WITH_FREELISTS - /* Dictionary reuse scheme to save calls to malloc and free */ - PyDictObject *items[PyDict_MAXFREELIST]; - int numfree; + return &tstate->interp->object_state.freelists; #endif -}; +} -struct _Py_dictkeys_freelist { -#ifdef WITH_FREELISTS - /* Dictionary keys reuse scheme to save calls to malloc and free */ - PyDictKeysObject *items[PyDict_MAXFREELIST]; - int numfree; -#endif -}; +#ifndef WITH_FREELISTS +#define _Py_FREELIST_FREE(NAME, op, freefunc) freefunc(op) +#define _Py_FREELIST_PUSH(NAME, op, limit) (0) +#define _Py_FREELIST_POP(TYPE, NAME) (NULL) +#define _Py_FREELIST_POP_MEM(NAME) (NULL) +#define _Py_FREELIST_SIZE(NAME) (0) +#else +// Pushes `op` to the freelist, calls `freefunc` if the freelist is full +#define _Py_FREELIST_FREE(NAME, op, freefunc) \ + _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \ + Py_ ## NAME ## _MAXFREELIST, freefunc) +// Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full +#define _Py_FREELIST_PUSH(NAME, op, limit) \ + _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit) + +// Pops a PyObject from the freelist, returns NULL if the freelist is empty. +#define _Py_FREELIST_POP(TYPE, NAME) \ + _Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME)) + +// Pops a non-PyObject data structure from the freelist, returns NULL if the +// freelist is empty. +#define _Py_FREELIST_POP_MEM(NAME) \ + _PyFreeList_PopMem(&_Py_freelists_GET()->NAME) + +#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size) + +static inline int +_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize) +{ + if (fl->size < maxsize && fl->size >= 0) { + *(void **)obj = fl->freelist; + fl->freelist = obj; + fl->size++; + OBJECT_STAT_INC(to_freelist); + return 1; + } + return 0; +} -struct _Py_slice_freelist { -#ifdef WITH_FREELISTS - /* Using a cache is very effective since typically only a single slice is - created and then deleted again. */ - PySliceObject *slice_cache; -#endif -}; +static inline void +_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize, + freefunc dofree) +{ + if (!_PyFreeList_Push(fl, obj, maxsize)) { + dofree(obj); + } +} -struct _Py_context_freelist { -#ifdef WITH_FREELISTS - // List of free PyContext objects - PyContext *items; - int numfree; -#endif -}; +static inline void * +_PyFreeList_PopNoStats(struct _Py_freelist *fl) +{ + void *obj = fl->freelist; + if (obj != NULL) { + assert(fl->size > 0); + fl->freelist = *(void **)obj; + fl->size--; + } + return obj; +} -struct _Py_async_gen_freelist { -#ifdef WITH_FREELISTS - /* Freelists boost performance 6-10%; they also reduce memory - fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend - are short-living objects that are instantiated for every - __anext__() call. */ - struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST]; - int numfree; -#endif -}; +static inline PyObject * +_PyFreeList_Pop(struct _Py_freelist *fl) +{ + PyObject *op = _PyFreeList_PopNoStats(fl); + if (op != NULL) { + OBJECT_STAT_INC(from_freelist); + _Py_NewReference(op); + } + return op; +} -struct _Py_async_gen_asend_freelist { -#ifdef WITH_FREELISTS - struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST]; - int numfree; +static inline void * +_PyFreeList_PopMem(struct _Py_freelist *fl) +{ + void *op = _PyFreeList_PopNoStats(fl); + if (op != NULL) { + OBJECT_STAT_INC(from_freelist); + } + return op; +} #endif -}; - -struct _PyObjectStackChunk; - -struct _Py_object_stack_freelist { - struct _PyObjectStackChunk *items; - Py_ssize_t numfree; -}; - -struct _Py_object_freelists { - struct _Py_float_freelist floats; - struct _Py_tuple_freelist tuples; - struct _Py_list_freelist lists; - struct _Py_dict_freelist dicts; - struct _Py_dictkeys_freelist dictkeys; - struct _Py_slice_freelist slices; - struct _Py_context_freelist contexts; - struct _Py_async_gen_freelist async_gens; - struct _Py_async_gen_asend_freelist async_gen_asends; - struct _Py_object_stack_freelist object_stacks; -}; -extern void _PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); -extern void _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization); +extern void _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization); #ifdef __cplusplus } diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h new file mode 100644 index 00000000000000..edf79dd7521c41 --- /dev/null +++ b/Include/internal/pycore_freelist_state.h @@ -0,0 +1,61 @@ +#ifndef Py_INTERNAL_FREELIST_STATE_H +#define Py_INTERNAL_FREELIST_STATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef WITH_FREELISTS +// with freelists +# define PyTuple_MAXSAVESIZE 20 // Largest tuple to save on freelist +# define Py_tuple_MAXFREELIST 2000 // Maximum number of tuples of each size to save +# define Py_lists_MAXFREELIST 80 +# define Py_dicts_MAXFREELIST 80 +# define Py_dictkeys_MAXFREELIST 80 +# define Py_floats_MAXFREELIST 100 +# define Py_slices_MAXFREELIST 1 +# define Py_contexts_MAXFREELIST 255 +# define Py_async_gens_MAXFREELIST 80 +# define Py_async_gen_asends_MAXFREELIST 80 +# define Py_futureiters_MAXFREELIST 255 +# define Py_object_stack_chunks_MAXFREELIST 4 +#else +# define PyTuple_MAXSAVESIZE 0 +#endif + +// A generic freelist of either PyObjects or other data structures. +struct _Py_freelist { + // Entries are linked together using the first word of the the object. + // For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid` + // field. + void *freelist; + + // The number of items in the free list or -1 if the free list is disabled + Py_ssize_t size; +}; + +struct _Py_freelists { +#ifdef WITH_FREELISTS + struct _Py_freelist floats; + struct _Py_freelist tuples[PyTuple_MAXSAVESIZE]; + struct _Py_freelist lists; + struct _Py_freelist dicts; + struct _Py_freelist dictkeys; + struct _Py_freelist slices; + struct _Py_freelist contexts; + struct _Py_freelist async_gens; + struct _Py_freelist async_gen_asends; + struct _Py_freelist futureiters; + struct _Py_freelist object_stack_chunks; +#else + char _unused; // Empty structs are not allowed. +#endif +}; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FREELIST_STATE_H */ diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 28e34d3809634c..b4bf36e82e376a 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -8,8 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState - /* GC information is stored BEFORE the object structure. */ typedef struct { // Pointer to next object in the list. diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 327fcc24cb29f1..913dce6f1ec0fe 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -81,6 +81,7 @@ struct _Py_interp_cached_objects { PyTypeObject *paramspec_type; PyTypeObject *paramspecargs_type; PyTypeObject *paramspeckwargs_type; + PyTypeObject *constevaluator_type; }; #define _Py_INTERP_STATIC_OBJECT(interp, NAME) \ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index c0840f9eb7eca2..d9b46df507dfd7 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -588,7 +588,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__args__)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__asyncio_running_event_loop__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bases__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bool__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 51735a8a726e11..10773d7a6c7e3f 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -77,7 +77,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(__annotate__) STRUCT_FOR_ID(__annotations__) STRUCT_FOR_ID(__args__) - STRUCT_FOR_ID(__asyncio_running_event_loop__) STRUCT_FOR_ID(__await__) STRUCT_FOR_ID(__bases__) STRUCT_FOR_ID(__bool__) diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 73695d10e0c372..56ddb545d37b55 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -8,8 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState - PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 3824434f3f375d..e6da083b807ce5 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -228,12 +228,12 @@ PyAPI_FUNC(void) _PySeqLock_AbandonWrite(_PySeqLock *seqlock); PyAPI_FUNC(uint32_t) _PySeqLock_BeginRead(_PySeqLock *seqlock); // End the read operation and confirm that the sequence number has not changed. -// Returns 1 if the read was successful or 0 if the read should be re-tried. -PyAPI_FUNC(uint32_t) _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous); +// Returns 1 if the read was successful or 0 if the read should be retried. +PyAPI_FUNC(int) _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous); // Check if the lock was held during a fork and clear the lock. Returns 1 -// if the lock was held and any associated datat should be cleared. -PyAPI_FUNC(uint32_t) _PySeqLock_AfterFork(_PySeqLock *seqlock); +// if the lock was held and any associated data should be cleared. +PyAPI_FUNC(int) _PySeqLock_AfterFork(_PySeqLock *seqlock); #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index fa789611133a6f..155810d00bef5b 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -15,6 +15,30 @@ extern "C" { #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED #include "pycore_pystate.h" // _PyInterpreterState_GET() + +#define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1) + +// gh-121528, gh-118997: Similar to _Py_IsImmortal() but be more loose when +// comparing the reference count to stay compatible with C extensions built +// with the stable ABI 3.11 or older. Such extensions implement INCREF/DECREF +// as refcnt++ and refcnt-- without taking in account immortal objects. For +// example, the reference count of an immortal object can change from +// _Py_IMMORTAL_REFCNT to _Py_IMMORTAL_REFCNT+1 (INCREF) or +// _Py_IMMORTAL_REFCNT-1 (DECREF). +// +// This function should only be used in assertions. Otherwise, _Py_IsImmortal() +// must be used instead. +static inline int _Py_IsImmortalLoose(PyObject *op) +{ +#if defined(Py_GIL_DISABLED) + return _Py_IsImmortal(op); +#else + return (op->ob_refcnt >= _Py_IMMORTAL_REFCNT_LOOSE); +#endif +} +#define _Py_IsImmortalLoose(op) _Py_IsImmortalLoose(_PyObject_CAST(op)) + + /* Check if an object is consistent. For example, ensure that the reference counter is greater than or equal to 1, and ensure that ob_type is not NULL. @@ -134,7 +158,7 @@ PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op); static inline void _Py_SetMortal(PyObject *op, Py_ssize_t refcnt) { if (op) { - assert(_Py_IsImmortal(op)); + assert(_Py_IsImmortalLoose(op)); #ifdef Py_GIL_DISABLED op->ob_tid = _Py_UNOWNED_TID; op->ob_ref_local = 0; @@ -266,7 +290,7 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) { assert(op != NULL); Py_SET_TYPE(op, typeobj); - assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj)); + assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortalLoose(typeobj)); Py_INCREF(typeobj); _Py_NewReference(op); } @@ -733,13 +757,7 @@ PyAPI_FUNC(PyObject*) _PyObject_GetState(PyObject *); * Third party code unintentionally rely on problematic fpcasts. The call * trampoline mitigates common occurrences of bad fpcasts on Emscripten. */ -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -#define _PyCFunction_TrampolineCall(meth, self, args) \ - _PyCFunctionWithKeywords_TrampolineCall( \ - (*(PyCFunctionWithKeywords)(void(*)(void))(meth)), (self), (args), NULL) -extern PyObject* _PyCFunctionWithKeywords_TrampolineCall( - PyCFunctionWithKeywords meth, PyObject *, PyObject *, PyObject *); -#else +#if !(defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE)) #define _PyCFunction_TrampolineCall(meth, self, args) \ (meth)((self), (args)) #define _PyCFunctionWithKeywords_TrampolineCall(meth, self, args, kw) \ diff --git a/Include/internal/pycore_object_stack.h b/Include/internal/pycore_object_stack.h index fc130b1e9920b4..c607ea8bc52545 100644 --- a/Include/internal/pycore_object_stack.h +++ b/Include/internal/pycore_object_stack.h @@ -1,8 +1,6 @@ #ifndef Py_INTERNAL_OBJECT_STACK_H #define Py_INTERNAL_OBJECT_STACK_H -#include "pycore_freelist.h" // _PyFreeListState - #ifdef __cplusplus extern "C" { #endif diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index cd7c9335b3e611..e7fa7c1f10d6d1 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -8,8 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyObject_freelists -#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_freelist_state.h" // _Py_freelists +#include "pycore_hashtable.h" // _Py_hashtable_t struct _py_object_runtime_state { #ifdef Py_REF_DEBUG @@ -20,7 +20,7 @@ struct _py_object_runtime_state { struct _py_object_state { #if !defined(Py_GIL_DISABLED) - struct _Py_object_freelists freelists; + struct _Py_freelists freelists; #endif #ifdef Py_REF_DEBUG Py_ssize_t reftotal; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 7b495238d7a9f3..eaba280f1bf1cd 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -65,8 +65,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case BINARY_SUBSCR_TUPLE_INT: return 2; - case BUILD_CONST_KEY_MAP: - return 1 + oparg; case BUILD_LIST: return oparg; case BUILD_MAP: @@ -216,7 +214,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case IMPORT_NAME: return 2; case INSTRUMENTED_CALL: - return 0; + return 2 + oparg; case INSTRUMENTED_CALL_FUNCTION_EX: return 0; case INSTRUMENTED_CALL_KW: @@ -233,6 +231,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case INSTRUMENTED_JUMP_FORWARD: return 0; + case INSTRUMENTED_LINE: + return 0; case INSTRUMENTED_LOAD_SUPER_ATTR: return 3; case INSTRUMENTED_POP_JUMP_IF_FALSE: @@ -512,8 +512,6 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case BINARY_SUBSCR_TUPLE_INT: return 1; - case BUILD_CONST_KEY_MAP: - return 1; case BUILD_LIST: return 1; case BUILD_MAP: @@ -557,7 +555,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_LEN: return 1; case CALL_LIST_APPEND: - return 1; + return 0; case CALL_METHOD_DESCRIPTOR_FAST: return 1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -663,7 +661,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case IMPORT_NAME: return 1; case INSTRUMENTED_CALL: - return 0; + return 1; case INSTRUMENTED_CALL_FUNCTION_EX: return 0; case INSTRUMENTED_CALL_KW: @@ -680,6 +678,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case INSTRUMENTED_JUMP_FORWARD: return 0; + case INSTRUMENTED_LINE: + return 0; case INSTRUMENTED_LOAD_SUPER_ATTR: return 1 + (oparg & 1); case INSTRUMENTED_POP_JUMP_IF_FALSE: @@ -693,9 +693,9 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case INSTRUMENTED_RESUME: return 0; case INSTRUMENTED_RETURN_CONST: - return 0; + return 1; case INSTRUMENTED_RETURN_VALUE: - return 0; + return 1; case INSTRUMENTED_YIELD_VALUE: return 1; case INTERPRETER_EXIT: @@ -737,7 +737,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: return 1; case LOAD_ATTR_PROPERTY: - return 1; + return 0; case LOAD_ATTR_SLOT: return 1 + (oparg & 1); case LOAD_ATTR_WITH_HINT: @@ -835,7 +835,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case SEND: return 2; case SEND_GEN: - return 2; + return 1; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -1004,10 +1004,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, @@ -1026,7 +1025,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1079,7 +1078,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, @@ -1088,6 +1087,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, @@ -1114,7 +1114,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1131,7 +1131,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1223,6 +1223,7 @@ _PyOpcode_macro_expansion[256] = { [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } }, [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_INPLACE_ADD_UNICODE, 0, 0 } } }, [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, 0, 0 } } }, [BINARY_OP_MULTIPLY_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_MULTIPLY_INT, 0, 0 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } }, @@ -1233,9 +1234,9 @@ _PyOpcode_macro_expansion[256] = { [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_LIST_INT, 0, 0 } } }, [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_STR_INT, 0, 0 } } }, [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { _BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, - [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { _BUILD_CONST_KEY_MAP, 0, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, 0, 0 } } }, [BUILD_MAP] = { .nuops = 1, .uops = { { _BUILD_MAP, 0, 0 } } }, + [BUILD_SET] = { .nuops = 1, .uops = { { _BUILD_SET, 0, 0 } } }, [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, @@ -1249,6 +1250,7 @@ _PyOpcode_macro_expansion[256] = { [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, + [CALL_LIST_APPEND] = { .nuops = 1, .uops = { { _CALL_LIST_APPEND, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, @@ -1295,6 +1297,8 @@ _PyOpcode_macro_expansion[256] = { [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, 0, 0 } } }, [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, 0, 0 } } }, [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, 0, 0 } } }, + [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, 0, 0 } } }, + [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, 0, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, 0, 0 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, 0, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, 0, 0 } } }, @@ -1307,6 +1311,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_PROPERTY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_PROPERTY_FRAME, 4, 5 }, { _SAVE_RETURN_OFFSET, 7, 9 }, { _PUSH_FRAME, 0, 0 } } }, [LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } }, [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, 0, 0 } } }, @@ -1322,6 +1327,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, + [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, 0, 0 } } }, [LOAD_SPECIAL] = { .nuops = 1, .uops = { { _LOAD_SPECIAL, 0, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, @@ -1345,6 +1351,7 @@ _PyOpcode_macro_expansion[256] = { [RETURN_CONST] = { .nuops = 2, .uops = { { _LOAD_CONST, 0, 0 }, { _RETURN_VALUE, 0, 0 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, 0, 0 } } }, [RETURN_VALUE] = { .nuops = 1, .uops = { { _RETURN_VALUE, 0, 0 } } }, + [SEND_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, 0, 0 }, { _SEND_GEN_FRAME, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, 0, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, 0, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, @@ -1403,7 +1410,6 @@ const char *_PyOpcode_OpName[264] = { [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_LIST] = "BUILD_LIST", [BUILD_MAP] = "BUILD_MAP", [BUILD_SET] = "BUILD_SET", @@ -1653,7 +1659,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, [BINARY_SUBSCR_STR_INT] = BINARY_SUBSCR, [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, - [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, [BUILD_LIST] = BUILD_LIST, [BUILD_MAP] = BUILD_MAP, [BUILD_SET] = BUILD_SET, @@ -1853,6 +1858,7 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 117: \ case 118: \ case 119: \ case 120: \ diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 15071638203457..9835e495d176e7 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -161,7 +161,8 @@ extern PyObject* _Py_Offer_Suggestions(PyObject* exception); PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, Py_ssize_t max_cost); -void _PyErr_FormatNote(const char *format, ...); +// Export for '_json' shared extension +PyAPI_FUNC(void) _PyErr_FormatNote(const char *format, ...); /* Context manipulation (PEP 3134) */ diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index b0e72523f58ed8..fade55945b7dbf 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,11 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_freelist.h" // _PyFreeListState #include "pycore_runtime.h" // _PyRuntime #include "pycore_tstate.h" // _PyThreadStateImpl - // Values for PyThreadState.state. A thread must be in the "attached" state // before calling most Python APIs. If the GIL is enabled, then "attached" // implies that the thread holds the GIL and "detached" implies that the @@ -279,20 +277,6 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // See also PyInterpreterState_Get() and _PyInterpreterState_GET(). extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void); -static inline struct _Py_object_freelists* _Py_object_freelists_GET(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); -#ifdef Py_DEBUG - _Py_EnsureTstateNotNULL(tstate); -#endif - -#ifdef Py_GIL_DISABLED - return &((_PyThreadStateImpl*)tstate)->freelists; -#else - return &tstate->interp->object_state.freelists; -#endif -} - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index c5be67c6d80b9d..618f8d0a36b6c3 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -586,7 +586,6 @@ extern "C" { INIT_ID(__annotate__), \ INIT_ID(__annotations__), \ INIT_ID(__args__), \ - INIT_ID(__asyncio_running_event_loop__), \ INIT_ID(__await__), \ INIT_ID(__bases__), \ INIT_ID(__bool__), \ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 8d3d559814bfd9..1b35a3e3269257 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -11,6 +11,7 @@ extern "C" { #include "pycore_object_deferred.h" #include +#include /* This file introduces a new API for handling references on the stack, called @@ -237,6 +238,38 @@ _PyObjectStack_FromStackRefStack(PyObject **dst, const _PyStackRef *src, size_t } } +// StackRef type checks + +static inline bool +PyStackRef_GenCheck(_PyStackRef stackref) +{ + return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_BoolCheck(_PyStackRef stackref) +{ + return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_LongCheck(_PyStackRef stackref) +{ + return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + +static inline bool +PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) +{ + return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} + + +static inline bool +PyStackRef_FunctionCheck(_PyStackRef stackref) +{ + return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref)); +} #ifdef __cplusplus } diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 15806552e0a384..205ac5d3781ddd 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -6,7 +6,7 @@ // Time formats: // // * Seconds. -// * Seconds as a floating point number (C double). +// * Seconds as a floating-point number (C double). // * Milliseconds (10^-3 seconds). // * Microseconds (10^-6 seconds). // * 100 nanoseconds (10^-7 seconds), used on Windows. diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index befca950920bac..18c972bd367599 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -8,10 +8,10 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_brc.h" // struct _brc_thread_state -#include "pycore_freelist.h" // struct _Py_freelist_state -#include "pycore_mimalloc.h" // struct _mimalloc_thread_state -#include "pycore_qsbr.h" // struct qsbr +#include "pycore_brc.h" // struct _brc_thread_state +#include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_mimalloc.h" // struct _mimalloc_thread_state +#include "pycore_qsbr.h" // struct qsbr // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The @@ -21,13 +21,15 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + PyObject *asyncio_running_loop; // Strong reference + struct _qsbr_thread_state *qsbr; // only used by free-threaded build struct llist_node mem_free_queue; // delayed free queue #ifdef Py_GIL_DISABLED struct _gc_thread_state gc; struct _mimalloc_thread_state mimalloc; - struct _Py_object_freelists freelists; + struct _Py_freelists freelists; struct _brc_thread_state brc; #endif diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 32bd19d968b917..df6bfef715dd34 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -33,6 +33,7 @@ struct _types_runtime_state { struct { struct { PyTypeObject *type; + PyTypeObject def; int64_t interp_count; } types[_Py_MAX_MANAGED_STATIC_TYPES]; } managed_static; diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h index a368edebd622a1..4d7556e68cdaee 100644 --- a/Include/internal/pycore_typevarobject.h +++ b/Include/internal/pycore_typevarobject.h @@ -16,6 +16,7 @@ extern PyObject *_Py_subscript_generic(PyThreadState *, PyObject *); extern PyObject *_Py_set_typeparam_default(PyThreadState *, PyObject *, PyObject *); extern int _Py_initialize_generic(PyInterpreterState *); extern void _Py_clear_generic_types(PyInterpreterState *); +extern int _Py_typing_type_repr(PyUnicodeWriter *, PyObject *); extern PyTypeObject _PyTypeAlias_Type; extern PyTypeObject _PyNoDefault_Type; diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 0e0ad6518771e9..f848a002c3b5d1 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -108,10 +108,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(__asyncio_running_event_loop__); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__await__); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index bd1d27b03b3d00..d6c910255eb87b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -15,25 +15,24 @@ extern "C" { #define _BINARY_OP_ADD_FLOAT 303 #define _BINARY_OP_ADD_INT 304 #define _BINARY_OP_ADD_UNICODE 305 -#define _BINARY_OP_MULTIPLY_FLOAT 306 -#define _BINARY_OP_MULTIPLY_INT 307 -#define _BINARY_OP_SUBTRACT_FLOAT 308 -#define _BINARY_OP_SUBTRACT_INT 309 +#define _BINARY_OP_INPLACE_ADD_UNICODE 306 +#define _BINARY_OP_MULTIPLY_FLOAT 307 +#define _BINARY_OP_MULTIPLY_INT 308 +#define _BINARY_OP_SUBTRACT_FLOAT 309 +#define _BINARY_OP_SUBTRACT_INT 310 #define _BINARY_SLICE BINARY_SLICE -#define _BINARY_SUBSCR 310 +#define _BINARY_SUBSCR 311 #define _BINARY_SUBSCR_DICT BINARY_SUBSCR_DICT #define _BINARY_SUBSCR_GETITEM BINARY_SUBSCR_GETITEM #define _BINARY_SUBSCR_LIST_INT BINARY_SUBSCR_LIST_INT #define _BINARY_SUBSCR_STR_INT BINARY_SUBSCR_STR_INT #define _BINARY_SUBSCR_TUPLE_INT BINARY_SUBSCR_TUPLE_INT -#define _BUILD_CONST_KEY_MAP BUILD_CONST_KEY_MAP #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP #define _BUILD_SET BUILD_SET #define _BUILD_SLICE BUILD_SLICE #define _BUILD_STRING BUILD_STRING #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL 311 #define _CALL_ALLOC_AND_ENTER_INIT CALL_ALLOC_AND_ENTER_INIT #define _CALL_BUILTIN_CLASS 312 #define _CALL_BUILTIN_FAST 313 @@ -45,6 +44,7 @@ extern "C" { #define _CALL_ISINSTANCE CALL_ISINSTANCE #define _CALL_KW CALL_KW #define _CALL_LEN CALL_LEN +#define _CALL_LIST_APPEND CALL_LIST_APPEND #define _CALL_METHOD_DESCRIPTOR_FAST 316 #define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 317 #define _CALL_METHOD_DESCRIPTOR_NOARGS 318 @@ -91,125 +91,125 @@ extern "C" { #define _DEOPT 345 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DYNAMIC_EXIT 346 +#define _DO_CALL 346 +#define _DYNAMIC_EXIT 347 #define _END_SEND END_SEND -#define _ERROR_POP_N 347 +#define _ERROR_POP_N 348 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 348 -#define _FATAL_ERROR 349 +#define _EXPAND_METHOD 349 +#define _FATAL_ERROR 350 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 350 -#define _FOR_ITER_GEN_FRAME 351 -#define _FOR_ITER_TIER_TWO 352 +#define _FOR_ITER 351 +#define _FOR_ITER_GEN_FRAME 352 +#define _FOR_ITER_TIER_TWO 353 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 353 -#define _GUARD_BOTH_INT 354 -#define _GUARD_BOTH_UNICODE 355 -#define _GUARD_BUILTINS_VERSION 356 -#define _GUARD_DORV_NO_DICT 357 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 358 -#define _GUARD_GLOBALS_VERSION 359 -#define _GUARD_IS_FALSE_POP 360 -#define _GUARD_IS_NONE_POP 361 -#define _GUARD_IS_NOT_NONE_POP 362 -#define _GUARD_IS_TRUE_POP 363 -#define _GUARD_KEYS_VERSION 364 -#define _GUARD_NOS_FLOAT 365 -#define _GUARD_NOS_INT 366 -#define _GUARD_NOT_EXHAUSTED_LIST 367 -#define _GUARD_NOT_EXHAUSTED_RANGE 368 -#define _GUARD_NOT_EXHAUSTED_TUPLE 369 -#define _GUARD_TOS_FLOAT 370 -#define _GUARD_TOS_INT 371 -#define _GUARD_TYPE_VERSION 372 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 373 -#define _INIT_CALL_PY_EXACT_ARGS 374 -#define _INIT_CALL_PY_EXACT_ARGS_0 375 -#define _INIT_CALL_PY_EXACT_ARGS_1 376 -#define _INIT_CALL_PY_EXACT_ARGS_2 377 -#define _INIT_CALL_PY_EXACT_ARGS_3 378 -#define _INIT_CALL_PY_EXACT_ARGS_4 379 -#define _INSTRUMENTED_CALL INSTRUMENTED_CALL +#define _GUARD_BOTH_FLOAT 354 +#define _GUARD_BOTH_INT 355 +#define _GUARD_BOTH_UNICODE 356 +#define _GUARD_BUILTINS_VERSION 357 +#define _GUARD_DORV_NO_DICT 358 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 359 +#define _GUARD_GLOBALS_VERSION 360 +#define _GUARD_IS_FALSE_POP 361 +#define _GUARD_IS_NONE_POP 362 +#define _GUARD_IS_NOT_NONE_POP 363 +#define _GUARD_IS_TRUE_POP 364 +#define _GUARD_KEYS_VERSION 365 +#define _GUARD_NOS_FLOAT 366 +#define _GUARD_NOS_INT 367 +#define _GUARD_NOT_EXHAUSTED_LIST 368 +#define _GUARD_NOT_EXHAUSTED_RANGE 369 +#define _GUARD_NOT_EXHAUSTED_TUPLE 370 +#define _GUARD_TOS_FLOAT 371 +#define _GUARD_TOS_INT 372 +#define _GUARD_TYPE_VERSION 373 +#define _IMPORT_FROM IMPORT_FROM +#define _IMPORT_NAME IMPORT_NAME +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 374 +#define _INIT_CALL_PY_EXACT_ARGS 375 +#define _INIT_CALL_PY_EXACT_ARGS_0 376 +#define _INIT_CALL_PY_EXACT_ARGS_1 377 +#define _INIT_CALL_PY_EXACT_ARGS_2 378 +#define _INIT_CALL_PY_EXACT_ARGS_3 379 +#define _INIT_CALL_PY_EXACT_ARGS_4 380 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_BACKWARD INSTRUMENTED_JUMP_BACKWARD #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD +#define _INSTRUMENTED_LINE INSTRUMENTED_LINE #define _INSTRUMENTED_LOAD_SUPER_ATTR INSTRUMENTED_LOAD_SUPER_ATTR #define _INSTRUMENTED_POP_JUMP_IF_FALSE INSTRUMENTED_POP_JUMP_IF_FALSE #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE #define _INSTRUMENTED_RESUME INSTRUMENTED_RESUME -#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST -#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE -#define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 380 -#define _IS_NONE 381 +#define _INTERNAL_INCREMENT_OPT_COUNTER 381 +#define _IS_NONE 382 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 382 -#define _ITER_CHECK_RANGE 383 -#define _ITER_CHECK_TUPLE 384 -#define _ITER_JUMP_LIST 385 -#define _ITER_JUMP_RANGE 386 -#define _ITER_JUMP_TUPLE 387 -#define _ITER_NEXT_LIST 388 -#define _ITER_NEXT_RANGE 389 -#define _ITER_NEXT_TUPLE 390 -#define _JUMP_TO_TOP 391 +#define _ITER_CHECK_LIST 383 +#define _ITER_CHECK_RANGE 384 +#define _ITER_CHECK_TUPLE 385 +#define _ITER_JUMP_LIST 386 +#define _ITER_JUMP_RANGE 387 +#define _ITER_JUMP_TUPLE 388 +#define _ITER_NEXT_LIST 389 +#define _ITER_NEXT_RANGE 390 +#define _ITER_NEXT_TUPLE 391 +#define _JUMP_TO_TOP 392 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 392 -#define _LOAD_ATTR_CLASS 393 -#define _LOAD_ATTR_CLASS_0 394 -#define _LOAD_ATTR_CLASS_1 395 +#define _LOAD_ATTR 393 +#define _LOAD_ATTR_CLASS 394 +#define _LOAD_ATTR_CLASS_0 395 +#define _LOAD_ATTR_CLASS_1 396 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 396 -#define _LOAD_ATTR_INSTANCE_VALUE_0 397 -#define _LOAD_ATTR_INSTANCE_VALUE_1 398 -#define _LOAD_ATTR_METHOD_LAZY_DICT 399 -#define _LOAD_ATTR_METHOD_NO_DICT 400 -#define _LOAD_ATTR_METHOD_WITH_VALUES 401 -#define _LOAD_ATTR_MODULE 402 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 403 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 404 -#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 405 -#define _LOAD_ATTR_SLOT_0 406 -#define _LOAD_ATTR_SLOT_1 407 -#define _LOAD_ATTR_WITH_HINT 408 +#define _LOAD_ATTR_INSTANCE_VALUE 397 +#define _LOAD_ATTR_INSTANCE_VALUE_0 398 +#define _LOAD_ATTR_INSTANCE_VALUE_1 399 +#define _LOAD_ATTR_METHOD_LAZY_DICT 400 +#define _LOAD_ATTR_METHOD_NO_DICT 401 +#define _LOAD_ATTR_METHOD_WITH_VALUES 402 +#define _LOAD_ATTR_MODULE 403 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 404 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 405 +#define _LOAD_ATTR_PROPERTY_FRAME 406 +#define _LOAD_ATTR_SLOT 407 +#define _LOAD_ATTR_SLOT_0 408 +#define _LOAD_ATTR_SLOT_1 409 +#define _LOAD_ATTR_WITH_HINT 410 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 409 -#define _LOAD_CONST_INLINE_BORROW 410 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 411 -#define _LOAD_CONST_INLINE_WITH_NULL 412 +#define _LOAD_CONST_INLINE 411 +#define _LOAD_CONST_INLINE_BORROW 412 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 413 +#define _LOAD_CONST_INLINE_WITH_NULL 414 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 413 -#define _LOAD_FAST_0 414 -#define _LOAD_FAST_1 415 -#define _LOAD_FAST_2 416 -#define _LOAD_FAST_3 417 -#define _LOAD_FAST_4 418 -#define _LOAD_FAST_5 419 -#define _LOAD_FAST_6 420 -#define _LOAD_FAST_7 421 +#define _LOAD_FAST 415 +#define _LOAD_FAST_0 416 +#define _LOAD_FAST_1 417 +#define _LOAD_FAST_2 418 +#define _LOAD_FAST_3 419 +#define _LOAD_FAST_4 420 +#define _LOAD_FAST_5 421 +#define _LOAD_FAST_6 422 +#define _LOAD_FAST_7 423 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 422 -#define _LOAD_GLOBAL_BUILTINS 423 -#define _LOAD_GLOBAL_MODULE 424 +#define _LOAD_GLOBAL 424 +#define _LOAD_GLOBAL_BUILTINS 425 +#define _LOAD_GLOBAL_MODULE 426 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SPECIAL LOAD_SPECIAL @@ -222,53 +222,55 @@ extern "C" { #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE +#define _MAYBE_EXPAND_METHOD 427 +#define _MONITOR_CALL 428 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 425 -#define _POP_JUMP_IF_TRUE 426 +#define _POP_JUMP_IF_FALSE 429 +#define _POP_JUMP_IF_TRUE 430 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 427 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 431 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 428 +#define _PUSH_FRAME 432 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 429 -#define _REPLACE_WITH_TRUE 430 +#define _PY_FRAME_GENERAL 433 +#define _REPLACE_WITH_TRUE 434 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 431 -#define _SEND 432 -#define _SEND_GEN SEND_GEN +#define _SAVE_RETURN_OFFSET 435 +#define _SEND 436 +#define _SEND_GEN_FRAME 437 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 433 -#define _STORE_ATTR 434 -#define _STORE_ATTR_INSTANCE_VALUE 435 -#define _STORE_ATTR_SLOT 436 -#define _STORE_ATTR_WITH_HINT 437 +#define _START_EXECUTOR 438 +#define _STORE_ATTR 439 +#define _STORE_ATTR_INSTANCE_VALUE 440 +#define _STORE_ATTR_SLOT 441 +#define _STORE_ATTR_WITH_HINT 442 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 438 -#define _STORE_FAST_0 439 -#define _STORE_FAST_1 440 -#define _STORE_FAST_2 441 -#define _STORE_FAST_3 442 -#define _STORE_FAST_4 443 -#define _STORE_FAST_5 444 -#define _STORE_FAST_6 445 -#define _STORE_FAST_7 446 +#define _STORE_FAST 443 +#define _STORE_FAST_0 444 +#define _STORE_FAST_1 445 +#define _STORE_FAST_2 446 +#define _STORE_FAST_3 447 +#define _STORE_FAST_4 448 +#define _STORE_FAST_5 449 +#define _STORE_FAST_6 450 +#define _STORE_FAST_7 451 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 447 +#define _STORE_SUBSCR 452 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 448 -#define _TO_BOOL 449 +#define _TIER2_RESUME_CHECK 453 +#define _TO_BOOL 454 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -278,13 +280,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 450 +#define _UNPACK_SEQUENCE 455 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 450 +#define MAX_UOP_ID 455 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 2a2d6e923b7617..d23a4e2ea14345 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -72,6 +72,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -91,6 +92,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, @@ -107,6 +109,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, @@ -124,9 +127,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_CONST_KEY_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAP_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -149,6 +152,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_CLASS_0] = 0, [_LOAD_ATTR_CLASS_1] = 0, [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, + [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = 0, [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -163,6 +167,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IS_NONE] = 0, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -171,7 +177,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, [_ITER_NEXT_LIST] = 0, @@ -193,6 +199,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, + [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG, [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -222,6 +229,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -244,7 +252,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SET_IP] = 0, [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, @@ -253,8 +261,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, - [_DYNAMIC_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_START_EXECUTOR] = HAS_DEOPT_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, + [_START_EXECUTOR] = 0, [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, @@ -273,6 +281,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", @@ -283,9 +292,9 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_SUBSCR_LIST_INT] = "_BINARY_SUBSCR_LIST_INT", [_BINARY_SUBSCR_STR_INT] = "_BINARY_SUBSCR_STR_INT", [_BINARY_SUBSCR_TUPLE_INT] = "_BINARY_SUBSCR_TUPLE_INT", - [_BUILD_CONST_KEY_MAP] = "_BUILD_CONST_KEY_MAP", [_BUILD_LIST] = "_BUILD_LIST", [_BUILD_MAP] = "_BUILD_MAP", + [_BUILD_SET] = "_BUILD_SET", [_BUILD_SLICE] = "_BUILD_SLICE", [_BUILD_STRING] = "_BUILD_STRING", [_BUILD_TUPLE] = "_BUILD_TUPLE", @@ -297,6 +306,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", [_CALL_LEN] = "_CALL_LEN", + [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", @@ -380,6 +390,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_IMPORT_FROM] = "_IMPORT_FROM", + [_IMPORT_NAME] = "_IMPORT_NAME", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", @@ -412,6 +424,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", [_LOAD_ATTR_SLOT_0] = "_LOAD_ATTR_SLOT_0", [_LOAD_ATTR_SLOT_1] = "_LOAD_ATTR_SLOT_1", @@ -441,6 +454,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", [_LOAD_LOCALS] = "_LOAD_LOCALS", + [_LOAD_NAME] = "_LOAD_NAME", [_LOAD_SPECIAL] = "_LOAD_SPECIAL", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", @@ -451,6 +465,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_MATCH_KEYS] = "_MATCH_KEYS", [_MATCH_MAPPING] = "_MATCH_MAPPING", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", [_NOP] = "_NOP", [_POP_EXCEPT] = "_POP_EXCEPT", [_POP_TOP] = "_POP_TOP", @@ -464,6 +479,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_RETURN_GENERATOR] = "_RETURN_GENERATOR", [_RETURN_VALUE] = "_RETURN_VALUE", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", [_SET_ADD] = "_SET_ADD", [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", @@ -620,6 +636,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_ADD_UNICODE: return 2; + case _BINARY_OP_INPLACE_ADD_UNICODE: + return 2; case _BINARY_SUBSCR: return 2; case _BINARY_SLICE: @@ -658,6 +676,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GET_AWAITABLE: return 1; + case _SEND_GEN_FRAME: + return 2; case _YIELD_VALUE: return 1; case _POP_EXCEPT: @@ -690,6 +710,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_LOCALS: return 0; + case _LOAD_NAME: + return 0; case _LOAD_GLOBAL: return 0; case _GUARD_GLOBALS_VERSION: @@ -724,12 +746,12 @@ int _PyUop_num_popped(int opcode, int oparg) return 2 + (oparg-1); case _SET_UPDATE: return 2 + (oparg-1); + case _BUILD_SET: + return oparg; case _BUILD_MAP: return oparg*2; case _SETUP_ANNOTATIONS: return 0; - case _BUILD_CONST_KEY_MAP: - return 1 + oparg; case _DICT_UPDATE: return 2 + (oparg - 1); case _DICT_MERGE: @@ -774,6 +796,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_CLASS: return 1; + case _LOAD_ATTR_PROPERTY_FRAME: + return 1; case _GUARD_DORV_NO_DICT: return 1; case _STORE_ATTR_INSTANCE_VALUE: @@ -802,6 +826,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _CHECK_EXC_MATCH: return 2; + case _IMPORT_NAME: + return 2; + case _IMPORT_FROM: + return 1; case _IS_NONE: return 1; case _GET_LEN: @@ -862,6 +890,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_METHOD_LAZY_DICT: return 1; + case _MAYBE_EXPAND_METHOD: + return 2 + oparg; case _CHECK_PERIODIC: return 0; case _PY_FRAME_GENERAL: @@ -920,6 +950,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 2 + oparg; case _CALL_ISINSTANCE: return 2 + oparg; + case _CALL_LIST_APPEND: + return 3; case _CALL_METHOD_DESCRIPTOR_O: return 2 + oparg; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: diff --git a/Include/object.h b/Include/object.h index a1e5b33b0fdaae..abfdb6ce24df21 100644 --- a/Include/object.h +++ b/Include/object.h @@ -249,11 +249,7 @@ PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); #else static inline PyTypeObject* _Py_TYPE(PyObject *ob) { - #if defined(Py_GIL_DISABLED) - return (PyTypeObject *)_Py_atomic_load_ptr_relaxed(&ob->ob_type); - #else return ob->ob_type; - #endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST(ob)) @@ -284,11 +280,7 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { -#ifdef Py_GIL_DISABLED - _Py_atomic_store_ptr(&ob->ob_type, type); -#else ob->ob_type = type; -#endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 2ae6e5c1ba51ec..54dd76158bf84d 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -53,81 +53,80 @@ extern "C" { #define UNARY_NOT 40 #define WITH_EXCEPT_START 41 #define BINARY_OP 42 -#define BUILD_CONST_KEY_MAP 43 -#define BUILD_LIST 44 -#define BUILD_MAP 45 -#define BUILD_SET 46 -#define BUILD_SLICE 47 -#define BUILD_STRING 48 -#define BUILD_TUPLE 49 -#define CALL 50 -#define CALL_FUNCTION_EX 51 -#define CALL_INTRINSIC_1 52 -#define CALL_INTRINSIC_2 53 -#define CALL_KW 54 -#define COMPARE_OP 55 -#define CONTAINS_OP 56 -#define CONVERT_VALUE 57 -#define COPY 58 -#define COPY_FREE_VARS 59 -#define DELETE_ATTR 60 -#define DELETE_DEREF 61 -#define DELETE_FAST 62 -#define DELETE_GLOBAL 63 -#define DELETE_NAME 64 -#define DICT_MERGE 65 -#define DICT_UPDATE 66 -#define ENTER_EXECUTOR 67 -#define EXTENDED_ARG 68 -#define FOR_ITER 69 -#define GET_AWAITABLE 70 -#define IMPORT_FROM 71 -#define IMPORT_NAME 72 -#define IS_OP 73 -#define JUMP_BACKWARD 74 -#define JUMP_BACKWARD_NO_INTERRUPT 75 -#define JUMP_FORWARD 76 -#define LIST_APPEND 77 -#define LIST_EXTEND 78 -#define LOAD_ATTR 79 -#define LOAD_COMMON_CONSTANT 80 -#define LOAD_CONST 81 -#define LOAD_DEREF 82 -#define LOAD_FAST 83 -#define LOAD_FAST_AND_CLEAR 84 -#define LOAD_FAST_CHECK 85 -#define LOAD_FAST_LOAD_FAST 86 -#define LOAD_FROM_DICT_OR_DEREF 87 -#define LOAD_FROM_DICT_OR_GLOBALS 88 -#define LOAD_GLOBAL 89 -#define LOAD_NAME 90 -#define LOAD_SPECIAL 91 -#define LOAD_SUPER_ATTR 92 -#define MAKE_CELL 93 -#define MAP_ADD 94 -#define MATCH_CLASS 95 -#define POP_JUMP_IF_FALSE 96 -#define POP_JUMP_IF_NONE 97 -#define POP_JUMP_IF_NOT_NONE 98 -#define POP_JUMP_IF_TRUE 99 -#define RAISE_VARARGS 100 -#define RERAISE 101 -#define RETURN_CONST 102 -#define SEND 103 -#define SET_ADD 104 -#define SET_FUNCTION_ATTRIBUTE 105 -#define SET_UPDATE 106 -#define STORE_ATTR 107 -#define STORE_DEREF 108 -#define STORE_FAST 109 -#define STORE_FAST_LOAD_FAST 110 -#define STORE_FAST_STORE_FAST 111 -#define STORE_GLOBAL 112 -#define STORE_NAME 113 -#define SWAP 114 -#define UNPACK_EX 115 -#define UNPACK_SEQUENCE 116 -#define YIELD_VALUE 117 +#define BUILD_LIST 43 +#define BUILD_MAP 44 +#define BUILD_SET 45 +#define BUILD_SLICE 46 +#define BUILD_STRING 47 +#define BUILD_TUPLE 48 +#define CALL 49 +#define CALL_FUNCTION_EX 50 +#define CALL_INTRINSIC_1 51 +#define CALL_INTRINSIC_2 52 +#define CALL_KW 53 +#define COMPARE_OP 54 +#define CONTAINS_OP 55 +#define CONVERT_VALUE 56 +#define COPY 57 +#define COPY_FREE_VARS 58 +#define DELETE_ATTR 59 +#define DELETE_DEREF 60 +#define DELETE_FAST 61 +#define DELETE_GLOBAL 62 +#define DELETE_NAME 63 +#define DICT_MERGE 64 +#define DICT_UPDATE 65 +#define ENTER_EXECUTOR 66 +#define EXTENDED_ARG 67 +#define FOR_ITER 68 +#define GET_AWAITABLE 69 +#define IMPORT_FROM 70 +#define IMPORT_NAME 71 +#define IS_OP 72 +#define JUMP_BACKWARD 73 +#define JUMP_BACKWARD_NO_INTERRUPT 74 +#define JUMP_FORWARD 75 +#define LIST_APPEND 76 +#define LIST_EXTEND 77 +#define LOAD_ATTR 78 +#define LOAD_COMMON_CONSTANT 79 +#define LOAD_CONST 80 +#define LOAD_DEREF 81 +#define LOAD_FAST 82 +#define LOAD_FAST_AND_CLEAR 83 +#define LOAD_FAST_CHECK 84 +#define LOAD_FAST_LOAD_FAST 85 +#define LOAD_FROM_DICT_OR_DEREF 86 +#define LOAD_FROM_DICT_OR_GLOBALS 87 +#define LOAD_GLOBAL 88 +#define LOAD_NAME 89 +#define LOAD_SPECIAL 90 +#define LOAD_SUPER_ATTR 91 +#define MAKE_CELL 92 +#define MAP_ADD 93 +#define MATCH_CLASS 94 +#define POP_JUMP_IF_FALSE 95 +#define POP_JUMP_IF_NONE 96 +#define POP_JUMP_IF_NOT_NONE 97 +#define POP_JUMP_IF_TRUE 98 +#define RAISE_VARARGS 99 +#define RERAISE 100 +#define RETURN_CONST 101 +#define SEND 102 +#define SET_ADD 103 +#define SET_FUNCTION_ATTRIBUTE 104 +#define SET_UPDATE 105 +#define STORE_ATTR 106 +#define STORE_DEREF 107 +#define STORE_FAST 108 +#define STORE_FAST_LOAD_FAST 109 +#define STORE_FAST_STORE_FAST 110 +#define STORE_GLOBAL 111 +#define STORE_NAME 112 +#define SWAP 113 +#define UNPACK_EX 114 +#define UNPACK_SEQUENCE 115 +#define YIELD_VALUE 116 #define RESUME 149 #define BINARY_OP_ADD_FLOAT 150 #define BINARY_OP_ADD_INT 151 @@ -205,21 +204,21 @@ extern "C" { #define INSTRUMENTED_RESUME 236 #define INSTRUMENTED_END_FOR 237 #define INSTRUMENTED_END_SEND 238 -#define INSTRUMENTED_RETURN_VALUE 239 -#define INSTRUMENTED_RETURN_CONST 240 -#define INSTRUMENTED_YIELD_VALUE 241 -#define INSTRUMENTED_LOAD_SUPER_ATTR 242 -#define INSTRUMENTED_FOR_ITER 243 -#define INSTRUMENTED_CALL 244 -#define INSTRUMENTED_CALL_KW 245 -#define INSTRUMENTED_CALL_FUNCTION_EX 246 -#define INSTRUMENTED_INSTRUCTION 247 -#define INSTRUMENTED_JUMP_FORWARD 248 -#define INSTRUMENTED_JUMP_BACKWARD 249 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 251 -#define INSTRUMENTED_POP_JUMP_IF_NONE 252 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 253 +#define INSTRUMENTED_LOAD_SUPER_ATTR 239 +#define INSTRUMENTED_FOR_ITER 240 +#define INSTRUMENTED_CALL_KW 241 +#define INSTRUMENTED_CALL_FUNCTION_EX 242 +#define INSTRUMENTED_INSTRUCTION 243 +#define INSTRUMENTED_JUMP_FORWARD 244 +#define INSTRUMENTED_JUMP_BACKWARD 245 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 246 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 247 +#define INSTRUMENTED_POP_JUMP_IF_NONE 248 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 249 +#define INSTRUMENTED_RETURN_VALUE 250 +#define INSTRUMENTED_RETURN_CONST 251 +#define INSTRUMENTED_YIELD_VALUE 252 +#define INSTRUMENTED_CALL 253 #define INSTRUMENTED_LINE 254 #define JUMP 256 #define JUMP_NO_INTERRUPT 257 diff --git a/Include/pymacro.h b/Include/pymacro.h index a7945ef84a46fc..e0378f9d27a048 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -15,11 +15,11 @@ // MSVC makes static_assert a keyword in C11-17, contrary to the standards. // // In C++11 and C2x, static_assert is a keyword, redefining is undefined -// behaviour. So only define if building as C (if __STDC_VERSION__ is defined), -// not C++, and only for C11-17. +// behaviour. So only define if building as C, not C++ (if __cplusplus is +// not defined), and only for C11-17. #if !defined(static_assert) && (defined(__GNUC__) || defined(__clang__)) \ - && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ - && __STDC_VERSION__ <= 201710L + && !defined(__cplusplus) && defined(__STDC_VERSION__) \ + && __STDC_VERSION__ >= 201112L && __STDC_VERSION__ <= 201710L # define static_assert _Static_assert #endif @@ -47,7 +47,7 @@ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) #if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ - && !defined(_MSC_VER)) + && !defined(__cplusplus) && !defined(_MSC_VER)) # define Py_BUILD_ASSERT_EXPR(cond) \ ((void)sizeof(struct { int dummy; _Static_assert(cond, #cond); }), \ 0) diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 17fe0df6e1db10..b3dc0a48069969 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -623,8 +623,8 @@ Important files Objects ======= -* [Objects/locations.md](https://github.com/python/cpython/blob/main/Objects/locations.md): Describes the location table -* [Objects/frame_layout.md](https://github.com/python/cpython/blob/main/Objects/frame_layout.md): Describes the frame stack +* [Locations](locations.md): Describes the location table +* [Frames](frames.md): Describes frames and the frame stack * [Objects/object_layout.md](https://github.com/python/cpython/blob/main/Objects/object_layout.md): Describes object layout for 3.11 and later * [Exception Handling](exception_handling.md): Describes the exception table diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 930ea110d857ac..358e2c070cd5fa 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -8,51 +8,50 @@ This is used to optimize dict and attribute lookups, among other things. -Python uses three different mechanisms to intern strings: +Python uses two different mechanisms to intern strings: singletons and +dynamic interning. -- Singleton strings marked in C source with `_Py_STR` and `_Py_ID` macros. - These are statically allocated, and collected using `make regen-global-objects` - (`Tools/build/generate_global_objects.py`), which generates code - for declaration, initialization and finalization. +## Singletons - The difference between the two kinds is not important. (A `_Py_ID` string is - a valid C name, with which we can refer to it; a `_Py_STR` may e.g. contain - non-identifier characters, so it needs a separate C-compatible name.) +The 256 possible one-character latin-1 strings, which can be retrieved with +`_Py_LATIN1_CHR(c)`, are stored in statically allocated arrays, +`_PyRuntime.static_objects.strings.ascii` and +`_PyRuntime.static_objects.strings.latin1`. - The empty string is in this category (as `_Py_STR(empty)`). +Longer singleton strings are marked in C source with `_Py_ID` (if the string +is a valid C identifier fragment) or `_Py_STR` (if it needs a separate +C-compatible name.) +These are also stored in statically allocated arrays. +They are collected from CPython sources using `make regen-global-objects` +(`Tools/build/generate_global_objects.py`), which generates code +for declaration, initialization and finalization. - These singletons are interned in a runtime-global lookup table, - `_PyRuntime.cached_objects.interned_strings` (`INTERNED_STRINGS`), - at runtime initialization. +The empty string is one of the singletons: `_Py_STR(empty)`. -- The 256 possible one-character latin-1 strings are singletons, - which can be retrieved with `_Py_LATIN1_CHR(c)`, are stored in runtime-global - arrays, `_PyRuntime.static_objects.strings.ascii` and - `_PyRuntime.static_objects.strings.latin1`. +The three sets of singletons (`_Py_LATIN1_CHR`, `_Py_ID`, `_Py_STR`) +are disjoint. +If you have such a singleton, it (and no other copy) will be interned. - These are NOT interned at startup in the normal build. - In the free-threaded build, they are; this avoids modifying the - global lookup table after threads are started. +These singletons are interned in a runtime-global lookup table, +`_PyRuntime.cached_objects.interned_strings` (`INTERNED_STRINGS`), +at runtime initialization, and immutable until it's torn down +at runtime finalization. +It is shared across threads and interpreters without any synchronization. - Interning a one-char latin-1 string will always intern the corresponding - singleton. -- All other strings are allocated dynamically, and have their - `_PyUnicode_STATE(s).statically_allocated` flag set to zero. - When interned, such strings are added to an interpreter-wide dict, - `PyInterpreterState.cached_objects.interned_strings`. +## Dynamically allocated strings - The key and value of each entry in this dict reference the same object. +All other strings are allocated dynamically, and have their +`_PyUnicode_STATE(s).statically_allocated` flag set to zero. +When interned, such strings are added to an interpreter-wide dict, +`PyInterpreterState.cached_objects.interned_strings`. -The three sets of singletons (`_Py_STR`, `_Py_ID`, `_Py_LATIN1_CHR`) -are disjoint. -If you have such a singleton, it (and no other copy) will be interned. +The key and value of each entry in this dict reference the same object. ## Immortality and reference counting -Invariant: Every immortal string is interned, *except* the one-char latin-1 -singletons (which might but might not be interned). +Invariant: Every immortal string is interned. In practice, this means that you must not use `_Py_SetImmortal` on a string. (If you know it's already immortal, don't immortalize it; @@ -115,8 +114,5 @@ The valid transitions between these states are: Using `_PyUnicode_InternStatic` on these is an error; the other cases don't change the state. -- One-char latin-1 singletons can be interned (0 -> 3) using any interning - function; after that the functions don't change the state. - -- Other statically allocated strings are interned (0 -> 3) at runtime init; +- Singletons are interned (0 -> 3) at runtime init; after that all interning functions don't change the state. diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 2d0a5ba46b3e10..05ee1f29b58331 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -231,99 +231,98 @@ 'UNARY_NOT': 40, 'WITH_EXCEPT_START': 41, 'BINARY_OP': 42, - 'BUILD_CONST_KEY_MAP': 43, - 'BUILD_LIST': 44, - 'BUILD_MAP': 45, - 'BUILD_SET': 46, - 'BUILD_SLICE': 47, - 'BUILD_STRING': 48, - 'BUILD_TUPLE': 49, - 'CALL': 50, - 'CALL_FUNCTION_EX': 51, - 'CALL_INTRINSIC_1': 52, - 'CALL_INTRINSIC_2': 53, - 'CALL_KW': 54, - 'COMPARE_OP': 55, - 'CONTAINS_OP': 56, - 'CONVERT_VALUE': 57, - 'COPY': 58, - 'COPY_FREE_VARS': 59, - 'DELETE_ATTR': 60, - 'DELETE_DEREF': 61, - 'DELETE_FAST': 62, - 'DELETE_GLOBAL': 63, - 'DELETE_NAME': 64, - 'DICT_MERGE': 65, - 'DICT_UPDATE': 66, - 'ENTER_EXECUTOR': 67, - 'EXTENDED_ARG': 68, - 'FOR_ITER': 69, - 'GET_AWAITABLE': 70, - 'IMPORT_FROM': 71, - 'IMPORT_NAME': 72, - 'IS_OP': 73, - 'JUMP_BACKWARD': 74, - 'JUMP_BACKWARD_NO_INTERRUPT': 75, - 'JUMP_FORWARD': 76, - 'LIST_APPEND': 77, - 'LIST_EXTEND': 78, - 'LOAD_ATTR': 79, - 'LOAD_COMMON_CONSTANT': 80, - 'LOAD_CONST': 81, - 'LOAD_DEREF': 82, - 'LOAD_FAST': 83, - 'LOAD_FAST_AND_CLEAR': 84, - 'LOAD_FAST_CHECK': 85, - 'LOAD_FAST_LOAD_FAST': 86, - 'LOAD_FROM_DICT_OR_DEREF': 87, - 'LOAD_FROM_DICT_OR_GLOBALS': 88, - 'LOAD_GLOBAL': 89, - 'LOAD_NAME': 90, - 'LOAD_SPECIAL': 91, - 'LOAD_SUPER_ATTR': 92, - 'MAKE_CELL': 93, - 'MAP_ADD': 94, - 'MATCH_CLASS': 95, - 'POP_JUMP_IF_FALSE': 96, - 'POP_JUMP_IF_NONE': 97, - 'POP_JUMP_IF_NOT_NONE': 98, - 'POP_JUMP_IF_TRUE': 99, - 'RAISE_VARARGS': 100, - 'RERAISE': 101, - 'RETURN_CONST': 102, - 'SEND': 103, - 'SET_ADD': 104, - 'SET_FUNCTION_ATTRIBUTE': 105, - 'SET_UPDATE': 106, - 'STORE_ATTR': 107, - 'STORE_DEREF': 108, - 'STORE_FAST': 109, - 'STORE_FAST_LOAD_FAST': 110, - 'STORE_FAST_STORE_FAST': 111, - 'STORE_GLOBAL': 112, - 'STORE_NAME': 113, - 'SWAP': 114, - 'UNPACK_EX': 115, - 'UNPACK_SEQUENCE': 116, - 'YIELD_VALUE': 117, + 'BUILD_LIST': 43, + 'BUILD_MAP': 44, + 'BUILD_SET': 45, + 'BUILD_SLICE': 46, + 'BUILD_STRING': 47, + 'BUILD_TUPLE': 48, + 'CALL': 49, + 'CALL_FUNCTION_EX': 50, + 'CALL_INTRINSIC_1': 51, + 'CALL_INTRINSIC_2': 52, + 'CALL_KW': 53, + 'COMPARE_OP': 54, + 'CONTAINS_OP': 55, + 'CONVERT_VALUE': 56, + 'COPY': 57, + 'COPY_FREE_VARS': 58, + 'DELETE_ATTR': 59, + 'DELETE_DEREF': 60, + 'DELETE_FAST': 61, + 'DELETE_GLOBAL': 62, + 'DELETE_NAME': 63, + 'DICT_MERGE': 64, + 'DICT_UPDATE': 65, + 'ENTER_EXECUTOR': 66, + 'EXTENDED_ARG': 67, + 'FOR_ITER': 68, + 'GET_AWAITABLE': 69, + 'IMPORT_FROM': 70, + 'IMPORT_NAME': 71, + 'IS_OP': 72, + 'JUMP_BACKWARD': 73, + 'JUMP_BACKWARD_NO_INTERRUPT': 74, + 'JUMP_FORWARD': 75, + 'LIST_APPEND': 76, + 'LIST_EXTEND': 77, + 'LOAD_ATTR': 78, + 'LOAD_COMMON_CONSTANT': 79, + 'LOAD_CONST': 80, + 'LOAD_DEREF': 81, + 'LOAD_FAST': 82, + 'LOAD_FAST_AND_CLEAR': 83, + 'LOAD_FAST_CHECK': 84, + 'LOAD_FAST_LOAD_FAST': 85, + 'LOAD_FROM_DICT_OR_DEREF': 86, + 'LOAD_FROM_DICT_OR_GLOBALS': 87, + 'LOAD_GLOBAL': 88, + 'LOAD_NAME': 89, + 'LOAD_SPECIAL': 90, + 'LOAD_SUPER_ATTR': 91, + 'MAKE_CELL': 92, + 'MAP_ADD': 93, + 'MATCH_CLASS': 94, + 'POP_JUMP_IF_FALSE': 95, + 'POP_JUMP_IF_NONE': 96, + 'POP_JUMP_IF_NOT_NONE': 97, + 'POP_JUMP_IF_TRUE': 98, + 'RAISE_VARARGS': 99, + 'RERAISE': 100, + 'RETURN_CONST': 101, + 'SEND': 102, + 'SET_ADD': 103, + 'SET_FUNCTION_ATTRIBUTE': 104, + 'SET_UPDATE': 105, + 'STORE_ATTR': 106, + 'STORE_DEREF': 107, + 'STORE_FAST': 108, + 'STORE_FAST_LOAD_FAST': 109, + 'STORE_FAST_STORE_FAST': 110, + 'STORE_GLOBAL': 111, + 'STORE_NAME': 112, + 'SWAP': 113, + 'UNPACK_EX': 114, + 'UNPACK_SEQUENCE': 115, + 'YIELD_VALUE': 116, 'INSTRUMENTED_RESUME': 236, 'INSTRUMENTED_END_FOR': 237, 'INSTRUMENTED_END_SEND': 238, - 'INSTRUMENTED_RETURN_VALUE': 239, - 'INSTRUMENTED_RETURN_CONST': 240, - 'INSTRUMENTED_YIELD_VALUE': 241, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 242, - 'INSTRUMENTED_FOR_ITER': 243, - 'INSTRUMENTED_CALL': 244, - 'INSTRUMENTED_CALL_KW': 245, - 'INSTRUMENTED_CALL_FUNCTION_EX': 246, - 'INSTRUMENTED_INSTRUCTION': 247, - 'INSTRUMENTED_JUMP_FORWARD': 248, - 'INSTRUMENTED_JUMP_BACKWARD': 249, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 250, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 239, + 'INSTRUMENTED_FOR_ITER': 240, + 'INSTRUMENTED_CALL_KW': 241, + 'INSTRUMENTED_CALL_FUNCTION_EX': 242, + 'INSTRUMENTED_INSTRUCTION': 243, + 'INSTRUMENTED_JUMP_FORWARD': 244, + 'INSTRUMENTED_JUMP_BACKWARD': 245, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 246, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 247, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 248, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 249, + 'INSTRUMENTED_RETURN_VALUE': 250, + 'INSTRUMENTED_RETURN_CONST': 251, + 'INSTRUMENTED_YIELD_VALUE': 252, + 'INSTRUMENTED_CALL': 253, 'JUMP': 256, 'JUMP_NO_INTERRUPT': 257, 'LOAD_CLOSURE': 258, diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 613123ec7b4329..75df3db262470b 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -424,7 +424,7 @@ def sin(x): # numbers.py for more detail. class Decimal(object): - """Floating point class for decimal arithmetic.""" + """Floating-point class for decimal arithmetic.""" __slots__ = ('_exp','_int','_sign', '_is_special') # Generally, the value of the Decimal instance is given by diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index efb6d343cc9a7c..3fa992eee8eeff 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -1,3 +1,6 @@ +# Important: don't add things to this module, as they will end up in the REPL's +# default globals. Use _pyrepl.main instead. + if __name__ == "__main__": from .main import interactive_console as __pyrepl_interactive_console __pyrepl_interactive_console() diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index dd90912d1d67f8..7f4d0672d02094 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -264,6 +264,7 @@ def select_item(self, i: int) -> None: self.historyi = i self.pos = len(self.buffer) self.dirty = True + self.last_refresh_cache.invalidated = True def get_item(self, i: int) -> str: if i != len(self.history): diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py index 041a4009f42ed8..a6f824dcc4ad14 100644 --- a/Lib/_pyrepl/main.py +++ b/Lib/_pyrepl/main.py @@ -1,16 +1,32 @@ +import errno import os import sys + CAN_USE_PYREPL: bool -if sys.platform != "win32": - CAN_USE_PYREPL = True +FAIL_REASON: str +try: + if sys.platform == "win32" and sys.getwindowsversion().build < 10586: + raise RuntimeError("Windows 10 TH2 or later required") + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + from .simple_interact import check + if err := check(): + raise RuntimeError(err) +except Exception as e: + CAN_USE_PYREPL = False + FAIL_REASON = f"warning: can't use pyrepl: {e}" else: - CAN_USE_PYREPL = sys.getwindowsversion().build >= 10586 # Windows 10 TH2 + CAN_USE_PYREPL = True + FAIL_REASON = "" def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): - global CAN_USE_PYREPL if not CAN_USE_PYREPL: + if not os.getenv('PYTHON_BASIC_REPL') and FAIL_REASON: + from .trace import trace + trace(FAIL_REASON) + print(FAIL_REASON, file=sys.stderr) return sys._baserepl() if mainmodule: @@ -20,8 +36,11 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): namespace = __main__.__dict__ namespace.pop("__pyrepl_interactive_console", None) + # sys._baserepl() above does this internally, we do it here startup_path = os.getenv("PYTHONSTARTUP") if pythonstartup and startup_path: + sys.audit("cpython.run_startup", startup_path) + import tokenize with tokenize.open(startup_path) as f: startup_code = compile(f.read(), startup_path, "exec") @@ -34,22 +53,7 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): if not hasattr(sys, "ps2"): sys.ps2 = "... " - run_interactive = None - try: - import errno - if not os.isatty(sys.stdin.fileno()): - raise OSError(errno.ENOTTY, "tty required", "stdin") - from .simple_interact import check - if err := check(): - raise RuntimeError(err) - from .simple_interact import run_multiline_interactive_console - run_interactive = run_multiline_interactive_console - except Exception as e: - from .trace import trace - msg = f"warning: can't use pyrepl: {e}" - trace(msg) - print(msg, file=sys.stderr) - CAN_USE_PYREPL = False - if run_interactive is None: - return sys._baserepl() - run_interactive(namespace) + from .console import InteractiveColoredConsole + from .simple_interact import run_multiline_interactive_console + console = InteractiveColoredConsole(namespace, filename="") + run_multiline_interactive_console(console) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 63ae661968408e..8b282a382d374f 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -58,7 +58,6 @@ def disp_str(buffer: str) -> tuple[str, list[int]]: elif unicodedata.category(c).startswith("C"): c = r"\u%04x" % ord(c) s.append(c) - b.append(str_width(c)) b.extend([0] * (len(c) - 1)) else: s.append(c) @@ -131,7 +130,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: (r"\M-7", "digit-arg"), (r"\M-8", "digit-arg"), (r"\M-9", "digit-arg"), - # (r'\M-\n', 'insert-nl'), + (r"\M-\n", "accept"), ("\\\\", "self-insert"), (r"\x1b[200~", "enable_bracketed_paste"), (r"\x1b[201~", "disable_bracketed_paste"), @@ -254,6 +253,7 @@ class RefreshCache: pos: int = field(init=False) cxy: tuple[int, int] = field(init=False) dimensions: tuple[int, int] = field(init=False) + invalidated: bool = False def update_cache(self, reader: Reader, @@ -266,14 +266,19 @@ def update_cache(self, self.pos = reader.pos self.cxy = reader.cxy self.dimensions = reader.console.width, reader.console.height + self.invalidated = False def valid(self, reader: Reader) -> bool: + if self.invalidated: + return False dimensions = reader.console.width, reader.console.height dimensions_changed = dimensions != self.dimensions paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste return not (dimensions_changed or paste_changed) def get_cached_location(self, reader: Reader) -> tuple[int, int]: + if self.invalidated: + raise ValueError("Cache is invalidated") offset = 0 earliest_common_pos = min(reader.pos, self.pos) num_common_lines = len(self.line_end_offsets) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 28f592d80b1b03..3d94f91753587e 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -58,7 +58,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any + from typing import Any, Mapping MoreLinesCallable = Callable[[str], bool] @@ -559,7 +559,7 @@ def stub(*args: object, **kwds: object) -> None: # ____________________________________________________________ -def _setup(namespace: dict[str, Any]) -> None: +def _setup(namespace: Mapping[str, Any]) -> None: global raw_input if raw_input is not None: return # don't run _setup twice @@ -575,7 +575,9 @@ def _setup(namespace: dict[str, Any]) -> None: _wrapper.f_in = f_in _wrapper.f_out = f_out - # set up namespace in rlcompleter + # set up namespace in rlcompleter, which requires it to be a bona fide dict + if not isinstance(namespace, dict): + namespace = dict(namespace) _wrapper.config.readline_completer = RLCompleter(namespace).complete # this is not really what readline.c does. Better than nothing I guess diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index bc16c1f6a23159..2c3dffe070c629 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -27,12 +27,10 @@ import _sitebuiltins import linecache -import builtins +import functools import sys import code -from types import ModuleType -from .console import InteractiveColoredConsole from .readline import _get_reader, multiline_input TYPE_CHECKING = False @@ -81,21 +79,36 @@ def _clear_screen(): } +def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool: + # ooh, look at the hack: + src = _strip_final_indent(unicodetext) + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + lines = src.splitlines(keepends=True) + if len(lines) == 1: + return False + + last_line = lines[-1] + was_indented = last_line.startswith((" ", "\t")) + not_empty = last_line.strip() != "" + incomplete = not last_line.endswith("\n") + return (was_indented or not_empty) and incomplete + else: + return code is None + + def run_multiline_interactive_console( - namespace: dict[str, Any], + console: code.InteractiveConsole, + *, future_flags: int = 0, - console: code.InteractiveConsole | None = None, ) -> None: from .readline import _setup - _setup(namespace) - - if console is None: - console = InteractiveColoredConsole( - namespace, filename="" - ) + _setup(console.locals) if future_flags: console.compile.compiler.flags |= future_flags + more_lines = functools.partial(_more_lines, console) input_n = 0 def maybe_run_command(statement: str) -> bool: @@ -121,16 +134,6 @@ def maybe_run_command(statement: str) -> bool: return False - def more_lines(unicodetext: str) -> bool: - # ooh, look at the hack: - src = _strip_final_indent(unicodetext) - try: - code = console.compile(src, "", "single") - except (OverflowError, SyntaxError, ValueError): - return False - else: - return code is None - while 1: try: try: @@ -154,7 +157,15 @@ def more_lines(unicodetext: str) -> bool: assert not more input_n += 1 except KeyboardInterrupt: - console.write("KeyboardInterrupt\n") + r = _get_reader() + if r.last_command and 'isearch' in r.last_command.__name__: + r.isearch_direction = '' + r.console.forgetinput() + r.pop_input_trans() + r.dirty = True + r.refresh() + r.in_bracketed_paste = False + console.write("\nKeyboardInterrupt\n") console.resetbuffer() except MemoryError: console.write("\nMemoryError\n") diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index c4dedd97d1e13d..18b2bba91c8c9b 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -383,6 +383,8 @@ def get_event(self, block: bool = True) -> Event | None: Returns: - Event: Event object from the event queue. """ + if not block and not self.wait(timeout=0): + return None while self.event_queue.empty(): while True: try: @@ -397,8 +399,6 @@ def get_event(self, block: bool = True) -> Event | None: raise else: break - if not block: - break return self.event_queue.get() def wait(self, timeout: float | None = None) -> bool: diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py new file mode 100644 index 00000000000000..141e31bbf910e3 --- /dev/null +++ b/Lib/annotationlib.py @@ -0,0 +1,696 @@ +"""Helpers for introspecting and wrapping annotations.""" + +import ast +import enum +import functools +import sys +import types + +__all__ = ["Format", "ForwardRef", "call_annotate_function", "get_annotations"] + + +class Format(enum.IntEnum): + VALUE = 1 + FORWARDREF = 2 + SOURCE = 3 + + +_Union = None +_sentinel = object() + +# Slots shared by ForwardRef and _Stringifier. The __forward__ names must be +# preserved for compatibility with the old typing.ForwardRef class. The remaining +# names are private. +_SLOTS = ( + "__forward_evaluated__", + "__forward_value__", + "__forward_is_argument__", + "__forward_is_class__", + "__forward_module__", + "__weakref__", + "__arg__", + "__ast_node__", + "__code__", + "__globals__", + "__owner__", + "__cell__", +) + + +class ForwardRef: + """Wrapper that holds a forward reference.""" + + __slots__ = _SLOTS + + def __init__( + self, + arg, + *, + module=None, + owner=None, + is_argument=True, + is_class=False, + _globals=None, + _cell=None, + ): + if not isinstance(arg, str): + raise TypeError(f"Forward reference must be a string -- got {arg!r}") + + self.__arg__ = arg + self.__forward_evaluated__ = False + self.__forward_value__ = None + self.__forward_is_argument__ = is_argument + self.__forward_is_class__ = is_class + self.__forward_module__ = module + self.__code__ = None + self.__ast_node__ = None + self.__globals__ = _globals + self.__cell__ = _cell + self.__owner__ = owner + + def __init_subclass__(cls, /, *args, **kwds): + raise TypeError("Cannot subclass ForwardRef") + + def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None): + """Evaluate the forward reference and return the value. + + If the forward reference is not evaluatable, raise an exception. + """ + if self.__forward_evaluated__: + return self.__forward_value__ + if self.__cell__ is not None: + try: + value = self.__cell__.cell_contents + except ValueError: + pass + else: + self.__forward_evaluated__ = True + self.__forward_value__ = value + return value + if owner is None: + owner = self.__owner__ + if type_params is None and owner is None: + raise TypeError("Either 'type_params' or 'owner' must be provided") + + if self.__forward_module__ is not None: + globals = getattr( + sys.modules.get(self.__forward_module__, None), "__dict__", globals + ) + if globals is None: + globals = self.__globals__ + if globals is None: + if isinstance(owner, type): + module_name = getattr(owner, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + globals = getattr(module, "__dict__", None) + elif isinstance(owner, types.ModuleType): + globals = getattr(owner, "__dict__", None) + elif callable(owner): + globals = getattr(owner, "__globals__", None) + + if locals is None: + locals = {} + if isinstance(self.__owner__, type): + locals.update(vars(self.__owner__)) + + if type_params is None and self.__owner__ is not None: + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + type_params = getattr(self.__owner__, "__type_params__", None) + + # type parameters require some special handling, + # as they exist in their own scope + # but `eval()` does not have a dedicated parameter for that scope. + # For classes, names in type parameter scopes should override + # names in the global scope (which here are called `localns`!), + # but should in turn be overridden by names in the class scope + # (which here are called `globalns`!) + if type_params is not None: + globals, locals = dict(globals), dict(locals) + for param in type_params: + param_name = param.__name__ + if not self.__forward_is_class__ or param_name not in globals: + globals[param_name] = param + locals.pop(param_name, None) + + code = self.__forward_code__ + value = eval(code, globals=globals, locals=locals) + self.__forward_evaluated__ = True + self.__forward_value__ = value + return value + + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + import typing + import warnings + + if type_params is _sentinel: + typing._deprecation_warning_for_no_type_params_passed( + "typing.ForwardRef._evaluate" + ) + type_params = () + warnings._deprecated( + "ForwardRef._evaluate", + "{name} is a private API and is retained for compatibility, but will be removed" + " in Python 3.16. Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.", + remove=(3, 16), + ) + return typing.evaluate_forward_ref( + self, + globals=globalns, + locals=localns, + type_params=type_params, + _recursive_guard=recursive_guard, + ) + + @property + def __forward_arg__(self): + if self.__arg__ is not None: + return self.__arg__ + if self.__ast_node__ is not None: + self.__arg__ = ast.unparse(self.__ast_node__) + return self.__arg__ + raise AssertionError( + "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" + ) + + @property + def __forward_code__(self): + if self.__code__ is not None: + return self.__code__ + arg = self.__forward_arg__ + # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. + # Unfortunately, this isn't a valid expression on its own, so we + # do the unpacking manually. + if arg.startswith("*"): + arg_to_compile = f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + arg_to_compile = arg + try: + self.__code__ = compile(arg_to_compile, "", "eval") + except SyntaxError: + raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") + return self.__code__ + + def __eq__(self, other): + if not isinstance(other, ForwardRef): + return NotImplemented + if self.__forward_evaluated__ and other.__forward_evaluated__: + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_value__ == other.__forward_value__ + ) + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_module__ == other.__forward_module__ + ) + + def __hash__(self): + return hash((self.__forward_arg__, self.__forward_module__)) + + def __or__(self, other): + global _Union + if _Union is None: + from typing import Union as _Union + return _Union[self, other] + + def __ror__(self, other): + global _Union + if _Union is None: + from typing import Union as _Union + return _Union[other, self] + + def __repr__(self): + if self.__forward_module__ is None: + module_repr = "" + else: + module_repr = f", module={self.__forward_module__!r}" + return f"ForwardRef({self.__forward_arg__!r}{module_repr})" + + +class _Stringifier: + # Must match the slots on ForwardRef, so we can turn an instance of one into an + # instance of the other in place. + __slots__ = _SLOTS + + def __init__(self, node, globals=None, owner=None, is_class=False, cell=None): + assert isinstance(node, ast.AST) + self.__arg__ = None + self.__forward_evaluated__ = False + self.__forward_value__ = None + self.__forward_is_argument__ = False + self.__forward_is_class__ = is_class + self.__forward_module__ = None + self.__code__ = None + self.__ast_node__ = node + self.__globals__ = globals + self.__cell__ = cell + self.__owner__ = owner + + def __convert(self, other): + if isinstance(other, _Stringifier): + return other.__ast_node__ + elif isinstance(other, slice): + return ast.Slice( + lower=self.__convert(other.start) if other.start is not None else None, + upper=self.__convert(other.stop) if other.stop is not None else None, + step=self.__convert(other.step) if other.step is not None else None, + ) + else: + return ast.Constant(value=other) + + def __make_new(self, node): + return _Stringifier( + node, self.__globals__, self.__owner__, self.__forward_is_class__ + ) + + # Must implement this since we set __eq__. We hash by identity so that + # stringifiers in dict keys are kept separate. + def __hash__(self): + return id(self) + + def __getitem__(self, other): + # Special case, to avoid stringifying references to class-scoped variables + # as '__classdict__["x"]'. + if ( + isinstance(self.__ast_node__, ast.Name) + and self.__ast_node__.id == "__classdict__" + ): + raise KeyError + if isinstance(other, tuple): + elts = [self.__convert(elt) for elt in other] + other = ast.Tuple(elts) + else: + other = self.__convert(other) + assert isinstance(other, ast.AST), repr(other) + return self.__make_new(ast.Subscript(self.__ast_node__, other)) + + def __getattr__(self, attr): + return self.__make_new(ast.Attribute(self.__ast_node__, attr)) + + def __call__(self, *args, **kwargs): + return self.__make_new( + ast.Call( + self.__ast_node__, + [self.__convert(arg) for arg in args], + [ + ast.keyword(key, self.__convert(value)) + for key, value in kwargs.items() + ], + ) + ) + + def __iter__(self): + yield self.__make_new(ast.Starred(self.__ast_node__)) + + def __repr__(self): + return ast.unparse(self.__ast_node__) + + def __format__(self, format_spec): + raise TypeError("Cannot stringify annotation containing string formatting") + + def _make_binop(op: ast.AST): + def binop(self, other): + return self.__make_new( + ast.BinOp(self.__ast_node__, op, self.__convert(other)) + ) + + return binop + + __add__ = _make_binop(ast.Add()) + __sub__ = _make_binop(ast.Sub()) + __mul__ = _make_binop(ast.Mult()) + __matmul__ = _make_binop(ast.MatMult()) + __truediv__ = _make_binop(ast.Div()) + __mod__ = _make_binop(ast.Mod()) + __lshift__ = _make_binop(ast.LShift()) + __rshift__ = _make_binop(ast.RShift()) + __or__ = _make_binop(ast.BitOr()) + __xor__ = _make_binop(ast.BitXor()) + __and__ = _make_binop(ast.BitAnd()) + __floordiv__ = _make_binop(ast.FloorDiv()) + __pow__ = _make_binop(ast.Pow()) + + del _make_binop + + def _make_rbinop(op: ast.AST): + def rbinop(self, other): + return self.__make_new( + ast.BinOp(self.__convert(other), op, self.__ast_node__) + ) + + return rbinop + + __radd__ = _make_rbinop(ast.Add()) + __rsub__ = _make_rbinop(ast.Sub()) + __rmul__ = _make_rbinop(ast.Mult()) + __rmatmul__ = _make_rbinop(ast.MatMult()) + __rtruediv__ = _make_rbinop(ast.Div()) + __rmod__ = _make_rbinop(ast.Mod()) + __rlshift__ = _make_rbinop(ast.LShift()) + __rrshift__ = _make_rbinop(ast.RShift()) + __ror__ = _make_rbinop(ast.BitOr()) + __rxor__ = _make_rbinop(ast.BitXor()) + __rand__ = _make_rbinop(ast.BitAnd()) + __rfloordiv__ = _make_rbinop(ast.FloorDiv()) + __rpow__ = _make_rbinop(ast.Pow()) + + del _make_rbinop + + def _make_compare(op): + def compare(self, other): + return self.__make_new( + ast.Compare( + left=self.__ast_node__, + ops=[op], + comparators=[self.__convert(other)], + ) + ) + + return compare + + __lt__ = _make_compare(ast.Lt()) + __le__ = _make_compare(ast.LtE()) + __eq__ = _make_compare(ast.Eq()) + __ne__ = _make_compare(ast.NotEq()) + __gt__ = _make_compare(ast.Gt()) + __ge__ = _make_compare(ast.GtE()) + + del _make_compare + + def _make_unary_op(op): + def unary_op(self): + return self.__make_new(ast.UnaryOp(op, self.__ast_node__)) + + return unary_op + + __invert__ = _make_unary_op(ast.Invert()) + __pos__ = _make_unary_op(ast.UAdd()) + __neg__ = _make_unary_op(ast.USub()) + + del _make_unary_op + + +class _StringifierDict(dict): + def __init__(self, namespace, globals=None, owner=None, is_class=False): + super().__init__(namespace) + self.namespace = namespace + self.globals = globals + self.owner = owner + self.is_class = is_class + self.stringifiers = [] + + def __missing__(self, key): + fwdref = _Stringifier( + ast.Name(id=key), + globals=self.globals, + owner=self.owner, + is_class=self.is_class, + ) + self.stringifiers.append(fwdref) + return fwdref + + +def call_evaluate_function(evaluate, format, *, owner=None): + """Call an evaluate function. Evaluate functions are normally generated for + the value of type aliases and the bounds, constraints, and defaults of + type parameter objects. + """ + return call_annotate_function(evaluate, format, owner=owner, _is_evaluate=True) + + +def call_annotate_function(annotate, format, *, owner=None, + _is_evaluate=False): + """Call an __annotate__ function. __annotate__ functions are normally + generated by the compiler to defer the evaluation of annotations. They + can be called with any of the format arguments in the Format enum, but + compiler-generated __annotate__ functions only support the VALUE format. + This function provides additional functionality to call __annotate__ + functions with the FORWARDREF and SOURCE formats. + + *annotate* must be an __annotate__ function, which takes a single argument + and returns a dict of annotations. + + *format* must be a member of the Format enum or one of the corresponding + integer values. + + *owner* can be the object that owns the annotations (i.e., the module, + class, or function that the __annotate__ function derives from). With the + FORWARDREF format, it is used to provide better evaluation capabilities + on the generated ForwardRef objects. + + """ + try: + return annotate(format) + except NotImplementedError: + pass + if format == Format.SOURCE: + # SOURCE is implemented by calling the annotate function in a special + # environment where every name lookup results in an instance of _Stringifier. + # _Stringifier supports every dunder operation and returns a new _Stringifier. + # At the end, we get a dictionary that mostly contains _Stringifier objects (or + # possibly constants if the annotate function uses them directly). We then + # convert each of those into a string to get an approximation of the + # original source. + globals = _StringifierDict({}) + if annotate.__closure__: + freevars = annotate.__code__.co_freevars + new_closure = [] + for i, cell in enumerate(annotate.__closure__): + if i < len(freevars): + name = freevars[i] + else: + name = "__cell__" + fwdref = _Stringifier(ast.Name(id=name)) + new_closure.append(types.CellType(fwdref)) + closure = tuple(new_closure) + else: + closure = None + func = types.FunctionType(annotate.__code__, globals, closure=closure, + argdefs=annotate.__defaults__, kwdefaults=annotate.__kwdefaults__) + annos = func(Format.VALUE) + if _is_evaluate: + return annos if isinstance(annos, str) else repr(annos) + return { + key: val if isinstance(val, str) else repr(val) + for key, val in annos.items() + } + elif format == Format.FORWARDREF: + # FORWARDREF is implemented similarly to SOURCE, but there are two changes, + # at the beginning and the end of the process. + # First, while SOURCE uses an empty dictionary as the namespace, so that all + # name lookups result in _Stringifier objects, FORWARDREF uses the globals + # and builtins, so that defined names map to their real values. + # Second, instead of returning strings, we want to return either real values + # or ForwardRef objects. To do this, we keep track of all _Stringifier objects + # created while the annotation is being evaluated, and at the end we convert + # them all to ForwardRef objects by assigning to __class__. To make this + # technique work, we have to ensure that the _Stringifier and ForwardRef + # classes share the same attributes. + # We use this technique because while the annotations are being evaluated, + # we want to support all operations that the language allows, including even + # __getattr__ and __eq__, and return new _Stringifier objects so we can accurately + # reconstruct the source. But in the dictionary that we eventually return, we + # want to return objects with more user-friendly behavior, such as an __eq__ + # that returns a bool and an defined set of attributes. + namespace = {**annotate.__builtins__, **annotate.__globals__} + is_class = isinstance(owner, type) + globals = _StringifierDict(namespace, annotate.__globals__, owner, is_class) + if annotate.__closure__: + freevars = annotate.__code__.co_freevars + new_closure = [] + for i, cell in enumerate(annotate.__closure__): + try: + cell.cell_contents + except ValueError: + if i < len(freevars): + name = freevars[i] + else: + name = "__cell__" + fwdref = _Stringifier( + ast.Name(id=name), + cell=cell, + owner=owner, + globals=annotate.__globals__, + is_class=is_class, + ) + globals.stringifiers.append(fwdref) + new_closure.append(types.CellType(fwdref)) + else: + new_closure.append(cell) + closure = tuple(new_closure) + else: + closure = None + func = types.FunctionType(annotate.__code__, globals, closure=closure, + argdefs=annotate.__defaults__, kwdefaults=annotate.__kwdefaults__) + result = func(Format.VALUE) + for obj in globals.stringifiers: + obj.__class__ = ForwardRef + return result + elif format == Format.VALUE: + # Should be impossible because __annotate__ functions must not raise + # NotImplementedError for this format. + raise RuntimeError("annotate function does not support VALUE format") + else: + raise ValueError(f"Invalid format: {format!r}") + + +# We use the descriptors from builtins.type instead of accessing +# .__annotations__ and .__annotate__ directly on class objects, because +# otherwise we could get wrong results in some cases involving metaclasses. +# See PEP 749. +_BASE_GET_ANNOTATE = type.__dict__["__annotate__"].__get__ +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + +def get_annotate_function(obj): + """Get the __annotate__ function for an object. + + obj may be a function, class, or module, or a user-defined type with + an `__annotate__` attribute. + + Returns the __annotate__ function or None. + """ + if isinstance(obj, type): + return _BASE_GET_ANNOTATE(obj) + return getattr(obj, "__annotate__", None) + + +def get_annotations( + obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE +): + """Compute the annotations dict for an object. + + obj may be a callable, class, or module. + Passing in an object of any other type raises TypeError. + + Returns a dict. get_annotations() returns a new dict every time + it's called; calling it twice on the same object will return two + different but equivalent dicts. + + This function handles several details for you: + + * If eval_str is true, values of type str will + be un-stringized using eval(). This is intended + for use with stringized annotations + ("from __future__ import annotations"). + * If obj doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using getattr() and dict.get() for safety. + * Always, always, always returns a freshly-created dict. + + eval_str controls whether or not values of type str are replaced + with the result of calling eval() on those values: + + * If eval_str is true, eval() is called on values of type str. + * If eval_str is false (the default), values of type str are unchanged. + + globals and locals are passed in to eval(); see the documentation + for eval() for more information. If either globals or locals is + None, this function may replace that value with a context-specific + default, contingent on type(obj): + + * If obj is a module, globals defaults to obj.__dict__. + * If obj is a class, globals defaults to + sys.modules[obj.__module__].__dict__ and locals + defaults to the obj class namespace. + * If obj is a callable, globals defaults to obj.__globals__, + although if obj is a wrapped function (using + functools.update_wrapper()) it is first unwrapped. + """ + if eval_str and format != Format.VALUE: + raise ValueError("eval_str=True is only supported with format=Format.VALUE") + + # For VALUE format, we look at __annotations__ directly. + if format != Format.VALUE: + annotate = get_annotate_function(obj) + if annotate is not None: + ann = call_annotate_function(annotate, format, owner=obj) + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") + return dict(ann) + + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return {} + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return {} + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if isinstance(obj, type): + # class + obj_globals = None + module_name = getattr(obj, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, "__dict__", None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + obj_globals = getattr(obj, "__dict__") + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + obj_globals = getattr(obj, "__globals__", None) + obj_locals = None + unwrap = obj + elif ann is not None: + obj_globals = obj_locals = unwrap = None + else: + raise TypeError(f"{obj!r} is not a module, class, or callable.") + + if unwrap is not None: + while True: + if hasattr(unwrap, "__wrapped__"): + unwrap = unwrap.__wrapped__ + continue + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + if locals is None: + locals = {} + locals = {param.__name__: param for param in type_params} | locals + + return_value = { + key: value if not isinstance(value, str) else eval(value, globals, locals) + for key, value in ann.items() + } + return return_value diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 3e2fe93943d4ed..111b7d92367210 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -91,35 +91,24 @@ def run(self): console.write(banner) if startup_path := os.getenv("PYTHONSTARTUP"): + sys.audit("cpython.run_startup", startup_path) + import tokenize with tokenize.open(startup_path) as f: startup_code = compile(f.read(), startup_path, "exec") exec(startup_code, console.locals) ps1 = getattr(sys, "ps1", ">>> ") - if can_colorize(): + if can_colorize() and CAN_USE_PYREPL: ps1 = f"{ANSIColors.BOLD_MAGENTA}{ps1}{ANSIColors.RESET}" console.write(f"{ps1}import asyncio\n") - try: - import errno - if os.getenv("PYTHON_BASIC_REPL"): - raise RuntimeError("user environment requested basic REPL") - if not os.isatty(sys.stdin.fileno()): - raise OSError(errno.ENOTTY, "tty required", "stdin") - - # This import will fail on operating systems with no termios. + if CAN_USE_PYREPL: from _pyrepl.simple_interact import ( - check, run_multiline_interactive_console, ) - if err := check(): - raise RuntimeError(err) - except Exception as e: - console.interact(banner="", exitmsg="") - else: try: - run_multiline_interactive_console(console=console) + run_multiline_interactive_console(console) except SystemExit: # expected via the `exit` and `quit` commands pass @@ -128,6 +117,8 @@ def run(self): console.showtraceback() console.write("Internal error, ") return_code = 1 + else: + console.interact(banner="", exitmsg="") finally: warnings.filterwarnings( 'ignore', @@ -138,7 +129,12 @@ def run(self): if __name__ == '__main__': - CAN_USE_PYREPL = True + sys.audit("cpython.run_stdin") + + if os.getenv('PYTHON_BASIC_REPL'): + CAN_USE_PYREPL = False + else: + from _pyrepl.main import CAN_USE_PYREPL return_code = 0 loop = asyncio.new_event_loop() @@ -163,6 +159,7 @@ def run(self): interactive_hook = getattr(sys, "__interactivehook__", None) if interactive_hook is not None: + sys.audit("cpython.run_interactivehook", interactive_hook) interactive_hook() if interactive_hook is site.register_readline: diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index f0e690b61a73dd..e4a39f4d345c79 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -1028,8 +1028,7 @@ async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None): except OSError as exc: msg = ( f'error while attempting to bind on ' - f'address {laddr!r}: ' - f'{exc.strerror.lower()}' + f'address {laddr!r}: {str(exc).lower()}' ) exc = OSError(exc.errno, msg) my_exceptions.append(exc) @@ -1599,7 +1598,7 @@ async def create_server( except OSError as err: msg = ('error while attempting ' 'to bind on address %r: %s' - % (sa, err.strerror.lower())) + % (sa, str(err).lower())) if err.errno == errno.EADDRNOTAVAIL: # Assume the family is not enabled (bpo-30945) sockets.pop() diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 9c1b5e49e1a70b..5f6fa2348726cf 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -43,7 +43,6 @@ class Future: - This class is not compatible with the wait() and as_completed() methods in the concurrent.futures package. - (In Python 3.4 or later we may be able to unify the implementations.) """ # Class variables serving as defaults for instance variables. @@ -61,7 +60,7 @@ class Future: # the Future protocol (i.e. is intended to be duck-type compatible). # The value must also be not-None, to enable a subclass to declare # that it is not compatible by setting this to None. - # - It is set by __iter__() below so that Task._step() can tell + # - It is set by __iter__() below so that Task.__step() can tell # the difference between # `await Future()` or`yield from Future()` (correct) vs. # `yield Future()` (incorrect). diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index cd869931e01409..2f8f4f08a1e111 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -281,7 +281,7 @@ def __eager_start(self): def __step(self, exc=None): if self.done(): raise exceptions.InvalidStateError( - f'_step(): already done: {self!r}, {exc!r}') + f'__step(): already done: {self!r}, {exc!r}') if self._must_cancel: if not isinstance(exc, exceptions.CancelledError): exc = self._make_cancelled_error() @@ -379,7 +379,7 @@ def __wakeup(self, future): else: # Don't pass the value of `future.result()` explicitly, # as `Future.__iter__` and `Future.__await__` don't need it. - # If we call `_step(value, None)` instead of `_step()`, + # If we call `__step(value, None)` instead of `__step()`, # Python eval loop would use `.send(value)` method call, # instead of `__next__()`, which is slower for futures # that return non-generator iterators from their `__iter__`. diff --git a/Lib/bdb.py b/Lib/bdb.py index aa621053cfb4bc..d7543017940d4f 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -369,6 +369,7 @@ def set_trace(self, frame=None): If frame is not specified, debugging starts from caller's frame. """ + sys.settrace(None) if frame is None: frame = sys._getframe().f_back self.reset() diff --git a/Lib/codecs.py b/Lib/codecs.py index a887e5d4c94a38..e365e6cf22929f 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -1109,21 +1109,12 @@ def make_encoding_map(decoding_map): ### error handlers -try: - strict_errors = lookup_error("strict") - ignore_errors = lookup_error("ignore") - replace_errors = lookup_error("replace") - xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") - backslashreplace_errors = lookup_error("backslashreplace") - namereplace_errors = lookup_error("namereplace") -except LookupError: - # In --disable-unicode builds, these error handler are missing - strict_errors = None - ignore_errors = None - replace_errors = None - xmlcharrefreplace_errors = None - backslashreplace_errors = None - namereplace_errors = None +strict_errors = lookup_error("strict") +ignore_errors = lookup_error("ignore") +replace_errors = lookup_error("replace") +xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") +backslashreplace_errors = lookup_error("backslashreplace") +namereplace_errors = lookup_error("namereplace") # Tell modulefinder that using codecs probably needs the encodings # package diff --git a/Lib/colorsys.py b/Lib/colorsys.py index bc897bd0f99298..e97f91718a3a30 100644 --- a/Lib/colorsys.py +++ b/Lib/colorsys.py @@ -24,7 +24,7 @@ __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb", "rgb_to_hsv","hsv_to_rgb"] -# Some floating point constants +# Some floating-point constants ONE_THIRD = 1.0/3.0 ONE_SIXTH = 1.0/6.0 diff --git a/Lib/csv.py b/Lib/csv.py index 75e35b23236795..cd202659873811 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -47,7 +47,7 @@ class excel: field contains either the quotechar or the delimiter csv.QUOTE_ALL means that quotes are always placed around fields. csv.QUOTE_NONNUMERIC means that quotes are always placed around - fields which do not parse as integers or floating point + fields which do not parse as integers or floating-point numbers. csv.QUOTE_STRINGS means that quotes are always placed around fields which are strings. Note that the Python value None diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 74011b7e28b9f3..4cba606dd8dd4d 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -5,6 +5,7 @@ import inspect import keyword import itertools +import annotationlib import abc from reprlib import recursive_repr @@ -981,7 +982,8 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # actual default value. Pseudo-fields ClassVars and InitVars are # included, despite the fact that they're not real fields. That's # dealt with later. - cls_annotations = inspect.get_annotations(cls) + cls_annotations = annotationlib.get_annotations( + cls, format=annotationlib.Format.FORWARDREF) # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) where diff --git a/Lib/decimal.py b/Lib/decimal.py index 13a0dcb77f1267..f8c548eb1c6ecf 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -1,6 +1,6 @@ -"""Decimal fixed point and floating point arithmetic. +"""Decimal fixed-point and floating-point arithmetic. -This is an implementation of decimal floating point arithmetic based on +This is an implementation of decimal floating-point arithmetic based on the General Decimal Arithmetic Specification: http://speleotrove.com/decimal/decarith.html diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index ab3c3031ef590c..7da1bbaf8a80d7 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -2988,6 +2988,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, excess = len(encoded_word) - remaining_space lines[-1] += encoded_word to_encode = to_encode[len(to_encode_word):] + leading_whitespace = '' if to_encode: lines.append(' ') diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 6d897ca8eeee91..f276303197396b 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -241,7 +241,7 @@ def formatdate(timeval=None, localtime=False, usegmt=False): Fri, 09 Nov 2001 01:08:47 -0000 - Optional timeval if given is a floating point time value as accepted by + Optional timeval if given is a floating-point time value as accepted by gmtime() and localtime(), otherwise the current time is used. Optional localtime is a flag that when True, interprets timeval, and diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 6ffc71fc059a80..020ea694ca63e9 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -88,7 +88,7 @@ def _do_cmp(f1, f2): class dircmp: """A class that manages the comparison of 2 directories. - dircmp(a, b, ignore=None, hide=None, shallow=True) + dircmp(a, b, ignore=None, hide=None, *, shallow=True) A and B are directories. IGNORE is a list of names to ignore, defaults to DEFAULT_IGNORES. @@ -124,7 +124,7 @@ class dircmp: in common_dirs. """ - def __init__(self, a, b, ignore=None, hide=None, shallow=True): # Initialize + def __init__(self, a, b, ignore=None, hide=None, *, shallow=True): # Initialize self.left = a self.right = b if hide is None: @@ -201,7 +201,7 @@ def phase4(self): # Find out differences between common subdirectories a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide, - self.shallow) + shallow=self.shallow) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() diff --git a/Lib/fractions.py b/Lib/fractions.py index 565503911bbe97..34fd0803d1b1ab 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -3,7 +3,6 @@ """Fraction, infinite-precision, rational numbers.""" -from decimal import Decimal import functools import math import numbers @@ -244,7 +243,9 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = numerator.denominator return self - elif isinstance(numerator, (float, Decimal)): + elif (isinstance(numerator, float) or + (not isinstance(numerator, type) and + hasattr(numerator, 'as_integer_ratio'))): # Exact conversion self._numerator, self._denominator = numerator.as_integer_ratio() return self @@ -278,8 +279,7 @@ def __new__(cls, numerator=0, denominator=None): numerator = -numerator else: - raise TypeError("argument should be a string " - "or a Rational instance") + raise TypeError("argument should be a string or a number") elif type(numerator) is int is type(denominator): pass # *very* normal case diff --git a/Lib/functools.py b/Lib/functools.py index a10493f0e25360..49ea9a2f6999f5 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -32,7 +32,7 @@ # wrapper functions that can handle naive introspection WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__', - '__annotations__', '__type_params__') + '__annotate__', '__type_params__') WRAPPER_UPDATES = ('__dict__',) def update_wrapper(wrapper, wrapped, @@ -882,8 +882,8 @@ def register(cls, func=None): f"Invalid first argument to `register()`. " f"{cls!r} is not a class or union type." ) - ann = getattr(cls, '__annotations__', {}) - if not ann: + ann = getattr(cls, '__annotate__', None) + if ann is None: raise TypeError( f"Invalid first argument to `register()`: {cls!r}. " f"Use either `@register(some_class)` or plain `@register` " @@ -893,13 +893,19 @@ def register(cls, func=None): # only import typing if annotation parsing is necessary from typing import get_type_hints - argname, cls = next(iter(get_type_hints(func).items())) + from annotationlib import Format, ForwardRef + argname, cls = next(iter(get_type_hints(func, format=Format.FORWARDREF).items())) if not _is_valid_dispatch_type(cls): if _is_union_type(cls): raise TypeError( f"Invalid annotation for {argname!r}. " f"{cls!r} not all arguments are classes." ) + elif isinstance(cls, ForwardRef): + raise TypeError( + f"Invalid annotation for {argname!r}. " + f"{cls!r} is an unresolved forward reference." + ) else: raise TypeError( f"Invalid annotation for {argname!r}. " diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 35ac2dc6ae280c..351faf428a20cd 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -234,7 +234,7 @@ def _unquote(str): # header. By default, _getdate() returns the current time in the appropriate # "expires" format for a Set-Cookie header. The one optional argument is an # offset from now, in seconds. For example, an offset of -3600 means "one hour -# ago". The offset may be a floating point number. +# ago". The offset may be a floating-point number. # _weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index b1b652dc562c8e..68702ac8fb9157 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,9 @@ Released on 2024-10-xx ========================= +gh-78889: Stop Shell freezes by blocking user access to non-method +sys.stdout.shell attributes, which are all private. + gh-78955: Use user-selected color theme for Help => IDLE Doc. gh-96905: In idlelib code, stop redefining built-ins 'dict' and 'object'. diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 8974b52674fb8c..8f98e73258e778 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -443,6 +443,9 @@ class StdioFile(io.TextIOBase): def __init__(self, shell, tags, encoding='utf-8', errors='strict'): self.shell = shell + # GH-78889: accessing unpickleable attributes freezes Shell. + # IDLE only needs methods; allow 'width' for possible use. + self.shell._RPCProxy__attributes = {'width': 1} self.tags = tags self._encoding = encoding self._errors = errors diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index bf14d57b2503ea..2bb44b290e4a84 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -475,6 +475,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT) # Python 3.14a1 3601 (Fix miscompilation of private names in generic classes) # Python 3.14a1 3602 (Add LOAD_SPECIAL. Remove BEFORE_WITH and BEFORE_ASYNC_WITH) +# Python 3.14a1 3603 (Remove BUILD_CONST_KEY_MAP) # Python 3.15 will start with 3650 @@ -491,7 +492,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3602).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3603).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/inspect.py b/Lib/inspect.py index 0e7b40eb39bce8..ba3ecbb87c7026 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -142,6 +142,7 @@ import abc +from annotationlib import get_annotations import ast import dis import collections.abc @@ -173,121 +174,6 @@ TPFLAGS_IS_ABSTRACT = 1 << 20 -def get_annotations(obj, *, globals=None, locals=None, eval_str=False): - """Compute the annotations dict for an object. - - obj may be a callable, class, or module. - Passing in an object of any other type raises TypeError. - - Returns a dict. get_annotations() returns a new dict every time - it's called; calling it twice on the same object will return two - different but equivalent dicts. - - This function handles several details for you: - - * If eval_str is true, values of type str will - be un-stringized using eval(). This is intended - for use with stringized annotations - ("from __future__ import annotations"). - * If obj doesn't have an annotations dict, returns an - empty dict. (Functions and methods always have an - annotations dict; classes, modules, and other types of - callables may not.) - * Ignores inherited annotations on classes. If a class - doesn't have its own annotations dict, returns an empty dict. - * All accesses to object members and dict values are done - using getattr() and dict.get() for safety. - * Always, always, always returns a freshly-created dict. - - eval_str controls whether or not values of type str are replaced - with the result of calling eval() on those values: - - * If eval_str is true, eval() is called on values of type str. - * If eval_str is false (the default), values of type str are unchanged. - - globals and locals are passed in to eval(); see the documentation - for eval() for more information. If either globals or locals is - None, this function may replace that value with a context-specific - default, contingent on type(obj): - - * If obj is a module, globals defaults to obj.__dict__. - * If obj is a class, globals defaults to - sys.modules[obj.__module__].__dict__ and locals - defaults to the obj class namespace. - * If obj is a callable, globals defaults to obj.__globals__, - although if obj is a wrapped function (using - functools.update_wrapper()) it is first unwrapped. - """ - if isinstance(obj, type): - # class - ann = obj.__annotations__ - - obj_globals = None - module_name = getattr(obj, '__module__', None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - obj_globals = getattr(module, '__dict__', None) - obj_locals = dict(vars(obj)) - unwrap = obj - elif isinstance(obj, types.ModuleType): - # module - ann = getattr(obj, '__annotations__', None) - obj_globals = getattr(obj, '__dict__') - obj_locals = None - unwrap = None - elif callable(obj): - # this includes types.Function, types.BuiltinFunctionType, - # types.BuiltinMethodType, functools.partial, functools.singledispatch, - # "class funclike" from Lib/test/test_inspect... on and on it goes. - ann = getattr(obj, '__annotations__', None) - obj_globals = getattr(obj, '__globals__', None) - obj_locals = None - unwrap = obj - else: - raise TypeError(f"{obj!r} is not a module, class, or callable.") - - if ann is None: - return {} - - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") - - if not ann: - return {} - - if not eval_str: - return dict(ann) - - if unwrap is not None: - while True: - if hasattr(unwrap, '__wrapped__'): - unwrap = unwrap.__wrapped__ - continue - if isinstance(unwrap, functools.partial): - unwrap = unwrap.func - continue - break - if hasattr(unwrap, "__globals__"): - obj_globals = unwrap.__globals__ - - if globals is None: - globals = obj_globals - if locals is None: - locals = obj_locals or {} - - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - if type_params := getattr(obj, "__type_params__", ()): - locals = {param.__name__: param for param in type_params} | locals - - return_value = {key: - value if not isinstance(value, str) else eval(value, globals, locals) - for key, value in ann.items() } - return return_value - - # ----------------------------------------------------------- type-checking def ismodule(object): """Return true if the object is a module.""" diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 323332f064edf8..b804224098e14f 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -293,37 +293,40 @@ def _iterencode_list(lst, _current_indent_level): else: newline_indent = None separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: + for i, value in enumerate(lst): + if i: buf = separator - if isinstance(value, str): - yield buf + _encoder(value) - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, int): - # Subclasses of int/float may override __repr__, but we still - # want to encode them as integers/floats in JSON. One example - # within the standard library is IntEnum. - yield buf + _intstr(value) - elif isinstance(value, float): - # see comment above for int - yield buf + _floatstr(value) - else: - yield buf - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + try: + if isinstance(value, str): + yield buf + _encoder(value) + elif value is None: + yield buf + 'null' + elif value is True: + yield buf + 'true' + elif value is False: + yield buf + 'false' + elif isinstance(value, int): + # Subclasses of int/float may override __repr__, but we still + # want to encode them as integers/floats in JSON. One example + # within the standard library is IntEnum. + yield buf + _intstr(value) + elif isinstance(value, float): + # see comment above for int + yield buf + _floatstr(value) else: - chunks = _iterencode(value, _current_indent_level) - yield from chunks + yield buf + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + yield from chunks + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(lst).__name__} item {i}') + raise if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -382,28 +385,34 @@ def _iterencode_dict(dct, _current_indent_level): yield item_separator yield _encoder(key) yield _key_separator - if isinstance(value, str): - yield _encoder(value) - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, int): - # see comment for int/float in _make_iterencode - yield _intstr(value) - elif isinstance(value, float): - # see comment for int/float in _make_iterencode - yield _floatstr(value) - else: - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + try: + if isinstance(value, str): + yield _encoder(value) + elif value is None: + yield 'null' + elif value is True: + yield 'true' + elif value is False: + yield 'false' + elif isinstance(value, int): + # see comment for int/float in _make_iterencode + yield _intstr(value) + elif isinstance(value, float): + # see comment for int/float in _make_iterencode + yield _floatstr(value) else: - chunks = _iterencode(value, _current_indent_level) - yield from chunks + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + yield from chunks + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(dct).__name__} item {key!r}') + raise if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -436,8 +445,14 @@ def _iterencode(o, _current_indent_level): if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = o - o = _default(o) - yield from _iterencode(o, _current_indent_level) + newobj = _default(o) + try: + yield from _iterencode(newobj, _current_indent_level) + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(o).__name__} object') + raise if markers is not None: del markers[markerid] return _iterencode diff --git a/Lib/linecache.py b/Lib/linecache.py index 3462f1c451ba29..4b38a0464d8747 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -70,7 +70,7 @@ def checkcache(filename=None): return try: stat = os.stat(fullname) - except OSError: + except (OSError, ValueError): cache.pop(filename, None) continue if size != stat.st_size or mtime != stat.st_mtime: @@ -135,10 +135,12 @@ def updatecache(filename, module_globals=None): try: stat = os.stat(fullname) break - except OSError: + except (OSError, ValueError): pass else: return [] + except ValueError: # may be raised by os.stat() + return [] try: with tokenize.open(fullname) as fp: lines = fp.readlines() diff --git a/Lib/os.py b/Lib/os.py index 4b48afb040e565..aaa758d955fe4c 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -373,61 +373,45 @@ def walk(top, topdown=True, onerror=None, followlinks=False): # minor reason when (say) a thousand readable directories are still # left to visit. try: - scandir_it = scandir(top) + with scandir(top) as entries: + for entry in entries: + try: + if followlinks is _walk_symlinks_as_files: + is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() + else: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) except OSError as error: if onerror is not None: onerror(error) continue - cont = False - with scandir_it: - while True: - try: - try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if onerror is not None: - onerror(error) - cont = True - break - - try: - if followlinks is _walk_symlinks_as_files: - is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() - else: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider the entry not to - # be a directory, same behaviour as os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) - - if not topdown and is_dir: - # Bottom-up: traverse into sub-directory, but exclude - # symlinks to directories if followlinks is False - if followlinks: - walk_into = True - else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider the - # entry not to be a symbolic link, same behaviour - # as os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - if cont: - continue - if topdown: # Yield before sub-directory traversal if going top down yield top, dirs, nondirs diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 05f55badd77c58..c32e7762cefea3 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -835,7 +835,8 @@ def copy(self, target, *, follow_symlinks=True, preserve_metadata=False): if preserve_metadata: self._copy_metadata(target) - def copytree(self, target, *, follow_symlinks=True, dirs_exist_ok=False, + def copytree(self, target, *, follow_symlinks=True, + preserve_metadata=False, dirs_exist_ok=False, ignore=None, on_error=None): """ Recursively copy this directory tree to the given destination. @@ -851,6 +852,8 @@ def on_error(err): try: sources = source_dir.iterdir() target_dir.mkdir(exist_ok=dirs_exist_ok) + if preserve_metadata: + source_dir._copy_metadata(target_dir) for source in sources: if ignore and ignore(source): continue @@ -859,7 +862,8 @@ def on_error(err): stack.append((source, target_dir.joinpath(source.name))) else: source.copy(target_dir.joinpath(source.name), - follow_symlinks=follow_symlinks) + follow_symlinks=follow_symlinks, + preserve_metadata=preserve_metadata) except OSError as err: on_error(err) except OSError as err: @@ -915,6 +919,47 @@ def rmdir(self): """ raise UnsupportedOperation(self._unsupported_msg('rmdir()')) + def rmtree(self, ignore_errors=False, on_error=None): + """ + Recursively delete this directory tree. + + If *ignore_errors* is true, exceptions raised from scanning the tree + and removing files and directories are ignored. Otherwise, if + *on_error* is set, it will be called to handle the error. If neither + *ignore_errors* nor *on_error* are set, exceptions are propagated to + the caller. + """ + if ignore_errors: + def on_error(err): + pass + elif on_error is None: + def on_error(err): + raise err + try: + if self.is_symlink(): + raise OSError("Cannot call rmtree on a symbolic link") + elif self.is_junction(): + raise OSError("Cannot call rmtree on a junction") + results = self.walk( + on_error=on_error, + top_down=False, # Bottom-up so we rmdir() empty directories. + follow_symlinks=False) + for dirpath, dirnames, filenames in results: + for name in filenames: + try: + dirpath.joinpath(name).unlink() + except OSError as err: + on_error(err) + for name in dirnames: + try: + dirpath.joinpath(name).rmdir() + except OSError as err: + on_error(err) + self.rmdir() + except OSError as err: + err.filename = str(self) + on_error(err) + def owner(self, *, follow_symlinks=True): """ Return the login name of the file owner. diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index eae8a30c876f19..4fd5279f9fe9ce 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -830,6 +830,25 @@ def rmdir(self): """ os.rmdir(self) + def rmtree(self, ignore_errors=False, on_error=None): + """ + Recursively delete this directory tree. + + If *ignore_errors* is true, exceptions raised from scanning the tree + and removing files and directories are ignored. Otherwise, if + *on_error* is set, it will be called to handle the error. If neither + *ignore_errors* nor *on_error* are set, exceptions are propagated to + the caller. + """ + if on_error: + def onexc(func, filename, err): + err.filename = filename + on_error(err) + else: + onexc = None + import shutil + shutil.rmtree(str(self), ignore_errors, onexc=onexc) + def rename(self, target): """ Rename this path to the target path. diff --git a/Lib/pdb.py b/Lib/pdb.py index 85a3aa2e37996f..7ff973149b167b 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -306,6 +306,8 @@ class Pdb(bdb.Bdb, cmd.Cmd): _file_mtime_table = {} + _last_pdb_instance = None + def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True): bdb.Bdb.__init__(self, skip=skip) @@ -359,6 +361,12 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self._chained_exceptions = tuple() self._chained_exception_index = 0 + def set_trace(self, frame=None): + Pdb._last_pdb_instance = self + if frame is None: + frame = sys._getframe().f_back + super().set_trace(frame) + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt @@ -2350,7 +2358,10 @@ def set_trace(*, header=None): an assertion fails). If given, *header* is printed to the console just before debugging begins. """ - pdb = Pdb() + if Pdb._last_pdb_instance is not None: + pdb = Pdb._last_pdb_instance + else: + pdb = Pdb() if header is not None: pdb.message(header) pdb.set_trace(sys._getframe().f_back) diff --git a/Lib/pickle.py b/Lib/pickle.py index d719ceb7a0b8e8..299c9e0e5e5641 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -51,7 +51,7 @@ bytes_types = (bytes, bytearray) # These are purely informational; no code uses these. -format_version = "4.0" # File format version we write +format_version = "5.0" # File format version we write compatible_formats = ["1.0", # Original protocol 0 "1.1", # Protocol 0 with INST added "1.2", # Original protocol 1 @@ -68,7 +68,7 @@ # The protocol we write by default. May be less than HIGHEST_PROTOCOL. # Only bump this if the oldest still supported version of Python already # includes it. -DEFAULT_PROTOCOL = 4 +DEFAULT_PROTOCOL = 5 class PickleError(Exception): """A common base class for the other pickling exceptions.""" @@ -314,16 +314,17 @@ def load_frame(self, frame_size): # Tools used for pickling. def _getattribute(obj, name): + top = obj for subpath in name.split('.'): if subpath == '': raise AttributeError("Can't get local attribute {!r} on {!r}" - .format(name, obj)) + .format(name, top)) try: parent = obj obj = getattr(obj, subpath) except AttributeError: raise AttributeError("Can't get attribute {!r} on {!r}" - .format(name, obj)) from None + .format(name, top)) from None return obj, parent def whichmodule(obj, name): @@ -408,7 +409,7 @@ def __init__(self, file, protocol=None, *, fix_imports=True, The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. - The default protocol is 4. It was introduced in Python 3.4, and + The default protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest @@ -832,7 +833,7 @@ def save_bytearray(self, obj): if _HAVE_PICKLE_BUFFER: def save_picklebuffer(self, obj): if self.proto < 5: - raise PicklingError("PickleBuffer can only pickled with " + raise PicklingError("PickleBuffer can only be pickled with " "protocol >= 5") with obj.raw() as m: if not m.contiguous: @@ -1110,11 +1111,35 @@ def save_global(self, obj, name=None): self.save(module_name) self.save(name) write(STACK_GLOBAL) - elif parent is not module: - self.save_reduce(getattr, (parent, lastname)) - elif self.proto >= 3: - write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + - bytes(name, "utf-8") + b'\n') + elif '.' in name: + # In protocol < 4, objects with multi-part __qualname__ + # are represented as + # getattr(getattr(..., attrname1), attrname2). + dotted_path = name.split('.') + name = dotted_path.pop(0) + save = self.save + for attrname in dotted_path: + save(getattr) + if self.proto < 2: + write(MARK) + self._save_toplevel_by_name(module_name, name) + for attrname in dotted_path: + save(attrname) + if self.proto < 2: + write(TUPLE) + else: + write(TUPLE2) + write(REDUCE) + else: + self._save_toplevel_by_name(module_name, name) + + self.memoize(obj) + + def _save_toplevel_by_name(self, module_name, name): + if self.proto >= 3: + # Non-ASCII identifiers are supported only with protocols >= 3. + self.write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + + bytes(name, "utf-8") + b'\n') else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING @@ -1124,14 +1149,12 @@ def save_global(self, obj, name=None): elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: - write(GLOBAL + bytes(module_name, "ascii") + b'\n' + - bytes(name, "ascii") + b'\n') + self.write(GLOBAL + bytes(module_name, "ascii") + b'\n' + + bytes(name, "ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " - "pickle protocol %i" % (module, name, self.proto)) from None - - self.memoize(obj) + "pickle protocol %i" % (module_name, name, self.proto)) from None def save_type(self, obj): if obj is type(None): diff --git a/Lib/pstats.py b/Lib/pstats.py index a174a545456e1a..46e18fb7592a77 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -83,7 +83,7 @@ class Stats: method now take arbitrarily many file names as arguments. All the print methods now take an argument that indicates how many lines - to print. If the arg is a floating point number between 0 and 1.0, then + to print. If the arg is a floating-point number between 0 and 1.0, then it is taken as a decimal percentage of the available lines to be printed (e.g., .1 means print 10% of all available lines). If it is an integer, it is taken to mean the number of lines of data that you wish to have diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 768c3dcb11ec59..d376592d69d40d 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1682,6 +1682,13 @@ def describe(thing): return 'function ' + thing.__name__ if inspect.ismethod(thing): return 'method ' + thing.__name__ + if inspect.ismethodwrapper(thing): + return 'method wrapper ' + thing.__name__ + if inspect.ismethoddescriptor(thing): + try: + return 'method descriptor ' + thing.__name__ + except AttributeError: + pass return type(thing).__name__ def locate(path, forceload=0): diff --git a/Lib/random.py b/Lib/random.py index bcc11c7cd3c208..f5a482b28dec78 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -1013,7 +1013,7 @@ def _parse_args(arg_list: list[str] | None): help="print a random integer between 1 and N inclusive") group.add_argument( "-f", "--float", type=float, metavar="N", - help="print a random floating point number between 1 and N inclusive") + help="print a random floating-point number between 1 and N inclusive") group.add_argument( "--test", type=int, const=10_000, nargs="?", help=argparse.SUPPRESS) diff --git a/Lib/sched.py b/Lib/sched.py index 14613cf29874da..fb20639d459967 100644 --- a/Lib/sched.py +++ b/Lib/sched.py @@ -11,7 +11,7 @@ implement simulated time by writing your own functions. This can also be used to integrate scheduling with STDWIN events; the delay function is allowed to modify the queue. Time can be expressed as -integers or floating point numbers, as long as it is consistent. +integers or floating-point numbers, as long as it is consistent. Events are specified by tuples (time, priority, action, argument, kwargs). As in UNIX, lower priority numbers mean higher priority; in this diff --git a/Lib/site.py b/Lib/site.py index daa56e158949db..cafd3ab70b2cac 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -312,6 +312,10 @@ def joinuser(*args): # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' implementation = _get_implementation() implementation_lower = implementation.lower() @@ -322,7 +326,7 @@ def _get_path(userbase): if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' + return f'{userbase}/lib/python{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): @@ -390,6 +394,10 @@ def getsitepackages(prefixes=None): implementation = _get_implementation().lower() ver = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -397,7 +405,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - f"{implementation}{ver[0]}.{ver[1]}", + f"{implementation}{ver[0]}.{ver[1]}{abi_thread}", "site-packages") sitepackages.append(path) else: @@ -509,6 +517,7 @@ def register_readline(): pass if readline.get_current_history_length() == 0: + from _pyrepl.main import CAN_USE_PYREPL # If no history was loaded, default to .python_history, # or PYTHON_HISTORY. # The guard is necessary to avoid doubling history size at @@ -516,25 +525,18 @@ def register_readline(): # through a PYTHONSTARTUP hook, see: # http://bugs.python.org/issue5845#msg198636 history = gethistoryfile() + if os.getenv("PYTHON_BASIC_REPL") or not CAN_USE_PYREPL: + readline_module = readline + else: + readline_module = _pyrepl.readline try: - if os.getenv("PYTHON_BASIC_REPL"): - readline.read_history_file(history) - else: - _pyrepl.readline.read_history_file(history) + readline_module.read_history_file(history) except (OSError,* _pyrepl.unix_console._error): pass def write_history(): try: - from _pyrepl.main import CAN_USE_PYREPL - except ImportError: - CAN_USE_PYREPL = False - - try: - if os.getenv("PYTHON_BASIC_REPL") or not CAN_USE_PYREPL: - readline.write_history_file(history) - else: - _pyrepl.readline.write_history_file(history) + readline_module.write_history_file(history) except (FileNotFoundError, PermissionError): # home directory does not exist or is not writable # https://bugs.python.org/issue19891 diff --git a/Lib/statistics.py b/Lib/statistics.py index c64c6fae4ab010..d3dd0d530c31cf 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -11,7 +11,7 @@ Function Description ================== ================================================== mean Arithmetic mean (average) of data. -fmean Fast, floating point arithmetic mean. +fmean Fast, floating-point arithmetic mean. geometric_mean Geometric mean of data. harmonic_mean Harmonic mean of data. median Median (middle value) of data. diff --git a/Lib/symtable.py b/Lib/symtable.py index 221aaf88d41670..7a30e1ac4ca378 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -237,6 +237,12 @@ class Class(SymbolTable): def get_methods(self): """Return a tuple of methods declared in the class. """ + import warnings + typename = f'{self.__class__.__module__}.{self.__class__.__name__}' + warnings.warn(f'{typename}.get_methods() is deprecated ' + f'and will be removed in Python 3.16.', + DeprecationWarning, stacklevel=2) + if self.__methods is None: d = {} @@ -249,6 +255,11 @@ def is_local_symbol(ident): if is_local_symbol(st.name): match st.type: case _symtable.TYPE_FUNCTION: + # generators are of type TYPE_FUNCTION with a ".0" + # parameter as a first parameter (which makes them + # distinguishable from a function named 'genexpr') + if st.name == 'genexpr' and '.0' in st.varnames: + continue d[st.name] = 1 case _symtable.TYPE_TYPE_PARAMETERS: # Get the function-def block in the annotation @@ -256,7 +267,14 @@ def is_local_symbol(ident): scope_name = st.name for c in st.children: if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION: - d[st.name] = 1 + # A generic generator of type TYPE_FUNCTION + # cannot be a direct child of 'st' (but it + # can be a descendant), e.g.: + # + # class A: + # type genexpr[genexpr] = (x for x in []) + assert scope_name != 'genexpr' or '.0' not in c.varnames + d[scope_name] = 1 break self.__methods = tuple(d) return self.__methods diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 98a14e5d3a3187..80aef3447117e5 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -27,10 +27,10 @@ _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': @@ -77,10 +77,10 @@ # Downstream distributors who patch posix_prefix/nt scheme are encouraged to # leave the following schemes unchanged 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': @@ -148,11 +148,11 @@ def joinuser(*args): 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}', - 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages', - 'include': '{userbase}/include/{implementation_lower}{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -487,6 +487,9 @@ def _init_config_vars(): # the init-function. _CONFIG_VARS['userbase'] = _getuserbase() + # e.g., 't' for free-threaded or '' for default build + _CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else '' + # Always convert srcdir to an absolute path srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) if os.name == 'posix': @@ -639,7 +642,7 @@ def get_platform(): release = m.group() elif osname[:6] == "darwin": if sys.platform == "ios": - release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "12.0") + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") osname = sys.platform machine = sys.implementation._multiarch else: @@ -655,6 +658,10 @@ def get_python_version(): return _PY_VERSION_SHORT +def _get_python_version_abi(): + return _PY_VERSION_SHORT + get_config_var("abi_thread") + + def expand_makefile_vars(s, vars): """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in 'string' according to 'vars' (a dictionary mapping variable names to diff --git a/Lib/tarfile.py b/Lib/tarfile.py index d5d8a469779f50..4fa7bb6740adbb 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2248,13 +2248,7 @@ def _get_filter_function(self, filter): if filter is None: filter = self.extraction_filter if filter is None: - import warnings - warnings.warn( - 'Python 3.14 will, by default, filter extracted tar ' - + 'archives and reject files or modify their metadata. ' - + 'Use the filter argument to control this behavior.', - DeprecationWarning, stacklevel=3) - return fully_trusted_filter + return data_filter if isinstance(filter, str): raise TypeError( 'String names are not supported for ' diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py index 15586f15dfab30..493932d6c6d441 100644 --- a/Lib/test/_test_eintr.py +++ b/Lib/test/_test_eintr.py @@ -18,6 +18,7 @@ import socket import subprocess import sys +import textwrap import time import unittest @@ -492,29 +493,31 @@ def test_devpoll(self): self.check_elapsed_time(dt) -class FNTLEINTRTest(EINTRBaseTest): +class FCNTLEINTRTest(EINTRBaseTest): def _lock(self, lock_func, lock_name): self.addCleanup(os_helper.unlink, os_helper.TESTFN) - code = '\n'.join(( - "import fcntl, time", - "with open('%s', 'wb') as f:" % os_helper.TESTFN, - " fcntl.%s(f, fcntl.LOCK_EX)" % lock_name, - " time.sleep(%s)" % self.sleep_time)) - start_time = time.monotonic() - proc = self.subprocess(code) + rd1, wr1 = os.pipe() + rd2, wr2 = os.pipe() + for fd in (rd1, wr1, rd2, wr2): + self.addCleanup(os.close, fd) + code = textwrap.dedent(f""" + import fcntl, os, time + with open('{os_helper.TESTFN}', 'wb') as f: + fcntl.{lock_name}(f, fcntl.LOCK_EX) + os.write({wr1}, b"ok") + _ = os.read({rd2}, 2) # wait for parent process + time.sleep({self.sleep_time}) + """) + proc = self.subprocess(code, pass_fds=[wr1, rd2]) with kill_on_error(proc): with open(os_helper.TESTFN, 'wb') as f: # synchronize the subprocess + ok = os.read(rd1, 2) + self.assertEqual(ok, b"ok") + + # notify the child that the parent is ready start_time = time.monotonic() - for _ in support.sleeping_retry(support.LONG_TIMEOUT, error=False): - try: - lock_func(f, fcntl.LOCK_EX | fcntl.LOCK_NB) - lock_func(f, fcntl.LOCK_UN) - except BlockingIOError: - break - else: - dt = time.monotonic() - start_time - raise Exception("failed to sync child in %.1f sec" % dt) + os.write(wr2, b"go") # the child locked the file just a moment ago for 'sleep_time' seconds # that means that the lock below will block for 'sleep_time' minus some diff --git a/Lib/test/crashers/README b/Lib/test/crashers/README index d844385113eb45..7111946b93b280 100644 --- a/Lib/test/crashers/README +++ b/Lib/test/crashers/README @@ -15,7 +15,3 @@ what the variables are. Once the crash is fixed, the test case should be moved into an appropriate test (even if it was originally from the test suite). This ensures the regression doesn't happen again. And if it does, it should be easier to track down. - -Also see Lib/test_crashers.py which exercises the crashers in this directory. -In particular, make sure to add any new infinite loop crashers to the black -list so it doesn't try to run them. diff --git a/Lib/test/crashers/bogus_code_obj.py b/Lib/test/crashers/bogus_code_obj.py index e71b3582cf2d76..b3ff07995c95ed 100644 --- a/Lib/test/crashers/bogus_code_obj.py +++ b/Lib/test/crashers/bogus_code_obj.py @@ -12,8 +12,8 @@ """ -import types +def f(): + pass -co = types.CodeType(0, 0, 0, 0, 0, 0, b'\x04\x00\x71\x00', - (), (), (), '', '', 1, b'') -exec(co) +f.__code__ = f.__code__.replace(co_code=b"") +f() diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index a125706927393c..fa1d4d575c8fd4 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -43,7 +43,10 @@ def log(self, line: str = '') -> None: def get_load_avg(self) -> float | None: if hasattr(os, 'getloadavg'): - return os.getloadavg()[0] + try: + return os.getloadavg()[0] + except OSError: + pass if self.win_load_tracker is not None: return self.win_load_tracker.getloadavg() return None diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 9922591ce7114a..3c936b3bc4029e 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -144,6 +144,14 @@ class E(C): def __getinitargs__(self): return () +import __main__ +__main__.C = C +C.__module__ = "__main__" +__main__.D = D +D.__module__ = "__main__" +__main__.E = E +E.__module__ = "__main__" + # Simple mutable object. class Object: pass @@ -157,14 +165,6 @@ def __reduce__(self): # Shouldn't support the recursion itself return K, (self.value,) -import __main__ -__main__.C = C -C.__module__ = "__main__" -__main__.D = D -D.__module__ = "__main__" -__main__.E = E -E.__module__ = "__main__" - class myint(int): def __init__(self, x): self.str = str(x) @@ -1179,6 +1179,138 @@ def test_compat_unpickle(self): self.assertIs(type(unpickled), collections.UserDict) self.assertEqual(unpickled, collections.UserDict({1: 2})) + def test_load_global(self): + self.assertIs(self.loads(b'cbuiltins\nstr\n.'), str) + self.assertIs(self.loads(b'cmath\nlog\n.'), math.log) + self.assertIs(self.loads(b'cos.path\njoin\n.'), os.path.join) + self.assertIs(self.loads(b'\x80\x04cbuiltins\nstr.upper\n.'), str.upper) + with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)): + self.assertEqual(self.loads(b'\x80\x04cm\xc3\xb6dule\ngl\xc3\xb6bal\n.'), 42) + + self.assertRaises(UnicodeDecodeError, self.loads, b'c\xff\nlog\n.') + self.assertRaises(UnicodeDecodeError, self.loads, b'cmath\n\xff\n.') + self.assertRaises(self.truncated_errors, self.loads, b'c\nlog\n.') + self.assertRaises(self.truncated_errors, self.loads, b'cmath\n\n.') + self.assertRaises(self.truncated_errors, self.loads, b'\x80\x04cmath\n\n.') + + def test_load_stack_global(self): + self.assertIs(self.loads(b'\x8c\x08builtins\x8c\x03str\x93.'), str) + self.assertIs(self.loads(b'\x8c\x04math\x8c\x03log\x93.'), math.log) + self.assertIs(self.loads(b'\x8c\x07os.path\x8c\x04join\x93.'), + os.path.join) + self.assertIs(self.loads(b'\x80\x04\x8c\x08builtins\x8c\x09str.upper\x93.'), + str.upper) + with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)): + self.assertEqual(self.loads(b'\x80\x04\x8c\x07m\xc3\xb6dule\x8c\x07gl\xc3\xb6bal\x93.'), 42) + + self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x01\xff\x8c\x03log\x93.') + self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x04math\x8c\x01\xff\x93.') + self.assertRaises(ValueError, self.loads, b'\x8c\x00\x8c\x03log\x93.') + self.assertRaises(AttributeError, self.loads, b'\x8c\x04math\x8c\x00\x93.') + self.assertRaises(AttributeError, self.loads, b'\x80\x04\x8c\x04math\x8c\x00\x93.') + + self.assertRaises(pickle.UnpicklingError, self.loads, b'N\x8c\x03log\x93.') + self.assertRaises(pickle.UnpicklingError, self.loads, b'\x8c\x04mathN\x93.') + self.assertRaises(pickle.UnpicklingError, self.loads, b'\x80\x04\x8c\x04mathN\x93.') + + def test_find_class(self): + unpickler = self.unpickler(io.BytesIO()) + unpickler_nofix = self.unpickler(io.BytesIO(), fix_imports=False) + unpickler4 = self.unpickler(io.BytesIO(b'\x80\x04N.')) + unpickler4.load() + + self.assertIs(unpickler.find_class('__builtin__', 'str'), str) + self.assertRaises(ModuleNotFoundError, + unpickler_nofix.find_class, '__builtin__', 'str') + self.assertIs(unpickler.find_class('builtins', 'str'), str) + self.assertIs(unpickler_nofix.find_class('builtins', 'str'), str) + self.assertIs(unpickler.find_class('math', 'log'), math.log) + self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) + self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) + + self.assertIs(unpickler4.find_class('builtins', 'str.upper'), str.upper) + with self.assertRaisesRegex(AttributeError, + r"module 'builtins' has no attribute 'str\.upper'|" + r"Can't get attribute 'str\.upper' on \.spam'|" + r"Can't get attribute 'log\.\.spam' on .spam') + with self.assertRaisesRegex(AttributeError, + r"Can't get local attribute 'log\.\.spam' on .spam') + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''|" + "Can't get attribute '' on = 5') + + def test_non_continuous_buffer(self): + if self.pickler is pickle._Pickler: + self.skipTest('CRASHES (see gh-122306)') + for proto in protocols[5:]: + with self.subTest(proto=proto): + pb = pickle.PickleBuffer(memoryview(b"foobar")[::2]) + with self.assertRaises(pickle.PicklingError): + self.dumps(pb, proto) + + def test_buffer_callback_error(self): + def buffer_callback(buffers): + raise CustomError + pb = pickle.PickleBuffer(b"foobar") + with self.assertRaises(CustomError): + self.dumps(pb, 5, buffer_callback=buffer_callback) + + def test_evil_pickler_mutating_collection(self): + # https://github.com/python/cpython/issues/92930 + global Clearer + class Clearer: + pass + + def check(collection): + class EvilPickler(self.pickler): + def persistent_id(self, obj): + if isinstance(obj, Clearer): + collection.clear() + return None + pickler = EvilPickler(io.BytesIO(), proto) + try: + pickler.dump(collection) + except RuntimeError as e: + expected = "changed size during iteration" + self.assertIn(expected, str(e)) + + for proto in protocols: + check([Clearer()]) + check([Clearer(), Clearer()]) + check({Clearer()}) + check({Clearer(), Clearer()}) + check({Clearer(): 1}) + check({Clearer(): 1, Clearer(): 2}) + check({1: Clearer(), 2: Clearer()}) + class AbstractPickleTests: # Subclass must define self.dumps, self.loads. @@ -2453,55 +3173,12 @@ def test_reduce_calls_base(self): y = self.loads(s) self.assertEqual(y._reduce_called, 1) - def test_reduce_ex_None(self): - c = REX_None() - with self.assertRaises(TypeError): - self.dumps(c) - - def test_reduce_None(self): - c = R_None() - with self.assertRaises(TypeError): - self.dumps(c) - def test_pickle_setstate_None(self): c = C_None_setstate() p = self.dumps(c) with self.assertRaises(TypeError): self.loads(p) - @no_tracing - def test_bad_getattr(self): - # Issue #3514: crash when there is an infinite loop in __getattr__ - x = BadGetattr() - for proto in range(2): - with support.infinite_recursion(25): - self.assertRaises(RuntimeError, self.dumps, x, proto) - for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): - s = self.dumps(x, proto) - - def test_reduce_bad_iterator(self): - # Issue4176: crash when 4th and 5th items of __reduce__() - # are not iterators - class C(object): - def __reduce__(self): - # 4th item is not an iterator - return list, (), None, [], None - class D(object): - def __reduce__(self): - # 5th item is not an iterator - return dict, (), None, None, [] - - # Python implementation is less strict and also accepts iterables. - for proto in protocols: - try: - self.dumps(C(), proto) - except pickle.PicklingError: - pass - try: - self.dumps(D(), proto) - except pickle.PicklingError: - pass - def test_many_puts_and_gets(self): # Test that internal data structures correctly deal with lots of # puts/gets. @@ -2818,6 +3495,18 @@ class Recursive: self.assertIs(unpickled, Recursive) del Recursive.mod # break reference loop + def test_recursive_nested_names2(self): + global Recursive + class Recursive: + pass + Recursive.ref = Recursive + Recursive.__qualname__ = 'Recursive.ref' + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(Recursive, proto)) + self.assertIs(unpickled, Recursive) + del Recursive.ref # break reference loop + def test_py_methods(self): global PyMethodsTest class PyMethodsTest: @@ -2938,27 +3627,6 @@ def test_compat_pickle(self): self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled) self.assertIs(type(self.loads(pickled)), type(val)) - def test_local_lookup_error(self): - # Test that whichmodule() errors out cleanly when looking up - # an assumed globally-reachable object fails. - def f(): - pass - # Since the function is local, lookup will fail - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # Same without a __module__ attribute (exercises a different path - # in _pickle.c). - del f.__module__ - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # Yet a different path. - f.__name__ = f.__qualname__ - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # # PEP 574 tests below # @@ -3069,20 +3737,6 @@ def test_oob_buffers_writable_to_readonly(self): self.assertIs(type(new), type(obj)) self.assertEqual(new, obj) - def test_picklebuffer_error(self): - # PickleBuffer forbidden with protocol < 5 - pb = pickle.PickleBuffer(b"foobar") - for proto in range(0, 5): - with self.assertRaises(pickle.PickleError): - self.dumps(pb, proto) - - def test_buffer_callback_error(self): - def buffer_callback(buffers): - 1/0 - pb = pickle.PickleBuffer(b"foobar") - with self.assertRaises(ZeroDivisionError): - self.dumps(pb, 5, buffer_callback=buffer_callback) - def test_buffers_error(self): pb = pickle.PickleBuffer(b"foobar") for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): @@ -3174,37 +3828,6 @@ def __reduce__(self): expected = "changed size during iteration" self.assertIn(expected, str(e)) - def test_evil_pickler_mutating_collection(self): - # https://github.com/python/cpython/issues/92930 - if not hasattr(self, "pickler"): - raise self.skipTest(f"{type(self)} has no associated pickler type") - - global Clearer - class Clearer: - pass - - def check(collection): - class EvilPickler(self.pickler): - def persistent_id(self, obj): - if isinstance(obj, Clearer): - collection.clear() - return None - pickler = EvilPickler(io.BytesIO(), proto) - try: - pickler.dump(collection) - except RuntimeError as e: - expected = "changed size during iteration" - self.assertIn(expected, str(e)) - - for proto in protocols: - check([Clearer()]) - check([Clearer(), Clearer()]) - check({Clearer()}) - check({Clearer(), Clearer()}) - check({Clearer(): 1}) - check({Clearer(): 1, Clearer(): 2}) - check({1: Clearer(), 2: Clearer()}) - class BigmemPickleTests: @@ -3335,6 +3958,18 @@ def test_huge_str_64b(self, size): # Test classes for reduce_ex +class R: + def __init__(self, reduce=None): + self.reduce = reduce + def __reduce__(self, proto): + return self.reduce + +class REX: + def __init__(self, reduce_ex=None): + self.reduce_ex = reduce_ex + def __reduce_ex__(self, proto): + return self.reduce_ex + class REX_one(object): """No __reduce_ex__ here, but inheriting it from object""" _reduce_called = 0 @@ -3425,6 +4060,19 @@ def __getstate__(self): __setstate__ = None +class CustomError(Exception): + pass + +class Unpickleable: + def __reduce__(self): + raise CustomError + +UNPICKLEABLE = Unpickleable() + +class UnpickleableCallable(Unpickleable): + def __call__(self, *args, **kwargs): + pass + # Test classes for newobj @@ -3493,6 +4141,12 @@ class BadGetattr: def __getattr__(self, key): self.foo +class NoNew: + def __getattribute__(self, name): + if name == '__new__': + raise AttributeError + return super().__getattribute__(name) + class AbstractPickleModuleTests: @@ -3565,7 +4219,7 @@ def raises_oserror(self, *args, **kwargs): raise OSError @property def bad_property(self): - 1/0 + raise CustomError # File without read and readline class F: @@ -3586,23 +4240,23 @@ class F: class F: read = bad_property readline = raises_oserror - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad readline class F: readline = bad_property read = raises_oserror - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad readline, no read class F: readline = bad_property - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad read, no readline class F: read = bad_property - self.assertRaises((AttributeError, ZeroDivisionError), self.Unpickler, F()) + self.assertRaises((AttributeError, CustomError), self.Unpickler, F()) # File with bad peek class F: @@ -3611,7 +4265,7 @@ class F: readline = raises_oserror try: self.Unpickler(F()) - except ZeroDivisionError: + except CustomError: pass # File with bad readinto @@ -3621,7 +4275,7 @@ class F: readline = raises_oserror try: self.Unpickler(F()) - except ZeroDivisionError: + except CustomError: pass def test_pickler_bad_file(self): @@ -3634,8 +4288,8 @@ class F: class F: @property def write(self): - 1/0 - self.assertRaises(ZeroDivisionError, self.Pickler, F()) + raise CustomError + self.assertRaises(CustomError, self.Pickler, F()) def check_dumps_loads_oob_buffers(self, dumps, loads): # No need to do the full gamut of tests here, just enough to @@ -3743,9 +4397,15 @@ def test_return_correct_type(self): def test_protocol0_is_ascii_only(self): non_ascii_str = "\N{EMPTY SET}" - self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0) + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(non_ascii_str, 0) + self.assertEqual(str(cm.exception), + 'persistent IDs in protocol 0 must be ASCII strings') pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.' - self.assertRaises(pickle.UnpicklingError, self.loads, pickled) + with self.assertRaises(pickle.UnpicklingError) as cm: + self.loads(pickled) + self.assertEqual(str(cm.exception), + 'persistent IDs in protocol 0 must be ASCII strings') class AbstractPicklerUnpicklerObjectTests: @@ -4015,8 +4675,11 @@ class MyClass: # NotImplemented self.assertIs(math_log, math.log) - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: p.dump(g) + self.assertRegex(str(cm.exception), + r'(__reduce__|)' + r' must return (a )?string or tuple') with self.assertRaisesRegex( ValueError, 'The reducer just failed'): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 18455bb6e0ff52..37e3305036f499 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -387,7 +387,7 @@ def skip_if_buildbot(reason=None): reason = 'not suitable for buildbots' try: isbuildbot = getpass.getuser().lower() == 'buildbot' - except (KeyError, EnvironmentError) as err: + except (KeyError, OSError) as err: warnings.warn(f'getpass.getuser() failed {err}.', RuntimeWarning) isbuildbot = False return unittest.skipIf(isbuildbot, reason) @@ -2614,14 +2614,18 @@ def force_not_colorized(func): def wrapper(*args, **kwargs): import _colorize original_fn = _colorize.can_colorize - variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} + variables: dict[str, str | None] = { + "PYTHON_COLORS": None, "FORCE_COLOR": None, "NO_COLOR": None + } try: for key in variables: variables[key] = os.environ.pop(key, None) + os.environ["NO_COLOR"] = "1" _colorize.can_colorize = lambda: False return func(*args, **kwargs) finally: _colorize.can_colorize = original_fn + del os.environ["NO_COLOR"] for key, value in variables.items(): if value is not None: os.environ[key] = value diff --git a/Lib/test/support/interpreters/_crossinterp.py b/Lib/test/support/interpreters/_crossinterp.py new file mode 100644 index 00000000000000..544e197ba4c028 --- /dev/null +++ b/Lib/test/support/interpreters/_crossinterp.py @@ -0,0 +1,102 @@ +"""Common code between queues and channels.""" + + +class ItemInterpreterDestroyed(Exception): + """Raised when trying to get an item whose interpreter was destroyed.""" + + +class classonly: + """A non-data descriptor that makes a value only visible on the class. + + This is like the "classmethod" builtin, but does not show up on + instances of the class. It may be used as a decorator. + """ + + def __init__(self, value): + self.value = value + self.getter = classmethod(value).__get__ + self.name = None + + def __set_name__(self, cls, name): + if self.name is not None: + raise TypeError('already used') + self.name = name + + def __get__(self, obj, cls): + if obj is not None: + raise AttributeError(self.name) + # called on the class + return self.getter(None, cls) + + +class UnboundItem: + """Represents a cross-interpreter item no longer bound to an interpreter. + + An item is unbound when the interpreter that added it to the + cross-interpreter container is destroyed. + """ + + __slots__ = () + + @classonly + def singleton(cls, kind, module, name='UNBOUND'): + doc = cls.__doc__.replace('cross-interpreter container', kind) + doc = doc.replace('cross-interpreter', kind) + subclass = type( + f'Unbound{kind.capitalize()}Item', + (cls,), + dict( + _MODULE=module, + _NAME=name, + __doc__=doc, + ), + ) + return object.__new__(subclass) + + _MODULE = __name__ + _NAME = 'UNBOUND' + + def __new__(cls): + raise Exception(f'use {cls._MODULE}.{cls._NAME}') + + def __repr__(self): + return f'{self._MODULE}.{self._NAME}' +# return f'interpreters.queues.UNBOUND' + + +UNBOUND = object.__new__(UnboundItem) +UNBOUND_ERROR = object() +UNBOUND_REMOVE = object() + +_UNBOUND_CONSTANT_TO_FLAG = { + UNBOUND_REMOVE: 1, + UNBOUND_ERROR: 2, + UNBOUND: 3, +} +_UNBOUND_FLAG_TO_CONSTANT = {v: k + for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()} + + +def serialize_unbound(unbound): + op = unbound + try: + flag = _UNBOUND_CONSTANT_TO_FLAG[op] + except KeyError: + raise NotImplementedError(f'unsupported unbound replacement op {op!r}') + return flag, + + +def resolve_unbound(flag, exctype_destroyed): + try: + op = _UNBOUND_FLAG_TO_CONSTANT[flag] + except KeyError: + raise NotImplementedError(f'unsupported unbound replacement op {flag!r}') + if op is UNBOUND_REMOVE: + # "remove" not possible here + raise NotImplementedError + elif op is UNBOUND_ERROR: + raise exctype_destroyed("item's original interpreter destroyed") + elif op is UNBOUND: + return UNBOUND + else: + raise NotImplementedError(repr(op)) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py index fbae7e634cf34d..d2bd93d77f7169 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/interpreters/channels.py @@ -2,35 +2,68 @@ import time import _interpchannels as _channels +from . import _crossinterp # aliases: from _interpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, ChannelEmptyError, ChannelNotEmptyError, ) +from ._crossinterp import ( + UNBOUND_ERROR, UNBOUND_REMOVE, +) __all__ = [ + 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', 'create', 'list_all', 'SendChannel', 'RecvChannel', 'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError', + 'ItemInterpreterDestroyed', ] -def create(): +class ItemInterpreterDestroyed(ChannelError, + _crossinterp.ItemInterpreterDestroyed): + """Raised from get() and get_nowait().""" + + +UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) + + +def _serialize_unbound(unbound): + if unbound is UNBOUND: + unbound = _crossinterp.UNBOUND + return _crossinterp.serialize_unbound(unbound) + + +def _resolve_unbound(flag): + resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) + if resolved is _crossinterp.UNBOUND: + resolved = UNBOUND + return resolved + + +def create(*, unbounditems=UNBOUND): """Return (recv, send) for a new cross-interpreter channel. The channel may be used to pass data safely between interpreters. + + "unbounditems" sets the default for the send end of the channel. + See SendChannel.send() for supported values. The default value + is UNBOUND, which replaces the unbound item when received. """ - cid = _channels.create() - recv, send = RecvChannel(cid), SendChannel(cid) + unbound = _serialize_unbound(unbounditems) + unboundop, = unbound + cid = _channels.create(unboundop) + recv, send = RecvChannel(cid), SendChannel(cid, _unbound=unbound) return recv, send def list_all(): """Return a list of (recv, send) for all open channels.""" - return [(RecvChannel(cid), SendChannel(cid)) - for cid in _channels.list_all()] + return [(RecvChannel(cid), SendChannel(cid, _unbound=unbound)) + for cid, unbound in _channels.list_all()] class _ChannelEnd: @@ -106,12 +139,15 @@ def recv(self, timeout=None, *, if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout - obj = _channels.recv(self._id, _sentinel) + obj, unboundop = _channels.recv(self._id, _sentinel) while obj is _sentinel: time.sleep(_delay) if timeout is not None and time.time() >= end: raise TimeoutError - obj = _channels.recv(self._id, _sentinel) + obj, unboundop = _channels.recv(self._id, _sentinel) + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) return obj def recv_nowait(self, default=_NOT_SET): @@ -122,9 +158,13 @@ def recv_nowait(self, default=_NOT_SET): is the same as recv(). """ if default is _NOT_SET: - return _channels.recv(self._id) + obj, unboundop = _channels.recv(self._id) else: - return _channels.recv(self._id, default) + obj, unboundop = _channels.recv(self._id, default) + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + return obj def close(self): _channels.close(self._id, recv=True) @@ -135,43 +175,79 @@ class SendChannel(_ChannelEnd): _end = 'send' + def __new__(cls, cid, *, _unbound=None): + if _unbound is None: + try: + op = _channels.get_channel_defaults(cid) + _unbound = (op,) + except ChannelNotFoundError: + _unbound = _serialize_unbound(UNBOUND) + self = super().__new__(cls, cid) + self._unbound = _unbound + return self + @property def is_closed(self): info = self._info return info.closed or info.closing - def send(self, obj, timeout=None): + def send(self, obj, timeout=None, *, + unbound=None, + ): """Send the object (i.e. its data) to the channel's receiving end. This blocks until the object is received. """ - _channels.send(self._id, obj, timeout=timeout, blocking=True) + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True) - def send_nowait(self, obj): + def send_nowait(self, obj, *, + unbound=None, + ): """Send the object to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) # XXX Note that at the moment channel_send() only ever returns # None. This should be fixed when channel_send_wait() is added. # See bpo-32604 and gh-19829. - return _channels.send(self._id, obj, blocking=False) + return _channels.send(self._id, obj, unboundop, blocking=False) - def send_buffer(self, obj, timeout=None): + def send_buffer(self, obj, timeout=None, *, + unbound=None, + ): """Send the object's buffer to the channel's receiving end. This blocks until the object is received. """ - _channels.send_buffer(self._id, obj, timeout=timeout, blocking=True) + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + _channels.send_buffer(self._id, obj, unboundop, + timeout=timeout, blocking=True) - def send_buffer_nowait(self, obj): + def send_buffer_nowait(self, obj, *, + unbound=None, + ): """Send the object's buffer to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - return _channels.send_buffer(self._id, obj, blocking=False) + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + return _channels.send_buffer(self._id, obj, unboundop, blocking=False) def close(self): _channels.close(self._id, send=True) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index 1b9e7481f2e313..deb8e8613af731 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -5,16 +5,22 @@ import time import weakref import _interpqueues as _queues +from . import _crossinterp # aliases: from _interpqueues import ( QueueError, QueueNotFoundError, ) +from ._crossinterp import ( + UNBOUND_ERROR, UNBOUND_REMOVE, +) __all__ = [ + 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', 'create', 'list_all', 'Queue', 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', + 'ItemInterpreterDestroyed', ] @@ -32,26 +38,54 @@ class QueueFull(QueueError, queue.Full): """ +class ItemInterpreterDestroyed(QueueError, + _crossinterp.ItemInterpreterDestroyed): + """Raised from get() and get_nowait().""" + + _SHARED_ONLY = 0 _PICKLED = 1 -def create(maxsize=0, *, syncobj=False): + +UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) + + +def _serialize_unbound(unbound): + if unbound is UNBOUND: + unbound = _crossinterp.UNBOUND + return _crossinterp.serialize_unbound(unbound) + + +def _resolve_unbound(flag): + resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) + if resolved is _crossinterp.UNBOUND: + resolved = UNBOUND + return resolved + + +def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND): """Return a new cross-interpreter queue. The queue may be used to pass data safely between interpreters. "syncobj" sets the default for Queue.put() and Queue.put_nowait(). + + "unbounditems" likewise sets the default. See Queue.put() for + supported values. The default value is UNBOUND, which replaces + the unbound item. """ fmt = _SHARED_ONLY if syncobj else _PICKLED - qid = _queues.create(maxsize, fmt) - return Queue(qid, _fmt=fmt) + unbound = _serialize_unbound(unbounditems) + unboundop, = unbound + qid = _queues.create(maxsize, fmt, unboundop) + return Queue(qid, _fmt=fmt, _unbound=unbound) def list_all(): """Return a list of all open queues.""" - return [Queue(qid, _fmt=fmt) - for qid, fmt in _queues.list_all()] + return [Queue(qid, _fmt=fmt, _unbound=(unboundop,)) + for qid, fmt, unboundop in _queues.list_all()] _known_queues = weakref.WeakValueDictionary() @@ -59,20 +93,28 @@ def list_all(): class Queue: """A cross-interpreter queue.""" - def __new__(cls, id, /, *, _fmt=None): + def __new__(cls, id, /, *, _fmt=None, _unbound=None): # There is only one instance for any given ID. if isinstance(id, int): id = int(id) else: raise TypeError(f'id must be an int, got {id!r}') if _fmt is None: - _fmt, = _queues.get_queue_defaults(id) + if _unbound is None: + _fmt, op = _queues.get_queue_defaults(id) + _unbound = (op,) + else: + _fmt, _ = _queues.get_queue_defaults(id) + elif _unbound is None: + _, op = _queues.get_queue_defaults(id) + _unbound = (op,) try: self = _known_queues[id] except KeyError: self = super().__new__(cls) self._id = id self._fmt = _fmt + self._unbound = _unbound _known_queues[id] = self _queues.bind(id) return self @@ -124,6 +166,7 @@ def qsize(self): def put(self, obj, timeout=None, *, syncobj=None, + unbound=None, _delay=10 / 1000, # 10 milliseconds ): """Add the object to the queue. @@ -131,7 +174,7 @@ def put(self, obj, timeout=None, *, This blocks while the queue is full. If "syncobj" is None (the default) then it uses the - queue's default, set with create_queue().. + queue's default, set with create_queue(). If "syncobj" is false then all objects are supported, at the expense of worse performance. @@ -152,11 +195,37 @@ def put(self, obj, timeout=None, *, actually is. That's a slightly different and stronger promise than just (initial) equality, which is all "syncobj=False" can promise. + + "unbound" controls the behavior of Queue.get() for the given + object if the current interpreter (calling put()) is later + destroyed. + + If "unbound" is None (the default) then it uses the + queue's default, set with create_queue(), + which is usually UNBOUND. + + If "unbound" is UNBOUND_ERROR then get() will raise an + ItemInterpreterDestroyed exception if the original interpreter + has been destroyed. This does not otherwise affect the queue; + the next call to put() will work like normal, returning the next + item in the queue. + + If "unbound" is UNBOUND_REMOVE then the item will be removed + from the queue as soon as the original interpreter is destroyed. + Be aware that this will introduce an imbalance between put() + and get() calls. + + If "unbound" is UNBOUND then it is returned by get() in place + of the unbound item. """ if syncobj is None: fmt = self._fmt else: fmt = _SHARED_ONLY if syncobj else _PICKLED + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) if timeout is not None: timeout = int(timeout) if timeout < 0: @@ -166,7 +235,7 @@ def put(self, obj, timeout=None, *, obj = pickle.dumps(obj) while True: try: - _queues.put(self._id, obj, fmt) + _queues.put(self._id, obj, fmt, unboundop) except QueueFull as exc: if timeout is not None and time.time() >= end: raise # re-raise @@ -174,14 +243,18 @@ def put(self, obj, timeout=None, *, else: break - def put_nowait(self, obj, *, syncobj=None): + def put_nowait(self, obj, *, syncobj=None, unbound=None): if syncobj is None: fmt = self._fmt else: fmt = _SHARED_ONLY if syncobj else _PICKLED + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) if fmt is _PICKLED: obj = pickle.dumps(obj) - _queues.put(self._id, obj, fmt) + _queues.put(self._id, obj, fmt, unboundop) def get(self, timeout=None, *, _delay=10 / 1000, # 10 milliseconds @@ -189,6 +262,10 @@ def get(self, timeout=None, *, """Return the next object from the queue. This blocks while the queue is empty. + + If the next item's original interpreter has been destroyed + then the "next object" is determined by the value of the + "unbound" argument to put(). """ if timeout is not None: timeout = int(timeout) @@ -197,13 +274,16 @@ def get(self, timeout=None, *, end = time.time() + timeout while True: try: - obj, fmt = _queues.get(self._id) + obj, fmt, unboundop = _queues.get(self._id) except QueueEmpty as exc: if timeout is not None and time.time() >= end: raise # re-raise time.sleep(_delay) else: break + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) if fmt == _PICKLED: obj = pickle.loads(obj) else: @@ -217,9 +297,12 @@ def get_nowait(self): is the same as get(). """ try: - obj, fmt = _queues.get(self._id) + obj, fmt, unboundop = _queues.get(self._id) except QueueEmpty as exc: raise # re-raise + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) if fmt == _PICKLED: obj = pickle.loads(obj) else: diff --git a/Lib/test/support/smtpd.py b/Lib/test/support/smtpd.py index c2e17cad422861..6537679db9ad24 100755 --- a/Lib/test/support/smtpd.py +++ b/Lib/test/support/smtpd.py @@ -633,7 +633,8 @@ def __init__(self, localaddr, remoteaddr, " be set to True at the same time") asyncore.dispatcher.__init__(self, map=map) try: - gai_results = socket.getaddrinfo(*localaddr, + family = 0 if socket.has_ipv6 else socket.AF_INET + gai_results = socket.getaddrinfo(*localaddr, family=family, type=socket.SOCK_STREAM) self.create_socket(gai_results[0][0], gai_results[0][1]) # try to re-use a server port if possible diff --git a/Lib/test/test__interpchannels.py b/Lib/test/test__interpchannels.py index b76c58917c0b9c..4a7f04b9df9843 100644 --- a/Lib/test/test__interpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -8,6 +8,8 @@ from test.support import import_helper +_channels = import_helper.import_module('_interpchannels') +from test.support.interpreters import _crossinterp from test.test__interpreters import ( _interpreters, _run_output, @@ -15,7 +17,7 @@ ) -_channels = import_helper.import_module('_interpchannels') +REPLACE = _crossinterp._UNBOUND_CONSTANT_TO_FLAG[_crossinterp.UNBOUND] # Additional tests are found in Lib/test/test_interpreters/test_channels.py. @@ -29,9 +31,19 @@ def recv_wait(cid): while True: try: - return _channels.recv(cid) + obj, unboundop = _channels.recv(cid) except _channels.ChannelEmptyError: time.sleep(0.1) + else: + assert unboundop is None, repr(unboundop) + return obj + + +def recv_nowait(cid, *args, unbound=False): + obj, unboundop = _channels.recv(cid, *args) + assert (unboundop is None) != unbound, repr(unboundop) + return obj + #@contextmanager #def run_threaded(id, source, **shared): @@ -212,7 +224,7 @@ def _run_action(cid, action, end, state): else: raise Exception('expected ChannelEmptyError') else: - _channels.recv(cid) + recv_nowait(cid) return state.decr() else: raise ValueError(end) @@ -235,7 +247,7 @@ def _run_action(cid, action, end, state): def clean_up_channels(): - for cid in _channels.list_all(): + for cid, _ in _channels.list_all(): try: _channels.destroy(cid) except _channels.ChannelNotFoundError: @@ -297,7 +309,7 @@ def test_bad_kwargs(self): _channels._channel_id(10, send=False, recv=False) def test_does_not_exist(self): - cid = _channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(_channels.ChannelNotFoundError): _channels._channel_id(int(cid) + 1) # unforced @@ -319,9 +331,9 @@ def test_repr(self): self.assertEqual(repr(cid), 'ChannelID(10)') def test_equality(self): - cid1 = _channels.create() + cid1 = _channels.create(REPLACE) cid2 = _channels._channel_id(int(cid1)) - cid3 = _channels.create() + cid3 = _channels.create(REPLACE) self.assertTrue(cid1 == cid1) self.assertTrue(cid1 == cid2) @@ -341,11 +353,11 @@ def test_equality(self): self.assertTrue(cid1 != cid3) def test_shareable(self): - chan = _channels.create() + chan = _channels.create(REPLACE) - obj = _channels.create() + obj = _channels.create(REPLACE) _channels.send(chan, obj, blocking=False) - got = _channels.recv(chan) + got = recv_nowait(chan) self.assertEqual(got, obj) self.assertIs(type(got), type(obj)) @@ -356,15 +368,15 @@ def test_shareable(self): class ChannelTests(TestBase): def test_create_cid(self): - cid = _channels.create() + cid = _channels.create(REPLACE) self.assertIsInstance(cid, _channels.ChannelID) def test_sequential_ids(self): - before = _channels.list_all() - id1 = _channels.create() - id2 = _channels.create() - id3 = _channels.create() - after = _channels.list_all() + before = [cid for cid, _ in _channels.list_all()] + id1 = _channels.create(REPLACE) + id2 = _channels.create(REPLACE) + id3 = _channels.create(REPLACE) + after = [cid for cid, _ in _channels.list_all()] self.assertEqual(id2, int(id1) + 1) self.assertEqual(id3, int(id2) + 1) @@ -374,7 +386,7 @@ def test_ids_global(self): id1 = _interpreters.create() out = _run_output(id1, dedent(""" import _interpchannels as _channels - cid = _channels.create() + cid = _channels.create(3) print(cid) """)) cid1 = int(out.strip()) @@ -382,7 +394,7 @@ def test_ids_global(self): id2 = _interpreters.create() out = _run_output(id2, dedent(""" import _interpchannels as _channels - cid = _channels.create() + cid = _channels.create(3) print(cid) """)) cid2 = int(out.strip()) @@ -392,7 +404,7 @@ def test_ids_global(self): def test_channel_list_interpreters_none(self): """Test listing interpreters for a channel with no associations.""" # Test for channel with no associated _interpreters. - cid = _channels.create() + cid = _channels.create(REPLACE) send_interps = _channels.list_interpreters(cid, send=True) recv_interps = _channels.list_interpreters(cid, send=False) self.assertEqual(send_interps, []) @@ -401,7 +413,7 @@ def test_channel_list_interpreters_none(self): def test_channel_list_interpreters_basic(self): """Test basic listing channel _interpreters.""" interp0, *_ = _interpreters.get_main() - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, "send", blocking=False) # Test for a channel that has one end associated to an interpreter. send_interps = _channels.list_interpreters(cid, send=True) @@ -412,7 +424,7 @@ def test_channel_list_interpreters_basic(self): interp1 = _interpreters.create() _run_output(interp1, dedent(f""" import _interpchannels as _channels - obj = _channels.recv({cid}) + _channels.recv({cid}) """)) # Test for channel that has both ends associated to an interpreter. send_interps = _channels.list_interpreters(cid, send=True) @@ -426,7 +438,7 @@ def test_channel_list_interpreters_multiple(self): interp1 = _interpreters.create() interp2 = _interpreters.create() interp3 = _interpreters.create() - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, "send", blocking=False) _run_output(interp1, dedent(f""" @@ -435,11 +447,11 @@ def test_channel_list_interpreters_multiple(self): """)) _run_output(interp2, dedent(f""" import _interpchannels as _channels - obj = _channels.recv({cid}) + _channels.recv({cid}) """)) _run_output(interp3, dedent(f""" import _interpchannels as _channels - obj = _channels.recv({cid}) + _channels.recv({cid}) """)) send_interps = _channels.list_interpreters(cid, send=True) recv_interps = _channels.list_interpreters(cid, send=False) @@ -450,11 +462,11 @@ def test_channel_list_interpreters_destroyed(self): """Test listing channel interpreters with a destroyed interpreter.""" interp0, *_ = _interpreters.get_main() interp1 = _interpreters.create() - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, "send", blocking=False) _run_output(interp1, dedent(f""" import _interpchannels as _channels - obj = _channels.recv({cid}) + _channels.recv({cid}) """)) # Should be one interpreter associated with each end. send_interps = _channels.list_interpreters(cid, send=True) @@ -476,16 +488,16 @@ def test_channel_list_interpreters_released(self): interp0, *_ = _interpreters.get_main() interp1 = _interpreters.create() interp2 = _interpreters.create() - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, "data", blocking=False) _run_output(interp1, dedent(f""" import _interpchannels as _channels - obj = _channels.recv({cid}) + _channels.recv({cid}) """)) _channels.send(cid, "data", blocking=False) _run_output(interp2, dedent(f""" import _interpchannels as _channels - obj = _channels.recv({cid}) + _channels.recv({cid}) """)) # Check the setup. send_interps = _channels.list_interpreters(cid, send=True) @@ -516,7 +528,7 @@ def test_channel_list_interpreters_closed(self): """Test listing channel interpreters with a closed channel.""" interp0, *_ = _interpreters.get_main() interp1 = _interpreters.create() - cid = _channels.create() + cid = _channels.create(REPLACE) # Put something in the channel so that it's not empty. _channels.send(cid, "send", blocking=False) @@ -538,7 +550,7 @@ def test_channel_list_interpreters_closed_send_end(self): """Test listing channel interpreters with a channel's send end closed.""" interp0, *_ = _interpreters.get_main() interp1 = _interpreters.create() - cid = _channels.create() + cid = _channels.create(REPLACE) # Put something in the channel so that it's not empty. _channels.send(cid, "send", blocking=False) @@ -570,7 +582,7 @@ def test_channel_list_interpreters_closed_send_end(self): _channels.list_interpreters(cid, send=False) def test_allowed_types(self): - cid = _channels.create() + cid = _channels.create(REPLACE) objects = [ None, 'spam', @@ -580,7 +592,7 @@ def test_allowed_types(self): for obj in objects: with self.subTest(obj): _channels.send(cid, obj, blocking=False) - got = _channels.recv(cid) + got = recv_nowait(cid) self.assertEqual(got, obj) self.assertIs(type(got), type(obj)) @@ -589,7 +601,7 @@ def test_allowed_types(self): # XXX What about between interpreters? def test_run_string_arg_unresolved(self): - cid = _channels.create() + cid = _channels.create(REPLACE) interp = _interpreters.create() _interpreters.set___main___attrs(interp, dict(cid=cid.send)) @@ -598,7 +610,7 @@ def test_run_string_arg_unresolved(self): print(cid.end) _channels.send(cid, b'spam', blocking=False) """)) - obj = _channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') self.assertEqual(out.strip(), 'send') @@ -608,7 +620,7 @@ def test_run_string_arg_unresolved(self): # Note: this test caused crashes on some buildbots (bpo-33615). @unittest.skip('disabled until high-level channels exist') def test_run_string_arg_resolved(self): - cid = _channels.create() + cid = _channels.create(REPLACE) cid = _channels._channel_id(cid, _resolve=True) interp = _interpreters.create() @@ -618,7 +630,7 @@ def test_run_string_arg_resolved(self): _channels.send(chan.id, b'spam', blocking=False) """), dict(chan=cid.send)) - obj = _channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') self.assertEqual(out.strip(), 'send') @@ -627,10 +639,10 @@ def test_run_string_arg_resolved(self): # send/recv def test_send_recv_main(self): - cid = _channels.create() + cid = _channels.create(REPLACE) orig = b'spam' _channels.send(cid, orig, blocking=False) - obj = _channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, orig) self.assertIsNot(obj, orig) @@ -639,27 +651,27 @@ def test_send_recv_same_interpreter(self): id1 = _interpreters.create() out = _run_output(id1, dedent(""" import _interpchannels as _channels - cid = _channels.create() + cid = _channels.create(REPLACE) orig = b'spam' _channels.send(cid, orig, blocking=False) - obj = _channels.recv(cid) + obj, _ = _channels.recv(cid) assert obj is not orig assert obj == orig """)) def test_send_recv_different_interpreters(self): - cid = _channels.create() + cid = _channels.create(REPLACE) id1 = _interpreters.create() out = _run_output(id1, dedent(f""" import _interpchannels as _channels _channels.send({cid}, b'spam', blocking=False) """)) - obj = _channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') def test_send_recv_different_threads(self): - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): obj = recv_wait(cid) @@ -674,7 +686,7 @@ def f(): self.assertEqual(obj, b'spam') def test_send_recv_different_interpreters_and_threads(self): - cid = _channels.create() + cid = _channels.create(REPLACE) id1 = _interpreters.create() out = None @@ -685,7 +697,7 @@ def f(): import _interpchannels as _channels while True: try: - obj = _channels.recv({cid}) + obj, _ = _channels.recv({cid}) break except _channels.ChannelEmptyError: time.sleep(0.1) @@ -710,23 +722,23 @@ def test_recv_not_found(self): _channels.recv(10) def test_recv_empty(self): - cid = _channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(_channels.ChannelEmptyError): _channels.recv(cid) def test_recv_default(self): default = object() - cid = _channels.create() - obj1 = _channels.recv(cid, default) + cid = _channels.create(REPLACE) + obj1 = recv_nowait(cid, default) _channels.send(cid, None, blocking=False) _channels.send(cid, 1, blocking=False) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'eggs', blocking=False) - obj2 = _channels.recv(cid, default) - obj3 = _channels.recv(cid, default) - obj4 = _channels.recv(cid) - obj5 = _channels.recv(cid, default) - obj6 = _channels.recv(cid, default) + obj2 = recv_nowait(cid, default) + obj3 = recv_nowait(cid, default) + obj4 = recv_nowait(cid) + obj5 = recv_nowait(cid, default) + obj6 = recv_nowait(cid, default) self.assertIs(obj1, default) self.assertIs(obj2, None) @@ -737,7 +749,7 @@ def test_recv_default(self): def test_recv_sending_interp_destroyed(self): with self.subTest('closed'): - cid1 = _channels.create() + cid1 = _channels.create(REPLACE) interp = _interpreters.create() _interpreters.run_string(interp, dedent(f""" import _interpchannels as _channels @@ -750,7 +762,7 @@ def test_recv_sending_interp_destroyed(self): _channels.recv(cid1) del cid1 with self.subTest('still open'): - cid2 = _channels.create() + cid2 = _channels.create(REPLACE) interp = _interpreters.create() _interpreters.run_string(interp, dedent(f""" import _interpchannels as _channels @@ -759,7 +771,8 @@ def test_recv_sending_interp_destroyed(self): _channels.send(cid2, b'eggs', blocking=False) _interpreters.destroy(interp) - _channels.recv(cid2) + recv_nowait(cid2, unbound=True) + recv_nowait(cid2, unbound=False) with self.assertRaisesRegex(RuntimeError, f'channel {cid2} is empty'): _channels.recv(cid2) @@ -770,9 +783,9 @@ def test_recv_sending_interp_destroyed(self): def test_send_buffer(self): buf = bytearray(b'spamspamspam') - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send_buffer(cid, buf, blocking=False) - obj = _channels.recv(cid) + obj = recv_nowait(cid) self.assertIsNot(obj, buf) self.assertIsInstance(obj, memoryview) @@ -794,12 +807,12 @@ def build_send_waiter(self, obj, *, buffer=False): else: send = _channels.send - cid = _channels.create() + cid = _channels.create(REPLACE) try: started = time.monotonic() send(cid, obj, blocking=False) stopped = time.monotonic() - _channels.recv(cid) + recv_nowait(cid) finally: _channels.destroy(cid) delay = stopped - started # seconds @@ -813,7 +826,7 @@ def test_send_blocking_waiting(self): received = None obj = b'spam' wait = self.build_send_waiter(obj) - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received wait() @@ -829,7 +842,7 @@ def test_send_buffer_blocking_waiting(self): received = None obj = bytearray(b'spam') wait = self.build_send_waiter(obj, buffer=True) - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received wait() @@ -844,7 +857,7 @@ def f(): def test_send_blocking_no_wait(self): received = None obj = b'spam' - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received received = recv_wait(cid) @@ -858,7 +871,7 @@ def f(): def test_send_buffer_blocking_no_wait(self): received = None obj = bytearray(b'spam') - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): nonlocal received received = recv_wait(cid) @@ -873,20 +886,20 @@ def test_send_timeout(self): obj = b'spam' with self.subTest('non-blocking with timeout'): - cid = _channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(ValueError): _channels.send(cid, obj, blocking=False, timeout=0.1) with self.subTest('timeout hit'): - cid = _channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(TimeoutError): _channels.send(cid, obj, blocking=True, timeout=0.1) with self.assertRaises(_channels.ChannelEmptyError): - received = _channels.recv(cid) + received = recv_nowait(cid) print(repr(received)) with self.subTest('timeout not hit'): - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): recv_wait(cid) t = threading.Thread(target=f) @@ -910,20 +923,20 @@ def test_send_buffer_timeout(self): obj = bytearray(b'spam') with self.subTest('non-blocking with timeout'): - cid = _channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(ValueError): _channels.send_buffer(cid, obj, blocking=False, timeout=0.1) with self.subTest('timeout hit'): - cid = _channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(TimeoutError): _channels.send_buffer(cid, obj, blocking=True, timeout=0.1) with self.assertRaises(_channels.ChannelEmptyError): - received = _channels.recv(cid) + received = recv_nowait(cid) print(repr(received)) with self.subTest('timeout not hit'): - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): recv_wait(cid) t = threading.Thread(target=f) @@ -936,7 +949,7 @@ def test_send_closed_while_waiting(self): wait = self.build_send_waiter(obj) with self.subTest('without timeout'): - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): wait() _channels.close(cid, force=True) @@ -947,7 +960,7 @@ def f(): t.join() with self.subTest('with timeout'): - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): wait() _channels.close(cid, force=True) @@ -974,7 +987,7 @@ def test_send_buffer_closed_while_waiting(self): wait = self.build_send_waiter(obj, buffer=True) with self.subTest('without timeout'): - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): wait() _channels.close(cid, force=True) @@ -985,7 +998,7 @@ def f(): t.join() with self.subTest('with timeout'): - cid = _channels.create() + cid = _channels.create(REPLACE) def f(): wait() _channels.close(cid, force=True) @@ -999,9 +1012,9 @@ def f(): # close def test_close_single_user(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) - _channels.recv(cid) + recv_nowait(cid) _channels.close(cid) with self.assertRaises(_channels.ChannelClosedError): @@ -1010,7 +1023,7 @@ def test_close_single_user(self): _channels.recv(cid) def test_close_multiple_users(self): - cid = _channels.create() + cid = _channels.create(REPLACE) id1 = _interpreters.create() id2 = _interpreters.create() _interpreters.run_string(id1, dedent(f""" @@ -1034,9 +1047,9 @@ def test_close_multiple_users(self): self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') def test_close_multiple_times(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) - _channels.recv(cid) + recv_nowait(cid) _channels.close(cid) with self.assertRaises(_channels.ChannelClosedError): @@ -1051,9 +1064,9 @@ def test_close_empty(self): ] for send, recv in tests: with self.subTest((send, recv)): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) - _channels.recv(cid) + recv_nowait(cid) _channels.close(cid, send=send, recv=recv) with self.assertRaises(_channels.ChannelClosedError): @@ -1062,56 +1075,56 @@ def test_close_empty(self): _channels.recv(cid) def test_close_defaults_with_unused_items(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'ham', blocking=False) with self.assertRaises(_channels.ChannelNotEmptyError): _channels.close(cid) - _channels.recv(cid) + recv_nowait(cid) _channels.send(cid, b'eggs', blocking=False) def test_close_recv_with_unused_items_unforced(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'ham', blocking=False) with self.assertRaises(_channels.ChannelNotEmptyError): _channels.close(cid, recv=True) - _channels.recv(cid) + recv_nowait(cid) _channels.send(cid, b'eggs', blocking=False) - _channels.recv(cid) - _channels.recv(cid) + recv_nowait(cid) + recv_nowait(cid) _channels.close(cid, recv=True) def test_close_send_with_unused_items_unforced(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'ham', blocking=False) _channels.close(cid, send=True) with self.assertRaises(_channels.ChannelClosedError): _channels.send(cid, b'eggs') - _channels.recv(cid) - _channels.recv(cid) + recv_nowait(cid) + recv_nowait(cid) with self.assertRaises(_channels.ChannelClosedError): _channels.recv(cid) def test_close_both_with_unused_items_unforced(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'ham', blocking=False) with self.assertRaises(_channels.ChannelNotEmptyError): _channels.close(cid, recv=True, send=True) - _channels.recv(cid) + recv_nowait(cid) _channels.send(cid, b'eggs', blocking=False) - _channels.recv(cid) - _channels.recv(cid) + recv_nowait(cid) + recv_nowait(cid) _channels.close(cid, recv=True) def test_close_recv_with_unused_items_forced(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'ham', blocking=False) _channels.close(cid, recv=True, force=True) @@ -1122,7 +1135,7 @@ def test_close_recv_with_unused_items_forced(self): _channels.recv(cid) def test_close_send_with_unused_items_forced(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'ham', blocking=False) _channels.close(cid, send=True, force=True) @@ -1133,7 +1146,7 @@ def test_close_send_with_unused_items_forced(self): _channels.recv(cid) def test_close_both_with_unused_items_forced(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'ham', blocking=False) _channels.close(cid, send=True, recv=True, force=True) @@ -1144,7 +1157,7 @@ def test_close_both_with_unused_items_forced(self): _channels.recv(cid) def test_close_never_used(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.close(cid) with self.assertRaises(_channels.ChannelClosedError): @@ -1153,7 +1166,7 @@ def test_close_never_used(self): _channels.recv(cid) def test_close_by_unassociated_interp(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) interp = _interpreters.create() _interpreters.run_string(interp, dedent(f""" @@ -1166,11 +1179,11 @@ def test_close_by_unassociated_interp(self): _channels.close(cid) def test_close_used_multiple_times_by_single_user(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'spam', blocking=False) - _channels.recv(cid) + recv_nowait(cid) _channels.close(cid, force=True) with self.assertRaises(_channels.ChannelClosedError): @@ -1179,7 +1192,7 @@ def test_close_used_multiple_times_by_single_user(self): _channels.recv(cid) def test_channel_list_interpreters_invalid_channel(self): - cid = _channels.create() + cid = _channels.create(REPLACE) # Test for invalid channel ID. with self.assertRaises(_channels.ChannelNotFoundError): _channels.list_interpreters(1000, send=True) @@ -1191,7 +1204,7 @@ def test_channel_list_interpreters_invalid_channel(self): def test_channel_list_interpreters_invalid_args(self): # Tests for invalid arguments passed to the API. - cid = _channels.create() + cid = _channels.create(REPLACE) with self.assertRaises(TypeError): _channels.list_interpreters(cid) @@ -1240,9 +1253,9 @@ class ChannelReleaseTests(TestBase): """ def test_single_user(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) - _channels.recv(cid) + recv_nowait(cid) _channels.release(cid, send=True, recv=True) with self.assertRaises(_channels.ChannelClosedError): @@ -1251,7 +1264,7 @@ def test_single_user(self): _channels.recv(cid) def test_multiple_users(self): - cid = _channels.create() + cid = _channels.create(REPLACE) id1 = _interpreters.create() id2 = _interpreters.create() _interpreters.run_string(id1, dedent(f""" @@ -1260,7 +1273,7 @@ def test_multiple_users(self): """)) out = _run_output(id2, dedent(f""" import _interpchannels as _channels - obj = _channels.recv({cid}) + obj, _ = _channels.recv({cid}) _channels.release({cid}) print(repr(obj)) """)) @@ -1271,9 +1284,9 @@ def test_multiple_users(self): self.assertEqual(out.strip(), "b'spam'") def test_no_kwargs(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) - _channels.recv(cid) + recv_nowait(cid) _channels.release(cid) with self.assertRaises(_channels.ChannelClosedError): @@ -1282,16 +1295,16 @@ def test_no_kwargs(self): _channels.recv(cid) def test_multiple_times(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) - _channels.recv(cid) + recv_nowait(cid) _channels.release(cid, send=True, recv=True) with self.assertRaises(_channels.ChannelClosedError): _channels.release(cid, send=True, recv=True) def test_with_unused_items(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'ham', blocking=False) _channels.release(cid, send=True, recv=True) @@ -1300,7 +1313,7 @@ def test_with_unused_items(self): _channels.recv(cid) def test_never_used(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.release(cid) with self.assertRaises(_channels.ChannelClosedError): @@ -1309,14 +1322,14 @@ def test_never_used(self): _channels.recv(cid) def test_by_unassociated_interp(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) interp = _interpreters.create() _interpreters.run_string(interp, dedent(f""" import _interpchannels as _channels _channels.release({cid}) """)) - obj = _channels.recv(cid) + obj = recv_nowait(cid) _channels.release(cid) with self.assertRaises(_channels.ChannelClosedError): @@ -1325,7 +1338,7 @@ def test_by_unassociated_interp(self): def test_close_if_unassociated(self): # XXX Something's not right with this test... - cid = _channels.create() + cid = _channels.create(REPLACE) interp = _interpreters.create() _interpreters.run_string(interp, dedent(f""" import _interpchannels as _channels @@ -1338,21 +1351,21 @@ def test_close_if_unassociated(self): def test_partially(self): # XXX Is partial close too weird/confusing? - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, None, blocking=False) - _channels.recv(cid) + recv_nowait(cid) _channels.send(cid, b'spam', blocking=False) _channels.release(cid, send=True) - obj = _channels.recv(cid) + obj = recv_nowait(cid) self.assertEqual(obj, b'spam') def test_used_multiple_times_by_single_user(self): - cid = _channels.create() + cid = _channels.create(REPLACE) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'spam', blocking=False) _channels.send(cid, b'spam', blocking=False) - _channels.recv(cid) + recv_nowait(cid) _channels.release(cid, send=True, recv=True) with self.assertRaises(_channels.ChannelClosedError): @@ -1428,9 +1441,9 @@ def clean_up(self): def _new_channel(self, creator): if creator.name == 'main': - return _channels.create() + return _channels.create(REPLACE) else: - ch = _channels.create() + ch = _channels.create(REPLACE) run_interp(creator.id, f""" import _interpreters cid = _xxsubchannels.create() @@ -1439,7 +1452,7 @@ def _new_channel(self, creator): _xxsubchannels.send({ch}, int(cid), blocking=False) del _interpreters """) - self._cid = _channels.recv(ch) + self._cid = recv_nowait(ch) return self._cid def _get_interpreter(self, interp): @@ -1657,7 +1670,7 @@ def run_action(self, fix, action, *, hideclosed=True): ) fix.record_action(action, result) else: - _cid = _channels.create() + _cid = _channels.create(REPLACE) run_interp(interp.id, f""" result = helpers.run_action( {fix.cid}, @@ -1670,8 +1683,8 @@ def run_action(self, fix, action, *, hideclosed=True): _channels.send({_cid}, b'X' if result.closed else b'', blocking=False) """) result = ChannelState( - pending=int.from_bytes(_channels.recv(_cid), 'little'), - closed=bool(_channels.recv(_cid)), + pending=int.from_bytes(recv_nowait(_cid), 'little'), + closed=bool(recv_nowait(_cid)), ) fix.record_action(action, result) @@ -1729,7 +1742,7 @@ def _assert_closed(self, fix): self.assertTrue(fix.state.closed) for _ in range(fix.state.pending): - _channels.recv(fix.cid) + recv_nowait(fix.cid) self._assert_closed_in_interp(fix) for interp in ('same', 'other'): diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py new file mode 100644 index 00000000000000..e4dcdb6b58d009 --- /dev/null +++ b/Lib/test/test_annotationlib.py @@ -0,0 +1,872 @@ +"""Tests for the annotations module.""" + +import annotationlib +import functools +import itertools +import pickle +import unittest +from annotationlib import Format, get_annotations, get_annotate_function +from typing import Unpack + +from test.test_inspect import inspect_stock_annotations +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_stringized_annotations_2 +from test.test_inspect import inspect_stringized_annotations_pep695 + + +def times_three(fn): + @functools.wraps(fn) + def wrapper(a, b): + return fn(a * 3, b * 3) + + return wrapper + + +class TestFormat(unittest.TestCase): + def test_enum(self): + self.assertEqual(annotationlib.Format.VALUE.value, 1) + self.assertEqual(annotationlib.Format.VALUE, 1) + + self.assertEqual(annotationlib.Format.FORWARDREF.value, 2) + self.assertEqual(annotationlib.Format.FORWARDREF, 2) + + self.assertEqual(annotationlib.Format.SOURCE.value, 3) + self.assertEqual(annotationlib.Format.SOURCE, 3) + + +class TestForwardRefFormat(unittest.TestCase): + def test_closure(self): + def inner(arg: x): + pass + + anno = annotationlib.get_annotations( + inner, format=annotationlib.Format.FORWARDREF + ) + fwdref = anno["arg"] + self.assertIsInstance(fwdref, annotationlib.ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = 1 + self.assertEqual(fwdref.evaluate(), x) + + anno = annotationlib.get_annotations( + inner, format=annotationlib.Format.FORWARDREF + ) + self.assertEqual(anno["arg"], x) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF) + self.assertIs(anno["x"], int) + fwdref = anno["y"] + self.assertIsInstance(fwdref, annotationlib.ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "doesntexist") + with self.assertRaises(NameError): + fwdref.evaluate() + self.assertEqual(fwdref.evaluate(globals={"doesntexist": 1}), 1) + + +class TestSourceFormat(unittest.TestCase): + def test_closure(self): + x = 0 + + def inner(arg: x): + pass + + anno = annotationlib.get_annotations(inner, format=annotationlib.Format.SOURCE) + self.assertEqual(anno, {"arg": "x"}) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + self.assertEqual(anno, {"x": "int", "y": "doesntexist"}) + + def test_expressions(self): + def f( + add: a + b, + sub: a - b, + mul: a * b, + matmul: a @ b, + truediv: a / b, + mod: a % b, + lshift: a << b, + rshift: a >> b, + or_: a | b, + xor: a ^ b, + and_: a & b, + floordiv: a // b, + pow_: a**b, + lt: a < b, + le: a <= b, + eq: a == b, + ne: a != b, + gt: a > b, + ge: a >= b, + invert: ~a, + neg: -a, + pos: +a, + getitem: a[b], + getattr: a.b, + call: a(b, *c, d=e), # **kwargs are not supported + *args: *a, + ): + pass + + anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + self.assertEqual( + anno, + { + "add": "a + b", + "sub": "a - b", + "mul": "a * b", + "matmul": "a @ b", + "truediv": "a / b", + "mod": "a % b", + "lshift": "a << b", + "rshift": "a >> b", + "or_": "a | b", + "xor": "a ^ b", + "and_": "a & b", + "floordiv": "a // b", + "pow_": "a ** b", + "lt": "a < b", + "le": "a <= b", + "eq": "a == b", + "ne": "a != b", + "gt": "a > b", + "ge": "a >= b", + "invert": "~a", + "neg": "-a", + "pos": "+a", + "getitem": "a[b]", + "getattr": "a.b", + "call": "a(b, *c, d=e)", + "args": "*a", + }, + ) + + def test_reverse_ops(self): + def f( + radd: 1 + a, + rsub: 1 - a, + rmul: 1 * a, + rmatmul: 1 @ a, + rtruediv: 1 / a, + rmod: 1 % a, + rlshift: 1 << a, + rrshift: 1 >> a, + ror: 1 | a, + rxor: 1 ^ a, + rand: 1 & a, + rfloordiv: 1 // a, + rpow: 1**a, + ): + pass + + anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + self.assertEqual( + anno, + { + "radd": "1 + a", + "rsub": "1 - a", + "rmul": "1 * a", + "rmatmul": "1 @ a", + "rtruediv": "1 / a", + "rmod": "1 % a", + "rlshift": "1 << a", + "rrshift": "1 >> a", + "ror": "1 | a", + "rxor": "1 ^ a", + "rand": "1 & a", + "rfloordiv": "1 // a", + "rpow": "1 ** a", + }, + ) + + def test_nested_expressions(self): + def f( + nested: list[Annotated[set[int], "set of ints", 4j]], + set: {a + b}, # single element because order is not guaranteed + dict: {a + b: c + d, "key": e + g}, + list: [a, b, c], + tuple: (a, b, c), + slice: (a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d]), + extended_slice: a[:, :, c:d], + unpack1: [*a], + unpack2: [*a, b, c], + ): + pass + + anno = annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + self.assertEqual( + anno, + { + "nested": "list[Annotated[set[int], 'set of ints', 4j]]", + "set": "{a + b}", + "dict": "{a + b: c + d, 'key': e + g}", + "list": "[a, b, c]", + "tuple": "(a, b, c)", + "slice": "(a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d])", + "extended_slice": "a[:, :, c:d]", + "unpack1": "[*a]", + "unpack2": "[*a, b, c]", + }, + ) + + def test_unsupported_operations(self): + format_msg = "Cannot stringify annotation containing string formatting" + + def f(fstring: f"{a}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + + def f(fstring_format: f"{a:02d}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + + +class TestForwardRefClass(unittest.TestCase): + def test_special_attrs(self): + # Forward refs provide a different introspection API. __name__ and + # __qualname__ make little sense for forward refs as they can store + # complex typing expressions. + fr = annotationlib.ForwardRef("set[Any]") + self.assertFalse(hasattr(fr, "__name__")) + self.assertFalse(hasattr(fr, "__qualname__")) + self.assertEqual(fr.__module__, "annotationlib") + # Forward refs are currently unpicklable once they contain a code object. + fr.__forward_code__ # fill the cache + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(fr, proto) + + +class TestGetAnnotations(unittest.TestCase): + def test_builtin_type(self): + self.assertEqual(annotationlib.get_annotations(int), {}) + self.assertEqual(annotationlib.get_annotations(object), {}) + + def test_custom_metaclass(self): + class Meta(type): + pass + + class C(metaclass=Meta): + x: int + + self.assertEqual(annotationlib.get_annotations(C), {"x": int}) + + def test_missing_dunder_dict(self): + class NoDict(type): + @property + def __dict__(cls): + raise AttributeError + + b: str + + class C1(metaclass=NoDict): + a: int + + self.assertEqual(annotationlib.get_annotations(C1), {"a": int}) + self.assertEqual( + annotationlib.get_annotations(C1, format=annotationlib.Format.FORWARDREF), + {"a": int}, + ) + self.assertEqual( + annotationlib.get_annotations(C1, format=annotationlib.Format.SOURCE), + {"a": "int"}, + ) + self.assertEqual(annotationlib.get_annotations(NoDict), {"b": str}) + self.assertEqual( + annotationlib.get_annotations(NoDict, format=annotationlib.Format.FORWARDREF), + {"b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(NoDict, format=annotationlib.Format.SOURCE), + {"b": "str"}, + ) + + def test_format(self): + def f1(a: int): + pass + + def f2(a: undefined): + pass + + self.assertEqual( + annotationlib.get_annotations(f1, format=annotationlib.Format.VALUE), + {"a": int}, + ) + self.assertEqual(annotationlib.get_annotations(f1, format=1), {"a": int}) + + fwd = annotationlib.ForwardRef("undefined") + self.assertEqual( + annotationlib.get_annotations(f2, format=annotationlib.Format.FORWARDREF), + {"a": fwd}, + ) + self.assertEqual(annotationlib.get_annotations(f2, format=2), {"a": fwd}) + + self.assertEqual( + annotationlib.get_annotations(f1, format=annotationlib.Format.SOURCE), + {"a": "int"}, + ) + self.assertEqual(annotationlib.get_annotations(f1, format=3), {"a": "int"}) + + with self.assertRaises(ValueError): + annotationlib.get_annotations(f1, format=0) + + with self.assertRaises(ValueError): + annotationlib.get_annotations(f1, format=4) + + def test_custom_object_with_annotations(self): + class C: + def __init__(self): + self.__annotations__ = {"x": int, "y": str} + + self.assertEqual(annotationlib.get_annotations(C()), {"x": int, "y": str}) + + def test_custom_format_eval_str(self): + def foo(): + pass + + with self.assertRaises(ValueError): + annotationlib.get_annotations( + foo, format=annotationlib.Format.FORWARDREF, eval_str=True + ) + annotationlib.get_annotations( + foo, format=annotationlib.Format.SOURCE, eval_str=True + ) + + def test_stock_annotations(self): + def foo(a: int, b: str): + pass + + for format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): + with self.subTest(format=format): + self.assertEqual( + annotationlib.get_annotations(foo, format=format), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(foo, format=annotationlib.Format.SOURCE), + {"a": "int", "b": "str"}, + ) + + foo.__annotations__ = {"a": "foo", "b": "str"} + for format in annotationlib.Format: + with self.subTest(format=format): + self.assertEqual( + annotationlib.get_annotations(foo, format=format), + {"a": "foo", "b": "str"}, + ) + + self.assertEqual( + annotationlib.get_annotations(foo, eval_str=True, locals=locals()), + {"a": foo, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(foo, eval_str=True, globals=locals()), + {"a": foo, "b": str}, + ) + + def test_stock_annotations_in_module(self): + isa = inspect_stock_annotations + + for kwargs in [ + {}, + {"eval_str": False}, + {"format": annotationlib.Format.VALUE}, + {"format": annotationlib.Format.FORWARDREF}, + {"format": annotationlib.Format.VALUE, "eval_str": False}, + {"format": annotationlib.Format.FORWARDREF, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), {"a": int, "b": str} + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(annotationlib, **kwargs), {} + ) # annotations module has no annotations + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"format": annotationlib.Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), {"a": int, "b": str} + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": int, "b": str, "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": int, "b": str, "c": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(annotationlib, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + self.assertEqual( + annotationlib.get_annotations(isa, format=annotationlib.Format.SOURCE), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.MyClass, format=annotationlib.Format.SOURCE + ), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.function, format=annotationlib.Format.SOURCE + ), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.function2, format=annotationlib.Format.SOURCE + ), + {"a": "int", "b": "str", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.function3, format=annotationlib.Format.SOURCE + ), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations( + annotationlib, format=annotationlib.Format.SOURCE + ), + {}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.UnannotatedClass, format=annotationlib.Format.SOURCE + ), + {}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.unannotated_function, format=annotationlib.Format.SOURCE + ), + {}, + ) + + def test_stock_annotations_on_wrapper(self): + isa = inspect_stock_annotations + + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + annotationlib.get_annotations(wrapped), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations( + wrapped, format=annotationlib.Format.FORWARDREF + ), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, format=annotationlib.Format.SOURCE), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=False), + {"a": int, "b": str, "return": isa.MyClass}, + ) + + def test_stringized_annotations_in_module(self): + isa = inspect_stringized_annotations + for kwargs in [ + {}, + {"eval_str": False}, + {"format": annotationlib.Format.VALUE}, + {"format": annotationlib.Format.FORWARDREF}, + {"format": annotationlib.Format.SOURCE}, + {"format": annotationlib.Format.VALUE, "eval_str": False}, + {"format": annotationlib.Format.FORWARDREF, "eval_str": False}, + {"format": annotationlib.Format.SOURCE, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": "int", "b": "'str'", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": "'int'", "b": "'str'", "c": "'MyClass'"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"format": annotationlib.Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + annotationlib.get_annotations(isa, **kwargs), {"a": int, "b": str} + ) + self.assertEqual( + annotationlib.get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(isa.UnannotatedClass, **kwargs), {} + ) + self.assertEqual( + annotationlib.get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + def test_stringized_annotations_in_empty_module(self): + isa2 = inspect_stringized_annotations_2 + self.assertEqual(annotationlib.get_annotations(isa2), {}) + self.assertEqual(annotationlib.get_annotations(isa2, eval_str=True), {}) + self.assertEqual(annotationlib.get_annotations(isa2, eval_str=False), {}) + + def test_stringized_annotations_on_wrapper(self): + isa = inspect_stringized_annotations + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + annotationlib.get_annotations(wrapped), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + annotationlib.get_annotations(wrapped, eval_str=False), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + + def test_stringized_annotations_on_class(self): + isa = inspect_stringized_annotations + # test that local namespace lookups work + self.assertEqual( + annotationlib.get_annotations(isa.MyClassWithLocalAnnotations), + {"x": "mytype"}, + ) + self.assertEqual( + annotationlib.get_annotations( + isa.MyClassWithLocalAnnotations, eval_str=True + ), + {"x": int}, + ) + + def test_modify_annotations(self): + def f(x: int): + pass + + self.assertEqual(annotationlib.get_annotations(f), {"x": int}) + self.assertEqual( + annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF), + {"x": int}, + ) + + f.__annotations__["x"] = str + # The modification is reflected in VALUE (the default) + self.assertEqual(annotationlib.get_annotations(f), {"x": str}) + # ... but not in FORWARDREF, which uses __annotate__ + self.assertEqual( + annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF), + {"x": int}, + ) + + def test_pep695_generic_class_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + A_annotations = annotationlib.get_annotations(ann_module695.A, eval_str=True) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(A_annotations["x"], A_type_params[0]) + self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + B_annotations = annotationlib.get_annotations( + inspect_stringized_annotations_pep695.B, eval_str=True + ) + self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): + ann_module695 = inspect_stringized_annotations_pep695 + C_annotations = annotationlib.get_annotations(ann_module695.C, eval_str=True) + self.assertEqual( + set(C_annotations.values()), + set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_func_annotations = annotationlib.get_annotations( + ann_module695.generic_function, eval_str=True + ) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(generic_func_annotations["x"], func_t_params[0]) + self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) + self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) + self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set( + annotationlib.get_annotations( + inspect_stringized_annotations_pep695.generic_function_2, + eval_str=True + ).values() + ), + set( + inspect_stringized_annotations_pep695.generic_function_2.__type_params__ + ) + ) + + def test_pep_695_generic_method_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_method_annotations = annotationlib.get_annotations( + ann_module695.D.generic_method, eval_str=True + ) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + generic_method_annotations, + {"x": params["Foo"], "y": params["Bar"], "return": None} + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set( + annotationlib.get_annotations( + inspect_stringized_annotations_pep695.D.generic_method_2, + eval_str=True + ).values() + ), + set( + inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ + ) + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars(self): + self.assertEqual( + annotationlib.get_annotations( + inspect_stringized_annotations_pep695.E, eval_str=True + ), + {"x": str}, + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = inspect_stringized_annotations_pep695.nested() + + self.assertEqual( + set(results.F_annotations.values()), + set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()), + set(results.F.generic_method.__type_params__) + ) + self.assertNotEqual( + set(results.F_meth_annotations.values()), + set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()).intersection(results.F.__type_params__), + set() + ) + + self.assertEqual(results.G_annotations, {"x": str}) + + self.assertEqual( + set(results.generic_func_annotations.values()), + set(results.generic_func.__type_params__), + ) + + +class TestCallEvaluateFunction(unittest.TestCase): + def test_evaluation(self): + def evaluate(format, exc=NotImplementedError): + if format != 1: + raise exc + return undefined + + with self.assertRaises(NameError): + annotationlib.call_evaluate_function(evaluate, annotationlib.Format.VALUE) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, annotationlib.Format.FORWARDREF), + annotationlib.ForwardRef("undefined"), + ) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, annotationlib.Format.SOURCE), + "undefined", + ) + + +class MetaclassTests(unittest.TestCase): + def test_annotated_meta(self): + class Meta(type): + a: int + + class X(metaclass=Meta): + pass + + class Y(metaclass=Meta): + b: float + + self.assertEqual(get_annotations(Meta), {"a": int}) + self.assertEqual(get_annotate_function(Meta)(Format.VALUE), {"a": int}) + + self.assertEqual(get_annotations(X), {}) + self.assertIs(get_annotate_function(X), None) + + self.assertEqual(get_annotations(Y), {"b": float}) + self.assertEqual(get_annotate_function(Y)(Format.VALUE), {"b": float}) + + def test_unannotated_meta(self): + class Meta(type): pass + + class X(metaclass=Meta): + a: str + + class Y(X): pass + + self.assertEqual(get_annotations(Meta), {}) + self.assertIs(get_annotate_function(Meta), None) + + self.assertEqual(get_annotations(Y), {}) + self.assertIs(get_annotate_function(Y), None) + + self.assertEqual(get_annotations(X), {"a": str}) + self.assertEqual(get_annotate_function(X)(Format.VALUE), {"a": str}) + + def test_ordering(self): + # Based on a sample by David Ellis + # https://discuss.python.org/t/pep-749-implementing-pep-649/54974/38 + + def make_classes(): + class Meta(type): + a: int + expected_annotations = {"a": int} + + class A(type, metaclass=Meta): + b: float + expected_annotations = {"b": float} + + class B(metaclass=A): + c: str + expected_annotations = {"c": str} + + class C(B): + expected_annotations = {} + + class D(metaclass=Meta): + expected_annotations = {} + + return Meta, A, B, C, D + + classes = make_classes() + class_count = len(classes) + for order in itertools.permutations(range(class_count), class_count): + names = ", ".join(classes[i].__name__ for i in order) + with self.subTest(names=names): + classes = make_classes() # Regenerate classes + for i in order: + get_annotations(classes[i]) + for c in classes: + with self.subTest(c=c): + self.assertEqual(get_annotations(c), c.expected_annotations) + annotate_func = get_annotate_function(c) + if c.expected_annotations: + self.assertEqual(annotate_func(Format.VALUE), c.expected_annotations) + else: + self.assertIs(annotate_func, None) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 95383be9659eb9..47cbe60bfca4e4 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1493,7 +1493,7 @@ def test_byteswap(self): self.assertEqual(a, b) else: # On alphas treating the byte swapped bit patters as - # floats/doubles results in floating point exceptions + # floats/doubles results in floating-point exceptions # => compare the 8bit string values instead self.assertNotEqual(a.tobytes(), b.tobytes()) b.byteswap() diff --git a/Lib/test/test_ast/__init__.py b/Lib/test/test_ast/__init__.py new file mode 100644 index 00000000000000..9a89d27ba9f979 --- /dev/null +++ b/Lib/test/test_ast/__init__.py @@ -0,0 +1,7 @@ +import os + +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_ast/snippets.py b/Lib/test/test_ast/snippets.py new file mode 100644 index 00000000000000..95dc3ca03cd38b --- /dev/null +++ b/Lib/test/test_ast/snippets.py @@ -0,0 +1,602 @@ +import ast +import sys + +from test.test_ast.utils import to_tuple + + +# These tests are compiled through "exec" +# There should be at least one test per statement +exec_tests = [ + # Module docstring + "'module docstring'", + # FunctionDef + "def f(): pass", + # FunctionDef with docstring + "def f(): 'function docstring'", + # FunctionDef with arg + "def f(a): pass", + # FunctionDef with arg and default value + "def f(a=0): pass", + # FunctionDef with varargs + "def f(*args): pass", + # FunctionDef with varargs as TypeVarTuple + "def f(*args: *Ts): pass", + # FunctionDef with varargs as unpacked Tuple + "def f(*args: *tuple[int, ...]): pass", + # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple + "def f(*args: *tuple[int, *Ts]): pass", + # FunctionDef with kwargs + "def f(**kwargs): pass", + # FunctionDef with all kind of args and docstring + "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", + # FunctionDef with type annotation on return involving unpacking + "def f() -> tuple[*Ts]: pass", + "def f() -> tuple[int, *Ts]: pass", + "def f() -> tuple[int, *tuple[int, ...]]: pass", + # ClassDef + "class C:pass", + # ClassDef with docstring + "class C: 'docstring for class C'", + # ClassDef, new style class + "class C(object): pass", + # Classdef with multiple bases + "class C(A, B): pass", + # Return + "def f():return 1", + "def f():return", + # Delete + "del v", + # Assign + "v = 1", + "a,b = c", + "(a,b) = c", + "[a,b] = c", + "a[b] = c", + # AnnAssign with unpacked types + "x: tuple[*Ts]", + "x: tuple[int, *Ts]", + "x: tuple[int, *tuple[str, ...]]", + # AugAssign + "v += 1", + "v -= 1", + "v *= 1", + "v @= 1", + "v /= 1", + "v %= 1", + "v **= 1", + "v <<= 1", + "v >>= 1", + "v |= 1", + "v ^= 1", + "v &= 1", + "v //= 1", + # For + "for v in v:pass", + # For-Else + "for v in v:\n pass\nelse:\n pass", + # While + "while v:pass", + # While-Else + "while v:\n pass\nelse:\n pass", + # If-Elif-Else + "if v:pass", + "if a:\n pass\nelif b:\n pass", + "if a:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelif b:\n pass\nelif b:\n pass\nelse:\n pass", + # With + "with x: pass", + "with x, y: pass", + "with x as y: pass", + "with x as y, z as q: pass", + "with (x as y): pass", + "with (x, y): pass", + # Raise + "raise", + "raise Exception('string')", + "raise Exception", + "raise Exception('string') from None", + # TryExcept + "try:\n pass\nexcept Exception:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass", + # TryFinally + "try:\n pass\nfinally:\n pass", + # TryStarExcept + "try:\n pass\nexcept* Exception:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass", + # TryExceptFinallyElse + "try:\n pass\nexcept Exception:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + # Assert + "assert v", + # Assert with message + "assert v, 'message'", + # Import + "import sys", + "import foo as bar", + # ImportFrom + "from sys import x as y", + "from sys import v", + # Global + "global v", + # Expr + "1", + # Pass, + "pass", + # Break + "for v in v:break", + # Continue + "for v in v:continue", + # for statements with naked tuples (see http://bugs.python.org/issue6704) + "for a,b in c: pass", + "for (a,b) in c: pass", + "for [a,b] in c: pass", + # Multiline generator expression (test for .lineno & .col_offset) + """( + ( + Aa + , + Bb + ) + for + Aa + , + Bb in Cc + )""", + # dictcomp + "{a : b for w in x for m in p if g}", + # dictcomp with naked tuple + "{a : b for v,w in x}", + # setcomp + "{r for l in x if g}", + # setcomp with naked tuple + "{r for l,m in x}", + # AsyncFunctionDef + "async def f():\n 'async function'\n await something()", + # AsyncFor + "async def f():\n async for e in i: 1\n else: 2", + # AsyncWith + "async def f():\n async with a as b: 1", + # PEP 448: Additional Unpacking Generalizations + "{**{1:2}, 2:3}", + "{*{1, 2}, 3}", + # Function with yield (from) + "def f(): yield 1", + "def f(): yield from []", + # Asynchronous comprehensions + "async def f():\n [i async for b in c]", + # Decorated FunctionDef + "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", + # Decorated AsyncFunctionDef + "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", + # Decorated ClassDef + "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", + # Decorator with generator argument + "@deco(a for a in b)\ndef f(): pass", + # Decorator with attribute + "@a.b.c\ndef f(): pass", + # Simple assignment expression + "(a := 1)", + # Assignment expression in if statement + "if a := foo(): pass", + # Assignment expression in while + "while a := foo(): pass", + # Positional-only arguments + "def f(a, /,): pass", + "def f(a, /, c, d, e): pass", + "def f(a, /, c, *, d, e): pass", + "def f(a, /, c, *, d, e, **kwargs): pass", + # Positional-only arguments with defaults + "def f(a=1, /,): pass", + "def f(a=1, /, b=2, c=4): pass", + "def f(a=1, /, b=2, *, c=4): pass", + "def f(a=1, /, b=2, *, c): pass", + "def f(a=1, /, b=2, *, c=4, **kwargs): pass", + "def f(a=1, /, b=2, *, c, **kwargs): pass", + # Type aliases + "type X = int", + "type X[T] = int", + "type X[T, *Ts, **P] = (T, Ts, P)", + "type X[T: int, *Ts, **P] = (T, Ts, P)", + "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", + "type X[T: int = 1, *Ts = 2, **P =3] = (T, Ts, P)", + # Generic classes + "class X[T]: pass", + "class X[T, *Ts, **P]: pass", + "class X[T: int, *Ts, **P]: pass", + "class X[T: (int, str), *Ts, **P]: pass", + "class X[T: int = 1, *Ts = 2, **P = 3]: pass", + # Generic functions + "def f[T](): pass", + "def f[T, *Ts, **P](): pass", + "def f[T: int, *Ts, **P](): pass", + "def f[T: (int, str), *Ts, **P](): pass", + "def f[T: int = 1, *Ts = 2, **P = 3](): pass", + # Match + "match x:\n\tcase 1:\n\t\tpass", + # Match with _ + "match x:\n\tcase 1:\n\t\tpass\n\tcase _:\n\t\tpass", +] + +# These are compiled through "single" +# because of overlap with "eval", it just tests what +# can't be tested with "eval" +single_tests = [ + "1+2" +] + +# These are compiled through "eval" +# It should test all expressions +eval_tests = [ + # Constant(value=None) + "None", + # True + "True", + # False + "False", + # BoolOp + "a and b", + "a or b", + # BinOp + "a + b", + "a - b", + "a * b", + "a / b", + "a @ b", + "a // b", + "a ** b", + "a % b", + "a >> b", + "a << b", + "a ^ b", + "a | b", + "a & b", + # UnaryOp + "not v", + "+v", + "-v", + "~v", + # Lambda + "lambda:None", + # Dict + "{ 1:2 }", + # Empty dict + "{}", + # Set + "{None,}", + # Multiline dict (test for .lineno & .col_offset) + """{ + 1 + : + 2 + }""", + # Multiline list + """[ + 1 + , + 1 + ]""", + # Multiline tuple + """( + 1 + , + )""", + # Multiline set + """{ + 1 + , + 1 + }""", + # ListComp + "[a for b in c if d]", + # GeneratorExp + "(a for b in c if d)", + # SetComp + "{a for b in c if d}", + # DictComp + "{k: v for k, v in c if d}", + # Comprehensions with multiple for targets + "[(a,b) for a,b in c]", + "[(a,b) for (a,b) in c]", + "[(a,b) for [a,b] in c]", + "{(a,b) for a,b in c}", + "{(a,b) for (a,b) in c}", + "{(a,b) for [a,b] in c}", + "((a,b) for a,b in c)", + "((a,b) for (a,b) in c)", + "((a,b) for [a,b] in c)", + # Async comprehensions - async comprehensions can't work outside an asynchronous function + # + # Yield - yield expressions can't work outside a function + # + # Compare + "1 < 2 < 3", + "a == b", + "a <= b", + "a >= b", + "a != b", + "a is b", + "a is not b", + "a in b", + "a not in b", + # Call without argument + "f()", + # Call + "f(1,2,c=3,*d,**e)", + # Call with multi-character starred + "f(*[0, 1])", + # Call with a generator argument + "f(a for a in b)", + # Constant(value=int()) + "10", + # Complex num + "1j", + # Constant(value=str()) + "'string'", + # Attribute + "a.b", + # Subscript + "a[b:c]", + # Name + "v", + # List + "[1,2,3]", + # Empty list + "[]", + # Tuple + "1,2,3", + # Tuple + "(1,2,3)", + # Empty tuple + "()", + # Combination + "a.b.c.d(a.b[1:2])", + # Slice + "[5][1:]", + "[5][:1]", + "[5][::1]", + "[5][1:1:1]", + # IfExp + "foo() if x else bar()", + # JoinedStr and FormattedValue + "f'{a}'", + "f'{a:.2f}'", + "f'{a!r}'", + "f'foo({a})'", +] + + +def main(): + if __name__ != '__main__': + return + if sys.argv[1:] == ['-g']: + for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), + (eval_tests, "eval")): + print(kind+"_results = [") + for statement in statements: + tree = ast.parse(statement, "?", kind) + print("%r," % (to_tuple(tree),)) + print("]") + print("main()") + raise SystemExit + unittest.main() + +#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast/snippets.py -g ##### +exec_results = [ +('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), +('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 19), 'C', [('Name', (1, 8, 1, 9), 'A', ('Load',)), ('Name', (1, 11, 1, 12), 'B', ('Load',))], [], [('Pass', (1, 15, 1, 19))], [], [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 14), None)], [], None, None, [])], []), +('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), +('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), +('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 8), [('Subscript', (1, 0, 1, 4), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Store',))], ('Name', (1, 7, 1, 8), 'c', ('Load',)), None)], []), +('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Sub',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('MatMult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Div',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mod',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Pow',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('LShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('RShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitOr',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitXor',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitAnd',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('FloorDiv',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), +('Module', [('For', (1, 0, 4, 6), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))], None)], []), +('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), +('Module', [('While', (1, 0, 4, 6), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), +('Module', [('If', (1, 0, 10, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 10, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('If', (5, 0, 10, 6), ('Name', (5, 5, 5, 6), 'b', ('Load',)), [('Pass', (6, 2, 6, 6))], [('If', (7, 0, 10, 6), ('Name', (7, 5, 7, 6), 'b', ('Load',)), [('Pass', (8, 2, 8, 6))], [('Pass', (10, 2, 10, 6))])])])])], []), +('Module', [('With', (1, 0, 1, 12), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None)], [('Pass', (1, 8, 1, 12))], None)], []), +('Module', [('With', (1, 0, 1, 15), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None), ('withitem', ('Name', (1, 8, 1, 9), 'y', ('Load',)), None)], [('Pass', (1, 11, 1, 15))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), +('Module', [('With', (1, 0, 1, 19), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), ('Name', (1, 11, 1, 12), 'y', ('Store',)))], [('Pass', (1, 15, 1, 19))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), None), ('withitem', ('Name', (1, 9, 1, 10), 'y', ('Load',)), None)], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('Raise', (1, 0, 1, 5), None, None)], []), +('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), +('Module', [('Raise', (1, 0, 1, 15), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), None)], []), +('Module', [('Raise', (1, 0, 1, 35), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), ('Constant', (1, 31, 1, 35), None, None))], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('TryStar', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), +('Module', [('Assert', (1, 0, 1, 19), ('Name', (1, 7, 1, 8), 'v', ('Load',)), ('Constant', (1, 10, 1, 19), 'message', None))], []), +('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), +('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')])], []), +('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0)], []), +('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), +('Module', [('Global', (1, 0, 1, 8), ['v'])], []), +('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), +('Module', [('Pass', (1, 0, 1, 4))], []), +('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), +('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), +('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), +('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), +('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), +('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), +('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), +('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), +('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 16), ('Yield', (1, 9, 1, 16), ('Constant', (1, 15, 1, 16), 1, None)))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 22), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 22), ('YieldFrom', (1, 9, 1, 22), ('List', (1, 20, 1, 22), [], ('Load',))))], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), +('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), +('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), +('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], [])], []), +('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), +('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), +('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), +('Module', [('If', (1, 0, 1, 19), ('NamedExpr', (1, 3, 1, 13), ('Name', (1, 3, 1, 4), 'a', ('Store',)), ('Call', (1, 8, 1, 13), ('Name', (1, 8, 1, 11), 'foo', ('Load',)), [], [])), [('Pass', (1, 15, 1, 19))], [])], []), +('Module', [('While', (1, 0, 1, 22), ('NamedExpr', (1, 6, 1, 16), ('Name', (1, 6, 1, 7), 'a', ('Store',)), ('Call', (1, 11, 1, 16), ('Name', (1, 11, 1, 14), 'foo', ('Load',)), [], [])), [('Pass', (1, 18, 1, 22))], [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), +('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts', None), ('ParamSpec', (1, 15, 1, 18), 'P', None)], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), None), ('TypeVarTuple', (1, 15, 1, 18), 'Ts', None), ('ParamSpec', (1, 20, 1, 23), 'P', None)], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 22, 1, 25), 'Ts', None), ('ParamSpec', (1, 27, 1, 30), 'P', None)], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 48), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 17), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Constant', (1, 16, 1, 17), 1, None)), ('TypeVarTuple', (1, 19, 1, 26), 'Ts', ('Constant', (1, 25, 1, 26), 2, None)), ('ParamSpec', (1, 28, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))], ('Tuple', (1, 38, 1, 48), [('Name', (1, 39, 1, 40), 'T', ('Load',)), ('Name', (1, 42, 1, 44), 'Ts', ('Load',)), ('Name', (1, 46, 1, 47), 'P', ('Load',))], ('Load',)))], []), +('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []), +('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 43), 'X', [], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts', None), ('ParamSpec', (1, 14, 1, 17), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), None), ('TypeVarTuple', (1, 14, 1, 17), 'Ts', None), ('ParamSpec', (1, 19, 1, 22), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 21, 1, 24), 'Ts', None), ('ParamSpec', (1, 26, 1, 29), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 43), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 39, 1, 43))], [], None, None, [('TypeVar', (1, 6, 1, 16), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Constant', (1, 15, 1, 16), 1, None)), ('TypeVarTuple', (1, 18, 1, 25), 'Ts', ('Constant', (1, 24, 1, 25), 2, None)), ('ParamSpec', (1, 27, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))])], []), +('Module', [('Match', (1, 0, 3, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))])])], []), +('Module', [('Match', (1, 0, 5, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))]), ('match_case', ('MatchAs', (4, 6, 4, 7), None, None), None, [('Pass', (5, 2, 5, 6))])])], []), +] +single_results = [ +('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), +] +eval_results = [ +('Expression', ('Constant', (1, 0, 1, 4), None, None)), +('Expression', ('Constant', (1, 0, 1, 4), True, None)), +('Expression', ('Constant', (1, 0, 1, 5), False, None)), +('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), +('Expression', ('BoolOp', (1, 0, 1, 6), ('Or',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Sub',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Div',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('MatMult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('FloorDiv',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Pow',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mod',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('RShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('LShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitXor',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitOr',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitAnd',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('UAdd',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('USub',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('Invert',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), +('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), +('Expression', ('Dict', (1, 0, 1, 2), [], [])), +('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), +('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), +('Expression', ('List', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 4, 6), [('Constant', (2, 6, 2, 7), 1, None)], ('Load',))), +('Expression', ('Set', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)])), +('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('DictComp', (1, 0, 1, 25), ('Name', (1, 1, 1, 2), 'k', ('Load',)), ('Name', (1, 4, 1, 5), 'v', ('Load',)), [('comprehension', ('Tuple', (1, 10, 1, 14), [('Name', (1, 10, 1, 11), 'k', ('Store',)), ('Name', (1, 13, 1, 14), 'v', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [('Name', (1, 23, 1, 24), 'd', ('Load',))], 0)])), +('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Eq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('LtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('GtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotEq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Is',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('IsNot',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('In',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotIn',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Call', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [], [])), +('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), +('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), +('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), +('Expression', ('Constant', (1, 0, 1, 2), 10, None)), +('Expression', ('Constant', (1, 0, 1, 2), 1j, None)), +('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), +('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), +('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), +('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), +('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), +('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), ('Constant', (1, 4, 1, 5), 1, None), None, None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), None, ('Constant', (1, 5, 1, 6), 1, None), None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 8), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 7), None, None, ('Constant', (1, 6, 1, 7), 1, None)), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 10), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 9), ('Constant', (1, 4, 1, 5), 1, None), ('Constant', (1, 6, 1, 7), 1, None), ('Constant', (1, 8, 1, 9), 1, None)), ('Load',))), +('Expression', ('IfExp', (1, 0, 1, 21), ('Name', (1, 9, 1, 10), 'x', ('Load',)), ('Call', (1, 0, 1, 5), ('Name', (1, 0, 1, 3), 'foo', ('Load',)), [], []), ('Call', (1, 16, 1, 21), ('Name', (1, 16, 1, 19), 'bar', ('Load',)), [], []))), +('Expression', ('JoinedStr', (1, 0, 1, 6), [('FormattedValue', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), +('Expression', ('JoinedStr', (1, 0, 1, 8), [('FormattedValue', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 114, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('FormattedValue', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), +] +main() diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast/test_ast.py similarity index 68% rename from Lib/test/test_ast.py rename to Lib/test/test_ast/test_ast.py index eb3aefd5c262f6..0a3edef4678546 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -19,389 +19,16 @@ from test import support from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin +from test.test_ast.utils import to_tuple +from test.test_ast.snippets import ( + eval_tests, eval_results, exec_tests, exec_results, single_tests, single_results +) -def to_tuple(t): - if t is None or isinstance(t, (str, int, complex, float, bytes)) or t is Ellipsis: - return t - elif isinstance(t, list): - return [to_tuple(e) for e in t] - result = [t.__class__.__name__] - if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): - result.append((t.lineno, t.col_offset)) - if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): - result[-1] += (t.end_lineno, t.end_col_offset) - if t._fields is None: - return tuple(result) - for f in t._fields: - result.append(to_tuple(getattr(t, f))) - return tuple(result) STDLIB = os.path.dirname(ast.__file__) STDLIB_FILES = [fn for fn in os.listdir(STDLIB) if fn.endswith(".py")] STDLIB_FILES.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) -# These tests are compiled through "exec" -# There should be at least one test per statement -exec_tests = [ - # Module docstring - "'module docstring'", - # FunctionDef - "def f(): pass", - # FunctionDef with docstring - "def f(): 'function docstring'", - # FunctionDef with arg - "def f(a): pass", - # FunctionDef with arg and default value - "def f(a=0): pass", - # FunctionDef with varargs - "def f(*args): pass", - # FunctionDef with varargs as TypeVarTuple - "def f(*args: *Ts): pass", - # FunctionDef with varargs as unpacked Tuple - "def f(*args: *tuple[int, ...]): pass", - # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple - "def f(*args: *tuple[int, *Ts]): pass", - # FunctionDef with kwargs - "def f(**kwargs): pass", - # FunctionDef with all kind of args and docstring - "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", - # FunctionDef with type annotation on return involving unpacking - "def f() -> tuple[*Ts]: pass", - "def f() -> tuple[int, *Ts]: pass", - "def f() -> tuple[int, *tuple[int, ...]]: pass", - # ClassDef - "class C:pass", - # ClassDef with docstring - "class C: 'docstring for class C'", - # ClassDef, new style class - "class C(object): pass", - # Classdef with multiple bases - "class C(A, B): pass", - # Return - "def f():return 1", - "def f():return", - # Delete - "del v", - # Assign - "v = 1", - "a,b = c", - "(a,b) = c", - "[a,b] = c", - "a[b] = c", - # AnnAssign with unpacked types - "x: tuple[*Ts]", - "x: tuple[int, *Ts]", - "x: tuple[int, *tuple[str, ...]]", - # AugAssign - "v += 1", - "v -= 1", - "v *= 1", - "v @= 1", - "v /= 1", - "v %= 1", - "v **= 1", - "v <<= 1", - "v >>= 1", - "v |= 1", - "v ^= 1", - "v &= 1", - "v //= 1", - # For - "for v in v:pass", - # For-Else - "for v in v:\n pass\nelse:\n pass", - # While - "while v:pass", - # While-Else - "while v:\n pass\nelse:\n pass", - # If-Elif-Else - "if v:pass", - "if a:\n pass\nelif b:\n pass", - "if a:\n pass\nelse:\n pass", - "if a:\n pass\nelif b:\n pass\nelse:\n pass", - "if a:\n pass\nelif b:\n pass\nelif b:\n pass\nelif b:\n pass\nelse:\n pass", - # With - "with x: pass", - "with x, y: pass", - "with x as y: pass", - "with x as y, z as q: pass", - "with (x as y): pass", - "with (x, y): pass", - # Raise - "raise", - "raise Exception('string')", - "raise Exception", - "raise Exception('string') from None", - # TryExcept - "try:\n pass\nexcept Exception:\n pass", - "try:\n pass\nexcept Exception as exc:\n pass", - # TryFinally - "try:\n pass\nfinally:\n pass", - # TryStarExcept - "try:\n pass\nexcept* Exception:\n pass", - "try:\n pass\nexcept* Exception as exc:\n pass", - # TryExceptFinallyElse - "try:\n pass\nexcept Exception:\n pass\nelse: pass\nfinally:\n pass", - "try:\n pass\nexcept Exception as exc:\n pass\nelse: pass\nfinally:\n pass", - "try:\n pass\nexcept* Exception as exc:\n pass\nelse: pass\nfinally:\n pass", - # Assert - "assert v", - # Assert with message - "assert v, 'message'", - # Import - "import sys", - "import foo as bar", - # ImportFrom - "from sys import x as y", - "from sys import v", - # Global - "global v", - # Expr - "1", - # Pass, - "pass", - # Break - "for v in v:break", - # Continue - "for v in v:continue", - # for statements with naked tuples (see http://bugs.python.org/issue6704) - "for a,b in c: pass", - "for (a,b) in c: pass", - "for [a,b] in c: pass", - # Multiline generator expression (test for .lineno & .col_offset) - """( - ( - Aa - , - Bb - ) - for - Aa - , - Bb in Cc - )""", - # dictcomp - "{a : b for w in x for m in p if g}", - # dictcomp with naked tuple - "{a : b for v,w in x}", - # setcomp - "{r for l in x if g}", - # setcomp with naked tuple - "{r for l,m in x}", - # AsyncFunctionDef - "async def f():\n 'async function'\n await something()", - # AsyncFor - "async def f():\n async for e in i: 1\n else: 2", - # AsyncWith - "async def f():\n async with a as b: 1", - # PEP 448: Additional Unpacking Generalizations - "{**{1:2}, 2:3}", - "{*{1, 2}, 3}", - # Function with yield (from) - "def f(): yield 1", - "def f(): yield from []", - # Asynchronous comprehensions - "async def f():\n [i async for b in c]", - # Decorated FunctionDef - "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", - # Decorated AsyncFunctionDef - "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", - # Decorated ClassDef - "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", - # Decorator with generator argument - "@deco(a for a in b)\ndef f(): pass", - # Decorator with attribute - "@a.b.c\ndef f(): pass", - # Simple assignment expression - "(a := 1)", - # Assignment expression in if statement - "if a := foo(): pass", - # Assignment expression in while - "while a := foo(): pass", - # Positional-only arguments - "def f(a, /,): pass", - "def f(a, /, c, d, e): pass", - "def f(a, /, c, *, d, e): pass", - "def f(a, /, c, *, d, e, **kwargs): pass", - # Positional-only arguments with defaults - "def f(a=1, /,): pass", - "def f(a=1, /, b=2, c=4): pass", - "def f(a=1, /, b=2, *, c=4): pass", - "def f(a=1, /, b=2, *, c): pass", - "def f(a=1, /, b=2, *, c=4, **kwargs): pass", - "def f(a=1, /, b=2, *, c, **kwargs): pass", - # Type aliases - "type X = int", - "type X[T] = int", - "type X[T, *Ts, **P] = (T, Ts, P)", - "type X[T: int, *Ts, **P] = (T, Ts, P)", - "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", - "type X[T: int = 1, *Ts = 2, **P =3] = (T, Ts, P)", - # Generic classes - "class X[T]: pass", - "class X[T, *Ts, **P]: pass", - "class X[T: int, *Ts, **P]: pass", - "class X[T: (int, str), *Ts, **P]: pass", - "class X[T: int = 1, *Ts = 2, **P = 3]: pass", - # Generic functions - "def f[T](): pass", - "def f[T, *Ts, **P](): pass", - "def f[T: int, *Ts, **P](): pass", - "def f[T: (int, str), *Ts, **P](): pass", - "def f[T: int = 1, *Ts = 2, **P = 3](): pass", - # Match - "match x:\n\tcase 1:\n\t\tpass", - # Match with _ - "match x:\n\tcase 1:\n\t\tpass\n\tcase _:\n\t\tpass", -] - -# These are compiled through "single" -# because of overlap with "eval", it just tests what -# can't be tested with "eval" -single_tests = [ - "1+2" -] - -# These are compiled through "eval" -# It should test all expressions -eval_tests = [ - # Constant(value=None) - "None", - # True - "True", - # False - "False", - # BoolOp - "a and b", - "a or b", - # BinOp - "a + b", - "a - b", - "a * b", - "a / b", - "a @ b", - "a // b", - "a ** b", - "a % b", - "a >> b", - "a << b", - "a ^ b", - "a | b", - "a & b", - # UnaryOp - "not v", - "+v", - "-v", - "~v", - # Lambda - "lambda:None", - # Dict - "{ 1:2 }", - # Empty dict - "{}", - # Set - "{None,}", - # Multiline dict (test for .lineno & .col_offset) - """{ - 1 - : - 2 - }""", - # Multiline list - """[ - 1 - , - 1 - ]""", - # Multiline tuple - """( - 1 - , - )""", - # Multiline set - """{ - 1 - , - 1 - }""", - # ListComp - "[a for b in c if d]", - # GeneratorExp - "(a for b in c if d)", - # SetComp - "{a for b in c if d}", - # DictComp - "{k: v for k, v in c if d}", - # Comprehensions with multiple for targets - "[(a,b) for a,b in c]", - "[(a,b) for (a,b) in c]", - "[(a,b) for [a,b] in c]", - "{(a,b) for a,b in c}", - "{(a,b) for (a,b) in c}", - "{(a,b) for [a,b] in c}", - "((a,b) for a,b in c)", - "((a,b) for (a,b) in c)", - "((a,b) for [a,b] in c)", - # Async comprehensions - async comprehensions can't work outside an asynchronous function - # - # Yield - yield expressions can't work outside a function - # - # Compare - "1 < 2 < 3", - "a == b", - "a <= b", - "a >= b", - "a != b", - "a is b", - "a is not b", - "a in b", - "a not in b", - # Call without argument - "f()", - # Call - "f(1,2,c=3,*d,**e)", - # Call with multi-character starred - "f(*[0, 1])", - # Call with a generator argument - "f(a for a in b)", - # Constant(value=int()) - "10", - # Complex num - "1j", - # Constant(value=str()) - "'string'", - # Attribute - "a.b", - # Subscript - "a[b:c]", - # Name - "v", - # List - "[1,2,3]", - # Empty list - "[]", - # Tuple - "1,2,3", - # Tuple - "(1,2,3)", - # Empty tuple - "()", - # Combination - "a.b.c.d(a.b[1:2])", - # Slice - "[5][1:]", - "[5][:1]", - "[5][::1]", - "[5][1:1:1]", - # IfExp - "foo() if x else bar()", - # JoinedStr and FormattedValue - "f'{a}'", - "f'{a:.2f}'", - "f'{a!r}'", - "f'foo({a})'", -] - class AST_Tests(unittest.TestCase): maxDiff = None @@ -1386,15 +1013,7 @@ class MyNode(ast.AST): self.assertEqual(node.y, 1) y = object() - # custom attributes are currently not supported and raise a warning - # because the allowed attributes are hard-coded ! - msg = ( - "MyNode.__init__ got an unexpected keyword argument 'y'. " - "Support for arbitrary keyword arguments is deprecated and " - "will be removed in Python 3.15" - ) - with self.assertWarnsRegex(DeprecationWarning, re.escape(msg)): - repl = copy.replace(node, y=y) + repl = copy.replace(node, y=y) # assert that there is no side-effect self.assertEqual(node.x, 0) self.assertEqual(node.y, 1) @@ -1829,6 +1448,12 @@ def test_get_docstring(self): node = ast.parse('async def foo():\n """spam\n ham"""') self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') + node = ast.parse('async def foo():\n """spam\n ham"""') + self.assertEqual(ast.get_docstring(node.body[0], clean=False), 'spam\n ham') + + node = ast.parse('x') + self.assertRaises(TypeError, ast.get_docstring, node.body[0]) + def test_get_docstring_none(self): self.assertIsNone(ast.get_docstring(ast.parse(''))) node = ast.parse('x = "not docstring"') @@ -1853,6 +1478,9 @@ def test_get_docstring_none(self): node = ast.parse('async def foo():\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('async def foo():\n 42') + self.assertIsNone(ast.get_docstring(node.body[0])) + def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): node = ast.parse( '"""line one\nline two"""\n\n' @@ -3250,6 +2878,18 @@ class FieldsAndTypes(ast.AST): obj = FieldsAndTypes(a=1) self.assertEqual(obj.a, 1) + def test_custom_attributes(self): + class MyAttrs(ast.AST): + _attributes = ("a", "b") + + obj = MyAttrs(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + with self.assertWarnsRegex(DeprecationWarning, + r"MyAttrs.__init__ got an unexpected keyword argument 'c'."): + obj = MyAttrs(c=3) + def test_fields_and_types_no_default(self): class FieldsAndTypesNoDefault(ast.AST): _fields = ('a',) @@ -3395,238 +3035,3 @@ def test_cli_file_input(self): self.assertEqual(expected.splitlines(), res.out.decode("utf8").splitlines()) self.assertEqual(res.rc, 0) - - -def main(): - if __name__ != '__main__': - return - if sys.argv[1:] == ['-g']: - for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), - (eval_tests, "eval")): - print(kind+"_results = [") - for statement in statements: - tree = ast.parse(statement, "?", kind) - print("%r," % (to_tuple(tree),)) - print("]") - print("main()") - raise SystemExit - unittest.main() - -#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g ##### -exec_results = [ -('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), -('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 19), 'C', [('Name', (1, 8, 1, 9), 'A', ('Load',)), ('Name', (1, 11, 1, 12), 'B', ('Load',))], [], [('Pass', (1, 15, 1, 19))], [], [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 14), None)], [], None, None, [])], []), -('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), -('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), -('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 8), [('Subscript', (1, 0, 1, 4), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Store',))], ('Name', (1, 7, 1, 8), 'c', ('Load',)), None)], []), -('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Sub',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mult',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('MatMult',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Div',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mod',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Pow',), ('Constant', (1, 6, 1, 7), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('LShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('RShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitOr',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitXor',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitAnd',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('FloorDiv',), ('Constant', (1, 6, 1, 7), 1, None))], []), -('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), -('Module', [('For', (1, 0, 4, 6), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))], None)], []), -('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), -('Module', [('While', (1, 0, 4, 6), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), -('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), -('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), -('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), -('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), -('Module', [('If', (1, 0, 10, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 10, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('If', (5, 0, 10, 6), ('Name', (5, 5, 5, 6), 'b', ('Load',)), [('Pass', (6, 2, 6, 6))], [('If', (7, 0, 10, 6), ('Name', (7, 5, 7, 6), 'b', ('Load',)), [('Pass', (8, 2, 8, 6))], [('Pass', (10, 2, 10, 6))])])])])], []), -('Module', [('With', (1, 0, 1, 12), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None)], [('Pass', (1, 8, 1, 12))], None)], []), -('Module', [('With', (1, 0, 1, 15), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None), ('withitem', ('Name', (1, 8, 1, 9), 'y', ('Load',)), None)], [('Pass', (1, 11, 1, 15))], None)], []), -('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), -('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), -('Module', [('With', (1, 0, 1, 19), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), ('Name', (1, 11, 1, 12), 'y', ('Store',)))], [('Pass', (1, 15, 1, 19))], None)], []), -('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), None), ('withitem', ('Name', (1, 9, 1, 10), 'y', ('Load',)), None)], [('Pass', (1, 13, 1, 17))], None)], []), -('Module', [('Raise', (1, 0, 1, 5), None, None)], []), -('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), -('Module', [('Raise', (1, 0, 1, 15), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), None)], []), -('Module', [('Raise', (1, 0, 1, 35), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), ('Constant', (1, 31, 1, 35), None, None))], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), -('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), -('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), -('Module', [('TryStar', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), -('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), -('Module', [('Assert', (1, 0, 1, 19), ('Name', (1, 7, 1, 8), 'v', ('Load',)), ('Constant', (1, 10, 1, 19), 'message', None))], []), -('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), -('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')])], []), -('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0)], []), -('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), -('Module', [('Global', (1, 0, 1, 8), ['v'])], []), -('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), -('Module', [('Pass', (1, 0, 1, 4))], []), -('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), -('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), -('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), -('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), -('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 16), ('Yield', (1, 9, 1, 16), ('Constant', (1, 15, 1, 16), 1, None)))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 22), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 22), ('YieldFrom', (1, 9, 1, 22), ('List', (1, 20, 1, 22), [], ('Load',))))], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), -('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), -('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), -('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], [])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), -('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), -('Module', [('If', (1, 0, 1, 19), ('NamedExpr', (1, 3, 1, 13), ('Name', (1, 3, 1, 4), 'a', ('Store',)), ('Call', (1, 8, 1, 13), ('Name', (1, 8, 1, 11), 'foo', ('Load',)), [], [])), [('Pass', (1, 15, 1, 19))], [])], []), -('Module', [('While', (1, 0, 1, 22), ('NamedExpr', (1, 6, 1, 16), ('Name', (1, 6, 1, 7), 'a', ('Store',)), ('Call', (1, 11, 1, 16), ('Name', (1, 11, 1, 14), 'foo', ('Load',)), [], [])), [('Pass', (1, 18, 1, 22))], [])], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), -('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts', None), ('ParamSpec', (1, 15, 1, 18), 'P', None)], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), None), ('TypeVarTuple', (1, 15, 1, 18), 'Ts', None), ('ParamSpec', (1, 20, 1, 23), 'P', None)], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 22, 1, 25), 'Ts', None), ('ParamSpec', (1, 27, 1, 30), 'P', None)], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 48), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 17), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Constant', (1, 16, 1, 17), 1, None)), ('TypeVarTuple', (1, 19, 1, 26), 'Ts', ('Constant', (1, 25, 1, 26), 2, None)), ('ParamSpec', (1, 28, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))], ('Tuple', (1, 38, 1, 48), [('Name', (1, 39, 1, 40), 'T', ('Load',)), ('Name', (1, 42, 1, 44), 'Ts', ('Load',)), ('Name', (1, 46, 1, 47), 'P', ('Load',))], ('Load',)))], []), -('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []), -('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []), -('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []), -('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []), -('Module', [('ClassDef', (1, 0, 1, 43), 'X', [], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts', None), ('ParamSpec', (1, 14, 1, 17), 'P', None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), None), ('TypeVarTuple', (1, 14, 1, 17), 'Ts', None), ('ParamSpec', (1, 19, 1, 22), 'P', None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 21, 1, 24), 'Ts', None), ('ParamSpec', (1, 26, 1, 29), 'P', None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 43), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 39, 1, 43))], [], None, None, [('TypeVar', (1, 6, 1, 16), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Constant', (1, 15, 1, 16), 1, None)), ('TypeVarTuple', (1, 18, 1, 25), 'Ts', ('Constant', (1, 24, 1, 25), 2, None)), ('ParamSpec', (1, 27, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))])], []), -('Module', [('Match', (1, 0, 3, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))])])], []), -('Module', [('Match', (1, 0, 5, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))]), ('match_case', ('MatchAs', (4, 6, 4, 7), None, None), None, [('Pass', (5, 2, 5, 6))])])], []), -] -single_results = [ -('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), -] -eval_results = [ -('Expression', ('Constant', (1, 0, 1, 4), None, None)), -('Expression', ('Constant', (1, 0, 1, 4), True, None)), -('Expression', ('Constant', (1, 0, 1, 5), False, None)), -('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), -('Expression', ('BoolOp', (1, 0, 1, 6), ('Or',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',))])), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Sub',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Div',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('MatMult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('FloorDiv',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Pow',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mod',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('RShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('LShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitXor',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitOr',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitAnd',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), -('Expression', ('UnaryOp', (1, 0, 1, 2), ('UAdd',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), -('Expression', ('UnaryOp', (1, 0, 1, 2), ('USub',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), -('Expression', ('UnaryOp', (1, 0, 1, 2), ('Invert',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), -('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), -('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), -('Expression', ('Dict', (1, 0, 1, 2), [], [])), -('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), -('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), -('Expression', ('List', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 4, 6), [('Constant', (2, 6, 2, 7), 1, None)], ('Load',))), -('Expression', ('Set', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)])), -('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('DictComp', (1, 0, 1, 25), ('Name', (1, 1, 1, 2), 'k', ('Load',)), ('Name', (1, 4, 1, 5), 'v', ('Load',)), [('comprehension', ('Tuple', (1, 10, 1, 14), [('Name', (1, 10, 1, 11), 'k', ('Store',)), ('Name', (1, 13, 1, 14), 'v', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [('Name', (1, 23, 1, 24), 'd', ('Load',))], 0)])), -('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), -('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Eq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), -('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('LtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), -('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('GtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), -('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotEq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), -('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Is',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), -('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('IsNot',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), -('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('In',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), -('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotIn',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), -('Expression', ('Call', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [], [])), -('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), -('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), -('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), -('Expression', ('Constant', (1, 0, 1, 2), 10, None)), -('Expression', ('Constant', (1, 0, 1, 2), 1j, None)), -('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), -('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), -('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), -('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), -('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), -('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), ('Constant', (1, 4, 1, 5), 1, None), None, None), ('Load',))), -('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), None, ('Constant', (1, 5, 1, 6), 1, None), None), ('Load',))), -('Expression', ('Subscript', (1, 0, 1, 8), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 7), None, None, ('Constant', (1, 6, 1, 7), 1, None)), ('Load',))), -('Expression', ('Subscript', (1, 0, 1, 10), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 9), ('Constant', (1, 4, 1, 5), 1, None), ('Constant', (1, 6, 1, 7), 1, None), ('Constant', (1, 8, 1, 9), 1, None)), ('Load',))), -('Expression', ('IfExp', (1, 0, 1, 21), ('Name', (1, 9, 1, 10), 'x', ('Load',)), ('Call', (1, 0, 1, 5), ('Name', (1, 0, 1, 3), 'foo', ('Load',)), [], []), ('Call', (1, 16, 1, 21), ('Name', (1, 16, 1, 19), 'bar', ('Load',)), [], []))), -('Expression', ('JoinedStr', (1, 0, 1, 6), [('FormattedValue', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, None)])), -('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), -('Expression', ('JoinedStr', (1, 0, 1, 8), [('FormattedValue', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 114, None)])), -('Expression', ('JoinedStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('FormattedValue', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), -] -main() diff --git a/Lib/test/test_ast/utils.py b/Lib/test/test_ast/utils.py new file mode 100644 index 00000000000000..145e89ee94e935 --- /dev/null +++ b/Lib/test/test_ast/utils.py @@ -0,0 +1,15 @@ +def to_tuple(t): + if t is None or isinstance(t, (str, int, complex, float, bytes)) or t is Ellipsis: + return t + elif isinstance(t, list): + return [to_tuple(e) for e in t] + result = [t.__class__.__name__] + if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): + result.append((t.lineno, t.col_offset)) + if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): + result[-1] += (t.end_lineno, t.end_col_offset) + if t._fields is None: + return tuple(result) + for f in t._fields: + result.append(to_tuple(getattr(t, f))) + return tuple(result) diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 0f8212dbec47be..0777f39b572486 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -241,6 +241,18 @@ class DummyLoop: _, out, err = assert_python_ok("-c", code) self.assertFalse(err) + def test_issue122332(self): + async def coro(): + pass + + async def run(): + task = self.loop.create_task(coro()) + await task + self.assertIsNone(task.get_coro()) + + self.run_coro(run()) + + class AsyncTaskCounter: def __init__(self, loop, *, task_class, eager): self.suspense_count = 0 diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 4966775acac7be..9ae54b6887010b 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1212,6 +1212,7 @@ async def test_fork_not_share_event_loop(self): wait_process(pid, exitcode=0) @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_signal_handling(self): self.addCleanup(multiprocessing_cleanup_tests) @@ -1258,6 +1259,7 @@ async def func(): self.assertTrue(child_handled.is_set()) @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_asyncio_run(self): self.addCleanup(multiprocessing_cleanup_tests) @@ -1277,6 +1279,7 @@ async def child_main(): self.assertEqual(result.value, 42) @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_asyncio_subprocess(self): self.addCleanup(multiprocessing_cleanup_tests) diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index dbb8d27c176950..35893ab3118e1e 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -301,12 +301,17 @@ def run_udp_echo_server(*, host='127.0.0.1', port=0): family, type, proto, _, sockaddr = addr_info[0] sock = socket.socket(family, type, proto) sock.bind((host, port)) + sockname = sock.getsockname() thread = threading.Thread(target=lambda: echo_datagrams(sock)) thread.start() try: - yield sock.getsockname() + yield sockname finally: - sock.sendto(b'STOP', sock.getsockname()) + # gh-122187: use a separate socket to send the stop message to avoid + # TSan reported race on the same socket. + sock2 = socket.socket(family, type, proto) + sock2.sendto(b'STOP', sockname) + sock2.close() thread.join() diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 5818e96d61f480..85f139db9bcd45 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -16,6 +16,7 @@ import random import re import sys +import textwrap import traceback import types import typing @@ -412,7 +413,7 @@ def test_compile_top_level_await_no_coro(self): "socket.accept is broken" ) def test_compile_top_level_await(self): - """Test whether code some top level await can be compiled. + """Test whether code with top level await can be compiled. Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set, and make sure the generated code object has the CO_COROUTINE flag @@ -426,6 +427,7 @@ async def arange(n): yield i modes = ('single', 'exec') + optimizations = (-1, 0, 1, 2) code_samples = [ '''a = await asyncio.sleep(0, result=1)''', '''async for i in arange(1): @@ -438,34 +440,52 @@ async def arange(n): '''a = [x async for x in arange(2) async for x in arange(2)][1]''', '''a = [x async for x in (x async for x in arange(5))][1]''', '''a, = [1 for x in {x async for x in arange(1)}]''', - '''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]''' + '''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]''', + # gh-121637: Make sure we correctly handle the case where the + # async code is optimized away + '''assert not await asyncio.sleep(0); a = 1''', + '''assert [x async for x in arange(1)]; a = 1''', + '''assert {x async for x in arange(1)}; a = 1''', + '''assert {x: x async for x in arange(1)}; a = 1''', + ''' + if (a := 1) and __debug__: + async with asyncio.Lock() as l: + pass + ''', + ''' + if (a := 1) and __debug__: + async for x in arange(2): + pass + ''', ] policy = maybe_get_event_loop_policy() try: - for mode, code_sample in product(modes, code_samples): - source = dedent(code_sample) - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) - - co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) - - self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, - msg=f"source={source} mode={mode}") + for mode, code_sample, optimize in product(modes, code_samples, optimizations): + with self.subTest(mode=mode, code_sample=code_sample, optimize=optimize): + source = dedent(code_sample) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + compile(source, '?', mode, optimize=optimize) - # test we can create and advance a function type - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - async_f = FunctionType(co, globals_) - asyncio.run(async_f()) - self.assertEqual(globals_['a'], 1) - - # test we can await-eval, - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - asyncio.run(eval(co, globals_)) - self.assertEqual(globals_['a'], 1) + co = compile(source, + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + optimize=optimize) + + self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, + msg=f"source={source} mode={mode}") + + # test we can create and advance a function type + globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} + async_f = FunctionType(co, globals_) + asyncio.run(async_f()) + self.assertEqual(globals_['a'], 1) + + # test we can await-eval, + globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} + asyncio.run(eval(co, globals_)) + self.assertEqual(globals_['a'], 1) finally: asyncio.set_event_loop_policy(policy) @@ -1758,6 +1778,8 @@ def test_sum(self): self.assertRaises(TypeError, sum, [], '') self.assertRaises(TypeError, sum, [], b'') self.assertRaises(TypeError, sum, [], bytearray()) + self.assertRaises(OverflowError, sum, [1.0, 10**1000]) + self.assertRaises(OverflowError, sum, [1j, 10**1000]) class BadSeq: def __getitem__(self, index): @@ -1783,6 +1805,9 @@ def test_sum_accuracy(self): self.assertEqual(sum([1.0, 10E100, 1.0, -10E100, 2j]), 2+2j) self.assertEqual(sum([2+1j, 10E100j, 1j, -10E100j]), 2+2j) self.assertEqual(sum([1j, 1, 10E100j, 1j, 1.0, -10E100j]), 2+2j) + self.assertEqual(sum([2j, 1., 10E100, 1., -10E100]), 2+2j) + self.assertEqual(sum([1.0, 10**100, 1.0, -10**100]), 2.0) + self.assertEqual(sum([2j, 1.0, 10**100, 1.0, -10**100]), 2+2j) self.assertEqual(sum([0.1j]*10 + [fractions.Fraction(1, 10)]), 0.1+1j) def test_type(self): diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index f14d5545c829e5..d5f047bcf18277 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -53,6 +53,8 @@ def test_fromstringandsize(self): self.assertEqual(fromstringandsize(b'abc'), b'abc') self.assertEqual(fromstringandsize(b'abc', 2), b'ab') self.assertEqual(fromstringandsize(b'abc\0def'), b'abc\0def') + self.assertEqual(fromstringandsize(b'a'), b'a') + self.assertEqual(fromstringandsize(b'a', 1), b'a') self.assertEqual(fromstringandsize(b'', 0), b'') self.assertEqual(fromstringandsize(NULL, 0), b'') self.assertEqual(len(fromstringandsize(NULL, 3)), 3) diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 7e8d571ae234d1..bdbdd7bcfe0f2a 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -162,224 +162,125 @@ def test_long_fromunicodeobject(self): # CRASHES fromunicodeobject(NULL, 0) # CRASHES fromunicodeobject(NULL, 16) + def check_long_asint(self, func, min_val, max_val, *, + use_index=True, + mask=False, + negative_value_error=OverflowError): + # round trip (object -> C integer -> object) + values = (0, 1, 1234, max_val) + if min_val < 0: + values += (-1, min_val) + for value in values: + with self.subTest(value=value): + self.assertEqual(func(value), value) + self.assertEqual(func(IntSubclass(value)), value) + if use_index: + self.assertEqual(func(Index(value)), value) + + if use_index: + self.assertEqual(func(MyIndexAndInt()), 10) + else: + self.assertRaises(TypeError, func, Index(42)) + self.assertRaises(TypeError, func, MyIndexAndInt()) + + if mask: + self.assertEqual(func(min_val - 1), max_val) + self.assertEqual(func(max_val + 1), min_val) + self.assertEqual(func(-1 << 1000), 0) + self.assertEqual(func(1 << 1000), 0) + else: + self.assertRaises(negative_value_error, func, min_val - 1) + self.assertRaises(negative_value_error, func, -1 << 1000) + self.assertRaises(OverflowError, func, max_val + 1) + self.assertRaises(OverflowError, func, 1 << 1000) + self.assertRaises(TypeError, func, 1.0) + self.assertRaises(TypeError, func, b'2') + self.assertRaises(TypeError, func, '3') + self.assertRaises(SystemError, func, NULL) + + def check_long_asintandoverflow(self, func, min_val, max_val): + # round trip (object -> C integer -> object) + for value in (min_val, max_val, -1, 0, 1, 1234): + with self.subTest(value=value): + self.assertEqual(func(value), (value, 0)) + self.assertEqual(func(IntSubclass(value)), (value, 0)) + self.assertEqual(func(Index(value)), (value, 0)) + + self.assertEqual(func(MyIndexAndInt()), (10, 0)) + + self.assertEqual(func(min_val - 1), (-1, -1)) + self.assertEqual(func(max_val + 1), (-1, +1)) + + # CRASHES func(1.0) + # CRASHES func(NULL) + def test_long_asint(self): # Test PyLong_AsInt() PyLong_AsInt = _testlimitedcapi.PyLong_AsInt from _testcapi import INT_MIN, INT_MAX - - # round trip (object -> int -> object) - for value in (INT_MIN, INT_MAX, -1, 0, 1, 123): - with self.subTest(value=value): - self.assertEqual(PyLong_AsInt(value), value) - self.assertEqual(PyLong_AsInt(IntSubclass(42)), 42) - self.assertEqual(PyLong_AsInt(Index(42)), 42) - self.assertEqual(PyLong_AsInt(MyIndexAndInt()), 10) - - # bound checking - self.assertRaises(OverflowError, PyLong_AsInt, INT_MIN - 1) - self.assertRaises(OverflowError, PyLong_AsInt, INT_MAX + 1) - - # invalid type - self.assertRaises(TypeError, PyLong_AsInt, 1.0) - self.assertRaises(TypeError, PyLong_AsInt, b'2') - self.assertRaises(TypeError, PyLong_AsInt, '3') - self.assertRaises(SystemError, PyLong_AsInt, NULL) + self.check_long_asint(PyLong_AsInt, INT_MIN, INT_MAX) def test_long_aslong(self): # Test PyLong_AsLong() and PyLong_FromLong() aslong = _testlimitedcapi.pylong_aslong from _testcapi import LONG_MIN, LONG_MAX - # round trip (object -> long -> object) - for value in (LONG_MIN, LONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslong(value), value) - - self.assertEqual(aslong(IntSubclass(42)), 42) - self.assertEqual(aslong(Index(42)), 42) - self.assertEqual(aslong(MyIndexAndInt()), 10) - - self.assertRaises(OverflowError, aslong, LONG_MIN - 1) - self.assertRaises(OverflowError, aslong, LONG_MAX + 1) - self.assertRaises(TypeError, aslong, 1.0) - self.assertRaises(TypeError, aslong, b'2') - self.assertRaises(TypeError, aslong, '3') - self.assertRaises(SystemError, aslong, NULL) + self.check_long_asint(aslong, LONG_MIN, LONG_MAX) def test_long_aslongandoverflow(self): # Test PyLong_AsLongAndOverflow() aslongandoverflow = _testlimitedcapi.pylong_aslongandoverflow from _testcapi import LONG_MIN, LONG_MAX - # round trip (object -> long -> object) - for value in (LONG_MIN, LONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslongandoverflow(value), (value, 0)) - - self.assertEqual(aslongandoverflow(IntSubclass(42)), (42, 0)) - self.assertEqual(aslongandoverflow(Index(42)), (42, 0)) - self.assertEqual(aslongandoverflow(MyIndexAndInt()), (10, 0)) - - self.assertEqual(aslongandoverflow(LONG_MIN - 1), (-1, -1)) - self.assertEqual(aslongandoverflow(LONG_MAX + 1), (-1, 1)) - # CRASHES aslongandoverflow(1.0) - # CRASHES aslongandoverflow(NULL) + self.check_long_asintandoverflow(aslongandoverflow, LONG_MIN, LONG_MAX) def test_long_asunsignedlong(self): # Test PyLong_AsUnsignedLong() and PyLong_FromUnsignedLong() asunsignedlong = _testlimitedcapi.pylong_asunsignedlong from _testcapi import ULONG_MAX - # round trip (object -> unsigned long -> object) - for value in (ULONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlong(value), value) - - self.assertEqual(asunsignedlong(IntSubclass(42)), 42) - self.assertRaises(TypeError, asunsignedlong, Index(42)) - self.assertRaises(TypeError, asunsignedlong, MyIndexAndInt()) - - self.assertRaises(OverflowError, asunsignedlong, -1) - self.assertRaises(OverflowError, asunsignedlong, ULONG_MAX + 1) - self.assertRaises(TypeError, asunsignedlong, 1.0) - self.assertRaises(TypeError, asunsignedlong, b'2') - self.assertRaises(TypeError, asunsignedlong, '3') - self.assertRaises(SystemError, asunsignedlong, NULL) + self.check_long_asint(asunsignedlong, 0, ULONG_MAX, + use_index=False) def test_long_asunsignedlongmask(self): # Test PyLong_AsUnsignedLongMask() asunsignedlongmask = _testlimitedcapi.pylong_asunsignedlongmask from _testcapi import ULONG_MAX - # round trip (object -> unsigned long -> object) - for value in (ULONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlongmask(value), value) - - self.assertEqual(asunsignedlongmask(IntSubclass(42)), 42) - self.assertEqual(asunsignedlongmask(Index(42)), 42) - self.assertEqual(asunsignedlongmask(MyIndexAndInt()), 10) - - self.assertEqual(asunsignedlongmask(-1), ULONG_MAX) - self.assertEqual(asunsignedlongmask(ULONG_MAX + 1), 0) - self.assertRaises(TypeError, asunsignedlongmask, 1.0) - self.assertRaises(TypeError, asunsignedlongmask, b'2') - self.assertRaises(TypeError, asunsignedlongmask, '3') - self.assertRaises(SystemError, asunsignedlongmask, NULL) + self.check_long_asint(asunsignedlongmask, 0, ULONG_MAX, mask=True) def test_long_aslonglong(self): # Test PyLong_AsLongLong() and PyLong_FromLongLong() aslonglong = _testlimitedcapi.pylong_aslonglong from _testcapi import LLONG_MIN, LLONG_MAX - # round trip (object -> long long -> object) - for value in (LLONG_MIN, LLONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslonglong(value), value) - - self.assertEqual(aslonglong(IntSubclass(42)), 42) - self.assertEqual(aslonglong(Index(42)), 42) - self.assertEqual(aslonglong(MyIndexAndInt()), 10) - - self.assertRaises(OverflowError, aslonglong, LLONG_MIN - 1) - self.assertRaises(OverflowError, aslonglong, LLONG_MAX + 1) - self.assertRaises(TypeError, aslonglong, 1.0) - self.assertRaises(TypeError, aslonglong, b'2') - self.assertRaises(TypeError, aslonglong, '3') - self.assertRaises(SystemError, aslonglong, NULL) + self.check_long_asint(aslonglong, LLONG_MIN, LLONG_MAX) def test_long_aslonglongandoverflow(self): # Test PyLong_AsLongLongAndOverflow() aslonglongandoverflow = _testlimitedcapi.pylong_aslonglongandoverflow from _testcapi import LLONG_MIN, LLONG_MAX - # round trip (object -> long long -> object) - for value in (LLONG_MIN, LLONG_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aslonglongandoverflow(value), (value, 0)) - - self.assertEqual(aslonglongandoverflow(IntSubclass(42)), (42, 0)) - self.assertEqual(aslonglongandoverflow(Index(42)), (42, 0)) - self.assertEqual(aslonglongandoverflow(MyIndexAndInt()), (10, 0)) - - self.assertEqual(aslonglongandoverflow(LLONG_MIN - 1), (-1, -1)) - self.assertEqual(aslonglongandoverflow(LLONG_MAX + 1), (-1, 1)) - # CRASHES aslonglongandoverflow(1.0) - # CRASHES aslonglongandoverflow(NULL) + self.check_long_asintandoverflow(aslonglongandoverflow, LLONG_MIN, LLONG_MAX) def test_long_asunsignedlonglong(self): # Test PyLong_AsUnsignedLongLong() and PyLong_FromUnsignedLongLong() asunsignedlonglong = _testlimitedcapi.pylong_asunsignedlonglong from _testcapi import ULLONG_MAX - # round trip (object -> unsigned long long -> object) - for value in (ULLONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlonglong(value), value) - - self.assertEqual(asunsignedlonglong(IntSubclass(42)), 42) - self.assertRaises(TypeError, asunsignedlonglong, Index(42)) - self.assertRaises(TypeError, asunsignedlonglong, MyIndexAndInt()) - - self.assertRaises(OverflowError, asunsignedlonglong, -1) - self.assertRaises(OverflowError, asunsignedlonglong, ULLONG_MAX + 1) - self.assertRaises(TypeError, asunsignedlonglong, 1.0) - self.assertRaises(TypeError, asunsignedlonglong, b'2') - self.assertRaises(TypeError, asunsignedlonglong, '3') - self.assertRaises(SystemError, asunsignedlonglong, NULL) + self.check_long_asint(asunsignedlonglong, 0, ULLONG_MAX, use_index=False) def test_long_asunsignedlonglongmask(self): # Test PyLong_AsUnsignedLongLongMask() asunsignedlonglongmask = _testlimitedcapi.pylong_asunsignedlonglongmask from _testcapi import ULLONG_MAX - # round trip (object -> unsigned long long -> object) - for value in (ULLONG_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(asunsignedlonglongmask(value), value) - - self.assertEqual(asunsignedlonglongmask(IntSubclass(42)), 42) - self.assertEqual(asunsignedlonglongmask(Index(42)), 42) - self.assertEqual(asunsignedlonglongmask(MyIndexAndInt()), 10) - - self.assertEqual(asunsignedlonglongmask(-1), ULLONG_MAX) - self.assertEqual(asunsignedlonglongmask(ULLONG_MAX + 1), 0) - self.assertRaises(TypeError, asunsignedlonglongmask, 1.0) - self.assertRaises(TypeError, asunsignedlonglongmask, b'2') - self.assertRaises(TypeError, asunsignedlonglongmask, '3') - self.assertRaises(SystemError, asunsignedlonglongmask, NULL) + self.check_long_asint(asunsignedlonglongmask, 0, ULLONG_MAX, mask=True) def test_long_as_ssize_t(self): # Test PyLong_AsSsize_t() and PyLong_FromSsize_t() as_ssize_t = _testlimitedcapi.pylong_as_ssize_t from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX - # round trip (object -> Py_ssize_t -> object) - for value in (PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(as_ssize_t(value), value) - - self.assertEqual(as_ssize_t(IntSubclass(42)), 42) - self.assertRaises(TypeError, as_ssize_t, Index(42)) - self.assertRaises(TypeError, as_ssize_t, MyIndexAndInt()) - - self.assertRaises(OverflowError, as_ssize_t, PY_SSIZE_T_MIN - 1) - self.assertRaises(OverflowError, as_ssize_t, PY_SSIZE_T_MAX + 1) - self.assertRaises(TypeError, as_ssize_t, 1.0) - self.assertRaises(TypeError, as_ssize_t, b'2') - self.assertRaises(TypeError, as_ssize_t, '3') - self.assertRaises(SystemError, as_ssize_t, NULL) + self.check_long_asint(as_ssize_t, PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, + use_index=False) def test_long_as_size_t(self): # Test PyLong_AsSize_t() and PyLong_FromSize_t() as_size_t = _testlimitedcapi.pylong_as_size_t from _testcapi import SIZE_MAX - # round trip (object -> size_t -> object) - for value in (SIZE_MAX, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(as_size_t(value), value) - - self.assertEqual(as_size_t(IntSubclass(42)), 42) - self.assertRaises(TypeError, as_size_t, Index(42)) - self.assertRaises(TypeError, as_size_t, MyIndexAndInt()) - - self.assertRaises(OverflowError, as_size_t, -1) - self.assertRaises(OverflowError, as_size_t, SIZE_MAX + 1) - self.assertRaises(TypeError, as_size_t, 1.0) - self.assertRaises(TypeError, as_size_t, b'2') - self.assertRaises(TypeError, as_size_t, '3') - self.assertRaises(SystemError, as_size_t, NULL) + self.check_long_asint(as_size_t, 0, SIZE_MAX, use_index=False) def test_long_asdouble(self): # Test PyLong_AsDouble() @@ -431,21 +332,7 @@ def _test_long_aspid(self, aspid): bits = 8 * SIZEOF_PID_T PID_T_MIN = -2**(bits-1) PID_T_MAX = 2**(bits-1) - 1 - # round trip (object -> long -> object) - for value in (PID_T_MIN, PID_T_MAX, -1, 0, 1, 1234): - with self.subTest(value=value): - self.assertEqual(aspid(value), value) - - self.assertEqual(aspid(IntSubclass(42)), 42) - self.assertEqual(aspid(Index(42)), 42) - self.assertEqual(aspid(MyIndexAndInt()), 10) - - self.assertRaises(OverflowError, aspid, PID_T_MIN - 1) - self.assertRaises(OverflowError, aspid, PID_T_MAX + 1) - self.assertRaises(TypeError, aspid, 1.0) - self.assertRaises(TypeError, aspid, b'2') - self.assertRaises(TypeError, aspid, '3') - self.assertRaises(SystemError, aspid, NULL) + self.check_long_asint(aspid, PID_T_MIN, PID_T_MAX) def test_long_aspid(self): self._test_long_aspid(_testcapi.pylong_aspid) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 9ef476a02de47d..e6f85427214958 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1736,7 +1736,7 @@ def test_basic(self): writer.write_char('=') # test PyUnicodeWriter_WriteSubstring() - writer.write_substring("[long]", 1, 5); + writer.write_substring("[long]", 1, 5) # test PyUnicodeWriter_WriteStr() writer.write_str(" value ") @@ -1862,6 +1862,10 @@ def test_ucs4(self): with self.assertRaises(ValueError): writer.write_ucs4("text", -1) + def test_substring_empty(self): + writer = self.create_writer(0) + writer.write_substring("abc", 1, 1) + self.assertEqual(writer.finish(), '') @unittest.skipIf(ctypes is None, 'need ctypes') diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 655d53b8d5bb6a..d1f828b1ed824d 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -882,6 +882,24 @@ class Foo: f.a = 3 self.assertEqual(f.a, 3) + def test_rematerialize_object_dict(self): + # gh-121860: rematerializing an object's managed dictionary after it + # had been deleted caused a crash. + class Foo: pass + f = Foo() + f.__dict__["attr"] = 1 + del f.__dict__ + + # Using a str subclass is a way to trigger the re-materialization + class StrSubclass(str): pass + self.assertFalse(hasattr(f, StrSubclass("attr"))) + + # Changing the __class__ also triggers the re-materialization + class Bar: pass + f.__class__ = Bar + self.assertIsInstance(f, Bar) + self.assertEqual(f.__dict__, {}) + def test_store_attr_type_cache(self): """Verifies that the type cache doesn't provide a value which is inconsistent from the dict.""" diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 155240e30f1ad1..c5a06c53771fca 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -36,6 +36,16 @@ def __float__(self): class ComplexSubclass(complex): pass +class OtherComplexSubclass(complex): + pass + +class MyInt: + def __init__(self, value): + self.value = value + + def __int__(self): + return self.value + class WithComplex: def __init__(self, value): self.value = value @@ -675,10 +685,39 @@ def test_underscores(self): if not any(ch in lit for ch in 'xXoObB'): self.assertRaises(ValueError, complex, lit) + def test_from_number(self, cls=complex): + def eq(actual, expected): + self.assertEqual(actual, expected) + self.assertIs(type(actual), cls) + + eq(cls.from_number(3.14), 3.14+0j) + eq(cls.from_number(3.14j), 3.14j) + eq(cls.from_number(314), 314.0+0j) + eq(cls.from_number(OtherComplexSubclass(3.14, 2.72)), 3.14+2.72j) + eq(cls.from_number(WithComplex(3.14+2.72j)), 3.14+2.72j) + eq(cls.from_number(WithFloat(3.14)), 3.14+0j) + eq(cls.from_number(WithIndex(314)), 314.0+0j) + + cNAN = complex(NAN, NAN) + x = cls.from_number(cNAN) + self.assertTrue(x != x) + self.assertIs(type(x), cls) + if cls is complex: + self.assertIs(cls.from_number(cNAN), cNAN) + + self.assertRaises(TypeError, cls.from_number, '3.14') + self.assertRaises(TypeError, cls.from_number, b'3.14') + self.assertRaises(TypeError, cls.from_number, MyInt(314)) + self.assertRaises(TypeError, cls.from_number, {}) + self.assertRaises(TypeError, cls.from_number) + + def test_from_number_subclass(self): + self.test_from_number(ComplexSubclass) + def test_hash(self): for x in range(-30, 30): self.assertEqual(hash(x), hash(complex(x, 0))) - x /= 3.0 # now check against floating point + x /= 3.0 # now check against floating-point self.assertEqual(hash(x), hash(complex(x, 0.))) self.assertNotEqual(hash(2000005 - 1j), -1) diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index a36f592b79b7cf..df640929309318 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -139,6 +139,7 @@ def _test(self, test_class): def test_spawn(self): self._test(ProcessPoolSpawnFailingInitializerTest) + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_forkserver(self): self._test(ProcessPoolForkserverFailingInitializerTest) diff --git a/Lib/test/test_contains.py b/Lib/test/test_contains.py index 471d04a76ca4e4..259c954aa1d616 100644 --- a/Lib/test/test_contains.py +++ b/Lib/test/test_contains.py @@ -24,8 +24,11 @@ def test_common_tests(self): self.assertNotIn(0, b) self.assertIn(1, c) self.assertNotIn(0, c) - self.assertRaises(TypeError, lambda: 1 in a) - self.assertRaises(TypeError, lambda: 1 not in a) + msg = "argument of type 'base_set' is not a container or iterable" + with self.assertRaisesRegex(TypeError, msg): + 1 in a + with self.assertRaisesRegex(TypeError, msg): + 1 not in a # test char in string self.assertIn('c', 'abc') diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 27e8a767903777..b2595eccc82f70 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -30,6 +30,43 @@ def test_bad_counter_during_dealloc(self): self.assertEqual(cm.unraisable.exc_type, TypeError) + def test_evil_external_timer(self): + # gh-120289 + # Disabling profiler in external timer should not crash + import _lsprof + class EvilTimer(): + def __init__(self, disable_count): + self.count = 0 + self.disable_count = disable_count + + def __call__(self): + self.count += 1 + if self.count == self.disable_count: + profiler_with_evil_timer.disable() + return self.count + + # this will trigger external timer to disable profiler at + # call event - in initContext in _lsprof.c + with support.catch_unraisable_exception() as cm: + profiler_with_evil_timer = _lsprof.Profiler(EvilTimer(1)) + profiler_with_evil_timer.enable() + # Make a call to trigger timer + (lambda: None)() + profiler_with_evil_timer.disable() + profiler_with_evil_timer.clear() + self.assertEqual(cm.unraisable.exc_type, RuntimeError) + + # this will trigger external timer to disable profiler at + # return event - in Stop in _lsprof.c + with support.catch_unraisable_exception() as cm: + profiler_with_evil_timer = _lsprof.Profiler(EvilTimer(2)) + profiler_with_evil_timer.enable() + # Make a call to trigger timer + (lambda: None)() + profiler_with_evil_timer.disable() + profiler_with_evil_timer.clear() + self.assertEqual(cm.unraisable.exc_type, RuntimeError) + def test_profile_enable_disable(self): prof = self.profilerclass() # Make sure we clean ourselves up if the test fails for some reason. diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index d74ab7e016f78c..c718ee1203cbe0 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -454,6 +454,10 @@ def test_read_quoting(self): quoting=csv.QUOTE_STRINGS) self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@') self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0') + self._read_test(['1\\.5,\\.5,.5'], [[1.5, 0.5, 0.5]], + quoting=csv.QUOTE_NONNUMERIC, escapechar='\\') + self._read_test(['1\\.5,\\.5,"\\.5"'], [[1.5, 0.5, ".5"]], + quoting=csv.QUOTE_STRINGS, escapechar='\\') def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], diff --git a/Lib/test/test_ctypes/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py index 3568cf97f40b50..6846773d7069ae 100644 --- a/Lib/test/test_ctypes/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -253,7 +253,7 @@ def test_empty_element_struct(self): class EmptyStruct(Structure): _fields_ = [] - obj = (EmptyStruct * 2)() # bpo37188: Floating point exception + obj = (EmptyStruct * 2)() # bpo37188: Floating-point exception self.assertEqual(sizeof(obj), 0) def test_empty_element_array(self): @@ -261,7 +261,7 @@ class EmptyArray(Array): _type_ = c_int _length_ = 0 - obj = (EmptyArray * 2)() # bpo37188: Floating point exception + obj = (EmptyArray * 2)() # bpo37188: Floating-point exception self.assertEqual(sizeof(obj), 0) def test_bpo36504_signed_int_overflow(self): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index ffb8bbe75c504f..b93c99d8c90bf3 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -4807,6 +4807,16 @@ def test_make_dataclass(self): self.assertTrue(fields(B)[0].kw_only) self.assertFalse(fields(B)[1].kw_only) + def test_deferred_annotations(self): + @dataclass + class A: + x: undefined + y: ClassVar[undefined] + + fs = fields(A) + self.assertEqual(len(fs), 1) + self.assertEqual(fs[0].name, 'x') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index ab1c48f9b25361..c8defde7b99ec7 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1605,204 +1605,204 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=93, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=93, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=92, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=92, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_TUPLE', opcode=49, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=48, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=109, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=82, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=82, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_LIST', opcode=44, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_MAP', opcode=45, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=108, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=False, line_number=2, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=True, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=81, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_LIST', opcode=43, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_MAP', opcode=44, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8, label=None, positions=None, cache_info=None), Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=59, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=93, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='MAKE_CELL', opcode=93, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='COPY_FREE_VARS', opcode=58, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=92, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='MAKE_CELL', opcode=92, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=True, line_number=2, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='BUILD_TUPLE', opcode=49, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='BUILD_TUPLE', opcode=48, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), Instruction(opname='MAKE_FUNCTION', opcode=23, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=105, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='STORE_FAST', opcode=109, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=82, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=82, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=82, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=82, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=104, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=108, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=81, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=True, line_number=6, label=None, positions=None, cache_info=None), Instruction(opname='RETURN_VALUE', opcode=33, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=6, label=None, positions=None, cache_info=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=59, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='COPY_FREE_VARS', opcode=58, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_DEREF', opcode=82, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=82, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=82, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_DEREF', opcode=82, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=86, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_DEREF', opcode=81, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_DEREF', opcode=81, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=85, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_CONST', opcode=102, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_CONST', opcode=101, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), ] expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=True, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='GET_ITER', opcode=16, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='FOR_ITER', opcode=69, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='FOR_ITER', opcode=68, arg=30, argval=88, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=108, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=4, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=False, line_number=4, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=55, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=74, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=55, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=74, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=True, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=False, line_number=5, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=54, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=2, argval=68, argrepr='to L2', offset=60, start_offset=60, starts_line=False, line_number=5, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=22, argval=24, argrepr='to L1', offset=64, start_offset=64, starts_line=True, line_number=6, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=68, start_offset=68, starts_line=True, line_number=7, label=2, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=6, argrepr='6', offset=70, start_offset=70, starts_line=False, line_number=7, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=54, arg=148, argval='>', argrepr='bool(>)', offset=72, start_offset=72, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=2, argval=84, argrepr='to L3', offset=76, start_offset=76, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=30, argval=24, argrepr='to L1', offset=80, start_offset=80, starts_line=False, line_number=7, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=True, line_number=8, label=3, positions=None, cache_info=None), - Instruction(opname='JUMP_FORWARD', opcode=76, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_FORWARD', opcode=75, arg=13, argval=114, argrepr='to L5', offset=86, start_offset=86, starts_line=False, line_number=8, label=None, positions=None, cache_info=None), Instruction(opname='END_FOR', opcode=9, arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=3, label=4, positions=None, cache_info=None), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=3, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=92, start_offset=92, starts_line=True, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=102, start_offset=102, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=104, start_offset=104, starts_line=False, line_number=10, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=112, start_offset=112, starts_line=False, line_number=10, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=85, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=84, arg=0, argval='i', argrepr='i', offset=114, start_offset=114, starts_line=True, line_number=11, label=5, positions=None, cache_info=None), Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=40, argval=208, argrepr='to L9', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=6, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=40, argval=208, argrepr='to L9', offset=124, start_offset=124, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=128, start_offset=128, starts_line=True, line_number=12, label=6, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=138, start_offset=138, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=140, start_offset=140, starts_line=False, line_number=12, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval=1, argrepr='1', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=True, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=5, argval=1, argrepr='1', offset=152, start_offset=152, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), Instruction(opname='BINARY_OP', opcode=42, arg=23, argval=23, argrepr='-=', offset=154, start_offset=154, starts_line=False, line_number=13, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=109, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=3, argval=6, argrepr='6', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=55, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=2, argval=176, argrepr='to L7', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=74, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=7, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=2, argval=4, argrepr='4', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), - Instruction(opname='COMPARE_OP', opcode=55, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=1, argval=190, argrepr='to L8', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_FORWARD', opcode=76, arg=20, argval=230, argrepr='to L10', offset=188, start_offset=188, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=190, start_offset=190, starts_line=True, line_number=11, label=8, positions=None, cache_info=None), + Instruction(opname='STORE_FAST', opcode=108, arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=False, line_number=13, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=160, start_offset=160, starts_line=True, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=3, argval=6, argrepr='6', offset=162, start_offset=162, starts_line=False, line_number=14, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=54, arg=148, argval='>', argrepr='bool(>)', offset=164, start_offset=164, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=2, argval=176, argrepr='to L7', offset=168, start_offset=168, starts_line=False, line_number=14, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=31, argval=114, argrepr='to L5', offset=172, start_offset=172, starts_line=True, line_number=15, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=16, label=7, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=2, argval=4, argrepr='4', offset=178, start_offset=178, starts_line=False, line_number=16, label=None, positions=None, cache_info=None), + Instruction(opname='COMPARE_OP', opcode=54, arg=18, argval='<', argrepr='bool(<)', offset=180, start_offset=180, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=1, argval=190, argrepr='to L8', offset=184, start_offset=184, starts_line=False, line_number=16, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_FORWARD', opcode=75, arg=20, argval=230, argrepr='to L10', offset=188, start_offset=188, starts_line=True, line_number=17, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=190, start_offset=190, starts_line=True, line_number=11, label=8, positions=None, cache_info=None), Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=192, start_offset=192, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=2, argval=208, argrepr='to L9', offset=200, start_offset=200, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='JUMP_BACKWARD', opcode=74, arg=40, argval=128, argrepr='to L6', offset=204, start_offset=204, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=208, start_offset=208, starts_line=True, line_number=19, label=9, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=2, argval=208, argrepr='to L9', offset=200, start_offset=200, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='JUMP_BACKWARD', opcode=73, arg=40, argval=128, argrepr='to L6', offset=204, start_offset=204, starts_line=False, line_number=11, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=208, start_offset=208, starts_line=True, line_number=19, label=9, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=218, start_offset=218, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=220, start_offset=220, starts_line=False, line_number=19, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=228, start_offset=228, starts_line=False, line_number=19, label=None, positions=None, cache_info=None), Instruction(opname='NOP', opcode=27, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=True, line_number=20, label=10, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=5, argval=1, argrepr='1', offset=232, start_offset=232, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=7, argval=0, argrepr='0', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=5, argval=1, argrepr='1', offset=232, start_offset=232, starts_line=True, line_number=21, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=7, argval=0, argrepr='0', offset=234, start_offset=234, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), Instruction(opname='BINARY_OP', opcode=42, arg=11, argval=11, argrepr='/', offset=236, start_offset=236, starts_line=False, line_number=21, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_FAST', opcode=83, arg=0, argval='i', argrepr='i', offset=242, start_offset=242, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=58, arg=1, argval=1, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=91, arg=1, argval=1, argrepr='__exit__', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=114, arg=2, argval=2, argrepr='', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='SWAP', opcode=114, arg=3, argval=3, argrepr='', offset=250, start_offset=250, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_SPECIAL', opcode=91, arg=0, argval=0, argrepr='__enter__', offset=252, start_offset=252, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=0, argval=0, argrepr='', offset=254, start_offset=254, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='STORE_FAST', opcode=109, arg=1, argval='dodgy', argrepr='dodgy', offset=262, start_offset=262, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=264, start_offset=264, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=274, start_offset=274, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_FAST', opcode=82, arg=0, argval='i', argrepr='i', offset=242, start_offset=242, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=57, arg=1, argval=1, argrepr='', offset=244, start_offset=244, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=90, arg=1, argval=1, argrepr='__exit__', offset=246, start_offset=246, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=113, arg=2, argval=2, argrepr='', offset=248, start_offset=248, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='SWAP', opcode=113, arg=3, argval=3, argrepr='', offset=250, start_offset=250, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_SPECIAL', opcode=90, arg=0, argval=0, argrepr='__enter__', offset=252, start_offset=252, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=0, argval=0, argrepr='', offset=254, start_offset=254, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='STORE_FAST', opcode=108, arg=1, argval='dodgy', argrepr='dodgy', offset=262, start_offset=262, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=264, start_offset=264, starts_line=True, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=274, start_offset=274, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=276, start_offset=276, starts_line=False, line_number=26, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=284, start_offset=284, starts_line=False, line_number=26, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=None, argrepr='None', offset=286, start_offset=286, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=None, argrepr='None', offset=288, start_offset=288, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_CONST', opcode=81, arg=0, argval=None, argrepr='None', offset=290, start_offset=290, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=3, argval=3, argrepr='', offset=292, start_offset=292, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=None, argrepr='None', offset=286, start_offset=286, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=None, argrepr='None', offset=288, start_offset=288, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='LOAD_CONST', opcode=80, arg=0, argval=None, argrepr='None', offset=290, start_offset=290, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=3, argval=3, argrepr='', offset=292, start_offset=292, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=302, start_offset=302, starts_line=True, line_number=28, label=11, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=312, start_offset=312, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=302, start_offset=302, starts_line=True, line_number=28, label=11, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=312, start_offset=312, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=322, start_offset=322, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RETURN_CONST', opcode=102, arg=0, argval=None, argrepr='None', offset=324, start_offset=324, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='RETURN_CONST', opcode=101, arg=0, argval=None, argrepr='None', offset=324, start_offset=324, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=True, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='WITH_EXCEPT_START', opcode=41, arg=None, argval=None, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='TO_BOOL', opcode=37, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=99, arg=1, argval=344, argrepr='to L12', offset=338, start_offset=338, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), - Instruction(opname='RERAISE', opcode=101, arg=2, argval=2, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=98, arg=1, argval=344, argrepr='to L12', offset=338, start_offset=338, starts_line=False, line_number=25, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='RERAISE', opcode=100, arg=2, argval=2, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, label=12, positions=None, cache_info=None), Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=348, start_offset=348, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=350, start_offset=350, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=352, start_offset=352, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=27, argval=302, argrepr='to L11', offset=354, start_offset=354, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=356, start_offset=356, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=74, arg=27, argval=302, argrepr='to L11', offset=354, start_offset=354, starts_line=False, line_number=25, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=356, start_offset=356, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=364, start_offset=364, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=364, start_offset=364, starts_line=True, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), Instruction(opname='CHECK_EXC_MATCH', opcode=5, arg=None, argval=None, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=96, arg=14, argval=408, argrepr='to L13', offset=376, start_offset=376, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=95, arg=14, argval=408, argrepr='to L13', offset=376, start_offset=376, starts_line=False, line_number=22, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=22, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=382, start_offset=382, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=392, start_offset=392, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=394, start_offset=394, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=382, start_offset=382, starts_line=True, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=392, start_offset=392, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=394, start_offset=394, starts_line=False, line_number=23, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=402, start_offset=402, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=404, start_offset=404, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=75, arg=53, argval=302, argrepr='to L11', offset=406, start_offset=406, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=True, line_number=22, label=13, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='JUMP_BACKWARD_NO_INTERRUPT', opcode=74, arg=53, argval=302, argrepr='to L11', offset=406, start_offset=406, starts_line=False, line_number=23, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=100, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=True, line_number=22, label=13, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='PUSH_EXC_INFO', opcode=30, arg=None, argval=None, argrepr='', offset=416, start_offset=416, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='LOAD_GLOBAL', opcode=89, arg=3, argval='print', argrepr='print + NULL', offset=418, start_offset=418, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - Instruction(opname='LOAD_CONST', opcode=81, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=428, start_offset=428, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='CALL', opcode=50, arg=1, argval=1, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + Instruction(opname='LOAD_GLOBAL', opcode=88, arg=3, argval='print', argrepr='print + NULL', offset=418, start_offset=418, starts_line=True, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + Instruction(opname='LOAD_CONST', opcode=80, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=428, start_offset=428, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='CALL', opcode=49, arg=1, argval=1, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=28, label=None, positions=None, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), Instruction(opname='POP_TOP', opcode=29, arg=None, argval=None, argrepr='', offset=438, start_offset=438, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=0, argval=0, argrepr='', offset=440, start_offset=440, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), - Instruction(opname='COPY', opcode=58, arg=3, argval=3, argrepr='', offset=442, start_offset=442, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=100, arg=0, argval=0, argrepr='', offset=440, start_offset=440, starts_line=False, line_number=28, label=None, positions=None, cache_info=None), + Instruction(opname='COPY', opcode=57, arg=3, argval=3, argrepr='', offset=442, start_offset=442, starts_line=True, line_number=None, label=None, positions=None, cache_info=None), Instruction(opname='POP_EXCEPT', opcode=28, arg=None, argval=None, argrepr='', offset=444, start_offset=444, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), - Instruction(opname='RERAISE', opcode=101, arg=1, argval=1, argrepr='', offset=446, start_offset=446, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), + Instruction(opname='RERAISE', opcode=100, arg=1, argval=1, argrepr='', offset=446, start_offset=446, starts_line=False, line_number=None, label=None, positions=None, cache_info=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ Instruction(opname='RESUME', opcode=149, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=simple.__code__.co_firstlineno, label=None, positions=None), - Instruction(opname='RETURN_CONST', opcode=102, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), + Instruction(opname='RETURN_CONST', opcode=101, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=False, line_number=simple.__code__.co_firstlineno, label=None), ] diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index bfff1051262079..bc6f734d4fd0a9 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -294,6 +294,19 @@ def test_defaults_handle_spaces_between_encoded_words_when_folded(self): g.flatten(msg) self.assertEqual(s.getvalue(), expected) + def test_defaults_handle_spaces_when_encoded_words_is_folded_in_middle(self): + source = ('A very long long long long long long long long long long long long ' + 'long long long long long long long long long long long súmmäry') + expected = ('Subject: A very long long long long long long long long long long long long\n' + ' long long long long long long long long long long long =?utf-8?q?s=C3=BAmm?=\n' + ' =?utf-8?q?=C3=A4ry?=\n\n').encode('ascii') + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + def test_defaults_handle_spaces_at_start_of_subject(self): source = " Уведомление" expected = b"Subject: =?utf-8?b?0KPQstC10LTQvtC80LvQtdC90LjQtQ==?=\n\n" diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 634513ec7a5812..fb7995e05152d2 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -5,6 +5,7 @@ from collections import namedtuple import contextlib +import io import json import os import os.path @@ -48,6 +49,8 @@ INIT_LOOPS = 4 MAX_HASH_SEED = 4294967295 +ABI_THREAD = 't' if sysconfig.get_config_var('Py_GIL_DISABLED') else '' + # If we are running from a build dir, but the stdlib has been installed, # some tests need to expect different results. @@ -413,6 +416,32 @@ def test_datetime_reset_strptime(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '20000101\n' * INIT_LOOPS) + def test_static_types_inherited_slots(self): + slots = [] + script = ['import sys'] + from test.test_types import iter_builtin_types, iter_own_slot_wrappers + for cls in iter_builtin_types(): + for slot in iter_own_slot_wrappers(cls): + slots.append((cls, slot)) + attr = f'{cls.__name__}.{slot}' + script.append(f'print("{attr}:", {attr}, file=sys.stderr)') + script.append('') + script = os.linesep.join(script) + + with contextlib.redirect_stderr(io.StringIO()) as stderr: + exec(script) + expected = stderr.getvalue().splitlines() + + out, err = self.run_embedded_interpreter("test_repeated_init_exec", script) + results = err.split('--- Loop #')[1:] + results = [res.rpartition(' ---\n')[-1] for res in results] + + self.maxDiff = None + for i, result in enumerate(results, start=1): + with self.subTest(loop=i): + self.assertEqual(result.splitlines(), expected) + self.assertEqual(out, '') + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): @@ -1285,11 +1314,11 @@ def module_search_paths(self, prefix=None, exec_prefix=None): ver = sys.version_info return [ os.path.join(prefix, sys.platlibdir, - f'python{ver.major}{ver.minor}.zip'), + f'python{ver.major}{ver.minor}{ABI_THREAD}.zip'), os.path.join(prefix, sys.platlibdir, - f'python{ver.major}.{ver.minor}'), + f'python{ver.major}.{ver.minor}{ABI_THREAD}'), os.path.join(exec_prefix, sys.platlibdir, - f'python{ver.major}.{ver.minor}', 'lib-dynload'), + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload'), ] @contextlib.contextmanager @@ -1343,7 +1372,7 @@ def test_init_setpythonhome(self): expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1384,7 +1413,7 @@ def test_init_is_python_build_with_home(self): expected_paths = [paths[0], os.path.join(home, 'DLLs'), stdlib] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + stdlib = os.path.join(home, sys.platlibdir, f'python{version}{ABI_THREAD}') expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { @@ -1515,7 +1544,7 @@ def test_init_pyvenv_cfg(self): if not MS_WINDOWS: lib_dynload = os.path.join(pyvenv_home, sys.platlibdir, - f'python{ver.major}.{ver.minor}', + f'python{ver.major}.{ver.minor}{ABI_THREAD}', 'lib-dynload') os.makedirs(lib_dynload) else: diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 61ec8fe3151af1..60815be96e14eb 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -236,7 +236,7 @@ def test_sigfpe(self): faulthandler._sigfpe() """, 3, - 'Floating point exception') + 'Floating-point exception') @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGBUS'), 'need signal.SIGBUS') diff --git a/Lib/test/test_filecmp.py b/Lib/test/test_filecmp.py index b5df71678264a8..1fb47163719ede 100644 --- a/Lib/test/test_filecmp.py +++ b/Lib/test/test_filecmp.py @@ -1,5 +1,6 @@ import filecmp import os +import re import shutil import tempfile import unittest @@ -277,6 +278,17 @@ def test_dircmp_shallow_same_file(self): ] self._assert_report(d.report, expected_report) + def test_dircmp_shallow_is_keyword_only(self): + with self.assertRaisesRegex( + TypeError, + re.escape("dircmp.__init__() takes from 3 to 5 positional arguments but 6 were given"), + ): + filecmp.dircmp(self.dir, self.dir_same, None, None, True) + self.assertIsInstance( + filecmp.dircmp(self.dir, self.dir_same, None, None, shallow=True), + filecmp.dircmp, + ) + def test_dircmp_subdirs_type(self): """Check that dircmp.subdirs respects subclassing.""" class MyDirCmp(filecmp.dircmp): diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 756cf9bd7719c0..36ba67ac7415e7 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -32,6 +32,28 @@ class FloatSubclass(float): class OtherFloatSubclass(float): pass +class MyIndex: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + +class MyInt: + def __init__(self, value): + self.value = value + + def __int__(self): + return self.value + +class FloatLike: + def __init__(self, value): + self.value = value + + def __float__(self): + return self.value + + class GeneralFloatCases(unittest.TestCase): def test_float(self): @@ -181,10 +203,6 @@ def test_float_with_comma(self): def test_floatconversion(self): # Make sure that calls to __float__() work properly - class Foo1(object): - def __float__(self): - return 42. - class Foo2(float): def __float__(self): return 42. @@ -206,45 +224,29 @@ class FooStr(str): def __float__(self): return float(str(self)) + 1 - self.assertEqual(float(Foo1()), 42.) + self.assertEqual(float(FloatLike(42.)), 42.) self.assertEqual(float(Foo2()), 42.) with self.assertWarns(DeprecationWarning): self.assertEqual(float(Foo3(21)), 42.) self.assertRaises(TypeError, float, Foo4(42)) self.assertEqual(float(FooStr('8')), 9.) - class Foo5: - def __float__(self): - return "" - self.assertRaises(TypeError, time.sleep, Foo5()) + self.assertRaises(TypeError, time.sleep, FloatLike("")) # Issue #24731 - class F: - def __float__(self): - return OtherFloatSubclass(42.) + f = FloatLike(OtherFloatSubclass(42.)) with self.assertWarns(DeprecationWarning): - self.assertEqual(float(F()), 42.) + self.assertEqual(float(f), 42.) with self.assertWarns(DeprecationWarning): - self.assertIs(type(float(F())), float) + self.assertIs(type(float(f)), float) with self.assertWarns(DeprecationWarning): - self.assertEqual(FloatSubclass(F()), 42.) + self.assertEqual(FloatSubclass(f), 42.) with self.assertWarns(DeprecationWarning): - self.assertIs(type(FloatSubclass(F())), FloatSubclass) - - class MyIndex: - def __init__(self, value): - self.value = value - def __index__(self): - return self.value + self.assertIs(type(FloatSubclass(f)), FloatSubclass) self.assertEqual(float(MyIndex(42)), 42.0) self.assertRaises(OverflowError, float, MyIndex(2**2000)) - - class MyInt: - def __int__(self): - return 42 - - self.assertRaises(TypeError, float, MyInt()) + self.assertRaises(TypeError, float, MyInt(42)) def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): @@ -277,6 +279,37 @@ def __new__(cls, arg, newarg=None): self.assertEqual(float(u), 2.5) self.assertEqual(u.newarg, 3) + def assertEqualAndType(self, actual, expected_value, expected_type): + self.assertEqual(actual, expected_value) + self.assertIs(type(actual), expected_type) + + def test_from_number(self, cls=float): + def eq(actual, expected): + self.assertEqual(actual, expected) + self.assertIs(type(actual), cls) + + eq(cls.from_number(3.14), 3.14) + eq(cls.from_number(314), 314.0) + eq(cls.from_number(OtherFloatSubclass(3.14)), 3.14) + eq(cls.from_number(FloatLike(3.14)), 3.14) + eq(cls.from_number(MyIndex(314)), 314.0) + + x = cls.from_number(NAN) + self.assertTrue(x != x) + self.assertIs(type(x), cls) + if cls is float: + self.assertIs(cls.from_number(NAN), NAN) + + self.assertRaises(TypeError, cls.from_number, '3.14') + self.assertRaises(TypeError, cls.from_number, b'3.14') + self.assertRaises(TypeError, cls.from_number, 3.14j) + self.assertRaises(TypeError, cls.from_number, MyInt(314)) + self.assertRaises(TypeError, cls.from_number, {}) + self.assertRaises(TypeError, cls.from_number) + + def test_from_number_subclass(self): + self.test_from_number(FloatSubclass) + def test_is_integer(self): self.assertFalse((1.1).is_integer()) self.assertTrue((1.).is_integer()) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index d2026152d8e747..9dde63e40d06db 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -36,7 +36,7 @@ def testformat(formatstr, args, output=None, limit=None, overflowok=False): # when 'limit' is specified, it determines how many characters # must match exactly; lengths must always match. # ex: limit=5, '12345678' matches '12345___' - # (mainly for floating point format tests for which an exact match + # (mainly for floating-point format tests for which an exact match # can't be guaranteed due to rounding and representation errors) elif output and limit is not None and ( len(result)!=len(output) or result[:limit]!=output[:limit]): diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 589669298e22e2..12c42126301265 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -354,6 +354,41 @@ def testInitFromDecimal(self): self.assertRaises(OverflowError, F, Decimal('inf')) self.assertRaises(OverflowError, F, Decimal('-inf')) + def testInitFromIntegerRatio(self): + class Ratio: + def __init__(self, ratio): + self._ratio = ratio + def as_integer_ratio(self): + return self._ratio + + self.assertEqual((7, 3), _components(F(Ratio((7, 3))))) + errmsg = "argument should be a string or a number" + # the type also has an "as_integer_ratio" attribute. + self.assertRaisesRegex(TypeError, errmsg, F, Ratio) + # bad ratio + self.assertRaises(TypeError, F, Ratio(7)) + self.assertRaises(ValueError, F, Ratio((7,))) + self.assertRaises(ValueError, F, Ratio((7, 3, 1))) + # only single-argument form + self.assertRaises(TypeError, F, Ratio((3, 7)), 11) + self.assertRaises(TypeError, F, 2, Ratio((-10, 9))) + + # as_integer_ratio not defined in a class + class A: + pass + a = A() + a.as_integer_ratio = lambda: (9, 5) + self.assertEqual((9, 5), _components(F(a))) + + # as_integer_ratio defined in a metaclass + class M(type): + def as_integer_ratio(self): + return (11, 9) + class B(metaclass=M): + pass + self.assertRaisesRegex(TypeError, errmsg, F, B) + self.assertRaisesRegex(TypeError, errmsg, F, B()) + def testFromString(self): self.assertEqual((5, 1), _components(F("5"))) self.assertEqual((3, 2), _components(F("3/2"))) diff --git a/Lib/test/test_free_threading/test_tokenize.py b/Lib/test/test_free_threading/test_tokenize.py new file mode 100644 index 00000000000000..860cfec4d710f4 --- /dev/null +++ b/Lib/test/test_free_threading/test_tokenize.py @@ -0,0 +1,57 @@ +import io +import time +import unittest +import tokenize +from functools import partial +from threading import Thread + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestTokenize(unittest.TestCase): + def test_tokenizer_iter(self): + source = io.StringIO("for _ in a:\n pass") + it = tokenize._tokenize.TokenizerIter(source.readline, extra_tokens=False) + + tokens = [] + def next_token(it): + while True: + try: + r = next(it) + tokens.append(tokenize.TokenInfo._make(r)) + time.sleep(0.03) + except StopIteration: + return + + threads = [] + for _ in range(5): + threads.append(Thread(target=partial(next_token, it))) + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + expected_tokens = [ + tokenize.TokenInfo(type=1, string='for', start=(1, 0), end=(1, 3), line='for _ in a:\n'), + tokenize.TokenInfo(type=1, string='_', start=(1, 4), end=(1, 5), line='for _ in a:\n'), + tokenize.TokenInfo(type=1, string='in', start=(1, 6), end=(1, 8), line='for _ in a:\n'), + tokenize.TokenInfo(type=1, string='a', start=(1, 9), end=(1, 10), line='for _ in a:\n'), + tokenize.TokenInfo(type=11, string=':', start=(1, 10), end=(1, 11), line='for _ in a:\n'), + tokenize.TokenInfo(type=4, string='', start=(1, 11), end=(1, 11), line='for _ in a:\n'), + tokenize.TokenInfo(type=5, string='', start=(2, -1), end=(2, -1), line=' pass'), + tokenize.TokenInfo(type=1, string='pass', start=(2, 2), end=(2, 6), line=' pass'), + tokenize.TokenInfo(type=4, string='', start=(2, 6), end=(2, 6), line=' pass'), + tokenize.TokenInfo(type=6, string='', start=(2, -1), end=(2, -1), line=' pass'), + tokenize.TokenInfo(type=0, string='', start=(2, -1), end=(2, -1), line=' pass'), + ] + + tokens.sort() + expected_tokens.sort() + self.assertListEqual(tokens, expected_tokens) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 75259795e81bcb..649676db9c08a5 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -106,7 +106,7 @@ class Bar: thing = Foo() def work(): foo = thing - for _ in range(10000): + for _ in range(5000): foo.__class__ = Bar type(foo) foo.__class__ = Foo diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 49c6f761e5b4f0..5da3c38014ebe4 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -8,6 +8,7 @@ # Unicode identifiers in tests is allowed by PEP 3131. import ast +import datetime import dis import os import re @@ -895,6 +896,7 @@ def test_missing_expression(self): "f'{:2}'", "f'''{\t\f\r\n:a}'''", "f'{:'", + "F'{[F'{:'}[F'{:'}]]]", ]) self.assertAllRaise(SyntaxError, @@ -1601,6 +1603,12 @@ def f(a): self.assertEqual(f'{f(a=4)}', '3=') self.assertEqual(x, 4) + # Check debug expressions in format spec + y = 20 + self.assertEqual(f"{2:{y=}}", "yyyyyyyyyyyyyyyyyyy2") + self.assertEqual(f"{datetime.datetime.now():h1{y=}h2{y=}h3{y=}}", + 'h1y=20h2y=20h3y=20') + # Make sure __format__ is being called. class C: def __format__(self, s): @@ -1614,9 +1622,11 @@ def __repr__(self): self.assertEqual(f'{C()=: }', 'C()=FORMAT- ') self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x') self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********') + self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20') self.assertRaises(SyntaxError, eval, "f'{C=]'") + # Make sure leading and following text works. x = 'foo' self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 492a16a8c7ff45..837f3795f0842d 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -741,6 +741,26 @@ def wrapper(*args): pass self.assertEqual(wrapper.__annotations__, {}) self.assertEqual(wrapper.__type_params__, ()) + def test_update_wrapper_annotations(self): + def inner(x: int): pass + def wrapper(*args): pass + + functools.update_wrapper(wrapper, inner) + self.assertEqual(wrapper.__annotations__, {'x': int}) + self.assertIs(wrapper.__annotate__, inner.__annotate__) + + def with_forward_ref(x: undefined): pass + def wrapper(*args): pass + + functools.update_wrapper(wrapper, with_forward_ref) + + self.assertIs(wrapper.__annotate__, with_forward_ref.__annotate__) + with self.assertRaises(NameError): + wrapper.__annotations__ + + undefined = str + self.assertEqual(wrapper.__annotations__, {'x': undefined}) + class TestWraps(TestUpdateWrapper): @@ -3059,6 +3079,27 @@ def _(arg: typing.List[float] | bytes): self.assertEqual(f(""), "default") self.assertEqual(f(b""), "default") + def test_forward_reference(self): + @functools.singledispatch + def f(arg, arg2=None): + return "default" + + @f.register + def _(arg: str, arg2: undefined = None): + return "forward reference" + + self.assertEqual(f(1), "default") + self.assertEqual(f(""), "forward reference") + + def test_unresolved_forward_reference(self): + @functools.singledispatch + def f(arg): + return "default" + + with self.assertRaisesRegex(TypeError, "is an unresolved forward reference"): + @f.register + def _(arg: undefined): + return "forward reference" class CachedCostItem: _cost = 1 diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 30e39e7720e6d1..a4cbdecdc9db11 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -139,7 +139,7 @@ def test_inst_no_args(self): def test_inst_one_pop(self): input = """ inst(OP, (value --)) { - spam(); + spam(value); } """ output = """ @@ -149,7 +149,7 @@ def test_inst_one_pop(self): INSTRUCTION_STATS(OP); _PyStackRef value; value = stack_pointer[-1]; - spam(); + spam(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -160,7 +160,7 @@ def test_inst_one_pop(self): def test_inst_one_push(self): input = """ inst(OP, (-- res)) { - spam(); + res = spam(); } """ output = """ @@ -169,7 +169,7 @@ def test_inst_one_push(self): next_instr += 1; INSTRUCTION_STATS(OP); _PyStackRef res; - spam(); + res = spam(); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -181,7 +181,7 @@ def test_inst_one_push(self): def test_inst_one_push_one_pop(self): input = """ inst(OP, (value -- res)) { - spam(); + res = spam(value); } """ output = """ @@ -192,7 +192,7 @@ def test_inst_one_push_one_pop(self): _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; - spam(); + res = spam(value); stack_pointer[-1] = res; DISPATCH(); } @@ -202,7 +202,7 @@ def test_inst_one_push_one_pop(self): def test_binary_op(self): input = """ inst(OP, (left, right -- res)) { - spam(); + res = spam(left, right); } """ output = """ @@ -210,12 +210,12 @@ def test_binary_op(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - spam(); + res = spam(left, right); stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -227,7 +227,7 @@ def test_binary_op(self): def test_overlap(self): input = """ inst(OP, (left, right -- left, result)) { - spam(); + result = spam(left, right); } """ output = """ @@ -235,12 +235,12 @@ def test_overlap(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef result; right = stack_pointer[-1]; left = stack_pointer[-2]; - spam(); + result = spam(left, right); stack_pointer[-1] = result; DISPATCH(); } @@ -253,6 +253,7 @@ def test_predictions_and_eval_breaker(self): } inst(OP3, (arg -- res)) { DEOPT_IF(xxx); + res = Py_None; CHECK_EVAL_BREAKER(); } family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; @@ -263,9 +264,6 @@ def test_predictions_and_eval_breaker(self): next_instr += 1; INSTRUCTION_STATS(OP1); PREDICTED(OP1); - _PyStackRef arg; - _PyStackRef rest; - arg = stack_pointer[-1]; stack_pointer[-1] = rest; DISPATCH(); } @@ -275,10 +273,9 @@ def test_predictions_and_eval_breaker(self): next_instr += 1; INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); - _PyStackRef arg; _PyStackRef res; - arg = stack_pointer[-1]; DEOPT_IF(xxx, OP1); + res = Py_None; stack_pointer[-1] = res; CHECK_EVAL_BREAKER(); DISPATCH(); @@ -324,6 +321,7 @@ def test_error_if_plain_with_comment(self): def test_error_if_pop(self): input = """ inst(OP, (left, right -- res)) { + res = spam(left, right); ERROR_IF(cond, label); } """ @@ -332,11 +330,12 @@ def test_error_if_pop(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; + res = spam(left, right); if (cond) goto pop_2_label; stack_pointer[-2] = res; stack_pointer += -1; @@ -357,8 +356,6 @@ def test_cache_effect(self): (void)this_instr; next_instr += 4; INSTRUCTION_STATS(OP); - _PyStackRef value; - value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; uint32_t extra = read_u32(&this_instr[2].cache); @@ -408,8 +405,8 @@ def test_macro_instruction(self): PREDICTED(OP); _Py_CODEUNIT *this_instr = next_instr - 6; (void)this_instr; - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef arg2; _PyStackRef res; // _OP1 @@ -439,8 +436,8 @@ def test_macro_instruction(self): (void)this_instr; next_instr += 2; INSTRUCTION_STATS(OP1); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; right = stack_pointer[-1]; left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); @@ -454,9 +451,9 @@ def test_macro_instruction(self): next_instr += 6; INSTRUCTION_STATS(OP3); static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); - _PyStackRef right; - _PyStackRef left; _PyStackRef arg2; + _PyStackRef left; + _PyStackRef right; _PyStackRef res; /* Skip 5 cache entries */ right = stack_pointer[-1]; @@ -531,7 +528,7 @@ def test_pseudo_instruction_with_flags(self): def test_array_input(self): input = """ inst(OP, (below, values[oparg*2], above --)) { - spam(); + spam(values, oparg); } """ output = """ @@ -539,13 +536,9 @@ def test_array_input(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - _PyStackRef above; _PyStackRef *values; - _PyStackRef below; - above = stack_pointer[-1]; values = &stack_pointer[-1 - oparg*2]; - below = stack_pointer[-2 - oparg*2]; - spam(); + spam(values, oparg); stack_pointer += -2 - oparg*2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -564,9 +557,7 @@ def test_array_output(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - _PyStackRef below; _PyStackRef *values; - _PyStackRef above; values = &stack_pointer[-1]; spam(values, oparg); stack_pointer[-2] = below; @@ -590,7 +581,6 @@ def test_array_input_output(self): next_instr += 1; INSTRUCTION_STATS(OP); _PyStackRef *values; - _PyStackRef above; values = &stack_pointer[-oparg]; spam(values, oparg); stack_pointer[0] = above; @@ -612,10 +602,6 @@ def test_array_error_if(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - _PyStackRef *values; - _PyStackRef extra; - values = &stack_pointer[-oparg]; - extra = stack_pointer[-1 - oparg]; if (oparg == 0) { stack_pointer += -1 - oparg; goto somewhere; } stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -627,7 +613,7 @@ def test_array_error_if(self): def test_cond_effect(self): input = """ inst(OP, (aa, input if ((oparg & 1) == 1), cc -- xx, output if (oparg & 2), zz)) { - output = spam(oparg, input); + output = spam(oparg, aa, cc, input); } """ output = """ @@ -635,16 +621,14 @@ def test_cond_effect(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - _PyStackRef cc; - _PyStackRef input = PyStackRef_NULL; _PyStackRef aa; - _PyStackRef xx; + _PyStackRef input = PyStackRef_NULL; + _PyStackRef cc; _PyStackRef output = PyStackRef_NULL; - _PyStackRef zz; cc = stack_pointer[-1]; if ((oparg & 1) == 1) { input = stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)]; } aa = stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)]; - output = spam(oparg, input); + output = spam(oparg, aa, cc, input); stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)] = xx; if (oparg & 2) stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)] = output; stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0) + ((oparg & 2) ? 1 : 0)] = zz; @@ -658,10 +642,11 @@ def test_cond_effect(self): def test_macro_cond_effect(self): input = """ op(A, (left, middle, right --)) { - # Body of A + use(left, middle, right); } op(B, (-- deep, extra if (oparg), res)) { - # Body of B + res = 0; + extra = 1; } macro(M) = A + B; """ @@ -670,10 +655,9 @@ def test_macro_cond_effect(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - _PyStackRef right; - _PyStackRef middle; _PyStackRef left; - _PyStackRef deep; + _PyStackRef middle; + _PyStackRef right; _PyStackRef extra = PyStackRef_NULL; _PyStackRef res; // A @@ -681,11 +665,12 @@ def test_macro_cond_effect(self): middle = stack_pointer[-2]; left = stack_pointer[-3]; { - # Body of A + use(left, middle, right); } // B { - # Body of B + res = 0; + extra = 1; } stack_pointer[-3] = deep; if (oparg) stack_pointer[-2] = extra; @@ -815,7 +800,6 @@ def test_annotated_op(self): """ self.run_cases_test(input, output) - def test_deopt_and_exit(self): input = """ pure op(OP, (arg1 -- out)) { @@ -827,6 +811,259 @@ def test_deopt_and_exit(self): with self.assertRaises(Exception): self.run_cases_test(input, output) + def test_array_of_one(self): + input = """ + inst(OP, (arg[1] -- out[1])) { + out[0] = arg[0]; + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef *out; + arg = &stack_pointer[-1]; + out = &stack_pointer[-1]; + out[0] = arg[0]; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pointer_to_stackref(self): + input = """ + inst(OP, (arg: _PyStackRef * -- out)) { + out = *arg; + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + _PyStackRef *arg; + _PyStackRef out; + arg = (_PyStackRef *)stack_pointer[-1].bits; + out = *arg; + stack_pointer[-1] = out; + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_cached_value(self): + input = """ + op(FIRST, (arg1 -- out)) { + out = arg1; + } + + op(SECOND, (unused -- unused)) { + } + + macro(BOTH) = FIRST + SECOND; + """ + output = """ + """ + with self.assertRaises(SyntaxError): + self.run_cases_test(input, output) + + def test_unused_named_values(self): + input = """ + op(OP, (named -- named)) { + } + + macro(INST) = OP; + """ + output = """ + TARGET(INST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INST); + DISPATCH(); + } + + """ + self.run_cases_test(input, output) + + def test_used_unused_used(self): + input = """ + op(FIRST, (w -- w)) { + use(w); + } + + op(SECOND, (x -- x)) { + } + + op(THIRD, (y -- y)) { + use(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef w; + _PyStackRef x; + _PyStackRef y; + // FIRST + w = stack_pointer[-1]; + { + use(w); + } + // SECOND + x = w; + { + } + // THIRD + y = x; + { + use(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_unused_used_used(self): + input = """ + op(FIRST, (w -- w)) { + } + + op(SECOND, (x -- x)) { + use(x); + } + + op(THIRD, (y -- y)) { + use(y); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + // FIRST + { + } + // SECOND + x = stack_pointer[-1]; + { + use(x); + } + // THIRD + y = x; + { + use(y); + } + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_flush(self): + input = """ + op(FIRST, ( -- a, b)) { + a = 0; + b = 1; + } + + op(SECOND, (a, b -- )) { + use(a, b); + } + + macro(TEST) = FIRST + flush + SECOND; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef a; + _PyStackRef b; + // FIRST + { + a = 0; + b = 1; + } + // flush + stack_pointer[0] = a; + stack_pointer[1] = b; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + // SECOND + b = stack_pointer[-1]; + a = stack_pointer[-2]; + { + use(a, b); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_pop_on_error_peeks(self): + + input = """ + op(FIRST, (x, y -- a, b)) { + a = x; + b = y; + } + + op(SECOND, (a, b -- a, b)) { + } + + op(THIRD, (j, k --)) { + ERROR_IF(cond, error); + } + + macro(TEST) = FIRST + SECOND + THIRD; + """ + output = """ + TARGET(TEST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST); + _PyStackRef x; + _PyStackRef y; + _PyStackRef a; + _PyStackRef b; + _PyStackRef j; + _PyStackRef k; + // FIRST + y = stack_pointer[-1]; + x = stack_pointer[-2]; + { + a = x; + b = y; + } + // SECOND + { + } + // THIRD + k = b; + j = a; + { + if (cond) goto pop_2_error; + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: super().setUp() @@ -914,7 +1151,6 @@ def test_overridden_abstract_args(self): case OP: { _Py_UopsSymbol *arg1; _Py_UopsSymbol *out; - arg1 = stack_pointer[-1]; eggs(); stack_pointer[-1] = out; break; @@ -954,7 +1190,6 @@ def test_no_overridden_case(self): case OP2: { _Py_UopsSymbol *arg1; _Py_UopsSymbol *out; - arg1 = stack_pointer[-1]; stack_pointer[-1] = out; break; } diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index daa65718bfc905..3cb4d51fff1eeb 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -2247,6 +2247,16 @@ def printsolution(self, x): ... SyntaxError: 'yield' outside function +>>> f=lambda: (yield from (1,2)), (yield from (3,4)) +Traceback (most recent call last): + ... +SyntaxError: 'yield from' outside function + +>>> yield from [1,2] +Traceback (most recent call last): + ... +SyntaxError: 'yield from' outside function + >>> def f(): x = yield = y Traceback (most recent call last): ... diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 2f7aa69efc184a..6c86c3d1c8c57e 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -844,6 +844,7 @@ def test_explicitly_set_stdlib_dir(self): PYDEBUGEXT="", VERSION_MAJOR=9, # fixed version number for ease VERSION_MINOR=8, # of testing + ABI_THREAD="", PYWINVER=None, EXE_SUFFIX=None, diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 5b7a639c025a0f..6a841587f49166 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -3,6 +3,7 @@ from test.support import check_syntax_error from test.support import import_helper +import annotationlib import inspect import unittest import sys @@ -459,7 +460,7 @@ def test_var_annot_simple_exec(self): gns = {}; lns = {} exec("'docstring'\n" "x: int = 5\n", gns, lns) - self.assertEqual(lns["__annotate__"](1), {'x': int}) + self.assertEqual(lns["__annotate__"](annotationlib.Format.VALUE), {'x': int}) with self.assertRaises(KeyError): gns['__annotate__'] diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index c10f689c4ea34b..e29097baaf53ae 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -3034,13 +3034,6 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): def test_basic_multiple_interpreters_reset_each(self): # resetting between each interpreter - if Py_TRACE_REFS: - # It's a Py_TRACE_REFS build. - # This test breaks interpreter isolation a little, - # which causes problems on Py_TRACE_REF builds. - # See gh-121110. - raise unittest.SkipTest('crashes on Py_TRACE_REFS builds') - # At this point: # * alive in 0 interpreters # * module def may or may not be loaded already diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index d39c3ccdc847bd..730c192d1aa260 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -45,10 +45,7 @@ from test.test_inspect import inspect_fodder as mod from test.test_inspect import inspect_fodder2 as mod2 -from test.test_inspect import inspect_stock_annotations from test.test_inspect import inspect_stringized_annotations -from test.test_inspect import inspect_stringized_annotations_2 -from test.test_inspect import inspect_stringized_annotations_pep695 # Functions tested in this suite: @@ -126,7 +123,7 @@ def istest(self, predicate, exp): self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp)) def test__all__(self): - support.check__all__(self, inspect, not_exported=("modulesbyfile",)) + support.check__all__(self, inspect, not_exported=("modulesbyfile",), extra=("get_annotations",)) def generator_function_example(self): for i in range(2): @@ -405,6 +402,8 @@ def test_isroutine(self): self.assertFalse(inspect.isroutine(type)) self.assertFalse(inspect.isroutine(int)) self.assertFalse(inspect.isroutine(type('some_class', (), {}))) + # partial + self.assertTrue(inspect.isroutine(functools.partial(mod.spam))) def test_isclass(self): self.istest(inspect.isclass, 'mod.StupidGit') @@ -1595,216 +1594,6 @@ class C(metaclass=M): attrs = [a[0] for a in inspect.getmembers(C)] self.assertNotIn('missing', attrs) - def test_get_annotations_with_stock_annotations(self): - def foo(a:int, b:str): pass - self.assertEqual(inspect.get_annotations(foo), {'a': int, 'b': str}) - - foo.__annotations__ = {'a': 'foo', 'b':'str'} - self.assertEqual(inspect.get_annotations(foo), {'a': 'foo', 'b': 'str'}) - - self.assertEqual(inspect.get_annotations(foo, eval_str=True, locals=locals()), {'a': foo, 'b': str}) - self.assertEqual(inspect.get_annotations(foo, eval_str=True, globals=locals()), {'a': foo, 'b': str}) - - isa = inspect_stock_annotations - self.assertEqual(inspect.get_annotations(isa), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(inspect), {}) # inspect module has no annotations - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=True), {'a': int, 'b': str, 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=True), {'a': int, 'b': str, 'c': isa.MyClass}) - self.assertEqual(inspect.get_annotations(inspect, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=True), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=False), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=False), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=False), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=False), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=False), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(inspect, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=False), {}) - - def times_three(fn): - @functools.wraps(fn) - def wrapper(a, b): - return fn(a*3, b*3) - return wrapper - - wrapped = times_three(isa.function) - self.assertEqual(wrapped(1, 'x'), isa.MyClass(3, 'xxx')) - self.assertIsNot(wrapped.__globals__, isa.function.__globals__) - self.assertEqual(inspect.get_annotations(wrapped), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=False), {'a': int, 'b': str, 'return': isa.MyClass}) - - def test_get_annotations_with_stringized_annotations(self): - isa = inspect_stringized_annotations - self.assertEqual(inspect.get_annotations(isa), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.MyClass), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.function), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function2), {'a': 'int', 'b': "'str'", 'c': 'MyClass', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function3), {'a': "'int'", 'b': "'str'", 'c': "'MyClass'"}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=True), {'a': int, 'b': str}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=True), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=True), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=True), {}) - - self.assertEqual(inspect.get_annotations(isa, eval_str=False), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=False), {'a': 'int', 'b': 'str'}) - self.assertEqual(inspect.get_annotations(isa.function, eval_str=False), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function2, eval_str=False), {'a': 'int', 'b': "'str'", 'c': 'MyClass', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(isa.function3, eval_str=False), {'a': "'int'", 'b': "'str'", 'c': "'MyClass'"}) - self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=False), {}) - self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=False), {}) - - isa2 = inspect_stringized_annotations_2 - self.assertEqual(inspect.get_annotations(isa2), {}) - self.assertEqual(inspect.get_annotations(isa2, eval_str=True), {}) - self.assertEqual(inspect.get_annotations(isa2, eval_str=False), {}) - - def times_three(fn): - @functools.wraps(fn) - def wrapper(a, b): - return fn(a*3, b*3) - return wrapper - - wrapped = times_three(isa.function) - self.assertEqual(wrapped(1, 'x'), isa.MyClass(3, 'xxx')) - self.assertIsNot(wrapped.__globals__, isa.function.__globals__) - self.assertEqual(inspect.get_annotations(wrapped), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) - self.assertEqual(inspect.get_annotations(wrapped, eval_str=False), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) - - # test that local namespace lookups work - self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations), {'x': 'mytype'}) - self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), {'x': int}) - - def test_pep695_generic_class_with_future_annotations(self): - ann_module695 = inspect_stringized_annotations_pep695 - A_annotations = inspect.get_annotations(ann_module695.A, eval_str=True) - A_type_params = ann_module695.A.__type_params__ - self.assertIs(A_annotations["x"], A_type_params[0]) - self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) - self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) - - def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): - B_annotations = inspect.get_annotations( - inspect_stringized_annotations_pep695.B, eval_str=True - ) - self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) - - def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): - ann_module695 = inspect_stringized_annotations_pep695 - C_annotations = inspect.get_annotations(ann_module695.C, eval_str=True) - self.assertEqual( - set(C_annotations.values()), - set(ann_module695.C.__type_params__) - ) - - def test_pep_695_generic_function_with_future_annotations(self): - ann_module695 = inspect_stringized_annotations_pep695 - generic_func_annotations = inspect.get_annotations( - ann_module695.generic_function, eval_str=True - ) - func_t_params = ann_module695.generic_function.__type_params__ - self.assertEqual( - generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} - ) - self.assertIs(generic_func_annotations["x"], func_t_params[0]) - self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) - self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) - self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) - - def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): - self.assertEqual( - set( - inspect.get_annotations( - inspect_stringized_annotations_pep695.generic_function_2, - eval_str=True - ).values() - ), - set( - inspect_stringized_annotations_pep695.generic_function_2.__type_params__ - ) - ) - - def test_pep_695_generic_method_with_future_annotations(self): - ann_module695 = inspect_stringized_annotations_pep695 - generic_method_annotations = inspect.get_annotations( - ann_module695.D.generic_method, eval_str=True - ) - params = { - param.__name__: param - for param in ann_module695.D.generic_method.__type_params__ - } - self.assertEqual( - generic_method_annotations, - {"x": params["Foo"], "y": params["Bar"], "return": None} - ) - - def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): - self.assertEqual( - set( - inspect.get_annotations( - inspect_stringized_annotations_pep695.D.generic_method_2, - eval_str=True - ).values() - ), - set( - inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ - ) - ) - - def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars(self): - self.assertEqual( - inspect.get_annotations( - inspect_stringized_annotations_pep695.E, eval_str=True - ), - {"x": str}, - ) - - def test_pep_695_generics_with_future_annotations_nested_in_function(self): - results = inspect_stringized_annotations_pep695.nested() - - self.assertEqual( - set(results.F_annotations.values()), - set(results.F.__type_params__) - ) - self.assertEqual( - set(results.F_meth_annotations.values()), - set(results.F.generic_method.__type_params__) - ) - self.assertNotEqual( - set(results.F_meth_annotations.values()), - set(results.F.__type_params__) - ) - self.assertEqual( - set(results.F_meth_annotations.values()).intersection(results.F.__type_params__), - set() - ) - - self.assertEqual(results.G_annotations, {"x": str}) - - self.assertEqual( - set(results.generic_func_annotations.values()), - set(results.generic_func.__type_params__) - ) - class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): @@ -1906,6 +1695,7 @@ def function(): self.assertFalse(inspect.ismethoddescriptor(Owner.static_method)) self.assertFalse(inspect.ismethoddescriptor(function)) self.assertFalse(inspect.ismethoddescriptor(a_lambda)) + self.assertTrue(inspect.ismethoddescriptor(functools.partial(function))) def test_descriptor_being_a_class(self): class MethodDescriptorMeta(type): diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 68cc45d1a5e09f..eada18f99d04db 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -48,6 +48,7 @@ def test_list_all(self): self.assertEqual(after, created) def test_shareable(self): + interp = interpreters.create() rch, sch = channels.create() self.assertTrue( @@ -60,8 +61,25 @@ def test_shareable(self): rch2 = rch.recv() sch2 = rch.recv() + interp.prepare_main(rch=rch, sch=sch) + sch.send_nowait(rch) + sch.send_nowait(sch) + interp.exec(dedent(""" + rch2 = rch.recv() + sch2 = rch.recv() + assert rch2 == rch + assert sch2 == sch + + sch.send_nowait(rch2) + sch.send_nowait(sch2) + """)) + rch3 = rch.recv() + sch3 = rch.recv() + self.assertEqual(rch2, rch) self.assertEqual(sch2, sch) + self.assertEqual(rch3, rch) + self.assertEqual(sch3, sch) def test_is_closed(self): rch, sch = channels.create() @@ -354,6 +372,228 @@ def test_send_buffer_nowait(self): obj[4:8] = b'ham.' self.assertEqual(obj, buf) + def test_send_cleared_with_subinterpreter(self): + def common(rch, sch, unbound=None, presize=0): + if not unbound: + extraargs = '' + elif unbound is channels.UNBOUND: + extraargs = ', unbound=channels.UNBOUND' + elif unbound is channels.UNBOUND_ERROR: + extraargs = ', unbound=channels.UNBOUND_ERROR' + elif unbound is channels.UNBOUND_REMOVE: + extraargs = ', unbound=channels.UNBOUND_REMOVE' + else: + raise NotImplementedError(repr(unbound)) + interp = interpreters.create() + + _run_output(interp, dedent(f""" + from test.support.interpreters import channels + sch = channels.SendChannel({sch.id}) + obj1 = b'spam' + obj2 = b'eggs' + sch.send_nowait(obj1{extraargs}) + sch.send_nowait(obj2{extraargs}) + """)) + self.assertEqual( + _channels.get_count(rch.id), + presize + 2, + ) + + if presize == 0: + obj1 = rch.recv() + self.assertEqual(obj1, b'spam') + self.assertEqual( + _channels.get_count(rch.id), + presize + 1, + ) + + return interp + + with self.subTest('default'): # UNBOUND + rch, sch = channels.create() + interp = common(rch, sch) + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + obj1 = rch.recv() + self.assertEqual(_channels.get_count(rch.id), 0) + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND'): + rch, sch = channels.create() + interp = common(rch, sch, channels.UNBOUND) + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + obj1 = rch.recv() + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND_ERROR'): + rch, sch = channels.create() + interp = common(rch, sch, channels.UNBOUND_ERROR) + + del interp + self.assertEqual(_channels.get_count(rch.id), 1) + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + with self.subTest('UNBOUND_REMOVE'): + rch, sch = channels.create() + + interp = common(rch, sch, channels.UNBOUND_REMOVE) + del interp + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + sch.send_nowait(b'ham', unbound=channels.UNBOUND_REMOVE) + self.assertEqual(_channels.get_count(rch.id), 1) + interp = common(rch, sch, channels.UNBOUND_REMOVE, 1) + self.assertEqual(_channels.get_count(rch.id), 3) + sch.send_nowait(42, unbound=channels.UNBOUND_REMOVE) + self.assertEqual(_channels.get_count(rch.id), 4) + del interp + self.assertEqual(_channels.get_count(rch.id), 2) + obj1 = rch.recv() + obj2 = rch.recv() + self.assertEqual(obj1, b'ham') + self.assertEqual(obj2, 42) + self.assertEqual(_channels.get_count(rch.id), 0) + with self.assertRaises(channels.ChannelEmptyError): + rch.recv_nowait() + + def test_send_cleared_with_subinterpreter_mixed(self): + rch, sch = channels.create() + interp = interpreters.create() + + # If we don't associate the main interpreter with the channel + # then the channel will be automatically closed when interp + # is destroyed. + sch.send_nowait(None) + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 0) + + _run_output(interp, dedent(f""" + from test.support.interpreters import channels + sch = channels.SendChannel({sch.id}) + sch.send_nowait(1, unbound=channels.UNBOUND) + sch.send_nowait(2, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(3) + sch.send_nowait(4, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(5, unbound=channels.UNBOUND) + """)) + self.assertEqual(_channels.get_count(rch.id), 5) + + del interp + self.assertEqual(_channels.get_count(rch.id), 4) + + obj1 = rch.recv() + self.assertIs(obj1, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 3) + + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 2) + + obj2 = rch.recv() + self.assertIs(obj2, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 1) + + obj3 = rch.recv() + self.assertIs(obj3, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + + def test_send_cleared_with_subinterpreter_multiple(self): + rch, sch = channels.create() + interp1 = interpreters.create() + interp2 = interpreters.create() + + sch.send_nowait(1) + _run_output(interp1, dedent(f""" + from test.support.interpreters import channels + rch = channels.RecvChannel({rch.id}) + sch = channels.SendChannel({sch.id}) + obj1 = rch.recv() + sch.send_nowait(2, unbound=channels.UNBOUND) + sch.send_nowait(obj1, unbound=channels.UNBOUND_REMOVE) + """)) + _run_output(interp2, dedent(f""" + from test.support.interpreters import channels + rch = channels.RecvChannel({rch.id}) + sch = channels.SendChannel({sch.id}) + obj2 = rch.recv() + obj1 = rch.recv() + """)) + self.assertEqual(_channels.get_count(rch.id), 0) + sch.send_nowait(3) + _run_output(interp1, dedent(""" + sch.send_nowait(4, unbound=channels.UNBOUND) + # interp closed here + sch.send_nowait(5, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(6, unbound=channels.UNBOUND) + """)) + _run_output(interp2, dedent(""" + sch.send_nowait(7, unbound=channels.UNBOUND_ERROR) + # interp closed here + sch.send_nowait(obj1, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(obj2, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(8, unbound=channels.UNBOUND) + """)) + _run_output(interp1, dedent(""" + sch.send_nowait(9, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(10, unbound=channels.UNBOUND) + """)) + self.assertEqual(_channels.get_count(rch.id), 10) + + obj3 = rch.recv() + self.assertEqual(obj3, 3) + self.assertEqual(_channels.get_count(rch.id), 9) + + obj4 = rch.recv() + self.assertEqual(obj4, 4) + self.assertEqual(_channels.get_count(rch.id), 8) + + del interp1 + self.assertEqual(_channels.get_count(rch.id), 6) + + # obj5 was removed + + obj6 = rch.recv() + self.assertIs(obj6, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 5) + + obj7 = rch.recv() + self.assertEqual(obj7, 7) + self.assertEqual(_channels.get_count(rch.id), 4) + + del interp2 + self.assertEqual(_channels.get_count(rch.id), 3) + + # obj1 + with self.assertRaises(channels.ItemInterpreterDestroyed): + rch.recv() + self.assertEqual(_channels.get_count(rch.id), 2) + + # obj2 was removed + + obj8 = rch.recv() + self.assertIs(obj8, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 1) + + # obj9 was removed + + obj10 = rch.recv() + self.assertIs(obj10, channels.UNBOUND) + self.assertEqual(_channels.get_count(rch.id), 0) + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 9ee7647ed79b4e..18f83d097eb360 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -8,17 +8,20 @@ # Raise SkipTest if subinterpreters not supported. _queues = import_helper.import_module('_interpqueues') from test.support import interpreters -from test.support.interpreters import queues +from test.support.interpreters import queues, _crossinterp from .utils import _run_output, TestBase as _TestBase +REPLACE = _crossinterp._UNBOUND_CONSTANT_TO_FLAG[_crossinterp.UNBOUND] + + def get_num_queues(): return len(_queues.list_all()) class TestBase(_TestBase): def tearDown(self): - for qid, _ in _queues.list_all(): + for qid, _, _ in _queues.list_all(): try: _queues.destroy(qid) except Exception: @@ -39,7 +42,7 @@ def test_highlevel_reloaded(self): importlib.reload(queues) def test_create_destroy(self): - qid = _queues.create(2, 0) + qid = _queues.create(2, 0, REPLACE) _queues.destroy(qid) self.assertEqual(get_num_queues(), 0) with self.assertRaises(queues.QueueNotFoundError): @@ -53,7 +56,7 @@ def test_not_destroyed(self): '-c', dedent(f""" import {_queues.__name__} as _queues - _queues.create(2, 0) + _queues.create(2, 0, {REPLACE}) """), ) self.assertEqual(stdout, '') @@ -64,13 +67,13 @@ def test_not_destroyed(self): def test_bind_release(self): with self.subTest('typical'): - qid = _queues.create(2, 0) + qid = _queues.create(2, 0, REPLACE) _queues.bind(qid) _queues.release(qid) self.assertEqual(get_num_queues(), 0) with self.subTest('bind too much'): - qid = _queues.create(2, 0) + qid = _queues.create(2, 0, REPLACE) _queues.bind(qid) _queues.bind(qid) _queues.release(qid) @@ -78,7 +81,7 @@ def test_bind_release(self): self.assertEqual(get_num_queues(), 0) with self.subTest('nested'): - qid = _queues.create(2, 0) + qid = _queues.create(2, 0, REPLACE) _queues.bind(qid) _queues.bind(qid) _queues.release(qid) @@ -86,7 +89,7 @@ def test_bind_release(self): self.assertEqual(get_num_queues(), 0) with self.subTest('release without binding'): - qid = _queues.create(2, 0) + qid = _queues.create(2, 0, REPLACE) with self.assertRaises(queues.QueueError): _queues.release(qid) @@ -426,26 +429,206 @@ def test_put_get_different_interpreters(self): self.assertNotEqual(id(obj2), int(out)) def test_put_cleared_with_subinterpreter(self): - interp = interpreters.create() - queue = queues.create() - - out = _run_output( - interp, - dedent(f""" + def common(queue, unbound=None, presize=0): + if not unbound: + extraargs = '' + elif unbound is queues.UNBOUND: + extraargs = ', unbound=queues.UNBOUND' + elif unbound is queues.UNBOUND_ERROR: + extraargs = ', unbound=queues.UNBOUND_ERROR' + elif unbound is queues.UNBOUND_REMOVE: + extraargs = ', unbound=queues.UNBOUND_REMOVE' + else: + raise NotImplementedError(repr(unbound)) + interp = interpreters.create() + + _run_output(interp, dedent(f""" from test.support.interpreters import queues queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' - queue.put(obj1, syncobj=True) - queue.put(obj2, syncobj=True) + queue.put(obj1, syncobj=True{extraargs}) + queue.put(obj2, syncobj=True{extraargs}) """)) - self.assertEqual(queue.qsize(), 2) + self.assertEqual(queue.qsize(), presize + 2) + + if presize == 0: + obj1 = queue.get() + self.assertEqual(obj1, b'spam') + self.assertEqual(queue.qsize(), presize + 1) + + return interp + + with self.subTest('default'): # UNBOUND + queue = queues.create() + interp = common(queue) + del interp + obj1 = queue.get() + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND'): + queue = queues.create() + interp = common(queue, queues.UNBOUND) + del interp + obj1 = queue.get() + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND_ERROR'): + queue = queues.create() + interp = common(queue, queues.UNBOUND_ERROR) + + del interp + self.assertEqual(queue.qsize(), 1) + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + with self.subTest('UNBOUND_REMOVE'): + queue = queues.create() + + interp = common(queue, queues.UNBOUND_REMOVE) + del interp + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + queue.put(b'ham', unbound=queues.UNBOUND_REMOVE) + self.assertEqual(queue.qsize(), 1) + interp = common(queue, queues.UNBOUND_REMOVE, 1) + self.assertEqual(queue.qsize(), 3) + queue.put(42, unbound=queues.UNBOUND_REMOVE) + self.assertEqual(queue.qsize(), 4) + del interp + self.assertEqual(queue.qsize(), 2) + obj1 = queue.get() + obj2 = queue.get() + self.assertEqual(obj1, b'ham') + self.assertEqual(obj2, 42) + self.assertEqual(queue.qsize(), 0) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + + def test_put_cleared_with_subinterpreter_mixed(self): + queue = queues.create() + interp = interpreters.create() + _run_output(interp, dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + queue.put(1, syncobj=True, unbound=queues.UNBOUND) + queue.put(2, syncobj=True, unbound=queues.UNBOUND_ERROR) + queue.put(3, syncobj=True) + queue.put(4, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(5, syncobj=True, unbound=queues.UNBOUND) + """)) + self.assertEqual(queue.qsize(), 5) + + del interp + self.assertEqual(queue.qsize(), 4) obj1 = queue.get() - self.assertEqual(obj1, b'spam') + self.assertIs(obj1, queues.UNBOUND) + self.assertEqual(queue.qsize(), 3) + + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + self.assertEqual(queue.qsize(), 2) + + obj2 = queue.get() + self.assertIs(obj2, queues.UNBOUND) self.assertEqual(queue.qsize(), 1) - del interp + obj3 = queue.get() + self.assertIs(obj3, queues.UNBOUND) + self.assertEqual(queue.qsize(), 0) + + def test_put_cleared_with_subinterpreter_multiple(self): + queue = queues.create() + interp1 = interpreters.create() + interp2 = interpreters.create() + + queue.put(1, syncobj=True) + _run_output(interp1, dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + obj1 = queue.get() + queue.put(2, syncobj=True, unbound=queues.UNBOUND) + queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_REMOVE) + """)) + _run_output(interp2, dedent(f""" + from test.support.interpreters import queues + queue = queues.Queue({queue.id}) + obj2 = queue.get() + obj1 = queue.get() + """)) + self.assertEqual(queue.qsize(), 0) + queue.put(3) + _run_output(interp1, dedent(""" + queue.put(4, syncobj=True, unbound=queues.UNBOUND) + # interp closed here + queue.put(5, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(6, syncobj=True, unbound=queues.UNBOUND) + """)) + _run_output(interp2, dedent(""" + queue.put(7, syncobj=True, unbound=queues.UNBOUND_ERROR) + # interp closed here + queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_ERROR) + queue.put(obj2, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(8, syncobj=True, unbound=queues.UNBOUND) + """)) + _run_output(interp1, dedent(""" + queue.put(9, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(10, syncobj=True, unbound=queues.UNBOUND) + """)) + self.assertEqual(queue.qsize(), 10) + + obj3 = queue.get() + self.assertEqual(obj3, 3) + self.assertEqual(queue.qsize(), 9) + + obj4 = queue.get() + self.assertEqual(obj4, 4) + self.assertEqual(queue.qsize(), 8) + + del interp1 + self.assertEqual(queue.qsize(), 6) + + # obj5 was removed + + obj6 = queue.get() + self.assertIs(obj6, queues.UNBOUND) + self.assertEqual(queue.qsize(), 5) + + obj7 = queue.get() + self.assertEqual(obj7, 7) + self.assertEqual(queue.qsize(), 4) + + del interp2 + self.assertEqual(queue.qsize(), 3) + + # obj1 + with self.assertRaises(queues.ItemInterpreterDestroyed): + queue.get() + self.assertEqual(queue.qsize(), 2) + + # obj2 was removed + + obj8 = queue.get() + self.assertIs(obj8, queues.UNBOUND) + self.assertEqual(queue.qsize(), 1) + + # obj9 was removed + + obj10 = queue.get() + self.assertIs(obj10, queues.UNBOUND) self.assertEqual(queue.qsize(), 0) def test_put_get_different_threads(self): diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 5fd6ecf37427f7..9c0c4b4de18cf1 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -15,26 +15,6 @@ import struct import threading import gc -import warnings - -def pickle_deprecated(testfunc): - """ Run the test three times. - First, verify that a Deprecation Warning is raised. - Second, run normally but with DeprecationWarnings temporarily disabled. - Third, run with warnings promoted to errors. - """ - def inner(self): - with self.assertWarns(DeprecationWarning): - testfunc(self) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - testfunc(self) - with warnings.catch_warnings(): - warnings.simplefilter("error", category=DeprecationWarning) - with self.assertRaises((DeprecationWarning, AssertionError, SystemError)): - testfunc(self) - - return inner maxsize = support.MAX_Py_ssize_t minsize = -maxsize-1 @@ -1278,7 +1258,7 @@ def test_tee(self): t3 = tnew(t1) self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) - # test that tee objects are weak referencable + # test that tee objects are weak referenceable a, b = tee(range(10)) p = weakref.proxy(a) self.assertEqual(getattr(p, '__class__'), type(b)) diff --git a/Lib/test/test_json/test_default.py b/Lib/test/test_json/test_default.py index 3ce16684a08272..811880a15c8020 100644 --- a/Lib/test/test_json/test_default.py +++ b/Lib/test/test_json/test_default.py @@ -8,6 +8,24 @@ def test_default(self): self.dumps(type, default=repr), self.dumps(repr(type))) + def test_bad_default(self): + def default(obj): + if obj is NotImplemented: + raise ValueError + if obj is ...: + return NotImplemented + if obj is type: + return collections + return [...] + + with self.assertRaises(ValueError) as cm: + self.dumps(type, default=default) + self.assertEqual(cm.exception.__notes__, + ['when serializing ellipsis object', + 'when serializing list item 0', + 'when serializing module object', + 'when serializing type object']) + def test_ordereddict(self): od = collections.OrderedDict(a=1, b=2, c=3, d=4) od.move_to_end('b') diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index a74240f1107de3..7c1696cc66d12b 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -100,8 +100,27 @@ def test_non_string_keys_dict(self): def test_not_serializable(self): import sys with self.assertRaisesRegex(TypeError, - 'Object of type module is not JSON serializable'): + 'Object of type module is not JSON serializable') as cm: self.dumps(sys) + self.assertFalse(hasattr(cm.exception, '__notes__')) + + with self.assertRaises(TypeError) as cm: + self.dumps([1, [2, 3, sys]]) + self.assertEqual(cm.exception.__notes__, + ['when serializing list item 2', + 'when serializing list item 1']) + + with self.assertRaises(TypeError) as cm: + self.dumps((1, (2, 3, sys))) + self.assertEqual(cm.exception.__notes__, + ['when serializing tuple item 2', + 'when serializing tuple item 1']) + + with self.assertRaises(TypeError) as cm: + self.dumps({'a': {'b': sys}}) + self.assertEqual(cm.exception.__notes__, + ["when serializing dict item 'b'", + "when serializing dict item 'a'"]) def test_truncated_input(self): test_cases = [ diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 164ff2013eb552..290207e9c15b88 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -12,8 +12,8 @@ def test_listrecursion(self): x.append(x) try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing list item 0"]) else: self.fail("didn't raise ValueError on list recursion") x = [] @@ -21,8 +21,8 @@ def test_listrecursion(self): x.append(y) try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing list item 0"]*2) else: self.fail("didn't raise ValueError on alternating list recursion") y = [] @@ -35,8 +35,8 @@ def test_dictrecursion(self): x["test"] = x try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing dict item 'test'"]) else: self.fail("didn't raise ValueError on dict recursion") x = {} @@ -60,8 +60,10 @@ def default(self, o): enc.recurse = True try: enc.encode(JSONTestObject) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, + ["when serializing list item 0", + "when serializing type object"]) else: self.fail("didn't raise ValueError on default recursion") diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 849b6cb3e50a19..41f7b70e5cfe81 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -141,6 +141,9 @@ def test_truncate(self): f.truncate(1) self.assertEqual(f.tell(), 0) # else pointer moved f.seek(0) + # Verify readall on a truncated file is well behaved. read() + # without a size can be unbounded, this should get just the byte + # that remains. self.assertEqual(len(f.read()), 1) # else wasn't truncated def test_seekable(self): @@ -151,6 +154,22 @@ def test_seekable(self): f.seek(pos) self.assertTrue(f.seekable()) + @bigmemtest(size=size, memuse=2, dry_run=False) + def test_seek_readall(self, _size): + # Seek which doesn't change position should readall successfully. + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(0, os.SEEK_CUR), 0) + self.assertEqual(len(f.read()), size + 1) + + # Seek which changes (or might change) position should readall + # successfully. + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(20, os.SEEK_SET), 20) + self.assertEqual(len(f.read()), size - 19) + + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(-3, os.SEEK_END), size - 2) + self.assertEqual(len(f.read()), 3) def skip_no_disk_space(path, required): def decorator(fun): diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 8ac521d72ef13e..6f5955791407ea 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -280,6 +280,37 @@ def test_loader(self): self.assertEqual(linecache.getlines(filename, module_globals), ['source for x.y.z\n']) + def test_invalid_names(self): + for name, desc in [ + ('\x00', 'NUL bytes filename'), + (__file__ + '\x00', 'filename with embedded NUL bytes'), + # A filename with surrogate codes. A UnicodeEncodeError is raised + # by os.stat() upon querying, which is a subclass of ValueError. + ("\uD834\uDD1E.py", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + # For POSIX platforms, an OSError will be raised but for Windows + # platforms, a ValueError is raised due to the path_t converter. + # See: https://github.com/python/cpython/issues/122170 + ('a' * 1_000_000, 'very long filename'), + ]: + with self.subTest(f'updatecache: {desc}'): + linecache.clearcache() + lines = linecache.updatecache(name) + self.assertListEqual(lines, []) + self.assertNotIn(name, linecache.cache) + + # hack into the cache (it shouldn't be allowed + # but we never know what people do...) + for key, fullname in [(name, 'ok'), ('key', name), (name, name)]: + with self.subTest(f'checkcache: {desc}', + key=key, fullname=fullname): + linecache.clearcache() + linecache.cache[key] = (0, 1234, [], fullname) + linecache.checkcache(key) + self.assertNotIn(key, linecache.cache) + + # just to be sure that we did not mess with cache + linecache.clearcache() + class LineCacheInvalidationTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 4d2d54705fc894..ad7accf2099f43 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -299,6 +299,15 @@ def __eq__(self, other): lst = [X(), X()] X() in lst + def test_tier2_invalidates_iterator(self): + # GH-121012 + for _ in range(100): + a = [1, 2, 3] + it = iter(a) + for _ in it: + pass + a.append(4) + self.assertEqual(list(it), []) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index e6daea2333b206..6d688d4b81bbf4 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3972,6 +3972,7 @@ def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_man ) manager.assert_not_called() + @skip_if_tsan_fork @support.requires_subprocess() def test_multiprocessing_queues(self): # See gh-119819 @@ -4284,6 +4285,7 @@ def test_queue_listener_with_multiple_handlers(self): import multiprocessing from unittest.mock import patch + @skip_if_tsan_fork @threading_helper.requires_working_threading() class QueueListenerTest(BaseTest): """ @@ -5184,6 +5186,7 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): else: return results + @skip_if_tsan_fork def test_multiprocessing(self): support.skip_if_broken_multiprocessing_synchronize() multiprocessing_imported = 'multiprocessing' in sys.modules diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index a07be306986b43..d7043cd4866a1c 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -832,20 +832,43 @@ def func1(): self.check_events(func1, [("raise", KeyError)]) - # gh-116090: This test doesn't really require specialization, but running - # it without specialization exposes a monitoring bug. - @requires_specialization def test_implicit_stop_iteration(self): + """Generators are documented as raising a StopIteration + when they terminate. + However, we don't do that if we can avoid it, for speed. + sys.monitoring handles that by injecting a STOP_ITERATION + event when we would otherwise have skip the RAISE event. + This test checks that both paths record an equivalent event. + """ def gen(): yield 1 return 2 - def implicit_stop_iteration(): - for _ in gen(): + def implicit_stop_iteration(iterator=None): + if iterator is None: + iterator = gen() + for _ in iterator: pass - self.check_events(implicit_stop_iteration, [("raise", StopIteration)], recorders=(StopiterationRecorder,)) + recorders=(ExceptionRecorder, StopiterationRecorder,) + expected = [("raise", StopIteration)] + + # Make sure that the loop is unspecialized, and that it will not + # re-specialize immediately, so that we can we can test the + # unspecialized version of the loop first. + # Note: this assumes that we don't specialize loops over sets. + implicit_stop_iteration(set(range(100))) + + # This will record a RAISE event for the StopIteration. + self.check_events(implicit_stop_iteration, expected, recorders=recorders) + + # Now specialize, so that we see a STOP_ITERATION event. + for _ in range(100): + implicit_stop_iteration() + + # This will record a STOP_ITERATION event for the StopIteration. + self.check_events(implicit_stop_iteration, expected, recorders=recorders) initial = [ ("raise", ZeroDivisionError), @@ -1575,7 +1598,7 @@ def f(): ('line', 'method', 2), ('line', 'method', 3), ('line', 'method', 2), - ('call', 'method', 1), + ('call', 'method', d["b"]), ('line', 'method', 1), ('line', 'method', 1), ('line', 'get_events', 11), diff --git a/Lib/test/test_multiprocessing_fork/__init__.py b/Lib/test/test_multiprocessing_fork/__init__.py index aa1fff50b28f5f..b35e82879d7fe2 100644 --- a/Lib/test/test_multiprocessing_fork/__init__.py +++ b/Lib/test/test_multiprocessing_fork/__init__.py @@ -12,5 +12,8 @@ if sys.platform == 'darwin': raise unittest.SkipTest("test may crash on macOS (bpo-33725)") +if support.check_sanitizer(thread=True): + raise unittest.SkipTest("TSAN doesn't support threads after fork") + def load_tests(*args): return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f93937fb587386..1e570c757fccbc 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -828,7 +828,7 @@ def ns_to_sec(ns): return (ns * 1e-9) + 0.5e-9 def test_utime_by_indexed(self): - # pass times as floating point seconds as the second indexed parameter + # pass times as floating-point seconds as the second indexed parameter def set_time(filename, ns): atime_ns, mtime_ns = ns atime = self.ns_to_sec(atime_ns) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 234e5746e544cd..7160e764dfb2fa 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -16,6 +16,7 @@ from test.support import import_helper from test.support import is_emscripten, is_wasi from test.support import infinite_recursion +from test.support import swap_attr from test.support import os_helper from test.support.os_helper import TESTFN, FakePath from test.test_pathlib import test_pathlib_abc @@ -31,6 +32,10 @@ if hasattr(os, 'geteuid'): root_in_posix = (os.geteuid() == 0) +rmtree_use_fd_functions = ( + {os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and + os.listdir in os.supports_fd and os.stat in os.supports_follow_symlinks) + # # Tests for the pure classes. # @@ -656,8 +661,6 @@ def test_open_unbuffered(self): def test_copy_file_preserve_metadata(self): base = self.cls(self.base) source = base / 'fileA' - if hasattr(os, 'setxattr'): - os.setxattr(source, b'user.foo', b'42') if hasattr(os, 'chmod'): os.chmod(source, stat.S_IRWXU | stat.S_IRWXO) if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): @@ -670,12 +673,19 @@ def test_copy_file_preserve_metadata(self): target_st = target.stat() self.assertLessEqual(source_st.st_atime, target_st.st_atime) self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) - if hasattr(os, 'getxattr'): - self.assertEqual(os.getxattr(target, b'user.foo'), b'42') self.assertEqual(source_st.st_mode, target_st.st_mode) if hasattr(source_st, 'st_flags'): self.assertEqual(source_st.st_flags, target_st.st_flags) + @os_helper.skip_unless_xattr + def test_copy_file_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'fileA' + os.setxattr(source, b'user.foo', b'42') + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertEqual(os.getxattr(target, b'user.foo'), b'42') + @needs_symlinks def test_copy_link_preserve_metadata(self): base = self.cls(self.base) @@ -711,6 +721,36 @@ def test_copytree_no_read_permission(self): self.assertIsInstance(errors[0], PermissionError) self.assertFalse(target.exists()) + def test_copytree_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'dirC' + if hasattr(os, 'chmod'): + os.chmod(source / 'dirD', stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): + os.chflags(source / 'fileC', stat.UF_NODUMP) + target = base / 'copyA' + source.copytree(target, preserve_metadata=True) + + for subpath in ['.', 'fileC', 'dirD', 'dirD/fileD']: + source_st = source.joinpath(subpath).stat() + target_st = target.joinpath(subpath).stat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @os_helper.skip_unless_xattr + def test_copytree_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'dirC' + source_file = source.joinpath('dirD', 'fileD') + os.setxattr(source_file, b'user.foo', b'42') + target = base / 'copyA' + source.copytree(target, preserve_metadata=True) + target_file = target.joinpath('dirD', 'fileD') + self.assertEqual(os.getxattr(target_file, b'user.foo'), b'42') + def test_resolve_nonexist_relative_issue38671(self): p = self.cls('non', 'exist') @@ -822,6 +862,250 @@ def test_group_no_follow_symlinks(self): self.assertEqual(expected_gid, gid_2) self.assertEqual(expected_name, link.group(follow_symlinks=False)) + def test_rmtree_uses_safe_fd_version_if_available(self): + if rmtree_use_fd_functions: + d = self.cls(self.base, 'a') + d.mkdir() + try: + real_open = os.open + + class Called(Exception): + pass + + def _raiser(*args, **kwargs): + raise Called + + os.open = _raiser + self.assertRaises(Called, d.rmtree) + finally: + os.open = real_open + + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_rmtree_unwritable(self): + tmp = self.cls(self.base, 'rmtree') + tmp.mkdir() + child_file_path = tmp / 'a' + child_dir_path = tmp / 'b' + child_file_path.write_text("") + child_dir_path.mkdir() + old_dir_mode = tmp.stat().st_mode + old_child_file_mode = child_file_path.stat().st_mode + old_child_dir_mode = child_dir_path.stat().st_mode + # Make unwritable. + new_mode = stat.S_IREAD | stat.S_IEXEC + try: + child_file_path.chmod(new_mode) + child_dir_path.chmod(new_mode) + tmp.chmod(new_mode) + + errors = [] + tmp.rmtree(on_error=errors.append) + # Test whether onerror has actually been called. + self.assertEqual(len(errors), 3) + finally: + tmp.chmod(old_dir_mode) + child_file_path.chmod(old_child_file_mode) + child_dir_path.chmod(old_child_dir_mode) + + @needs_windows + def test_rmtree_inner_junction(self): + import _winapi + tmp = self.cls(self.base, 'rmtree') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + _winapi.CreateJunction(str(dir2), str(link1)) + link2 = dir1 / 'link2' + _winapi.CreateJunction(str(dir3), str(link2)) + link3 = dir1 / 'link3' + _winapi.CreateJunction(str(file1), str(link3)) + # make sure junctions are removed but not followed + dir1.rmtree() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + + @needs_windows + def test_rmtree_outer_junction(self): + import _winapi + tmp = self.cls(self.base, 'rmtree') + tmp.mkdir() + try: + src = tmp / 'cheese' + dst = tmp / 'shop' + src.mkdir() + spam = src / 'spam' + spam.write_text('') + _winapi.CreateJunction(str(src), str(dst)) + self.assertRaises(OSError, dst.rmtree) + dst.rmtree(ignore_errors=True) + finally: + tmp.rmtree(ignore_errors=True) + + @needs_windows + def test_rmtree_outer_junction_on_error(self): + import _winapi + tmp = self.cls(self.base, 'rmtree') + tmp.mkdir() + dir_ = tmp / 'dir' + dir_.mkdir() + link = tmp / 'link' + _winapi.CreateJunction(str(dir_), str(link)) + try: + self.assertRaises(OSError, link.rmtree) + self.assertTrue(dir_.exists()) + self.assertTrue(link.exists(follow_symlinks=False)) + errors = [] + + def on_error(error): + errors.append(error) + + link.rmtree(on_error=on_error) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], OSError) + self.assertEqual(errors[0].filename, str(link)) + finally: + os.unlink(str(link)) + + @unittest.skipUnless(rmtree_use_fd_functions, "requires safe rmtree") + def test_rmtree_fails_on_close(self): + # Test that the error handler is called for failed os.close() and that + # os.close() is only called once for a file descriptor. + tmp = self.cls(self.base, 'rmtree') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir1.mkdir() + dir2 = dir1 / 'dir2' + dir2.mkdir() + + def close(fd): + orig_close(fd) + nonlocal close_count + close_count += 1 + raise OSError + + close_count = 0 + with swap_attr(os, 'close', close) as orig_close: + with self.assertRaises(OSError): + dir1.rmtree() + self.assertTrue(dir2.is_dir()) + self.assertEqual(close_count, 2) + + close_count = 0 + errors = [] + + with swap_attr(os, 'close', close) as orig_close: + dir1.rmtree(on_error=errors.append) + self.assertEqual(len(errors), 2) + self.assertEqual(errors[0].filename, str(dir2)) + self.assertEqual(errors[1].filename, str(dir1)) + self.assertEqual(close_count, 2) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_rmtree_on_named_pipe(self): + p = self.cls(self.base, 'pipe') + os.mkfifo(p) + try: + with self.assertRaises(NotADirectoryError): + p.rmtree() + self.assertTrue(p.exists()) + finally: + p.unlink() + + p = self.cls(self.base, 'dir') + p.mkdir() + os.mkfifo(p / 'mypipe') + p.rmtree() + self.assertFalse(p.exists()) + + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_rmtree_deleted_race_condition(self): + # bpo-37260 + # + # Test that a file or a directory deleted after it is enumerated + # by scandir() but before unlink() or rmdr() is called doesn't + # generate any errors. + def on_error(exc): + assert exc.filename + if not isinstance(exc, PermissionError): + raise + # Make the parent and the children writeable. + for p, mode in zip(paths, old_modes): + p.chmod(mode) + # Remove other dirs except one. + keep = next(p for p in dirs if str(p) != exc.filename) + for p in dirs: + if p != keep: + p.rmdir() + # Remove other files except one. + keep = next(p for p in files if str(p) != exc.filename) + for p in files: + if p != keep: + p.unlink() + + tmp = self.cls(self.base, 'rmtree') + tmp.mkdir() + paths = [tmp] + [tmp / f'child{i}' for i in range(6)] + dirs = paths[1::2] + files = paths[2::2] + for path in dirs: + path.mkdir() + for path in files: + path.write_text('') + + old_modes = [path.stat().st_mode for path in paths] + + # Make the parent and the children non-writeable. + new_mode = stat.S_IREAD | stat.S_IEXEC + for path in reversed(paths): + path.chmod(new_mode) + + try: + tmp.rmtree(on_error=on_error) + except: + # Test failed, so cleanup artifacts. + for path, mode in zip(paths, old_modes): + try: + path.chmod(mode) + except OSError: + pass + tmp.rmtree() + raise + + def test_rmtree_does_not_choke_on_failing_lstat(self): + try: + orig_lstat = os.lstat + tmp = self.cls(self.base, 'rmtree') + + def raiser(fn, *args, **kwargs): + if fn != str(tmp): + raise OSError() + else: + return orig_lstat(fn) + + os.lstat = raiser + + tmp.mkdir() + foo = tmp / 'foo' + foo.write_text('') + tmp.rmtree() + finally: + os.lstat = orig_lstat + @os_helper.skip_unless_hardlink def test_hardlink_to(self): P = self.cls(self.base) diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 28c9664cc90fe1..37678c5d799e9a 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -2641,6 +2641,105 @@ def test_rmdir(self): self.assertFileNotFound(p.stat) self.assertFileNotFound(p.unlink) + def test_rmtree(self): + base = self.cls(self.base) + base.joinpath('dirA').rmtree() + self.assertRaises(FileNotFoundError, base.joinpath('dirA').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirA', 'linkC').lstat) + base.joinpath('dirB').rmtree() + self.assertRaises(FileNotFoundError, base.joinpath('dirB').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'fileB').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'linkD').lstat) + base.joinpath('dirC').rmtree() + self.assertRaises(FileNotFoundError, base.joinpath('dirC').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD', 'fileD').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'fileC').stat) + self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'novel.txt').stat) + + def test_rmtree_errors(self): + tmp = self.cls(self.base, 'rmtree') + tmp.mkdir() + # filename is guaranteed not to exist + filename = tmp / 'foo' + self.assertRaises(FileNotFoundError, filename.rmtree) + # test that ignore_errors option is honored + filename.rmtree(ignore_errors=True) + + # existing file + filename = tmp / "tstfile" + filename.write_text("") + with self.assertRaises(NotADirectoryError) as cm: + filename.rmtree() + self.assertEqual(cm.exception.filename, str(filename)) + self.assertTrue(filename.exists()) + # test that ignore_errors option is honored + filename.rmtree(ignore_errors=True) + self.assertTrue(filename.exists()) + + def test_rmtree_on_error(self): + tmp = self.cls(self.base, 'rmtree') + tmp.mkdir() + filename = tmp / "tstfile" + filename.write_text("") + errors = [] + + def on_error(error): + errors.append(error) + + filename.rmtree(on_error=on_error) + self.assertEqual(len(errors), 2) + # First from scandir() + self.assertIsInstance(errors[0], NotADirectoryError) + self.assertEqual(errors[0].filename, str(filename)) + # Then from munlink() + self.assertIsInstance(errors[1], NotADirectoryError) + self.assertEqual(errors[1].filename, str(filename)) + + @needs_symlinks + def test_rmtree_outer_symlink(self): + tmp = self.cls(self.base, 'rmtree') + tmp.mkdir() + dir_ = tmp / 'dir' + dir_.mkdir() + link = tmp / 'link' + link.symlink_to(dir_) + self.assertRaises(OSError, link.rmtree) + self.assertTrue(dir_.exists()) + self.assertTrue(link.exists(follow_symlinks=False)) + errors = [] + + def on_error(error): + errors.append(error) + + link.rmtree(on_error=on_error) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], OSError) + self.assertEqual(errors[0].filename, str(link)) + + @needs_symlinks + def test_rmtree_inner_symlink(self): + tmp = self.cls(self.base, 'rmtree') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + link1.symlink_to(dir2) + link2 = dir1 / 'link2' + link2.symlink_to(dir3) + link3 = dir1 / 'link3' + link3.symlink_to(file1) + # make sure symlinks are removed but not followed + dir1.rmtree() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + def setUpWalk(self): # Build: # TESTFN/ diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 5c7445574f5d75..9395e9b133f50d 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2448,6 +2448,49 @@ def test_pdb_show_attribute_and_item(): (Pdb) c """ +# doctest will modify pdb.set_trace during the test, so we need to backup +# the original function to use it in the test +original_pdb_settrace = pdb.set_trace + +def test_pdb_with_inline_breakpoint(): + """Hard-coded breakpoint() calls should invoke the same debugger instance + + >>> def test_function(): + ... x = 1 + ... import pdb; pdb.Pdb().set_trace() + ... original_pdb_settrace() + ... x = 2 + + >>> with PdbTestInput(['display x', + ... 'n', + ... 'n', + ... 'n', + ... 'n', + ... 'undisplay', + ... 'c']): + ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb().set_trace() + (Pdb) display x + display x: 1 + (Pdb) n + > (4)test_function() + -> original_pdb_settrace() + (Pdb) n + > (4)test_function() + -> original_pdb_settrace() + (Pdb) n + > (5)test_function() + -> x = 2 + (Pdb) n + --Return-- + > (5)test_function()->None + -> x = 2 + display x: 2 [old: 1] + (Pdb) undisplay + (Pdb) c + """ + def test_pdb_issue_20766(): """Test for reference leaks when the SIGINT handler is set. @@ -3346,7 +3389,12 @@ def test_header(self): header = 'Nobody expects... blah, blah, blah' with ExitStack() as resources: resources.enter_context(patch('sys.stdout', stdout)) + # patch pdb.Pdb.set_trace() to avoid entering the debugger resources.enter_context(patch.object(pdb.Pdb, 'set_trace')) + # We need to manually clear pdb.Pdb._last_pdb_instance so a + # new instance with stdout redirected could be created when + # pdb.set_trace() is called. + pdb.Pdb._last_pdb_instance = None pdb.set_trace(header=header) self.assertEqual(stdout.getvalue(), header + '\n') @@ -3469,10 +3517,12 @@ def test_file_modified_after_execution(self): print("hello") """ + # the time.sleep is needed for low-resolution filesystems like HFS+ commands = """ filename = $_frame.f_code.co_filename f = open(filename, "w") f.write("print('goodbye')") + import time; time.sleep(1) f.close() ll """ @@ -3482,10 +3532,12 @@ def test_file_modified_after_execution(self): self.assertIn("was edited", stdout) def test_file_modified_after_execution_with_multiple_instances(self): + # the time.sleep is needed for low-resolution filesystems like HFS+ script = """ import pdb; pdb.Pdb().set_trace() with open(__file__, "w") as f: f.write("print('goodbye')\\n" * 5) + import time; time.sleep(1) import pdb; pdb.Pdb().set_trace() """ diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 49aa4b386039ec..c84e507cdf645f 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -16,6 +16,7 @@ from test.pickletester import AbstractHookTests from test.pickletester import AbstractUnpickleTests +from test.pickletester import AbstractPicklingErrorTests from test.pickletester import AbstractPickleTests from test.pickletester import AbstractPickleModuleTests from test.pickletester import AbstractPersistentPicklerTests @@ -55,6 +56,18 @@ def loads(self, buf, **kwds): return u.load() +class PyPicklingErrorTests(AbstractPicklingErrorTests, unittest.TestCase): + + pickler = pickle._Pickler + + def dumps(self, arg, proto=None, **kwargs): + f = io.BytesIO() + p = self.pickler(f, proto, **kwargs) + p.dump(arg) + f.seek(0) + return bytes(f.read()) + + class PyPicklerTests(AbstractPickleTests, unittest.TestCase): pickler = pickle._Pickler @@ -88,6 +101,8 @@ def loads(self, buf, **kwds): return pickle.loads(buf, **kwds) test_framed_write_sizes_with_delayed_writer = None + test_find_class = None + test_custom_find_class = None class PersistentPicklerUnpicklerMixin(object): @@ -267,6 +282,9 @@ class CUnpicklerTests(PyUnpicklerTests): bad_stack_errors = (pickle.UnpicklingError,) truncated_errors = (pickle.UnpicklingError,) + class CPicklingErrorTests(PyPicklingErrorTests): + pickler = _pickle.Pickler + class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index d095f440a99f63..20fba87e4ec120 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -522,7 +522,43 @@ def test_mixed_namespace(self): del sys.modules['foo.bar'] del sys.modules['foo.baz'] - # XXX: test .pkg files + + def test_extend_path_argument_types(self): + pkgname = 'foo' + dirname_0 = self.create_init(pkgname) + + # If the input path is not a list it is returned unchanged + self.assertEqual('notalist', pkgutil.extend_path('notalist', 'foo')) + self.assertEqual(('not', 'a', 'list'), pkgutil.extend_path(('not', 'a', 'list'), 'foo')) + self.assertEqual(123, pkgutil.extend_path(123, 'foo')) + self.assertEqual(None, pkgutil.extend_path(None, 'foo')) + + # Cleanup + shutil.rmtree(dirname_0) + del sys.path[0] + + + def test_extend_path_pkg_files(self): + pkgname = 'foo' + dirname_0 = self.create_init(pkgname) + + with open(os.path.join(dirname_0, 'bar.pkg'), 'w') as pkg_file: + pkg_file.write('\n'.join([ + 'baz', + '/foo/bar/baz', + '', + '#comment' + ])) + + extended_paths = pkgutil.extend_path(sys.path, 'bar') + + self.assertEqual(extended_paths[:-2], sys.path) + self.assertEqual(extended_paths[-2], 'baz') + self.assertEqual(extended_paths[-1], '/foo/bar/baz') + + # Cleanup + shutil.rmtree(dirname_0) + del sys.path[0] class NestedNamespacePackageTest(unittest.TestCase): diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 40b3aca25e0da8..2dba077cdea6a7 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -776,9 +776,16 @@ def run_pydoc_pager(request, what, expected_first_line): 'Help on function help in module pydoc:') run_pydoc_pager('str', 'str', 'Help on class str in module builtins:') run_pydoc_pager(str, 'str', 'Help on class str in module builtins:') - run_pydoc_pager('str.upper', 'str.upper', 'Help on method_descriptor in str:') - run_pydoc_pager(str.upper, 'str.upper', 'Help on method_descriptor:') - run_pydoc_pager(str.__add__, 'str.__add__', 'Help on wrapper_descriptor:') + run_pydoc_pager('str.upper', 'str.upper', + 'Help on method descriptor upper in str:') + run_pydoc_pager(str.upper, 'str.upper', + 'Help on method descriptor upper:') + run_pydoc_pager(''.upper, 'str.upper', + 'Help on built-in function upper:') + run_pydoc_pager(str.__add__, + 'str.__add__', 'Help on method descriptor __add__:') + run_pydoc_pager(''.__add__, + 'str.__add__', 'Help on method wrapper __add__:') run_pydoc_pager(int.numerator, 'int.numerator', 'Help on getset descriptor builtins.int.numerator:') run_pydoc_pager(list[int], 'list', diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py index 70e12286f7d781..cb5cb4ab20aa54 100644 --- a/Lib/test/test_pyrepl/support.py +++ b/Lib/test/test_pyrepl/support.py @@ -1,3 +1,4 @@ +import os from code import InteractiveConsole from functools import partial from typing import Iterable @@ -38,6 +39,20 @@ def code_to_events(code: str): yield Event(evt="key", data=c, raw=bytearray(c.encode("utf-8"))) +def clean_screen(screen: Iterable[str]): + """Cleans color and console characters out of a screen output. + + This is useful for screen testing, it increases the test readability since + it strips out all the unreadable side of the screen. + """ + output = [] + for line in screen: + if line.startswith(">>>") or line.startswith("..."): + line = line[3:] + output.append(line) + return "\n".join(output).strip() + + def prepare_reader(console: Console, **kwargs): config = ReadlineConfig(readline_completer=kwargs.pop("readline_completer", None)) reader = ReadlineAlikeReader(console=console, config=config) @@ -86,8 +101,18 @@ def handle_all_events( ) +def make_clean_env() -> dict[str, str]: + clean_env = os.environ.copy() + for k in clean_env.copy(): + if k.startswith("PYTHON"): + clean_env.pop(k) + clean_env.pop("FORCE_COLOR", None) + clean_env.pop("NO_COLOR", None) + return clean_env + + class FakeConsole(Console): - def __init__(self, events, encoding="utf-8"): + def __init__(self, events, encoding="utf-8") -> None: self.events = iter(events) self.encoding = encoding self.screen = [] diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index 31f08cdb25e078..369dab316af132 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -7,7 +7,7 @@ from test.support import force_not_colorized from _pyrepl.console import InteractiveColoredConsole - +from _pyrepl.simple_interact import _more_lines class TestSimpleInteract(unittest.TestCase): def test_multiple_statements(self): @@ -111,3 +111,104 @@ def test_no_active_future(self): result = console.runsource(source) self.assertFalse(result) self.assertEqual(f.getvalue(), "{'x': }\n") + + +class TestMoreLines(unittest.TestCase): + def test_invalid_syntax_single_line(self): + namespace = {} + code = "if foo" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_empty_line(self): + namespace = {} + code = "" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_valid_single_statement(self): + namespace = {} + code = "foo = 1" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiline_single_assignment(self): + namespace = {} + code = dedent("""\ + foo = [ + 1, + 2, + 3, + ]""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiline_single_block(self): + namespace = {} + code = dedent("""\ + def foo(): + '''docs''' + + return 1""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_statements_single_line(self): + namespace = {} + code = "foo = 1;bar = 2" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiple_statements(self): + namespace = {} + code = dedent("""\ + import time + + foo = 1""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_blocks(self): + namespace = {} + code = dedent("""\ + from dataclasses import dataclass + + @dataclass + class Point: + x: float + y: float""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_blocks_empty_newline(self): + namespace = {} + code = dedent("""\ + from dataclasses import dataclass + + @dataclass + class Point: + x: float + y: float + """) + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiple_blocks_indented_newline(self): + namespace = {} + code = ( + "from dataclasses import dataclass\n" + "\n" + "@dataclass\n" + "class Point:\n" + " x: float\n" + " y: float\n" + " " + ) + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_incomplete_statement(self): + namespace = {} + code = "if foo:" + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 015b690566223d..3a1bacef8a1756 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -2,6 +2,7 @@ import itertools import os import pathlib +import re import rlcompleter import select import subprocess @@ -21,6 +22,8 @@ more_lines, multiline_input, code_to_events, + clean_screen, + make_clean_env, ) from _pyrepl.console import Event from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig @@ -483,13 +486,35 @@ def prepare_reader(self, events): console = FakeConsole(events) config = ReadlineConfig(readline_completer=None) reader = ReadlineAlikeReader(console=console, config=config) + reader.can_colorize = False return reader + def test_stdin_is_tty(self): + # Used during test log analysis to figure out if a TTY was available. + try: + if os.isatty(sys.stdin.fileno()): + return + except OSError as ose: + self.skipTest(f"stdin tty check failed: {ose}") + else: + self.skipTest("stdin is not a tty") + + def test_stdout_is_tty(self): + # Used during test log analysis to figure out if a TTY was available. + try: + if os.isatty(sys.stdout.fileno()): + return + except OSError as ose: + self.skipTest(f"stdout tty check failed: {ose}") + else: + self.skipTest("stdout is not a tty") + def test_basic(self): reader = self.prepare_reader(code_to_events("1+1\n")) output = multiline_input(reader) self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") def test_multiline_edit(self): events = itertools.chain( @@ -519,8 +544,10 @@ def test_multiline_edit(self): output = multiline_input(reader) self.assertEqual(output, "def f():\n ...\n ") + self.assertEqual(clean_screen(reader.screen), "def f():\n ...") output = multiline_input(reader) self.assertEqual(output, "def g():\n pass\n ") + self.assertEqual(clean_screen(reader.screen), "def g():\n pass") def test_history_navigation_with_up_arrow(self): events = itertools.chain( @@ -539,12 +566,40 @@ def test_history_navigation_with_up_arrow(self): output = multiline_input(reader) self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") output = multiline_input(reader) self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") output = multiline_input(reader) self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") output = multiline_input(reader) self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") + + def test_history_with_multiline_entries(self): + code = "def foo():\nx = 1\ny = 2\nz = 3\n\ndef bar():\nreturn 42\n\n" + events = list(itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + )) + + reader = self.prepare_reader(events) + output = multiline_input(reader) + output = multiline_input(reader) + output = multiline_input(reader) + self.assertEqual( + clean_screen(reader.screen), + 'def foo():\n x = 1\n y = 2\n z = 3' + ) + self.assertEqual(output, "def foo():\n x = 1\n y = 2\n z = 3\n ") + def test_history_navigation_with_down_arrow(self): events = itertools.chain( @@ -562,6 +617,7 @@ def test_history_navigation_with_down_arrow(self): output = multiline_input(reader) self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") def test_history_search(self): events = itertools.chain( @@ -578,18 +634,23 @@ def test_history_search(self): output = multiline_input(reader) self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") output = multiline_input(reader) self.assertEqual(output, "2+2") + self.assertEqual(clean_screen(reader.screen), "2+2") output = multiline_input(reader) self.assertEqual(output, "3+3") + self.assertEqual(clean_screen(reader.screen), "3+3") output = multiline_input(reader) self.assertEqual(output, "1+1") + self.assertEqual(clean_screen(reader.screen), "1+1") def test_control_character(self): events = code_to_events("c\x1d\n") reader = self.prepare_reader(events) output = multiline_input(reader) self.assertEqual(output, "c\x1d") + self.assertEqual(clean_screen(reader.screen), "c") class TestPyReplCompleter(TestCase): @@ -845,11 +906,19 @@ def test_bracketed_paste_single_line(self): @skipUnless(pty, "requires pty") class TestMain(TestCase): + def setUp(self): + # Cleanup from PYTHON* variables to isolate from local + # user settings, see #121359. Such variables should be + # added later in test methods to patched os.environ. + patcher = patch('os.environ', new=make_clean_env()) + self.addCleanup(patcher.stop) + patcher.start() + @force_not_colorized def test_exposed_globals_in_repl(self): pre = "['__annotations__', '__builtins__'" post = "'__loader__', '__name__', '__package__', '__spec__']" - output, exit_code = self.run_repl(["sorted(dir())", "exit"]) + output, exit_code = self.run_repl(["sorted(dir())", "exit()"]) if "can't use pyrepl" in output: self.skipTest("pyrepl not available") self.assertEqual(exit_code, 0) @@ -868,6 +937,84 @@ def test_exposed_globals_in_repl(self): self.assertTrue(case1 or case2 or case3 or case4, output) + def _assertMatchOK( + self, var: str, expected: str | re.Pattern, actual: str + ) -> None: + if isinstance(expected, re.Pattern): + self.assertTrue( + expected.match(actual), + f"{var}={actual} does not match {expected.pattern}", + ) + else: + self.assertEqual( + actual, + expected, + f"expected {var}={expected}, got {var}={actual}", + ) + + @force_not_colorized + def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False): + clean_env = make_clean_env() + clean_env["NO_COLOR"] = "1" # force_not_colorized doesn't touch subprocesses + + with tempfile.TemporaryDirectory() as td: + blue = pathlib.Path(td) / "blue" + blue.mkdir() + mod = blue / "calx.py" + mod.write_text("FOO = 42", encoding="utf-8") + commands = [ + "print(f'^{" + var + "=}')" for var in expectations + ] + ["exit()"] + if as_file and as_module: + self.fail("as_file and as_module are mutually exclusive") + elif as_file: + output, exit_code = self.run_repl( + commands, + cmdline_args=[str(mod)], + env=clean_env, + ) + elif as_module: + output, exit_code = self.run_repl( + commands, + cmdline_args=["-m", "blue.calx"], + env=clean_env, + cwd=td, + ) + else: + self.fail("Choose one of as_file or as_module") + + if "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + + self.assertEqual(exit_code, 0) + for var, expected in expectations.items(): + with self.subTest(var=var, expected=expected): + if m := re.search(rf"\^{var}=(.+?)[\r\n]", output): + self._assertMatchOK(var, expected, actual=m.group(1)) + else: + self.fail(f"{var}= not found in output: {output!r}\n\n{output}") + + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + def test_inspect_keeps_globals_from_inspected_file(self): + expectations = { + "FOO": "42", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, as_file=True) + + def test_inspect_keeps_globals_from_inspected_module(self): + expectations = { + "FOO": "42", + "__name__": "'__main__'", + "__package__": "'blue'", + "__file__": re.compile(r"^'.*calx.py'$"), + } + self._run_repl_globals_test(expectations, as_module=True) + def test_dumb_terminal_exits_cleanly(self): env = os.environ.copy() env.update({"TERM": "dumb"}) @@ -929,36 +1076,54 @@ def test_not_wiping_history_file(self): self.assertIn("spam", output) self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) - def run_repl(self, repl_input: str | list[str], env: dict | None = None) -> tuple[str, int]: + def run_repl( + self, + repl_input: str | list[str], + env: dict | None = None, + *, + cmdline_args: list[str] | None = None, + cwd: str | None = None, + ) -> tuple[str, int]: + assert pty master_fd, slave_fd = pty.openpty() cmd = [sys.executable, "-i", "-u"] if env is None: cmd.append("-I") + if cmdline_args is not None: + cmd.extend(cmdline_args) process = subprocess.Popen( cmd, stdin=slave_fd, stdout=slave_fd, stderr=slave_fd, + cwd=cwd, text=True, close_fds=True, env=env if env else os.environ, - ) + ) + os.close(slave_fd) if isinstance(repl_input, list): repl_input = "\n".join(repl_input) + "\n" os.write(master_fd, repl_input.encode("utf-8")) output = [] - while select.select([master_fd], [], [], 0.5)[0]: - data = os.read(master_fd, 1024).decode("utf-8") - if not data: + while select.select([master_fd], [], [], SHORT_TIMEOUT)[0]: + try: + data = os.read(master_fd, 1024).decode("utf-8") + if not data: + break + except OSError: break output.append(data) + else: + os.close(master_fd) + process.kill() + self.fail(f"Timeout while waiting for output, got: {''.join(output)}") os.close(master_fd) - os.close(slave_fd) try: exit_code = process.wait(timeout=SHORT_TIMEOUT) except subprocess.TimeoutExpired: process.kill() exit_code = process.wait() - return "\n".join(output), exit_code + return "".join(output), exit_code diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 986bc36d9a1070..e82c3ca0bb5cc2 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -88,6 +88,12 @@ def test_setpos_for_xy_simple(self): reader.setpos_from_xy(0, 0) self.assertEqual(reader.pos, 0) + def test_control_characters(self): + code = 'flag = "🏳️‍🌈"' + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, 'flag = "🏳️\\u200d🌈"') + def test_setpos_from_xy_multiple_lines(self): # fmt: off code = ( diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 5e0e6f8dfac651..91fd7dd13f9063 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -132,6 +132,32 @@ def test_nonascii_history(self): self.assertEqual(readline.get_history_item(1), "entrée 1") self.assertEqual(readline.get_history_item(2), "entrée 22") + def test_write_read_limited_history(self): + previous_length = readline.get_history_length() + self.addCleanup(readline.set_history_length, previous_length) + + readline.clear_history() + readline.add_history("first line") + readline.add_history("second line") + readline.add_history("third line") + + readline.set_history_length(2) + self.assertEqual(readline.get_history_length(), 2) + readline.write_history_file(TESTFN) + self.addCleanup(os.remove, TESTFN) + + readline.clear_history() + self.assertEqual(readline.get_current_history_length(), 0) + self.assertEqual(readline.get_history_length(), 2) + + readline.read_history_file(TESTFN) + self.assertEqual(readline.get_history_item(1), "second line") + self.assertEqual(readline.get_history_item(2), "third line") + self.assertEqual(readline.get_history_item(3), None) + + # Readline seems to report an additional history element. + self.assertIn(readline.get_current_history_length(), (2, 3)) + class TestReadline(unittest.TestCase): @@ -323,6 +349,26 @@ def test_history_size(self): self.assertEqual(len(lines), history_size) self.assertEqual(lines[-1].strip(), b"last input") + def test_write_read_limited_history(self): + previous_length = readline.get_history_length() + self.addCleanup(readline.set_history_length, previous_length) + + readline.add_history("first line") + readline.add_history("second line") + readline.add_history("third line") + + readline.set_history_length(2) + self.assertEqual(readline.get_history_length(), 2) + readline.write_history_file(TESTFN) + self.addCleanup(os.remove, TESTFN) + + readline.read_history_file(TESTFN) + # Without clear_history() there's no good way to test if + # the correct entries are present (we're combining history limiting and + # possible deduplication with arbitrary previous content). + # So, we've only tested that the read did not fail. + # See TestHistoryManipulation for the full test. + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 1caf09ceaf10fc..0b938623856e4f 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -1,15 +1,27 @@ """Test the interactive interpreter.""" import os +import select import subprocess import sys import unittest from textwrap import dedent from test import support -from test.support import cpython_only, has_subprocess_support, SuppressCrashReport -from test.support.script_helper import kill_python, assert_python_ok +from test.support import ( + cpython_only, + has_subprocess_support, + os_helper, + SuppressCrashReport, + SHORT_TIMEOUT, +) +from test.support.script_helper import kill_python from test.support.import_helper import import_module +try: + import pty +except ImportError: + pty = None + if not has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") @@ -195,9 +207,56 @@ def bar(x): expected = "(30, None, [\'def foo(x):\\n\', \' return x + 1\\n\', \'\\n\'], \'\')" self.assertIn(expected, output, expected) + def test_asyncio_repl_reaches_python_startup_script(self): + with os_helper.temp_dir() as tmpdir: + script = os.path.join(tmpdir, "pythonstartup.py") + with open(script, "w") as f: + f.write("print('pythonstartup done!')" + os.linesep) + f.write("exit(0)" + os.linesep) + + env = os.environ.copy() + env["PYTHONSTARTUP"] = script + subprocess.check_call( + [sys.executable, "-m", "asyncio"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + timeout=SHORT_TIMEOUT, + ) + + @unittest.skipUnless(pty, "requires pty") def test_asyncio_repl_is_ok(self): - assert_python_ok("-m", "asyncio") + m, s = pty.openpty() + cmd = [sys.executable, "-m", "asyncio"] + proc = subprocess.Popen( + cmd, + stdin=s, + stdout=s, + stderr=s, + text=True, + close_fds=True, + env=os.environ, + ) + os.close(s) + os.write(m, b"await asyncio.sleep(0)\n") + os.write(m, b"exit()\n") + output = [] + while select.select([m], [], [], SHORT_TIMEOUT)[0]: + try: + data = os.read(m, 1024).decode("utf-8") + if not data: + break + except OSError: + break + output.append(data) + os.close(m) + try: + exit_code = proc.wait(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + exit_code = proc.wait() + self.assertEqual(exit_code, 0) class TestInteractiveModeSyntaxErrors(unittest.TestCase): diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 6e46dfa96a664f..24a366efc6ca05 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -810,6 +810,30 @@ def dig(self): gc_collect() # For PyPy or other GCs. self.assertIsNone(ref()) + def test_multiple_nesting(self): + # Regression test for https://github.com/python/cpython/issues/121863 + class MultiplyNested: + def f1(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g(_MultiplyNested__arg=2) + + def f2(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g + + inst = MultiplyNested() + with self.assertRaises(TypeError): + inst.f1() + + closure = inst.f2() + with self.assertRaises(TypeError): + closure(_MultiplyNested__arg=2) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index d9102eb98a54a6..a8531d466e56e7 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -635,6 +635,16 @@ def __le__(self, some_set): myset >= myobj self.assertTrue(myobj.le_called) + def test_set_membership(self): + myfrozenset = frozenset(range(3)) + myset = {myfrozenset, "abc", 1} + self.assertIn(set(range(3)), myset) + self.assertNotIn(set(range(1)), myset) + myset.discard(set(range(3))) + self.assertEqual(myset, {"abc", 1}) + self.assertRaises(KeyError, myset.remove, set(range(1))) + self.assertRaises(KeyError, myset.remove, set(range(3))) + class SetSubclass(set): pass diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 02ef172613bf94..c770be21b41c2b 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -905,10 +905,10 @@ def test_copytree_arg_types_of_ignore(self): os.mkdir(os.path.join(src_dir, 'test_dir', 'subdir')) write_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') - invokations = [] + invocations = [] def _ignore(src, names): - invokations.append(src) + invocations.append(src) self.assertIsInstance(src, str) self.assertIsInstance(names, list) self.assertEqual(len(names), len(set(names))) @@ -933,7 +933,7 @@ def _ignore(src, names): self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) - self.assertEqual(len(invokations), 9) + self.assertEqual(len(invocations), 9) def test_copytree_retains_permissions(self): tmp_dir = self.mkdtemp() @@ -2145,9 +2145,6 @@ def check_unpack_archive_with_converter(self, format, converter, **kwargs): def check_unpack_tarball(self, format): self.check_unpack_archive(format, filter='fully_trusted') self.check_unpack_archive(format, filter='data') - with warnings_helper.check_warnings( - ('Python 3.14', DeprecationWarning)): - self.check_unpack_archive(format) def test_unpack_archive_tar(self): self.check_unpack_tarball('tar') diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index bcdc232c712071..035913cdd05f34 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -328,13 +328,13 @@ def test_getsitepackages(self): if sys.platlibdir != "lib": self.assertEqual(len(dirs), 2) wanted = os.path.join('xoxo', sys.platlibdir, - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[0], wanted) else: self.assertEqual(len(dirs), 1) wanted = os.path.join('xoxo', 'lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[-1], wanted) else: diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 293baccaf1831d..488b401fb0054d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1434,7 +1434,7 @@ def test_blob_sequence_not_supported(self): self.blob + self.blob with self.assertRaisesRegex(TypeError, "unsupported operand"): self.blob * 5 - with self.assertRaisesRegex(TypeError, "is not iterable"): + with self.assertRaisesRegex(TypeError, "is not.+iterable"): b"a" in self.blob def test_blob_context_manager(self): diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index c374c947e02a6b..89201c647dfe51 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1072,7 +1072,7 @@ def test_no_inplace_modifications(self): def test_order_doesnt_matter(self): # Test that the order of data points doesn't change the result. - # CAUTION: due to floating point rounding errors, the result actually + # CAUTION: due to floating-point rounding errors, the result actually # may depend on the order. Consider this test representing an ideal. # To avoid this test failing, only test with exact values such as ints # or Fractions. diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 8b69cd03ba7f24..9412a2d737bb2e 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1407,7 +1407,7 @@ def open_fds(): t = threading.Thread(target=open_fds) t.start() try: - with self.assertRaises(EnvironmentError): + with self.assertRaises(OSError): subprocess.Popen(NONEXISTING_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 0cc192655931ba..bd367c1591c744 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -1,6 +1,9 @@ """ Test the API of the symtable module. """ + +import re +import textwrap import symtable import unittest @@ -356,26 +359,86 @@ def test_name(self): self.assertEqual(self.spam.lookup("x").get_name(), "x") self.assertEqual(self.Mine.get_name(), "Mine") - def test_class_info(self): - self.assertEqual(self.Mine.get_methods(), ('a_method',)) + def test_class_get_methods(self): + deprecation_mess = ( + re.escape('symtable.Class.get_methods() is deprecated ' + 'and will be removed in Python 3.16.') + ) + + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(self.Mine.get_methods(), ('a_method',)) top = symtable.symtable(TEST_COMPLEX_CLASS_CODE, "?", "exec") this = find_block(top, "ComplexClass") - self.assertEqual(this.get_methods(), ( - 'a_method', 'a_method_pep_695', - 'an_async_method', 'an_async_method_pep_695', - 'a_classmethod', 'a_classmethod_pep_695', - 'an_async_classmethod', 'an_async_classmethod_pep_695', - 'a_staticmethod', 'a_staticmethod_pep_695', - 'an_async_staticmethod', 'an_async_staticmethod_pep_695', - 'a_fakemethod', 'a_fakemethod_pep_695', - 'an_async_fakemethod', 'an_async_fakemethod_pep_695', - 'glob_unassigned_meth', 'glob_unassigned_meth_pep_695', - 'glob_unassigned_async_meth', 'glob_unassigned_async_meth_pep_695', - 'glob_assigned_meth', 'glob_assigned_meth_pep_695', - 'glob_assigned_async_meth', 'glob_assigned_async_meth_pep_695', - )) + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(this.get_methods(), ( + 'a_method', 'a_method_pep_695', + 'an_async_method', 'an_async_method_pep_695', + 'a_classmethod', 'a_classmethod_pep_695', + 'an_async_classmethod', 'an_async_classmethod_pep_695', + 'a_staticmethod', 'a_staticmethod_pep_695', + 'an_async_staticmethod', 'an_async_staticmethod_pep_695', + 'a_fakemethod', 'a_fakemethod_pep_695', + 'an_async_fakemethod', 'an_async_fakemethod_pep_695', + 'glob_unassigned_meth', 'glob_unassigned_meth_pep_695', + 'glob_unassigned_async_meth', 'glob_unassigned_async_meth_pep_695', + 'glob_assigned_meth', 'glob_assigned_meth_pep_695', + 'glob_assigned_async_meth', 'glob_assigned_async_meth_pep_695', + )) + + # Test generator expressions that are of type TYPE_FUNCTION + # but will not be reported by get_methods() since they are + # not functions per se. + # + # Other kind of comprehensions such as list, set or dict + # expressions do not have the TYPE_FUNCTION type. + + def check_body(body, expected_methods): + indented = textwrap.indent(body, ' ' * 4) + top = symtable.symtable(f"class A:\n{indented}", "?", "exec") + this = find_block(top, "A") + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(this.get_methods(), expected_methods) + + # statements with 'genexpr' inside it + GENEXPRS = ( + 'x = (x for x in [])', + 'x = (x async for x in [])', + 'type x[genexpr = (x for x in [])] = (x for x in [])', + 'type x[genexpr = (x async for x in [])] = (x async for x in [])', + 'genexpr = (x for x in [])', + 'genexpr = (x async for x in [])', + 'type genexpr[genexpr = (x for x in [])] = (x for x in [])', + 'type genexpr[genexpr = (x async for x in [])] = (x async for x in [])', + ) + + for gen in GENEXPRS: + # test generator expression + with self.subTest(gen=gen): + check_body(gen, ()) + + # test generator expression + variable named 'genexpr' + with self.subTest(gen=gen, isvar=True): + check_body('\n'.join((gen, 'genexpr = 1')), ()) + check_body('\n'.join(('genexpr = 1', gen)), ()) + + for paramlist in ('()', '(x)', '(x, y)', '(z: T)'): + for func in ( + f'def genexpr{paramlist}:pass', + f'async def genexpr{paramlist}:pass', + f'def genexpr[T]{paramlist}:pass', + f'async def genexpr[T]{paramlist}:pass', + ): + with self.subTest(func=func): + # test function named 'genexpr' + check_body(func, ('genexpr',)) + + for gen in GENEXPRS: + with self.subTest(gen=gen, func=func): + # test generator expression + function named 'genexpr' + check_body('\n'.join((gen, func)), ('genexpr',)) + check_body('\n'.join((func, gen)), ('genexpr',)) def test_filename_correct(self): ### Bug tickler: SyntaxError file name correct whether error raised diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index cdeb26adf34d89..206b7f0088a925 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -59,6 +59,18 @@ Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> def __debug__(): pass +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> async def __debug__(): pass +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> class __debug__: pass +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + >>> del __debug__ Traceback (most recent call last): SyntaxError: cannot delete __debug__ @@ -786,6 +798,9 @@ >>> __debug__: int Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> x.__debug__: int +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ >>> f(a=) Traceback (most recent call last): SyntaxError: expected argument value expression @@ -1182,6 +1197,24 @@ Traceback (most recent call last): SyntaxError: expected ':' + >>> match x: + ... case a, __debug__, b: + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> match x: + ... case a, b, *__debug__: + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> match x: + ... case Foo(a, __debug__=1, b=2): + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + >>> if x = 3: ... pass Traceback (most recent call last): @@ -1275,6 +1308,15 @@ Traceback (most recent call last): SyntaxError: expected 'except' or 'finally' block +Custom error message for __debug__ as exception variable + + >>> try: + ... pass + ... except TypeError as __debug__: + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + Custom error message for try block mixing except and except* >>> try: @@ -1522,6 +1564,19 @@ Traceback (most recent call last): IndentationError: expected an indented block after class definition on line 1 + >>> class C(__debug__=42): ... + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> class Meta(type): + ... def __new__(*args, **kwargs): + ... pass + + >>> class C(metaclass=Meta, __debug__=42): + ... pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + >>> match something: ... pass Traceback (most recent call last): @@ -1708,6 +1763,26 @@ Traceback (most recent call last): SyntaxError: Did you mean to use 'from ... import ...' instead? +>>> import __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> import a as __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> import a.b.c as __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> from a import __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + +>>> from a import b as __debug__ +Traceback (most recent call last): +SyntaxError: cannot assign to __debug__ + # Check that we dont raise the "trailing comma" error if there is more # input to the left of the valid part that we parsed. @@ -2186,6 +2261,14 @@ def f(x: *b) ... SyntaxError: yield expression cannot be used within a type alias + >>> type __debug__ = int + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + + >>> class A[__debug__]: pass + Traceback (most recent call last): + SyntaxError: cannot assign to __debug__ + >>> class A[T]((x := 3)): ... Traceback (most recent call last): ... diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 69dccf9a9322f6..7e4bc980b390f7 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1603,7 +1603,8 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi2cP7P2ic??2P')) + INTERPRETER_FRAME = '9PhcP' + check(x, size('3PiccPP' + INTERPRETER_FRAME + 'P')) # function def func(): pass check(func, size('16Pi')) @@ -1620,7 +1621,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('PP4P4c7P2ic??2P')) + check(get_gen(), size('6P4c' + INTERPRETER_FRAME + 'P')) # iterator check(iter('abc'), size('lP')) # callable-iterator diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index 32e03d7cd25dbe..b2e8e8a15b67ea 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -479,6 +479,20 @@ def f(): sys.setprofile(lambda *args: None) f() + def test_method_with_c_function(self): + # gh-122029 + # When we have a PyMethodObject whose im_func is a C function, we + # should record both the call and the return. f = classmethod(repr) + # is just a way to create a PyMethodObject with a C function. + class A: + f = classmethod(repr) + events = [] + sys.setprofile(lambda frame, event, args: events.append(event)) + A().f() + sys.setprofile(None) + # The last c_call is the call to sys.setprofile + self.assertEqual(events, ['c_call', 'c_return', 'c_call']) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index c622fd9ce7c466..95cf0d1ec2d9ab 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2857,7 +2857,7 @@ def test_no_jump_from_exception_event(output): output.append(1) 1 / 0 - @jump_test(3, 2, [2, 5], event='return') + @jump_test(3, 2, [2, 2, 5], event='return') def test_jump_from_yield(output): def gen(): output.append(2) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 9233304c6a5327..1ade49281b4e26 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -157,7 +157,7 @@ def test_posix_venv_scheme(self): binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') # Resolve the paths in an imaginary venv/ directory @@ -417,8 +417,8 @@ def test_user_similar(self): if name == 'platlib': # Replace "/lib64/python3.11/site-packages" suffix # with "/lib/python3.11/site-packages". - py_version_short = sysconfig.get_python_version() - suffix = f'python{py_version_short}/site-packages' + py_version_abi = sysconfig._get_python_version_abi() + suffix = f'python{py_version_abi}/site-packages' expected = expected.replace(f'/{sys.platlibdir}/{suffix}', f'/lib/{suffix}') self.assertEqual(user_path, expected) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index f715940de1d584..5600f0746770b8 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -722,6 +722,24 @@ def format_mtime(mtime): tar.close() os_helper.rmtree(DIR) + @staticmethod + def test_extractall_default_filter(): + # Test that the default filter is now "data", and the other filter types are not used. + DIR = pathlib.Path(TEMPDIR) / "extractall_default_filter" + with ( + os_helper.temp_dir(DIR), + tarfile.open(tarname, encoding="iso8859-1") as tar, + unittest.mock.patch("tarfile.data_filter", wraps=tarfile.data_filter) as mock_data_filter, + unittest.mock.patch("tarfile.tar_filter", wraps=tarfile.tar_filter) as mock_tar_filter, + unittest.mock.patch("tarfile.fully_trusted_filter", wraps=tarfile.fully_trusted_filter) as mock_ft_filter + ): + directories = [t for t in tar if t.isdir()] + tar.extractall(DIR, directories) + + mock_data_filter.assert_called() + mock_ft_filter.assert_not_called() + mock_tar_filter.assert_not_called() + @os_helper.skip_unless_working_chmod def test_extract_directory(self): dirtype = "ustar/dirtype" @@ -738,31 +756,6 @@ def test_extract_directory(self): finally: os_helper.rmtree(DIR) - def test_deprecation_if_no_filter_passed_to_extractall(self): - DIR = pathlib.Path(TEMPDIR) / "extractall" - with ( - os_helper.temp_dir(DIR), - tarfile.open(tarname, encoding="iso8859-1") as tar - ): - directories = [t for t in tar if t.isdir()] - with self.assertWarnsRegex(DeprecationWarning, "Use the filter argument") as cm: - tar.extractall(DIR, directories) - # check that the stacklevel of the deprecation warning is correct: - self.assertEqual(cm.filename, __file__) - - def test_deprecation_if_no_filter_passed_to_extract(self): - dirtype = "ustar/dirtype" - DIR = pathlib.Path(TEMPDIR) / "extractall" - with ( - os_helper.temp_dir(DIR), - tarfile.open(tarname, encoding="iso8859-1") as tar - ): - tarinfo = tar.getmember(dirtype) - with self.assertWarnsRegex(DeprecationWarning, "Use the filter argument") as cm: - tar.extract(tarinfo, path=DIR) - # check that the stacklevel of the deprecation warning is correct: - self.assertEqual(cm.filename, __file__) - def test_extractall_pathlike_dir(self): DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ @@ -4011,15 +4004,6 @@ def test_data_filter(self): self.assertIs(filtered.name, tarinfo.name) self.assertIs(filtered.type, tarinfo.type) - def test_default_filter_warns(self): - """Ensure the default filter warns""" - with ArchiveMaker() as arc: - arc.add('foo') - with warnings_helper.check_warnings( - ('Python 3.14', DeprecationWarning)): - with self.check_context(arc.open(), None): - self.expect_file('foo') - def test_change_default_filter_on_instance(self): tar = tarfile.TarFile(tarname, 'r') def strict_filter(tarinfo, path): diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 51aeb35f01065a..de0e0b430a21bf 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -228,7 +228,7 @@ def test_long(self): """) def test_float(self): - # Floating point numbers + # Floating-point numbers self.check_tokenize("x = 3.14159", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index a9be1f5aa84681..91082e6b23c04b 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,12 +1,9 @@ +import annotationlib import textwrap import types import unittest from test.support import run_code, check_syntax_error -VALUE = 1 -FORWARDREF = 2 -SOURCE = 3 - class TypeAnnotationTests(unittest.TestCase): @@ -376,12 +373,12 @@ class X: self.assertIsInstance(annotate, types.FunctionType) self.assertEqual(annotate.__name__, "__annotate__") with self.assertRaises(NotImplementedError): - annotate(FORWARDREF) + annotate(annotationlib.Format.FORWARDREF) with self.assertRaises(NotImplementedError): - annotate(SOURCE) + annotate(annotationlib.Format.SOURCE) with self.assertRaises(NotImplementedError): annotate(None) - self.assertEqual(annotate(VALUE), {"x": int}) + self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) def test_comprehension_in_annotation(self): # This crashed in an earlier version of the code @@ -398,7 +395,7 @@ def f(x: int) -> int: pass f = ns["f"] self.assertIsInstance(f.__annotate__, types.FunctionType) annos = {"x": "int", "return": "int"} - self.assertEqual(f.__annotate__(VALUE), annos) + self.assertEqual(f.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(f.__annotations__, annos) def test_name_clash_with_format(self): diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 89632a3abebfb5..66abe73f8d766d 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -161,8 +161,8 @@ def load_foo_2(type_): self._check_specialization(load_foo_2, A, "LOAD_ATTR", should_specialize=False) def test_class_load_attr_specialization_static_type(self): - self._assign_valid_version_or_skip(str) - self._assign_valid_version_or_skip(bytes) + self.assertNotEqual(type_get_version(str), 0) + self.assertNotEqual(type_get_version(bytes), 0) def get_capitalize_1(type_): return type_.capitalize @@ -170,25 +170,6 @@ def get_capitalize_1(type_): self._check_specialization(get_capitalize_1, str, "LOAD_ATTR", should_specialize=True) self.assertEqual(get_capitalize_1(str)('hello'), 'Hello') self.assertEqual(get_capitalize_1(bytes)(b'hello'), b'Hello') - del get_capitalize_1 - - # Permanently overflow the static type version counter, and force str and bytes - # to have tp_version_tag == 0 - for _ in range(2**16): - type_modified(str) - type_assign_version(str) - type_modified(bytes) - type_assign_version(bytes) - - self.assertEqual(type_get_version(str), 0) - self.assertEqual(type_get_version(bytes), 0) - - def get_capitalize_2(type_): - return type_.capitalize - - self._check_specialization(get_capitalize_2, str, "LOAD_ATTR", should_specialize=False) - self.assertEqual(get_capitalize_2(str)('hello'), 'Hello') - self.assertEqual(get_capitalize_2(bytes)(b'hello'), b'Hello') def test_property_load_attr_specialization_user_type(self): class G: diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index bf1a34b9fc82b3..d9c9ec4eddc368 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1,3 +1,4 @@ +import annotationlib import asyncio import textwrap import types @@ -6,7 +7,7 @@ import weakref from test.support import requires_working_socket, check_syntax_error, run_code -from typing import Generic, NoDefault, Sequence, TypeVar, TypeVarTuple, ParamSpec, get_args +from typing import Generic, NoDefault, Sequence, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, get_args class TypeParamsInvalidTest(unittest.TestCase): @@ -1394,3 +1395,43 @@ def test_symtable_key_regression_name(self): self.assertEqual(ns["X1"].__type_params__[0].__default__, "A") self.assertEqual(ns["X2"].__type_params__[0].__default__, "B") + + +class TestEvaluateFunctions(unittest.TestCase): + def test_general(self): + type Alias = int + Alias2 = TypeAliasType("Alias2", int) + def f[T: int = int, **P = int, *Ts = int](): pass + T, P, Ts = f.__type_params__ + T2 = TypeVar("T2", bound=int, default=int) + P2 = ParamSpec("P2", default=int) + Ts2 = TypeVarTuple("Ts2", default=int) + cases = [ + Alias.evaluate_value, + Alias2.evaluate_value, + T.evaluate_bound, + T.evaluate_default, + P.evaluate_default, + Ts.evaluate_default, + T2.evaluate_bound, + T2.evaluate_default, + P2.evaluate_default, + Ts2.evaluate_default, + ] + for case in cases: + with self.subTest(case=case): + self.assertIs(case(1), int) + self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.VALUE), int) + self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) + self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.SOURCE), 'int') + + def test_constraints(self): + def f[T: (int, str)](): pass + T, = f.__type_params__ + T2 = TypeVar("T2", int, str) + for case in [T, T2]: + with self.subTest(case=case): + self.assertEqual(case.evaluate_constraints(1), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.VALUE), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.FORWARDREF), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.SOURCE), '(int, str)') diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index fbca198aab5180..a84f43ba9b51fd 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1,6 +1,9 @@ # Python test set -- part 6, built-in types -from test.support import run_with_locale, cpython_only, MISSING_C_DOCSTRINGS +from test.support import ( + run_with_locale, is_apple_mobile, cpython_only, no_rerun, + MISSING_C_DOCSTRINGS, +) import collections.abc from collections import namedtuple, UserDict import copy @@ -10,6 +13,7 @@ import pickle import locale import sys +import textwrap import types import unittest.mock import weakref @@ -28,6 +32,26 @@ def clear_typing_caches(): f() +def iter_builtin_types(): + for obj in __builtins__.values(): + if not isinstance(obj, type): + continue + cls = obj + if cls.__module__ != 'builtins': + continue + yield cls + + +@cpython_only +def iter_own_slot_wrappers(cls): + for name, value in vars(cls).items(): + if not name.startswith('__') or not name.endswith('__'): + continue + if 'slot wrapper' not in str(value): + continue + yield name + + class TypesTests(unittest.TestCase): def test_truth_values(self): @@ -2345,5 +2369,50 @@ def ex(a, /, b, *, c): ) +class SubinterpreterTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + global interpreters + try: + from test.support import interpreters + except ModuleNotFoundError: + raise unittest.SkipTest('subinterpreters required') + import test.support.interpreters.channels + + @cpython_only + @no_rerun('channels (and queues) might have a refleak; see gh-122199') + def test_slot_wrappers(self): + rch, sch = interpreters.channels.create() + + slots = [] + script = '' + for cls in iter_builtin_types(): + for slot in iter_own_slot_wrappers(cls): + slots.append((cls, slot)) + script += textwrap.dedent(f""" + text = repr({cls.__name__}.{slot}) + sch.send_nowait(({cls.__name__!r}, {slot!r}, text)) + """) + + exec(script) + all_expected = [] + for cls, slot in slots: + result = rch.recv() + assert result == (cls.__name__, slot, result[2]), (cls, slot, result) + all_expected.append(result) + + interp = interpreters.create() + interp.exec('from test.support import interpreters') + interp.prepare_main(sch=sch) + interp.exec(script) + + for i, _ in enumerate(slots): + with self.subTest(cls=cls, slot=slot): + expected = all_expected[i] + result = rch.recv() + self.assertEqual(result, expected) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a931da55908236..290b3c63a762e9 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1,3 +1,4 @@ +import annotationlib import contextlib import collections import collections.abc @@ -45,7 +46,7 @@ import weakref import types -from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper +from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper, run_code from test.typinganndata import ann_module695, mod_generics_cache, _typed_dict_helper @@ -7812,6 +7813,48 @@ class XMethBad2(NamedTuple): def _source(self): return 'no chance for this as well' + def test_annotation_type_check(self): + # These are rejected by _type_check + with self.assertRaises(TypeError): + class X(NamedTuple): + a: Final + with self.assertRaises(TypeError): + class Y(NamedTuple): + a: (1, 2) + + # Conversion by _type_convert + class Z(NamedTuple): + a: None + b: "str" + annos = {'a': type(None), 'b': ForwardRef("str")} + self.assertEqual(Z.__annotations__, annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.FORWARDREF), annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.SOURCE), {"a": "None", "b": "str"}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + from typing import NamedTuple + class X(NamedTuple): + a: int + b: None + """ + ns = run_code(textwrap.dedent(code)) + X = ns['X'] + self.assertEqual(X.__annotations__, {'a': ForwardRef("int"), 'b': ForwardRef("None")}) + + def test_deferred_annotations(self): + class X(NamedTuple): + y: undefined + + self.assertEqual(X._fields, ('y',)) + with self.assertRaises(NameError): + X.__annotations__ + + undefined = int + self.assertEqual(X.__annotations__, {'y': int}) + def test_multiple_inheritance(self): class A: pass @@ -8126,7 +8169,11 @@ def test_basics_functional_syntax(self): self.assertEqual(Emp.__name__, 'Emp') self.assertEqual(Emp.__module__, __name__) self.assertEqual(Emp.__bases__, (dict,)) - self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) + annos = {'name': str, 'id': int} + self.assertEqual(Emp.__annotations__, annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.FORWARDREF), annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.SOURCE), {'name': 'str', 'id': 'int'}) self.assertEqual(Emp.__total__, True) self.assertEqual(Emp.__required_keys__, {'name', 'id'}) self.assertIsInstance(Emp.__required_keys__, frozenset) @@ -8487,6 +8534,8 @@ class A[T](TypedDict): self.assertEqual(A.__bases__, (Generic, dict)) self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__annotations__, {'a': T}) + self.assertEqual(A.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8498,6 +8547,8 @@ class A(TypedDict, Generic[T]): self.assertEqual(A.__bases__, (Generic, dict)) self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__annotations__, {'a': T}) + self.assertEqual(A.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) self.assertEqual(A.__parameters__, (T,)) self.assertEqual(A[str].__parameters__, ()) self.assertEqual(A[str].__args__, (str,)) @@ -8508,6 +8559,8 @@ class A2(Generic[T], TypedDict): self.assertEqual(A2.__bases__, (Generic, dict)) self.assertEqual(A2.__orig_bases__, (Generic[T], TypedDict)) self.assertEqual(A2.__mro__, (A2, Generic, dict, object)) + self.assertEqual(A2.__annotations__, {'a': T}) + self.assertEqual(A2.__annotate__(annotationlib.Format.SOURCE), {'a': 'T'}) self.assertEqual(A2.__parameters__, (T,)) self.assertEqual(A2[str].__parameters__, ()) self.assertEqual(A2[str].__args__, (str,)) @@ -8518,6 +8571,8 @@ class B(A[KT], total=False): self.assertEqual(B.__bases__, (Generic, dict)) self.assertEqual(B.__orig_bases__, (A[KT],)) self.assertEqual(B.__mro__, (B, Generic, dict, object)) + self.assertEqual(B.__annotations__, {'a': T, 'b': KT}) + self.assertEqual(B.__annotate__(annotationlib.Format.SOURCE), {'a': 'T', 'b': 'KT'}) self.assertEqual(B.__parameters__, (KT,)) self.assertEqual(B.__total__, False) self.assertEqual(B.__optional_keys__, frozenset(['b'])) @@ -8542,6 +8597,11 @@ class C(B[int]): 'b': KT, 'c': int, }) + self.assertEqual(C.__annotate__(annotationlib.Format.SOURCE), { + 'a': 'T', + 'b': 'KT', + 'c': 'int', + }) with self.assertRaises(TypeError): C[str] @@ -8561,6 +8621,11 @@ class Point3D(Point2DGeneric[T], Generic[T, KT]): 'b': T, 'c': KT, }) + self.assertEqual(Point3D.__annotate__(annotationlib.Format.SOURCE), { + 'a': 'T', + 'b': 'T', + 'c': 'KT', + }) self.assertEqual(Point3D[int, str].__origin__, Point3D) with self.assertRaises(TypeError): @@ -8592,6 +8657,11 @@ class WithImplicitAny(B): 'b': KT, 'c': int, }) + self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.SOURCE), { + 'a': 'T', + 'b': 'KT', + 'c': 'int', + }) with self.assertRaises(TypeError): WithImplicitAny[str] @@ -8748,6 +8818,54 @@ class AllTheThings(TypedDict): }, ) + def test_annotations(self): + # _type_check is applied + with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): + class X(TypedDict): + a: Final + + # _type_convert is applied + class Y(TypedDict): + a: None + b: "int" + fwdref = ForwardRef('int', module='test.test_typing') + self.assertEqual(Y.__annotations__, {'a': type(None), 'b': fwdref}) + self.assertEqual(Y.__annotate__(annotationlib.Format.FORWARDREF), {'a': type(None), 'b': fwdref}) + + # _type_check is also applied later + class Z(TypedDict): + a: undefined + + with self.assertRaises(NameError): + Z.__annotations__ + + undefined = Final + with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): + Z.__annotations__ + + undefined = None + self.assertEqual(Z.__annotations__, {'a': type(None)}) + + def test_deferred_evaluation(self): + class A(TypedDict): + x: NotRequired[undefined] + y: ReadOnly[undefined] + z: Required[undefined] + + self.assertEqual(A.__required_keys__, frozenset({'y', 'z'})) + self.assertEqual(A.__optional_keys__, frozenset({'x'})) + self.assertEqual(A.__readonly_keys__, frozenset({'y'})) + self.assertEqual(A.__mutable_keys__, frozenset({'x', 'z'})) + + with self.assertRaises(NameError): + A.__annotations__ + + self.assertEqual( + A.__annotate__(annotationlib.Format.SOURCE), + {'x': 'NotRequired[undefined]', 'y': 'ReadOnly[undefined]', + 'z': 'Required[undefined]'}, + ) + class RequiredTests(BaseTestCase): @@ -10075,7 +10193,6 @@ def test_special_attrs(self): typing.ClassVar: 'ClassVar', typing.Concatenate: 'Concatenate', typing.Final: 'Final', - typing.ForwardRef: 'ForwardRef', typing.Literal: 'Literal', typing.NewType: 'NewType', typing.NoReturn: 'NoReturn', @@ -10087,7 +10204,7 @@ def test_special_attrs(self): typing.TypeVar: 'TypeVar', typing.Union: 'Union', typing.Self: 'Self', - # Subscribed special forms + # Subscripted special forms typing.Annotated[Any, "Annotation"]: 'Annotated', typing.Annotated[int, 'Annotation']: 'Annotated', typing.ClassVar[Any]: 'ClassVar', @@ -10102,7 +10219,6 @@ def test_special_attrs(self): typing.Union[Any]: 'Any', typing.Union[int, float]: 'Union', # Incompatible special forms (tested in test_special_attrs2) - # - typing.ForwardRef('set[Any]') # - typing.NewType('TypeName', Any) # - typing.ParamSpec('SpecialAttrsP') # - typing.TypeVar('T') @@ -10121,18 +10237,6 @@ def test_special_attrs(self): TypeName = typing.NewType('SpecialAttrsTests.TypeName', Any) def test_special_attrs2(self): - # Forward refs provide a different introspection API. __name__ and - # __qualname__ make little sense for forward refs as they can store - # complex typing expressions. - fr = typing.ForwardRef('set[Any]') - self.assertFalse(hasattr(fr, '__name__')) - self.assertFalse(hasattr(fr, '__qualname__')) - self.assertEqual(fr.__module__, 'typing') - # Forward refs are currently unpicklable. - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(TypeError): - pickle.dumps(fr, proto) - self.assertEqual(SpecialAttrsTests.TypeName.__name__, 'TypeName') self.assertEqual( SpecialAttrsTests.TypeName.__qualname__, diff --git a/Lib/test/test_unittest/test_util.py b/Lib/test/test_unittest/test_util.py new file mode 100644 index 00000000000000..d590a333930278 --- /dev/null +++ b/Lib/test/test_unittest/test_util.py @@ -0,0 +1,33 @@ +import unittest +from unittest.util import safe_repr, sorted_list_difference, unorderable_list_difference + + +class TestUtil(unittest.TestCase): + def test_safe_repr(self): + class RaisingRepr: + def __repr__(self): + raise ValueError("Invalid repr()") + + class LongRepr: + def __repr__(self): + return 'x' * 100 + + safe_repr(RaisingRepr()) + self.assertEqual(safe_repr('foo'), "'foo'") + self.assertEqual(safe_repr(LongRepr(), short=True), 'x'*80 + ' [truncated]...') + + def test_sorted_list_difference(self): + self.assertEqual(sorted_list_difference([], []), ([], [])) + self.assertEqual(sorted_list_difference([1, 2], [2, 3]), ([1], [3])) + self.assertEqual(sorted_list_difference([1, 2], [1, 3]), ([2], [3])) + self.assertEqual(sorted_list_difference([1, 1, 1], [1, 2, 3]), ([], [2, 3])) + self.assertEqual(sorted_list_difference([4], [1, 2, 3, 4]), ([], [1, 2, 3])) + self.assertEqual(sorted_list_difference([1, 1], [2]), ([1], [2])) + self.assertEqual(sorted_list_difference([2], [1, 1]), ([2], [1])) + self.assertEqual(sorted_list_difference([1, 2], [1, 1]), ([2], [])) + + def test_unorderable_list_difference(self): + self.assertEqual(unorderable_list_difference([], []), ([], [])) + self.assertEqual(unorderable_list_difference([1, 2], []), ([2, 1], [])) + self.assertEqual(unorderable_list_difference([], [1, 2]), ([], [1, 2])) + self.assertEqual(unorderable_list_difference([1, 2], [1, 3]), ([2], [3])) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 1769ed61b94075..2b7d297f011741 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -75,7 +75,7 @@ def setUp(self): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) + self.lib = ('lib', f'python{sysconfig._get_python_version_abi()}') self.include = 'include' executable = sys._base_executable self.exe = os.path.split(executable)[-1] @@ -593,7 +593,8 @@ def test_zippath_from_non_installed_posix(self): libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) os.makedirs(libdir) landmark = os.path.join(libdir, "os.py") - stdlib_zip = "python%d%d.zip" % sys.version_info[:2] + abi_thread = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "" + stdlib_zip = f"python{sys.version_info.major}{sys.version_info.minor}{abi_thread}" zip_landmark = os.path.join(non_installed_dir, platlibdir, stdlib_zip) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index f9b2b07fbd6576..95515a4b694315 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1,6 +1,8 @@ from contextlib import contextmanager import linecache import os +import importlib +import inspect from io import StringIO import re import sys @@ -886,37 +888,46 @@ def test_issue31285(self): # warn_explicit() should neither raise a SystemError nor cause an # assertion failure, in case the return value of get_source() has a # bad splitlines() method. - def get_bad_loader(splitlines_ret_val): + get_source_called = [] + def get_module_globals(*, splitlines_ret_val): + class BadSource(str): + def splitlines(self): + return splitlines_ret_val + class BadLoader: def get_source(self, fullname): - class BadSource(str): - def splitlines(self): - return splitlines_ret_val + get_source_called.append(splitlines_ret_val) return BadSource('spam') - return BadLoader() + + loader = BadLoader() + spec = importlib.machinery.ModuleSpec('foobar', loader) + return {'__loader__': loader, + '__spec__': spec, + '__name__': 'foobar'} + wmod = self.module with original_warnings.catch_warnings(module=wmod): wmod.filterwarnings('default', category=UserWarning) + linecache.clearcache() with support.captured_stderr() as stderr: wmod.warn_explicit( 'foo', UserWarning, 'bar', 1, - module_globals={'__loader__': get_bad_loader(42), - '__name__': 'foobar'}) + module_globals=get_module_globals(splitlines_ret_val=42)) self.assertIn('UserWarning: foo', stderr.getvalue()) + self.assertEqual(get_source_called, [42]) - show = wmod._showwarnmsg - try: + linecache.clearcache() + with support.swap_attr(wmod, '_showwarnmsg', None): del wmod._showwarnmsg with support.captured_stderr() as stderr: wmod.warn_explicit( 'eggs', UserWarning, 'bar', 1, - module_globals={'__loader__': get_bad_loader([42]), - '__name__': 'foobar'}) + module_globals=get_module_globals(splitlines_ret_val=[42])) self.assertIn('UserWarning: eggs', stderr.getvalue()) - finally: - wmod._showwarnmsg = show + self.assertEqual(get_source_called, [42, [42]]) + linecache.clearcache() @support.cpython_only def test_issue31411(self): @@ -1684,6 +1695,29 @@ def d(): pass isinstance(cell.cell_contents, deprecated) for cell in d.__closure__ )) + def test_inspect(self): + @deprecated("depr") + def sync(): + pass + + @deprecated("depr") + async def coro(): + pass + + class Cls: + @deprecated("depr") + def sync(self): + pass + + @deprecated("depr") + async def coro(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(sync)) + self.assertTrue(inspect.iscoroutinefunction(coro)) + self.assertFalse(inspect.iscoroutinefunction(Cls.sync)) + self.assertTrue(inspect.iscoroutinefunction(Cls.coro)) + def setUpModule(): py_warnings.onceregistry.clear() c_warnings.onceregistry.clear() diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index d70fa3e863b98c..1861616d5ec3bf 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -52,8 +52,11 @@ def module_path_to_dotted_name(path): TESTMOD = "ziptestmodule" +TESTMOD2 = "ziptestmodule2" +TESTMOD3 = "ziptestmodule3" TESTPACK = "ziptestpackage" TESTPACK2 = "ziptestpackage2" +TESTPACK3 = "ziptestpackage3" TEMP_DIR = os.path.abspath("junk95142") TEMP_ZIP = os.path.abspath("junk95142.zip") TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "zipimport_data") @@ -95,8 +98,10 @@ def makeTree(self, files, dirName=TEMP_DIR): # defined by files under the directory dirName. self.addCleanup(os_helper.rmtree, dirName) - for name, (mtime, data) in files.items(): - path = os.path.join(dirName, name) + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + path = os.path.join(dirName, *name.split('/')) if path[-1] == os.sep: if not os.path.isdir(path): os.makedirs(path) @@ -107,22 +112,18 @@ def makeTree(self, files, dirName=TEMP_DIR): with open(path, 'wb') as fp: fp.write(data) - def makeZip(self, files, zipName=TEMP_ZIP, **kw): + def makeZip(self, files, zipName=TEMP_ZIP, *, + comment=None, file_comment=None, stuff=None, prefix='', **kw): # Create a zip archive based set of modules/packages - # defined by files in the zip file zipName. If the - # key 'stuff' exists in kw it is prepended to the archive. + # defined by files in the zip file zipName. + # If stuff is not None, it is prepended to the archive. self.addCleanup(os_helper.unlink, zipName) - with ZipFile(zipName, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - z.writestr(zinfo, data) - comment = kw.get("comment", None) + with ZipFile(zipName, "w", compression=self.compression) as z: + self.writeZip(z, files, file_comment=file_comment, prefix=prefix) if comment is not None: z.comment = comment - stuff = kw.get("stuff", None) if stuff is not None: # Prepend 'stuff' to the start of the zipfile with open(zipName, "rb") as f: @@ -131,26 +132,47 @@ def makeZip(self, files, zipName=TEMP_ZIP, **kw): f.write(stuff) f.write(data) + def writeZip(self, z, files, *, file_comment=None, prefix=''): + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + else: + mtime = NOW + name = name.replace(os.sep, '/') + zinfo = ZipInfo(prefix + name, time.localtime(mtime)) + zinfo.compress_type = self.compression + if file_comment is not None: + zinfo.comment = file_comment + if data is None: + zinfo.CRC = 0 + z.mkdir(zinfo) + else: + assert name[-1] != '/' + z.writestr(zinfo, data) + def getZip64Files(self): # This is the simplest way to make zipfile generate the zip64 EOCD block - return {f"f{n}.py": (NOW, test_src) for n in range(65537)} + return {f"f{n}.py": test_src for n in range(65537)} def doTest(self, expected_ext, files, *modules, **kw): + if 'prefix' not in kw: + kw['prefix'] = 'pre/fix/' self.makeZip(files, **kw) self.doTestWithPreBuiltZip(expected_ext, *modules, **kw) - def doTestWithPreBuiltZip(self, expected_ext, *modules, **kw): - sys.path.insert(0, TEMP_ZIP) + def doTestWithPreBuiltZip(self, expected_ext, *modules, + call=None, prefix='', **kw): + zip_path = os.path.join(TEMP_ZIP, *prefix.split('/')[:-1]) + sys.path.insert(0, zip_path) mod = importlib.import_module(".".join(modules)) - call = kw.get('call') if call is not None: call(mod) if expected_ext: file = mod.get_file() - self.assertEqual(file, os.path.join(TEMP_ZIP, + self.assertEqual(file, os.path.join(zip_path, *modules) + expected_ext) def testAFakeZlib(self): @@ -176,7 +198,7 @@ def testAFakeZlib(self): self.skipTest('zlib is a builtin module') if "zlib" in sys.modules: del sys.modules["zlib"] - files = {"zlib.py": (NOW, test_src)} + files = {"zlib.py": test_src} try: self.doTest(".py", files, "zlib") except ImportError: @@ -187,16 +209,16 @@ def testAFakeZlib(self): self.fail("expected test to raise ImportError") def testPy(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD) def testPyc(self): - files = {TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testBoth(self): - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testUncheckedHashBasedPyc(self): @@ -229,22 +251,22 @@ def check(mod): self.doTest(None, files, TESTMOD, call=check) def testEmptyPy(self): - files = {TESTMOD + ".py": (NOW, "")} + files = {TESTMOD + ".py": ""} self.doTest(None, files, TESTMOD) def testBadMagic(self): # make pyc magic word invalid, forcing loading from .py badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badmagic_pyc} self.doTest(".py", files, TESTMOD) def testBadMagic2(self): # make pyc magic word invalid, causing an ImportError badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + pyc_ext: badmagic_pyc} try: self.doTest(".py", files, TESTMOD) self.fail("This should not be reached") @@ -257,22 +279,22 @@ def testBadMTime(self): # flip the second bit -- not the first as that one isn't stored in the # .py's mtime in the zip archive. badtime_pyc[11] ^= 0x02 - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badtime_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badtime_pyc} self.doTest(".py", files, TESTMOD) def test2038MTime(self): # Make sure we can handle mtimes larger than what a 32-bit signed number # can hold. twenty_thirty_eight_pyc = make_pyc(test_co, 2**32 - 1, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, twenty_thirty_eight_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: twenty_thirty_eight_pyc} self.doTest(".py", files, TESTMOD) def testPackage(self): packdir = TESTPACK + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTMOD) def testSubPackage(self): @@ -280,9 +302,9 @@ def testSubPackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) def testSubNamespacePackage(self): @@ -291,9 +313,9 @@ def testSubNamespacePackage(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep # The first two files are just directory entries (so have no data). - files = {packdir: (NOW, ""), - packdir2: (NOW, ""), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir: None, + packdir2: None, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) def testPackageExplicitDirectories(self): @@ -376,19 +398,19 @@ def testMixedNamespacePackage(self): # real filesystem and a zip archive. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) @@ -421,8 +443,8 @@ def testMixedNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-3]) @@ -439,13 +461,13 @@ def testMixedNamespacePackage(self): self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testNamespacePackage(self): @@ -453,22 +475,22 @@ def testNamespacePackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip2 = os.path.abspath("path2.zip") self.makeZip(files2, zip2) @@ -497,8 +519,8 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual("path2.zip", mod.__file__.split(os.sep)[-3]) # One level deeper... @@ -513,29 +535,22 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((subpkg, TESTMOD))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testZipImporterMethods(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(zi.archive, TEMP_ZIP) @@ -591,18 +606,12 @@ def testZipImporterMethods(self): def testInvalidateCaches(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} extra_files = [packdir, packdir2] - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) @@ -610,14 +619,10 @@ def testInvalidateCaches(self): zi.invalidate_caches() self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Add a new file to the ZIP archive - newfile = {"spam2" + pyc_ext: (NOW, test_pyc)} + newfile = {"spam2" + pyc_ext: test_pyc} files.update(newfile) - with ZipFile(TEMP_ZIP, "a") as z: - for name, (mtime, data) in newfile.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") # Check that we can detect the new file after invalidating the cache zi.invalidate_caches() self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) @@ -634,18 +639,12 @@ def testInvalidateCaches(self): def testInvalidateCachesWithMultipleZipimports(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} extra_files = [packdir, packdir2] - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) @@ -653,14 +652,10 @@ def testInvalidateCachesWithMultipleZipimports(self): zi2 = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) # Add a new file to the ZIP archive to make the cache wrong. - newfile = {"spam2" + pyc_ext: (NOW, test_pyc)} + newfile = {"spam2" + pyc_ext: test_pyc} files.update(newfile) - with ZipFile(TEMP_ZIP, "a") as z: - for name, (mtime, data) in newfile.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") # Invalidate the cache of the first zipimporter. zi.invalidate_caches() # Check that the second zipimporter detects the new file and isn't using a stale cache. @@ -672,16 +667,9 @@ def testInvalidateCachesWithMultipleZipimports(self): def testZipImporterMethodsInSubDirectory(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} - - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"eggs" - z.writestr(zinfo, data) + files = {packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"eggs") zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir) self.assertEqual(zi.archive, TEMP_ZIP) @@ -762,9 +750,9 @@ def get_file(): if __loader__.get_data("some.data") != b"some data": raise AssertionError("bad data")\n""" pyc = make_pyc(compile(src, "", "exec"), NOW, len(src)) - files = {TESTMOD + pyc_ext: (NOW, pyc), - "some.data": (NOW, "some data")} - self.doTest(pyc_ext, files, TESTMOD) + files = {TESTMOD + pyc_ext: pyc, + "some.data": "some data"} + self.doTest(pyc_ext, files, TESTMOD, prefix='') def testDefaultOptimizationLevel(self): # zipimport should use the default optimization level (#28131) @@ -772,7 +760,7 @@ def testDefaultOptimizationLevel(self): def test(val): assert(val) return val\n""" - files = {TESTMOD + '.py': (NOW, src)} + files = {TESTMOD + '.py': src} self.makeZip(files) sys.path.insert(0, TEMP_ZIP) mod = importlib.import_module(TESTMOD) @@ -785,7 +773,7 @@ def test(val): def testImport_WithStuff(self): # try importing from a zipfile which contains additional # stuff at the beginning of the file - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"Some Stuff"*31) @@ -793,18 +781,18 @@ def assertModuleSource(self, module): self.assertEqual(inspect.getsource(module), test_src) def testGetSource(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, call=self.assertModuleSource) def testGetCompiledSource(self): pyc = make_pyc(compile(test_src, "", "exec"), NOW, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: pyc} self.doTest(pyc_ext, files, TESTMOD, call=self.assertModuleSource) def runDoctest(self, callback): - files = {TESTMOD + ".py": (NOW, test_src), - "xyz.txt": (NOW, ">>> log.append(True)\n")} + files = {TESTMOD + ".py": test_src, + "xyz.txt": ">>> log.append(True)\n"} self.doTest(".py", files, TESTMOD, call=callback) def doDoctestFile(self, module): @@ -856,29 +844,21 @@ def doTraceback(self, module): raise AssertionError("This ought to be impossible") def testTraceback(self): - files = {TESTMOD + ".py": (NOW, raise_src)} + files = {TESTMOD + ".py": raise_src} self.doTest(None, files, TESTMOD, call=self.doTraceback) @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, "need an unencodable filename") def testUnencodable(self): filename = os_helper.TESTFN_UNENCODABLE + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) spec = zipimport.zipimporter(filename).find_spec(TESTMOD) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) def testBytesPath(self): filename = os_helper.TESTFN + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) zipimport.zipimporter(filename) with self.assertRaises(TypeError): @@ -889,15 +869,15 @@ def testBytesPath(self): zipimport.zipimporter(memoryview(os.fsencode(filename))) def testComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"comment") def testBeginningCruftAndComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"cruft" * 64, comment=b"hi") def testLargestPossibleComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"c" * ((1 << 16) - 1)) @support.requires_resource('cpu') diff --git a/Lib/threading.py b/Lib/threading.py index 31ab77c92b1c20..2dcdd0c9e067b6 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -336,7 +336,7 @@ def wait(self, timeout=None): awakened or timed out, it re-acquires the lock and returns. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an RLock, it is not released using its @@ -646,7 +646,7 @@ def wait(self, timeout=None): the optional timeout occurs. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). This method returns the internal flag on exit, so it will always return @@ -1059,7 +1059,7 @@ def join(self, timeout=None): or until the optional timeout occurs. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). As join() always returns None, you must call is_alive() after join() to decide whether a timeout happened -- if the thread is still alive, the join() call timed out. diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py index 0f0dc66460f798..6e5b025a9f9d7d 100644 --- a/Lib/tkinter/simpledialog.py +++ b/Lib/tkinter/simpledialog.py @@ -357,7 +357,7 @@ def askinteger(title, prompt, **kw): class _QueryFloat(_QueryDialog): - errormessage = "Not a floating point value." + errormessage = "Not a floating-point value." def getresult(self): return self.getdouble(self.entry.get()) diff --git a/Lib/typing.py b/Lib/typing.py index bc17d136082891..626053d8166160 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -19,6 +19,8 @@ """ from abc import abstractmethod, ABCMeta +import annotationlib +from annotationlib import ForwardRef import collections from collections import defaultdict import collections.abc @@ -125,6 +127,7 @@ 'cast', 'clear_overloads', 'dataclass_transform', + 'evaluate_forward_ref', 'final', 'get_args', 'get_origin', @@ -165,7 +168,7 @@ def _type_convert(arg, module=None, *, allow_special_forms=False): if arg is None: return type(None) if isinstance(arg, str): - return ForwardRef(arg, module=module, is_class=allow_special_forms) + return _make_forward_ref(arg, module=module, is_class=allow_special_forms) return arg @@ -459,7 +462,8 @@ def __repr__(self): _sentinel = _Sentinel() -def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset()): +def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset(), + format=annotationlib.Format.VALUE, owner=None): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). @@ -470,11 +474,13 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f _deprecation_warning_for_no_type_params_passed("typing._eval_type") type_params = () if isinstance(t, ForwardRef): - return t._evaluate(globalns, localns, type_params, recursive_guard=recursive_guard) + return evaluate_forward_ref(t, globals=globalns, locals=localns, + type_params=type_params, owner=owner, + _recursive_guard=recursive_guard, format=format) if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): if isinstance(t, GenericAlias): args = tuple( - ForwardRef(arg) if isinstance(arg, str) else arg + _make_forward_ref(arg) if isinstance(arg, str) else arg for arg in t.__args__ ) is_unpacked = t.__unpacked__ @@ -487,7 +493,8 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f ev_args = tuple( _eval_type( - a, globalns, localns, type_params, recursive_guard=recursive_guard + a, globalns, localns, type_params, recursive_guard=recursive_guard, + format=format, owner=owner, ) for a in t.__args__ ) @@ -1011,111 +1018,77 @@ def run(arg: Child | Unrelated): return _GenericAlias(self, (item,)) -class ForwardRef(_Final, _root=True): - """Internal wrapper to hold a forward reference.""" +def _make_forward_ref(code, **kwargs): + forward_ref = ForwardRef(code, **kwargs) + # For compatibility, eagerly compile the forwardref's code. + forward_ref.__forward_code__ + return forward_ref - __slots__ = ('__forward_arg__', '__forward_code__', - '__forward_evaluated__', '__forward_value__', - '__forward_is_argument__', '__forward_is_class__', - '__forward_module__') - def __init__(self, arg, is_argument=True, module=None, *, is_class=False): - if not isinstance(arg, str): - raise TypeError(f"Forward reference must be a string -- got {arg!r}") - - # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. - # Unfortunately, this isn't a valid expression on its own, so we - # do the unpacking manually. - if arg.startswith('*'): - arg_to_compile = f'({arg},)[0]' # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] - else: - arg_to_compile = arg - try: - code = compile(arg_to_compile, '', 'eval') - except SyntaxError: - raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") - - self.__forward_arg__ = arg - self.__forward_code__ = code - self.__forward_evaluated__ = False - self.__forward_value__ = None - self.__forward_is_argument__ = is_argument - self.__forward_is_class__ = is_class - self.__forward_module__ = module - - def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): - if type_params is _sentinel: - _deprecation_warning_for_no_type_params_passed("typing.ForwardRef._evaluate") - type_params = () - if self.__forward_arg__ in recursive_guard: - return self - if not self.__forward_evaluated__ or localns is not globalns: - if globalns is None and localns is None: - globalns = localns = {} - elif globalns is None: - globalns = localns - elif localns is None: - localns = globalns - if self.__forward_module__ is not None: - globalns = getattr( - sys.modules.get(self.__forward_module__, None), '__dict__', globalns - ) - - # type parameters require some special handling, - # as they exist in their own scope - # but `eval()` does not have a dedicated parameter for that scope. - # For classes, names in type parameter scopes should override - # names in the global scope (which here are called `localns`!), - # but should in turn be overridden by names in the class scope - # (which here are called `globalns`!) - if type_params: - globalns, localns = dict(globalns), dict(localns) - for param in type_params: - param_name = param.__name__ - if not self.__forward_is_class__ or param_name not in globalns: - globalns[param_name] = param - localns.pop(param_name, None) - - type_ = _type_check( - eval(self.__forward_code__, globalns, localns), - "Forward references must evaluate to types.", - is_argument=self.__forward_is_argument__, - allow_special_forms=self.__forward_is_class__, - ) - self.__forward_value__ = _eval_type( - type_, - globalns, - localns, - type_params, - recursive_guard=(recursive_guard | {self.__forward_arg__}), - ) - self.__forward_evaluated__ = True - return self.__forward_value__ - - def __eq__(self, other): - if not isinstance(other, ForwardRef): - return NotImplemented - if self.__forward_evaluated__ and other.__forward_evaluated__: - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_value__ == other.__forward_value__) - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_module__ == other.__forward_module__) - - def __hash__(self): - return hash((self.__forward_arg__, self.__forward_module__)) - - def __or__(self, other): - return Union[self, other] +def evaluate_forward_ref( + forward_ref, + *, + owner=None, + globals=None, + locals=None, + type_params=None, + format=annotationlib.Format.VALUE, + _recursive_guard=frozenset(), +): + """Evaluate a forward reference as a type hint. + + This is similar to calling the ForwardRef.evaluate() method, + but unlike that method, evaluate_forward_ref() also: + + * Recursively evaluates forward references nested within the type hint. + * Rejects certain objects that are not valid type hints. + * Replaces type hints that evaluate to None with types.NoneType. + * Supports the *FORWARDREF* and *SOURCE* formats. + + *forward_ref* must be an instance of ForwardRef. *owner*, if given, + should be the object that holds the annotations that the forward reference + derived from, such as a module, class object, or function. It is used to + infer the namespaces to use for looking up names. *globals* and *locals* + can also be explicitly given to provide the global and local namespaces. + *type_params* is a tuple of type parameters that are in scope when + evaluating the forward reference. This parameter must be provided (though + it may be an empty tuple) if *owner* is not given and the forward reference + does not already have an owner set. *format* specifies the format of the + annotation and is a member of the annoations.Format enum. - def __ror__(self, other): - return Union[other, self] + """ + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing.evaluate_forward_ref") + type_params = () + if format == annotationlib.Format.SOURCE: + return forward_ref.__forward_arg__ + if forward_ref.__forward_arg__ in _recursive_guard: + return forward_ref - def __repr__(self): - if self.__forward_module__ is None: - module_repr = '' + try: + value = forward_ref.evaluate(globals=globals, locals=locals, + type_params=type_params, owner=owner) + except NameError: + if format == annotationlib.Format.FORWARDREF: + return forward_ref else: - module_repr = f', module={self.__forward_module__!r}' - return f'ForwardRef({self.__forward_arg__!r}{module_repr})' + raise + + type_ = _type_check( + value, + "Forward references must evaluate to types.", + is_argument=forward_ref.__forward_is_argument__, + allow_special_forms=forward_ref.__forward_is_class__, + ) + return _eval_type( + type_, + globals, + locals, + type_params, + recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, + format=format, + owner=owner, + ) def _is_unpacked_typevartuple(x: Any) -> bool: @@ -2196,7 +2169,7 @@ class _AnnotatedAlias(_NotIterable, _GenericAlias, _root=True): """Runtime representation of an annotated type. At its core 'Annotated[t, dec1, dec2, ...]' is an alias for the type 't' - with extra annotations. The alias behaves like a normal typing alias. + with extra metadata. The alias behaves like a normal typing alias. Instantiating is the same as instantiating the underlying type; binding it to types is also the same. @@ -2380,7 +2353,8 @@ def greet(name: str) -> None: WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) -def get_type_hints(obj, globalns=None, localns=None, include_extras=False): +def get_type_hints(obj, globalns=None, localns=None, include_extras=False, + *, format=annotationlib.Format.VALUE): """Return type hints for an object. This is often the same as obj.__annotations__, but it handles @@ -2417,13 +2391,14 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): if isinstance(obj, type): hints = {} for base in reversed(obj.__mro__): + ann = annotationlib.get_annotations(base, format=format) + if format is annotationlib.Format.SOURCE: + hints.update(ann) + continue if globalns is None: base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = getattr(base, '__annotations__', {}) - if isinstance(ann, types.GetSetDescriptorType): - ann = {} base_locals = dict(vars(base)) if localns is None else localns if localns is None and globalns is None: # This is surprising, but required. Before Python 3.10, @@ -2437,10 +2412,26 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): if value is None: value = type(None) if isinstance(value, str): - value = ForwardRef(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals, base.__type_params__) + value = _make_forward_ref(value, is_argument=False, is_class=True) + value = _eval_type(value, base_globals, base_locals, base.__type_params__, + format=format, owner=obj) hints[name] = value - return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} + if include_extras or format is annotationlib.Format.SOURCE: + return hints + else: + return {k: _strip_annotations(t) for k, t in hints.items()} + + hints = annotationlib.get_annotations(obj, format=format) + if ( + not hints + and not isinstance(obj, types.ModuleType) + and not callable(obj) + and not hasattr(obj, '__annotations__') + and not hasattr(obj, '__annotate__') + ): + raise TypeError(f"{obj!r} is not a module, class, or callable.") + if format is annotationlib.Format.SOURCE: + return hints if globalns is None: if isinstance(obj, types.ModuleType): @@ -2455,15 +2446,6 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): localns = globalns elif localns is None: localns = globalns - hints = getattr(obj, '__annotations__', None) - if hints is None: - # Return empty annotations for something that _could_ have them. - if isinstance(obj, _allowed_types): - return {} - else: - raise TypeError('{!r} is not a module, class, method, ' - 'or function.'.format(obj)) - hints = dict(hints) type_params = getattr(obj, "__type_params__", ()) for name, value in hints.items(): if value is None: @@ -2471,12 +2453,12 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): if isinstance(value, str): # class-level forward refs were handled above, this must be either # a module-level annotation or a function argument annotation - value = ForwardRef( + value = _make_forward_ref( value, is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - hints[name] = _eval_type(value, globalns, localns, type_params) + hints[name] = _eval_type(value, globalns, localns, type_params, format=format, owner=obj) return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} @@ -2953,22 +2935,34 @@ def __round__(self, ndigits: int = 0) -> T: pass -def _make_nmtuple(name, types, module, defaults = ()): - fields = [n for n, t in types] - types = {n: _type_check(t, f"field {n} annotation must be a type") - for n, t in types} +def _make_nmtuple(name, fields, annotate_func, module, defaults = ()): nm_tpl = collections.namedtuple(name, fields, defaults=defaults, module=module) - nm_tpl.__annotations__ = nm_tpl.__new__.__annotations__ = types + nm_tpl.__annotate__ = nm_tpl.__new__.__annotate__ = annotate_func return nm_tpl +def _make_eager_annotate(types): + checked_types = {key: _type_check(val, f"field {key} annotation must be a type") + for key, val in types.items()} + def annotate(format): + if format in (annotationlib.Format.VALUE, annotationlib.Format.FORWARDREF): + return checked_types + else: + return _convert_to_source(types) + return annotate + + +def _convert_to_source(types): + return {n: t if isinstance(t, str) else _type_repr(t) for n, t in types.items()} + + # attributes prohibited to set in NamedTuple class syntax _prohibited = frozenset({'__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', '_make', '_replace', '_asdict', '_source'}) -_special = frozenset({'__module__', '__name__', '__annotations__'}) +_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__'}) class NamedTupleMeta(type): @@ -2981,12 +2975,29 @@ def __new__(cls, typename, bases, ns): bases = tuple(tuple if base is _NamedTuple else base for base in bases) if "__annotations__" in ns: types = ns["__annotations__"] + field_names = list(types) + annotate = _make_eager_annotate(types) elif "__annotate__" in ns: - types = ns["__annotate__"](1) # VALUE + original_annotate = ns["__annotate__"] + types = annotationlib.call_annotate_function(original_annotate, annotationlib.Format.FORWARDREF) + field_names = list(types) + + # For backward compatibility, type-check all the types at creation time + for typ in types.values(): + _type_check(typ, "field annotation must be a type") + + def annotate(format): + annos = annotationlib.call_annotate_function(original_annotate, format) + if format != annotationlib.Format.SOURCE: + return {key: _type_check(val, f"field {key} annotation must be a type") + for key, val in annos.items()} + return annos else: - types = {} + # Empty NamedTuple + field_names = [] + annotate = lambda format: {} default_names = [] - for field_name in types: + for field_name in field_names: if field_name in ns: default_names.append(field_name) elif default_names: @@ -2994,7 +3005,7 @@ def __new__(cls, typename, bases, ns): f"cannot follow default field" f"{'s' if len(default_names) > 1 else ''} " f"{', '.join(default_names)}") - nm_tpl = _make_nmtuple(typename, types.items(), + nm_tpl = _make_nmtuple(typename, field_names, annotate, defaults=[ns[n] for n in default_names], module=ns['__module__']) nm_tpl.__bases__ = bases @@ -3085,7 +3096,11 @@ class Employee(NamedTuple): import warnings warnings._deprecated(deprecated_thing, message=deprecation_msg, remove=(3, 15)) fields = kwargs.items() - nt = _make_nmtuple(typename, fields, module=_caller()) + types = {n: _type_check(t, f"field {n} annotation must be a type") + for n, t in fields} + field_names = [n for n, _ in fields] + + nt = _make_nmtuple(typename, field_names, _make_eager_annotate(types), module=_caller()) nt.__orig_bases__ = (NamedTuple,) return nt @@ -3144,15 +3159,19 @@ def __new__(cls, name, bases, ns, total=True): if not hasattr(tp_dict, '__orig_bases__'): tp_dict.__orig_bases__ = bases - annotations = {} if "__annotations__" in ns: + own_annotate = None own_annotations = ns["__annotations__"] elif "__annotate__" in ns: - own_annotations = ns["__annotate__"](1) # VALUE + own_annotate = ns["__annotate__"] + own_annotations = annotationlib.call_annotate_function( + own_annotate, annotationlib.Format.FORWARDREF, owner=tp_dict + ) else: + own_annotate = None own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" - own_annotations = { + own_checked_annotations = { n: _type_check(tp, msg, module=tp_dict.__module__) for n, tp in own_annotations.items() } @@ -3162,13 +3181,6 @@ def __new__(cls, name, bases, ns, total=True): mutable_keys = set() for base in bases: - # TODO: Avoid eagerly evaluating annotations in VALUE format. - # Instead, evaluate in FORWARDREF format to figure out which - # keys have Required/NotRequired/ReadOnly qualifiers, and create - # a new __annotate__ function for the resulting TypedDict that - # combines the annotations from this class and its parents. - annotations.update(base.__annotations__) - base_required = base.__dict__.get('__required_keys__', set()) required_keys |= base_required optional_keys -= base_required @@ -3180,8 +3192,7 @@ def __new__(cls, name, bases, ns, total=True): readonly_keys.update(base.__dict__.get('__readonly_keys__', ())) mutable_keys.update(base.__dict__.get('__mutable_keys__', ())) - annotations.update(own_annotations) - for annotation_key, annotation_type in own_annotations.items(): + for annotation_key, annotation_type in own_checked_annotations.items(): qualifiers = set(_get_typeddict_qualifiers(annotation_type)) if Required in qualifiers: is_required = True @@ -3212,7 +3223,32 @@ def __new__(cls, name, bases, ns, total=True): f"Required keys overlap with optional keys in {name}:" f" {required_keys=}, {optional_keys=}" ) - tp_dict.__annotations__ = annotations + + def __annotate__(format): + annos = {} + for base in bases: + if base is Generic: + continue + base_annotate = base.__annotate__ + if base_annotate is None: + continue + base_annos = annotationlib.call_annotate_function(base.__annotate__, format, owner=base) + annos.update(base_annos) + if own_annotate is not None: + own = annotationlib.call_annotate_function(own_annotate, format, owner=tp_dict) + if format != annotationlib.Format.SOURCE: + own = { + n: _type_check(tp, msg, module=tp_dict.__module__) + for n, tp in own.items() + } + elif format == annotationlib.Format.SOURCE: + own = _convert_to_source(own_annotations) + else: + own = own_checked_annotations + annos.update(own) + return annos + + tp_dict.__annotate__ = __annotate__ tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) tp_dict.__readonly_keys__ = frozenset(readonly_keys) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index d50535dffeb5d1..2bbbcf40e21543 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1829,7 +1829,8 @@ def patch( class _patch_dict(object): """ Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. + to its original state after the test, where the restored dictionary is + a copy of the dictionary as it was before the test. `in_dict` can be a dictionary or a mapping like container. If it is a mapping then it must at least support getting, setting and deleting items diff --git a/Lib/warnings.py b/Lib/warnings.py index d344ceaebb0378..e83cde37ab2d1a 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -628,12 +628,16 @@ def __init_subclass__(*args, **kwargs): return arg elif callable(arg): import functools + import inspect @functools.wraps(arg) def wrapper(*args, **kwargs): warn(msg, category=category, stacklevel=stacklevel + 1) return arg(*args, **kwargs) + if inspect.iscoroutinefunction(arg): + wrapper = inspect.markcoroutinefunction(wrapper) + arg.__deprecated__ = wrapper.__deprecated__ = msg return wrapper else: diff --git a/Makefile.pre.in b/Makefile.pre.in index 94cfb74138a3d9..5608e593ac9aca 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -41,6 +41,7 @@ AR= @AR@ READELF= @READELF@ SOABI= @SOABI@ ABIFLAGS= @ABIFLAGS@ +ABI_THREAD= @ABI_THREAD@ LDVERSION= @LDVERSION@ MODULE_LDFLAGS=@MODULE_LDFLAGS@ GITVERSION= @GITVERSION@ @@ -158,7 +159,7 @@ WHEEL_PKG_DIR= @WHEEL_PKG_DIR@ # Detailed destination directories BINLIBDEST= @BINLIBDEST@ -LIBDEST= $(SCRIPTDIR)/python$(VERSION) +LIBDEST= $(SCRIPTDIR)/python$(VERSION)$(ABI_THREAD) INCLUDEPY= $(INCLUDEDIR)/python$(LDVERSION) CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) @@ -694,7 +695,7 @@ list-targets: @grep -E '^[A-Za-z][-A-Za-z0-9]+:' Makefile | awk -F : '{print $$1}' .PHONY: build_all -build_all: check-clean-src @APP_STORE_COMPLIANCE_PATCH_TARGET@ $(BUILDPYTHON) platform sharedmods \ +build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sharedmods \ gdbhooks Programs/_testembed scripts checksharedmods rundsymutil .PHONY: build_wasm @@ -717,6 +718,16 @@ check-clean-src: exit 1; \ fi +# Check that the app store compliance patch can be applied (if configured). +# This is checked as a dry-run against the original library sources; +# the patch will be actually applied during the install phase. +.PHONY: check-app-store-compliance +check-app-store-compliance: + @if [ "$(APP_STORE_COMPLIANCE_PATCH)" != "" ]; then \ + patch --dry-run --quiet --force --strip 1 --directory "$(abs_srcdir)" --input "$(abs_srcdir)/$(APP_STORE_COMPLIANCE_PATCH)"; \ + echo "App store compliance patch can be applied."; \ + fi + # Profile generation build must start from a clean tree. profile-clean-stamp: $(MAKE) clean @@ -930,18 +941,6 @@ SRC_GDB_HOOKS=$(srcdir)/Tools/gdb/libpython.py $(BUILDPYTHON)-gdb.py: $(SRC_GDB_HOOKS) $(INSTALL_DATA) $(SRC_GDB_HOOKS) $(BUILDPYTHON)-gdb.py -# Compliance with app stores (such as iOS and macOS) sometimes requires making -# modifications to the Python standard library. If enabled, apply the patch of -# known modifications to the source tree before building. The patch will be -# applied in a dry-run mode (validating, but not applying the patch) on builds -# that *have* a compliance patch, but where compliance has not been enabled. -build/app-store-compliant: - patch @APP_STORE_COMPLIANCE_PATCH_FLAGS@ --forward --strip=1 --directory="$(srcdir)" --input "$(APP_STORE_COMPLIANCE_PATCH)" - @if test "@APP_STORE_COMPLIANCE_PATCH_FLAGS@" == ""; then \ - mkdir -p build ; \ - echo "$(APP_STORE_COMPLIANCE_PATCH)" > build/app-store-compliant ; \ - fi - # This rule is here for OPENSTEP/Rhapsody/MacOSX. It builds a temporary # minimal framework (not including the Lib directory and such) in the current # directory. @@ -1184,6 +1183,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_format.h \ $(srcdir)/Include/internal/pycore_frame.h \ $(srcdir)/Include/internal/pycore_freelist.h \ + $(srcdir)/Include/internal/pycore_freelist_state.h \ $(srcdir)/Include/internal/pycore_function.h \ $(srcdir)/Include/internal/pycore_gc.h \ $(srcdir)/Include/internal/pycore_genobject.h \ @@ -2366,6 +2366,7 @@ LIBSUBDIRS= asyncio \ __phello__ TESTSUBDIRS= idlelib/idle_test \ test \ + test/test_ast \ test/archivetestdata \ test/audiodata \ test/certdata \ @@ -2584,6 +2585,14 @@ libinstall: all $(srcdir)/Modules/xxmodule.c $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt + @ # If app store compliance has been configured, apply the patch to the + @ # installed library code. The patch has been previously validated against + @ # the original source tree, so we can ignore any errors that are raised + @ # due to files that are missing because of --disable-test-modules etc. + @if [ "$(APP_STORE_COMPLIANCE_PATCH)" != "" ]; then \ + echo "Applying app store compliance patch"; \ + patch --force --reject-file "$(abs_builddir)/app-store-compliance.rej" --strip 2 --directory "$(DESTDIR)$(LIBDEST)" --input "$(abs_srcdir)/$(APP_STORE_COMPLIANCE_PATCH)" || true ; \ + fi @ # Build PYC files for the 3 optimization levels (0, 1, 2) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ @@ -2652,7 +2661,7 @@ inclinstall: $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(INCLUDEPY)/internal; \ else true; \ fi - @if test "$(INSTALL_MIMALLOC)" == "yes"; then \ + @if test "$(INSTALL_MIMALLOC)" = "yes"; then \ if test ! -d $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; then \ echo "Creating directory $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc"; \ $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; \ @@ -2673,7 +2682,7 @@ inclinstall: echo $(INSTALL_DATA) $$i $(INCLUDEPY)/internal; \ $(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal; \ done - @if test "$(INSTALL_MIMALLOC)" == "yes"; then \ + @if test "$(INSTALL_MIMALLOC)" = "yes"; then \ echo $(INSTALL_DATA) $(srcdir)/Include/internal/mimalloc/mimalloc.h $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc.h; \ $(INSTALL_DATA) $(srcdir)/Include/internal/mimalloc/mimalloc.h $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc.h; \ for i in $(srcdir)/Include/internal/mimalloc/mimalloc/*.h; \ diff --git a/Misc/ACKS b/Misc/ACKS index 53258dbfd9f478..6008f9e1770d1d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1670,6 +1670,7 @@ Fred Sells Jiwon Seo Iñigo Serna Joakim Sernbrant +Rodrigo Girão Serrão Roger D. Serwy Jerry Seutter Pete Sevander diff --git a/Misc/HISTORY b/Misc/HISTORY index 8ca35e1af62c05..a74d7e06acd071 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -3952,7 +3952,7 @@ Library - Issue #18626: the inspect module now offers a basic command line introspection interface (Initial patch by Claudiu Popa) -- Issue #3015: Fixed tkinter with wantobject=False. Any Tcl command call +- Issue #3015: Fixed tkinter with ``wantobjects=False``. Any Tcl command call returned empty string. - Issue #19037: The mailbox module now makes all changes to maildir files diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index 9a729a45b160eb..f30ed548e7e033 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -97,7 +97,7 @@ convention. Patch by Donghee Na. .. nonce: aJS9B3 .. section: Core and Builtins -Port the :mod:`_bisect` module to the multi-phase initialization API +Port the :mod:`!_bisect` module to the multi-phase initialization API (:pep:`489`). .. @@ -128,7 +128,7 @@ Taskaya. .. nonce: lh335O .. section: Core and Builtins -Port the :mod:`_lsprof` extension module to multi-phase initialization +Port the :mod:`!_lsprof` extension module to multi-phase initialization (:pep:`489`). .. @@ -148,7 +148,7 @@ Port the :mod:`cmath` extension module to multi-phase initialization .. nonce: jiXmyT .. section: Core and Builtins -Port the :mod:`_scproxy` extension module to multi-phase initialization +Port the :mod:`!_scproxy` extension module to multi-phase initialization (:pep:`489`). .. @@ -168,7 +168,7 @@ Port the :mod:`termios` extension module to multi-phase initialization .. nonce: QuDIut .. section: Core and Builtins -Convert the :mod:`_sha256` extension module types to heap types. +Convert the :mod:`!_sha256` extension module types to heap types. .. @@ -187,7 +187,7 @@ classes with a huge amount of arguments. Patch by Pablo Galindo. .. nonce: CnRME3 .. section: Core and Builtins -Port the :mod:`_overlapped` extension module to multi-phase initialization +Port the :mod:`!_overlapped` extension module to multi-phase initialization (:pep:`489`). .. @@ -197,7 +197,7 @@ Port the :mod:`_overlapped` extension module to multi-phase initialization .. nonce: X9CZgo .. section: Core and Builtins -Port the :mod:`_curses_panel` extension module to multi-phase initialization +Port the :mod:`!_curses_panel` extension module to multi-phase initialization (:pep:`489`). .. @@ -207,7 +207,7 @@ Port the :mod:`_curses_panel` extension module to multi-phase initialization .. nonce: 5jZymK .. section: Core and Builtins -Port the :mod:`_opcode` extension module to multi-phase initialization +Port the :mod:`!_opcode` extension module to multi-phase initialization (:pep:`489`). .. @@ -282,7 +282,7 @@ initialized ``_ast`` module. .. nonce: vcxSUa .. section: Core and Builtins -Convert :mod:`_operator` to use :c:func:`PyType_FromSpec`. +Convert :mod:`!_operator` to use :c:func:`PyType_FromSpec`. .. @@ -291,7 +291,7 @@ Convert :mod:`_operator` to use :c:func:`PyType_FromSpec`. .. nonce: fubBkb .. section: Core and Builtins -Port :mod:`_sha3` to multi-phase init. Convert static types to heap types. +Port :mod:`!_sha3` to multi-phase init. Convert static types to heap types. .. @@ -300,7 +300,7 @@ Port :mod:`_sha3` to multi-phase init. Convert static types to heap types. .. nonce: FC13e7 .. section: Core and Builtins -Port the :mod:`_blake2` extension module to the multi-phase initialization +Port the :mod:`!_blake2` extension module to the multi-phase initialization API (:pep:`489`). .. @@ -339,7 +339,7 @@ The output of ``python --help`` contains now only ASCII characters. .. nonce: O0d3ym .. section: Core and Builtins -Port the :mod:`_sha1`, :mod:`_sha512`, and :mod:`_md5` extension modules to +Port the :mod:`!_sha1`, :mod:`!_sha512`, and :mod:`!_md5` extension modules to multi-phase initialization API (:pep:`489`). .. @@ -636,7 +636,7 @@ Remove the remaining files from the old parser and the :mod:`symbol` module. .. nonce: _yI-ax .. section: Core and Builtins -Convert :mod:`_bz2` to use :c:func:`PyType_FromSpec`. +Convert :mod:`!_bz2` to use :c:func:`PyType_FromSpec`. .. @@ -666,7 +666,7 @@ by Brandt Bucher. .. nonce: 61iyYh .. section: Core and Builtins -Port :mod:`_gdbm` to multiphase initialization. +Port :mod:`!_gdbm` to multiphase initialization. .. @@ -696,7 +696,7 @@ for emitting syntax errors. Patch by Pablo Galindo. .. nonce: mmlp3Q .. section: Core and Builtins -Port :mod:`_dbm` to multiphase initialization. +Port :mod:`!_dbm` to multiphase initialization. .. @@ -1010,7 +1010,7 @@ Port :mod:`mmap` to multiphase initialization. .. nonce: Kfe9fT .. section: Core and Builtins -Port :mod:`_lzma` to multiphase initialization. +Port :mod:`!_lzma` to multiphase initialization. .. diff --git a/Misc/NEWS.d/3.10.0a2.rst b/Misc/NEWS.d/3.10.0a2.rst index 79f570439b52b8..bdf9488c81bae1 100644 --- a/Misc/NEWS.d/3.10.0a2.rst +++ b/Misc/NEWS.d/3.10.0a2.rst @@ -362,7 +362,7 @@ plistlib: fix parsing XML plists with hexadecimal integer values .. nonce: 85BsRA .. section: Library -Fix an incorrectly formatted error from :meth:`_codecs.charmap_decode` when +Fix an incorrectly formatted error from :meth:`!_codecs.charmap_decode` when called with a mapped value outside the range of valid Unicode code points. PR by Max Bernstein. diff --git a/Misc/NEWS.d/3.10.0a3.rst b/Misc/NEWS.d/3.10.0a3.rst index 179cf3e9cfb08c..2aef87ab929aab 100644 --- a/Misc/NEWS.d/3.10.0a3.rst +++ b/Misc/NEWS.d/3.10.0a3.rst @@ -1386,7 +1386,7 @@ Python already implicitly installs signal handlers: see The ``Py_TRASHCAN_BEGIN`` macro no longer accesses PyTypeObject attributes, but now can get the condition by calling the new private -:c:func:`_PyTrash_cond()` function which hides implementation details. +:c:func:`!_PyTrash_cond()` function which hides implementation details. .. diff --git a/Misc/NEWS.d/3.10.0a4.rst b/Misc/NEWS.d/3.10.0a4.rst index ae667f2bffe192..5cea16c259d5ee 100644 --- a/Misc/NEWS.d/3.10.0a4.rst +++ b/Misc/NEWS.d/3.10.0a4.rst @@ -193,7 +193,7 @@ subinterpreters. Patch by Victor Stinner. .. nonce: j7nl6A .. section: Core and Builtins -Make :c:func:`_PyUnicode_FromId` function compatible with subinterpreters. +Make :c:func:`!_PyUnicode_FromId` function compatible with subinterpreters. Each interpreter now has an array of identifier objects (interned strings decoded from UTF-8). Patch by Victor Stinner. @@ -367,7 +367,7 @@ uses "options" instead. .. nonce: Quy3zn .. section: Library -Port the :mod:`_thread` extension module to the multiphase initialization +Port the :mod:`!_thread` extension module to the multiphase initialization API (:pep:`489`) and convert its static types to heap types. .. @@ -960,8 +960,8 @@ explicitly and so not exported. .. nonce: Je08Ny .. section: C API -Remove the private :c:func:`_Py_fopen` function which is no longer needed. -Use :c:func:`_Py_wfopen` or :c:func:`_Py_fopen_obj` instead. Patch by Victor +Remove the private :c:func:`!_Py_fopen` function which is no longer needed. +Use :c:func:`!_Py_wfopen` or :c:func:`!_Py_fopen_obj` instead. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.10.0a5.rst b/Misc/NEWS.d/3.10.0a5.rst index dc95e8ce072fd9..a85ea1ff1c2817 100644 --- a/Misc/NEWS.d/3.10.0a5.rst +++ b/Misc/NEWS.d/3.10.0a5.rst @@ -108,7 +108,7 @@ a slice at the start of the ``bytearray`` to a shorter byte string). .. nonce: WfTdfg .. section: Core and Builtins -Fix the :c:func:`_PyUnicode_FromId` function (_Py_IDENTIFIER(var) API) when +Fix the :c:func:`!_PyUnicode_FromId` function (_Py_IDENTIFIER(var) API) when :c:func:`Py_Initialize` / :c:func:`Py_Finalize` is called multiple times: preserve ``_PyRuntime.unicode_ids.next_index`` value. diff --git a/Misc/NEWS.d/3.10.0a6.rst b/Misc/NEWS.d/3.10.0a6.rst index bad3528084897b..31b7df2c61158e 100644 --- a/Misc/NEWS.d/3.10.0a6.rst +++ b/Misc/NEWS.d/3.10.0a6.rst @@ -315,7 +315,7 @@ Adds :const:`resource.RLIMIT_KQUEUES` constant from FreeBSD to the .. section: Library Make the pure Python implementation of :mod:`xml.etree.ElementTree` behave -the same as the C implementation (:mod:`_elementree`) regarding default +the same as the C implementation (:mod:`!_elementree`) regarding default attribute values (by not setting ``specified_attributes=1``). .. diff --git a/Misc/NEWS.d/3.10.0a7.rst b/Misc/NEWS.d/3.10.0a7.rst index fe6213d95a88bb..32ee34d9a68910 100644 --- a/Misc/NEWS.d/3.10.0a7.rst +++ b/Misc/NEWS.d/3.10.0a7.rst @@ -83,7 +83,7 @@ instruction dispatch a bit. .. nonce: PhaT-B .. section: Core and Builtins -Fix reference leak in the :mod:`_hashopenssl` extension. Patch by Pablo +Fix reference leak in the :mod:`!_hashopenssl` extension. Patch by Pablo Galindo. .. diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index 640f3ee58adbae..306e987a41612e 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -182,7 +182,7 @@ normally be possible, but might occur in some unusual circumstances. .. nonce: u5Y6bS .. section: Core and Builtins -Importing the :mod:`_signal` module in a subinterpreter has no longer side +Importing the :mod:`!_signal` module in a subinterpreter has no longer side effects. .. @@ -776,11 +776,11 @@ builtins.open() is now io.open(). .. nonce: o1zEk_ .. section: Library -The Python :func:`_pyio.open` function becomes a static method to behave as +The Python :func:`!_pyio.open` function becomes a static method to behave as :func:`io.open` built-in function: don't become a bound method when stored as a class variable. It becomes possible since static methods are now -callable in Python 3.10. Moreover, :func:`_pyio.OpenWrapper` becomes a -simple alias to :func:`_pyio.open`. Patch by Victor Stinner. +callable in Python 3.10. Moreover, :func:`!_pyio.OpenWrapper` becomes a +simple alias to :func:`!_pyio.open`. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 40fbb9d42b7944..8a1391ef0515c3 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -613,7 +613,7 @@ Rename ``types.Union`` to ``types.UnionType``. .. section: Core and Builtins Expose specialization stats in python via -:func:`_opcode.get_specialization_stats`. +:func:`!_opcode.get_specialization_stats`. .. @@ -1701,7 +1701,7 @@ Remove many old deprecated :mod:`unittest` features: .. nonce: y1kEfP .. section: Library -Remove the deprecated ``split()`` method of :class:`_tkinter.TkappType`. +Remove the deprecated ``split()`` method of :class:`!_tkinter.TkappType`. Patch by Erlend E. Aasland. .. @@ -2298,9 +2298,9 @@ Adopt *binacii.a2b_base64*'s strict mode in *base64.b64decode*. .. nonce: ThuDMI .. section: Library -Fixed a bug in the :mod:`_ssl` module that was throwing :exc:`OverflowError` -when using :meth:`_ssl._SSLSocket.write` and :meth:`_ssl._SSLSocket.read` -for a big value of the ``len`` parameter. Patch by Pablo Galindo +Fixed a bug in the :mod:`!_ssl` module that was throwing :exc:`OverflowError` +when using :meth:`!_ssl._SSLSocket.write` and :meth:`!_ssl._SSLSocket.read` +for a big value of the ``len`` parameter. Patch by Pablo Galindo. .. @@ -2398,7 +2398,7 @@ class in the interactive session. Instead of :exc:`TypeError`, it should be .. nonce: R3IcM1 .. section: Library -Fix memory leak in :func:`_tkinter._flatten` if it is called with a sequence +Fix memory leak in :func:`!_tkinter._flatten` if it is called with a sequence or set, but not list or tuple. .. @@ -4187,7 +4187,7 @@ Add calls of :func:`gc.collect` in tests to support PyPy. .. nonce: mQZdXU .. section: Tests -Made tests relying on the :mod:`_asyncio` C extension module optional to +Made tests relying on the :mod:`!_asyncio` C extension module optional to allow running on alternative Python implementations. Patch by Serhiy Storchaka. @@ -4238,7 +4238,7 @@ harmless "malloc can't allocate region" messages spewed by test_decimal. .. nonce: KKsNOV .. section: Tests -Fixed floating point precision issue in turtle tests. +Fixed floating-point precision issue in turtle tests. .. diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index 05644d0a4639b1..48cf2c1e428d87 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -15,7 +15,7 @@ Improve the :exc:`SyntaxError` message when using ``True``, ``None`` or .. section: Core and Builtins :data:`sys.stdlib_module_names` now contains the macOS-specific module -:mod:`_scproxy`. +:mod:`!_scproxy`. .. @@ -1023,7 +1023,7 @@ compile shared modules. .. nonce: 61gM2A .. section: Build -:mod:`pyexpat` and :mod:`_elementtree` no longer define obsolete macros +:mod:`pyexpat` and :mod:`!_elementtree` no longer define obsolete macros ``HAVE_EXPAT_CONFIG_H`` and ``USE_PYEXPAT_CAPI``. ``XML_POOR_ENTROPY`` is now defined in ``expat_config.h``. diff --git a/Misc/NEWS.d/3.11.0a3.rst b/Misc/NEWS.d/3.11.0a3.rst index 2842aad0e163d6..6a0ae20d1fb5ed 100644 --- a/Misc/NEWS.d/3.11.0a3.rst +++ b/Misc/NEWS.d/3.11.0a3.rst @@ -27,7 +27,7 @@ invalid targets. Patch by Pablo Galindo .. nonce: 3TmTSw .. section: Core and Builtins -:c:func:`_PyErr_ChainStackItem` no longer normalizes ``exc_info`` (including +:c:func:`!_PyErr_ChainStackItem` no longer normalizes ``exc_info`` (including setting the traceback on the exception instance) because ``exc_info`` is always normalized. diff --git a/Misc/NEWS.d/3.11.0a4.rst b/Misc/NEWS.d/3.11.0a4.rst index a5ce7620016cc7..64e2f39ad9db18 100644 --- a/Misc/NEWS.d/3.11.0a4.rst +++ b/Misc/NEWS.d/3.11.0a4.rst @@ -258,7 +258,7 @@ instruction which performs the same operation, but without the loop. .. nonce: ADVaPT .. section: Core and Builtins -The code called from :c:func:`_PyErr_Display` was refactored to improve +The code called from :c:func:`!_PyErr_Display` was refactored to improve error handling. It now exits immediately upon an unrecoverable error. .. diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index 66ffa4ffba52e5..e88142e641f040 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -1054,7 +1054,7 @@ Patch by Victor Stinner. .. nonce: ajJjkh .. section: Build -Building Python now requires support for floating point Not-a-Number (NaN): +Building Python now requires support for floating-point Not-a-Number (NaN): remove the ``Py_NO_NAN`` macro. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.11.0a7.rst b/Misc/NEWS.d/3.11.0a7.rst index a376c8becea9f4..1254abfddcabc8 100644 --- a/Misc/NEWS.d/3.11.0a7.rst +++ b/Misc/NEWS.d/3.11.0a7.rst @@ -1401,7 +1401,7 @@ Christian's container image ``quay.io/tiran/cpython_autoconf:269``. .. nonce: fry4aK .. section: Build -Building Python now requires support of IEEE 754 floating point numbers. +Building Python now requires support of IEEE 754 floating-point numbers. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index c35e8e2c1caf07..a035d0f5addbf2 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -285,7 +285,7 @@ macros. .. nonce: 11YXHQ .. section: Core and Builtins -Add a new :c:func:`_PyFrame_IsEntryFrame` API function, to check if a +Add a new :c:func:`!_PyFrame_IsEntryFrame` API function, to check if a :c:type:`PyFrameObject` is an entry frame. Patch by Pablo Galindo. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 84d9d4e017609d..77a34124fb39e6 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -102,7 +102,7 @@ well as generator expressions. .. section: Core and Builtins Added unicode check for ``name`` attribute of ``spec`` argument passed in -:func:`_imp.create_builtin` function. +:func:`!_imp.create_builtin` function. .. @@ -483,7 +483,7 @@ Fix case of undefined behavior in ceval.c .. nonce: AfCi36 .. section: Core and Builtins -Convert :mod:`_functools` to argument clinic. +Convert :mod:`!_functools` to argument clinic. .. @@ -492,7 +492,7 @@ Convert :mod:`_functools` to argument clinic. .. nonce: wky0Fc .. section: Core and Builtins -Do not expose ``KeyWrapper`` in :mod:`_functools`. +Do not expose ``KeyWrapper`` in :mod:`!_functools`. .. @@ -1731,7 +1731,7 @@ tracing functions implemented in C. .. nonce: lenv9h .. section: Core and Builtins -:meth:`_warnings.warn_explicit` is ported to Argument Clinic. +:meth:`!_warnings.warn_explicit` is ported to Argument Clinic. .. @@ -3142,8 +3142,8 @@ test.test_codecs.EncodedFileTest`` instead. .. nonce: VhS1eS .. section: Library -Made :class:`_struct.Struct` GC-tracked in order to fix a reference leak in -the :mod:`_struct` module. +Made :class:`!_struct.Struct` GC-tracked in order to fix a reference leak in +the :mod:`!_struct` module. .. @@ -3258,7 +3258,7 @@ on the main thread Remove ``io.OpenWrapper`` and ``_pyio.OpenWrapper``, deprecated in Python 3.10: just use :func:`open` instead. The :func:`open` (:func:`io.open`) -function is a built-in function. Since Python 3.10, :func:`_pyio.open` is +function is a built-in function. Since Python 3.10, :func:`!_pyio.open` is also a static method. Patch by Victor Stinner. .. @@ -5610,7 +5610,7 @@ Accept os.PathLike for the argument to winsound.PlaySound Support native Windows case-insensitive path comparisons by using ``LCMapStringEx`` instead of :func:`str.lower` in :func:`ntpath.normcase`. -Add ``LCMapStringEx`` to the :mod:`_winapi` module. +Add ``LCMapStringEx`` to the :mod:`!_winapi` module. .. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 88d84ad93b35b5..3626f8b1e20809 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -527,7 +527,7 @@ Stinner. .. nonce: Ai2KDh .. section: Library -Now :mod:`_pyio` is consistent with :mod:`_io` in raising ``ValueError`` +Now :mod:`!_pyio` is consistent with :mod:`!_io` in raising ``ValueError`` when executing methods over closed buffers. .. @@ -537,7 +537,7 @@ when executing methods over closed buffers. .. nonce: 0v8iyw .. section: Library -Clean up refleak on failed module initialisation in :mod:`_zoneinfo` +Clean up refleak on failed module initialisation in :mod:`!_zoneinfo` .. @@ -546,7 +546,7 @@ Clean up refleak on failed module initialisation in :mod:`_zoneinfo` .. nonce: qc_KHr .. section: Library -Clean up refleaks on failed module initialisation in :mod:`_pickle` +Clean up refleaks on failed module initialisation in :mod:`!_pickle` .. @@ -555,7 +555,7 @@ Clean up refleaks on failed module initialisation in :mod:`_pickle` .. nonce: LBl79O .. section: Library -Clean up refleak on failed module initialisation in :mod:`_io`. +Clean up refleak on failed module initialisation in :mod:`!_io`. .. diff --git a/Misc/NEWS.d/3.12.0a3.rst b/Misc/NEWS.d/3.12.0a3.rst index 07593998d80891..f6a4dc75d456f4 100644 --- a/Misc/NEWS.d/3.12.0a3.rst +++ b/Misc/NEWS.d/3.12.0a3.rst @@ -70,7 +70,7 @@ Fix bug where compiler crashes on an if expression with an empty body block. .. nonce: DcKoBJ .. section: Core and Builtins -Fix a reference bug in :func:`_imp.create_builtin()` after the creation of +Fix a reference bug in :func:`!_imp.create_builtin` after the creation of the first sub-interpreter for modules ``builtins`` and ``sys``. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index d7af30f6c09b2b..53e1688b802bae 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -241,7 +241,7 @@ are now always dumped, even if switched off. Improve ``BUILD_LIST`` opcode so that it works similarly to the ``BUILD_TUPLE`` opcode, by stealing references from the stack rather than repeatedly using stack operations to set list elements. Implementation -details are in a new private API :c:func:`_PyList_FromArraySteal`. +details are in a new private API :c:func:`!_PyList_FromArraySteal`. .. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 9f3095b224233e..7126e08a20c7fd 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -1828,7 +1828,7 @@ is relative. .. nonce: 511Tbh .. section: Library -Convert private :meth:`_posixsubprocess.fork_exec` to use Argument Clinic. +Convert private :meth:`!_posixsubprocess.fork_exec` to use Argument Clinic. .. diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 9a321f779c24ff..0ba61b43411792 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -2888,9 +2888,9 @@ documented and were not intended to be used externally. .. nonce: vMbmj_ .. section: Library -:data:`opcode.ENABLE_SPECIALIZATION` (which was added in 3.12 but never +:data:`!opcode.ENABLE_SPECIALIZATION` (which was added in 3.12 but never documented or intended for external usage) is moved to -:data:`_opcode.ENABLE_SPECIALIZATION` where tests can access it. +:data:`!_opcode.ENABLE_SPECIALIZATION` where tests can access it. .. @@ -3053,7 +3053,7 @@ Donghee Na. .. nonce: U9nD_B .. section: Library -Optimize :meth:`_PollLikeSelector.select` for many iteration case. +Optimize :meth:`!_PollLikeSelector.select` for many iteration case. .. @@ -3173,7 +3173,7 @@ Disable tab completion in multiline mode of :mod:`pdb` .. nonce: pYSwMj .. section: Library -Expose opcode metadata through :mod:`_opcode`. +Expose opcode metadata through :mod:`!_opcode`. .. @@ -3735,7 +3735,7 @@ overwritten. .. nonce: _sZilh .. section: Library -Fix bugs in :mod:`_ctypes` where exceptions could end up being overwritten. +Fix bugs in :mod:`!_ctypes` where exceptions could end up being overwritten. .. diff --git a/Misc/NEWS.d/3.13.0a2.rst b/Misc/NEWS.d/3.13.0a2.rst index c6b2b1b263ffab..f4a637bf624d03 100644 --- a/Misc/NEWS.d/3.13.0a2.rst +++ b/Misc/NEWS.d/3.13.0a2.rst @@ -777,7 +777,7 @@ Add error checking during :mod:`!_socket` module init. .. nonce: urFYtn .. section: Library -Fix :mod:`_blake2` not checking for errors when initializing. +Fix :mod:`!_blake2` not checking for errors when initializing. .. diff --git a/Misc/NEWS.d/3.13.0a3.rst b/Misc/NEWS.d/3.13.0a3.rst index 2c660192dcd5b3..29fbe00efef76d 100644 --- a/Misc/NEWS.d/3.13.0a3.rst +++ b/Misc/NEWS.d/3.13.0a3.rst @@ -449,8 +449,8 @@ well-formed for surrogateescape encoding. Patch by Sidney Markowitz. .. nonce: N8E1zw .. section: Core and Builtins -Use the object's actual class name in :meth:`_io.FileIO.__repr__`, -:meth:`_io._WindowsConsoleIO` and :meth:`_io.TextIOWrapper.__repr__`, to +Use the object's actual class name in :meth:`!_io.FileIO.__repr__`, +:meth:`!_io._WindowsConsoleIO` and :meth:`!_io.TextIOWrapper.__repr__`, to make these methods subclass friendly. .. diff --git a/Misc/NEWS.d/3.13.0a5.rst b/Misc/NEWS.d/3.13.0a5.rst index 6d74c6bc5c4d55..d8cc88c8756a17 100644 --- a/Misc/NEWS.d/3.13.0a5.rst +++ b/Misc/NEWS.d/3.13.0a5.rst @@ -541,7 +541,7 @@ descriptors in :meth:`inspect.Signature.from_callable`. .. nonce: sGMKr0 .. section: Library -Isolate :mod:`_lsprof` (apply :pep:`687`). +Isolate :mod:`!_lsprof` (apply :pep:`687`). .. @@ -773,8 +773,8 @@ combination with unicode encoding. .. section: Library Fix :func:`io.BufferedReader.tell`, :func:`io.BufferedReader.seek`, -:func:`_pyio.BufferedReader.tell`, :func:`io.BufferedRandom.tell`, -:func:`io.BufferedRandom.seek` and :func:`_pyio.BufferedRandom.tell` being +:func:`!_pyio.BufferedReader.tell`, :func:`io.BufferedRandom.tell`, +:func:`io.BufferedRandom.seek` and :func:`!_pyio.BufferedRandom.tell` being able to return negative offsets. .. diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst index fff29083e0dab7..0cdbb8232250d7 100644 --- a/Misc/NEWS.d/3.13.0a6.rst +++ b/Misc/NEWS.d/3.13.0a6.rst @@ -550,7 +550,7 @@ or DuplicateOptionError. .. nonce: PBiRQB .. section: Library -:class:`_io.WindowsConsoleIO` now emit a warning if a boolean value is +:class:`!_io.WindowsConsoleIO` now emit a warning if a boolean value is passed as a filedescriptor argument. .. diff --git a/Misc/NEWS.d/3.13.0b1.rst b/Misc/NEWS.d/3.13.0b1.rst index ab5f24fe345af9..b09efa45cddc0c 100644 --- a/Misc/NEWS.d/3.13.0b1.rst +++ b/Misc/NEWS.d/3.13.0b1.rst @@ -354,16 +354,6 @@ asend().throw() .. -.. date: 2024-04-13-18-59-25 -.. gh-issue: 115874 -.. nonce: c3xG-E -.. section: Core and Builtins - -Fixed a possible segfault during garbage collection of -``_asyncio.FutureIter`` objects - -.. - .. date: 2024-04-13-16-55-53 .. gh-issue: 117536 .. nonce: xkVbfv @@ -666,7 +656,7 @@ by :pep:`738`. .. section: Library Allow to specify the signature of custom callable instances of extension -type by the :attr:`__text_signature__` attribute. Specify signatures of +type by the ``__text_signature__`` attribute. Specify signatures of :class:`operator.attrgetter`, :class:`operator.itemgetter`, and :class:`operator.methodcaller` instances. @@ -687,10 +677,10 @@ padding is not detected when no padding is necessary. .. nonce: 5N2Xcy .. section: Library -Add the :class:`!PhotoImage` methods :meth:`~tkinter.PhotoImage.read` to -read an image from a file and :meth:`~tkinter.PhotoImage.data` to get the +Add the :class:`!PhotoImage` methods :meth:`!read` to +read an image from a file and :meth:`!data` to get the image data. Add *background* and *grayscale* parameters to -:class:`!PhotoImage` method :meth:`~tkinter.PhotoImage.write`. +:class:`!PhotoImage` method :meth:`!write`. .. @@ -855,7 +845,7 @@ is used to bind indexed, nameless placeholders. See also :gh:`100668`. .. nonce: RstWg- .. section: Library -Fix TypeError in :func:`email.Message.get_payload` when the charset is +Fix TypeError in :func:`email.message.Message.get_payload` when the charset is :rfc:`2231` encoded. .. @@ -883,6 +873,16 @@ Alex Waygood. .. +.. date: 2024-04-13-18-59-25 +.. gh-issue: 115874 +.. nonce: c3xG-E +.. section: Library + +Fixed a possible segfault during garbage collection of +``_asyncio.FutureIter`` objects. Patch by Savannah Ostrowski. + +.. + .. date: 2024-04-13-01-45-15 .. gh-issue: 115060 .. nonce: IxoM03 @@ -953,7 +953,7 @@ Speed up :meth:`pathlib.Path.walk` by working with strings internally. .. nonce: oxIUEI .. section: Library -Change the new multi-separator support in :meth:`asyncio.Stream.readuntil` +Change the new multi-separator support in :meth:`asyncio.StreamReader.readuntil` to only accept tuples of separators rather than arbitrary iterables. .. @@ -1260,7 +1260,7 @@ Support opcode events in :mod:`bdb` .. nonce: YoI8TV .. section: Library -:mod:`ncurses`: fixed a crash that could occur on macOS 13 or earlier when +:mod:`!ncurses`: fixed a crash that could occur on macOS 13 or earlier when Python was built with Apple Xcode 15's SDK. .. @@ -1315,7 +1315,7 @@ Hamdan. .. section: Library Adjust ``logging.LogRecord`` to use ``time.time_ns()`` and fix minor bug -related to floating point math. +related to floating-point math. .. @@ -1347,13 +1347,13 @@ urllib. .. nonce: du4UKW .. section: Library -Setting the :mod:`!tkinter` module global :data:`~tkinter.wantobject` to ``2`` +Setting the :mod:`!tkinter` module global :data:`!wantobjects` to ``2`` before creating the :class:`~tkinter.Tk` object or call the -:meth:`~tkinter.Tk.wantobject` method of the :class:`!Tk` object with argument +:meth:`!wantobjects` method of the :class:`!Tk` object with argument ``2`` makes now arguments to callbacks registered in the :mod:`tkinter` module to be passed as various Python objects (``int``, ``float``, ``bytes``, ``tuple``), depending on their internal representation in Tcl, instead of always ``str``. -:data:`!tkinter.wantobject` is now set to ``2`` by default. +:data:`!tkinter.wantobjects` is now set to ``2`` by default. .. diff --git a/Misc/NEWS.d/3.5.0a1.rst b/Misc/NEWS.d/3.5.0a1.rst index 442ab62fee8185..8f9d8ce57caa05 100644 --- a/Misc/NEWS.d/3.5.0a1.rst +++ b/Misc/NEWS.d/3.5.0a1.rst @@ -3447,7 +3447,8 @@ tkinter.ttk now works when default root window is not set. .. nonce: FE_PII .. section: Library -_tkinter.create() now creates tkapp object with wantobject=1 by default. +``_tkinter.create()`` now creates ``tkapp`` object with ``wantobjects=1`` by +default. .. @@ -5467,7 +5468,7 @@ All resources are now allowed when tests are not run by regrtest.py. .. section: Tests Fix pystone micro-benchmark: use floor division instead of true division to -benchmark integers instead of floating point numbers. Set pystone version to +benchmark integers instead of floating-point numbers. Set pystone version to 1.2. Patch written by Lennart Regebro. .. diff --git a/Misc/NEWS.d/3.6.0a1.rst b/Misc/NEWS.d/3.6.0a1.rst index 5c9a6e5d64b469..803c9fc5925fa6 100644 --- a/Misc/NEWS.d/3.6.0a1.rst +++ b/Misc/NEWS.d/3.6.0a1.rst @@ -1484,9 +1484,9 @@ on UNIX signals (SIGSEGV, SIGFPE, SIGABRT). .. nonce: RWN1jR .. section: Library -Add C functions :c:func:`_PyTraceMalloc_Track` and -:c:func:`_PyTraceMalloc_Untrack` to track memory blocks using the -:mod:`tracemalloc` module. Add :c:func:`_PyTraceMalloc_GetTraceback` to get +Add C functions :c:func:`!_PyTraceMalloc_Track` and +:c:func:`!_PyTraceMalloc_Untrack` to track memory blocks using the +:mod:`tracemalloc` module. Add :c:func:`!_PyTraceMalloc_GetTraceback` to get the traceback of an object. .. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 9decc4034d6b87..c3533643bc0810 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -224,7 +224,7 @@ positives from posix, socket, time, test_io, and test_faulthandler. .. nonce: 9vMWSP .. section: Core and Builtins -Fix an assertion error in :func:`format` in debug build for floating point +Fix an assertion error in :func:`format` in debug build for floating-point formatting with "n" format, zero padding and small width. Release build is not impacted. Patch by Karthikeyan Singaravelan. @@ -2519,7 +2519,7 @@ non-Windows systems. .. nonce: dQS1ng .. section: Library -Fix incorrect parsing of :class:`_io.IncrementalNewlineDecoder`'s +Fix incorrect parsing of :class:`io.IncrementalNewlineDecoder`'s *translate* argument. .. @@ -8051,7 +8051,7 @@ Update macOS 10.9+ installer to Tcl/Tk 8.6.8. .. nonce: K6jCVG .. section: macOS -In :mod:`_scproxy`, drop the GIL when calling into ``SystemConfiguration`` +In :mod:`!_scproxy`, drop the GIL when calling into ``SystemConfiguration`` to avoid deadlocks. .. diff --git a/Misc/NEWS.d/3.8.0a2.rst b/Misc/NEWS.d/3.8.0a2.rst index c8620aeea7f133..0dbfa2758fe601 100644 --- a/Misc/NEWS.d/3.8.0a2.rst +++ b/Misc/NEWS.d/3.8.0a2.rst @@ -202,7 +202,7 @@ the mean and standard deviation of measurement data as single entity. .. nonce: V88MCD .. section: Library -Added statistics.fmean() as a faster, floating point variant of the existing +Added statistics.fmean() as a faster, floating-point variant of the existing mean() function. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 7bf0de1210935b..edce71b2555a89 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -945,7 +945,7 @@ P. Hemsley. .. nonce: __FTq9 .. section: Tests -Add a new :mod:`_testinternalcapi` module to test the internal C API. +Add a new :mod:`!_testinternalcapi` module to test the internal C API. .. @@ -1383,7 +1383,7 @@ Since Python 3.7.0, calling :c:func:`Py_DecodeLocale` before coerced and/or if the UTF-8 Mode is enabled by the user configuration. The LC_CTYPE coercion and UTF-8 Mode are now disabled by default to fix the mojibake issue. They must now be enabled explicitly (opt-in) using the new -:c:func:`_Py_PreInitialize` API with ``_PyPreConfig``. +:c:func:`!_Py_PreInitialize` API with ``_PyPreConfig``. .. diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst index 4174ab8fac6192..fc4e3a9bd887fb 100644 --- a/Misc/NEWS.d/3.8.0b1.rst +++ b/Misc/NEWS.d/3.8.0b1.rst @@ -600,7 +600,7 @@ default. .. nonce: sLULGQ .. section: Library -Fix destructor :class:`_pyio.BytesIO` and :class:`_pyio.TextIOWrapper`: +Fix destructor :class:`!_pyio.BytesIO` and :class:`!_pyio.TextIOWrapper`: initialize their ``_buffer`` attribute as soon as possible (in the class body), because it's used by ``__del__()`` which calls ``close()``. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index a38b93e4b76d17..705a0a32f0e861 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -299,7 +299,7 @@ Check the error from the system's underlying ``crypt`` or ``crypt_r``. .. section: Core and Builtins On FreeBSD, Python no longer calls ``fedisableexcept()`` at startup to -control the floating point control mode. The call became useless since +control the floating-point control mode. The call became useless since FreeBSD 6: it became the default mode. .. @@ -1384,7 +1384,7 @@ Nested subclasses of :class:`typing.NamedTuple` are now pickleable. .. nonce: hwrPN7 .. section: Library -Prevent :exc:`KeyError` thrown by :func:`_encoded_words.decode` when given +Prevent :exc:`KeyError` thrown by :func:`!_encoded_words.decode` when given an encoded-word with invalid content-type encoding from propagating all the way to :func:`email.message.get`. @@ -1395,7 +1395,7 @@ way to :func:`email.message.get`. .. nonce: S6Klvm .. section: Library -Deprecated the ``split()`` method in :class:`_tkinter.TkappType` in favour +Deprecated the ``split()`` method in :class:`!_tkinter.TkappType` in favour of the ``splitlist()`` method which has more consistent and predictable behavior. @@ -3013,7 +3013,7 @@ thread was still running. .. section: Library Allow pure Python implementation of :mod:`pickle` to work even when the C -:mod:`_pickle` module is unavailable. +:mod:`!_pickle` module is unavailable. .. @@ -3064,8 +3064,8 @@ internal tasks weak set is changed by another thread during iteration. .. nonce: ADqCkq .. section: Library -:class:`_pyio.IOBase` destructor now does nothing if getting the ``closed`` -attribute fails to better mimic :class:`_io.IOBase` finalizer. +:class:`!_pyio.IOBase` destructor now does nothing if getting the ``closed`` +attribute fails to better mimic :class:`!_io.IOBase` finalizer. .. @@ -4993,7 +4993,7 @@ Make :const:`winreg.REG_MULTI_SZ` support zero-length strings. .. section: Windows Replace use of :c:func:`strcasecmp` for the system function -:c:func:`_stricmp`. Patch by Minmin Gong. +:c:func:`!_stricmp`. Patch by Minmin Gong. .. @@ -5696,8 +5696,8 @@ Add :c:func:`PyConfig_SetWideStringList` function. .. section: C API Add fast functions for calling methods: -:c:func:`_PyObject_VectorcallMethod`, :c:func:`_PyObject_CallMethodNoArgs` -and :c:func:`_PyObject_CallMethodOneArg`. +:c:func:`!_PyObject_VectorcallMethod`, :c:func:`!_PyObject_CallMethodNoArgs` +and :c:func:`!_PyObject_CallMethodOneArg`. .. diff --git a/Misc/NEWS.d/3.9.0a6.rst b/Misc/NEWS.d/3.9.0a6.rst index b7ea1051c314f2..4ba4cfe818c2d0 100644 --- a/Misc/NEWS.d/3.9.0a6.rst +++ b/Misc/NEWS.d/3.9.0a6.rst @@ -111,7 +111,7 @@ str.decode(). .. nonce: m15TTX .. section: Core and Builtins -Fix possible refleaks in :mod:`_json`, memo of PyScannerObject should be +Fix possible refleaks in :mod:`!_json`, memo of PyScannerObject should be traversed. .. @@ -666,8 +666,8 @@ for _main_thread, instead of a _DummyThread instance. .. nonce: VTq_8s .. section: Library -Add a private ``_at_fork_reinit()`` method to :class:`_thread.Lock`, -:class:`_thread.RLock`, :class:`threading.RLock` and +Add a private ``_at_fork_reinit()`` method to :class:`!_thread.Lock`, +:class:`!_thread.RLock`, :class:`threading.RLock` and :class:`threading.Condition` classes: reinitialize the lock at fork in the child process, reset the lock to the unlocked state. Rename also the private ``_reset_internal_locks()`` method of :class:`threading.Event` to diff --git a/Misc/NEWS.d/next/Build/2024-07-02-12-42-25.gh-issue-120831.i3eIjg.rst b/Misc/NEWS.d/next/Build/2024-07-02-12-42-25.gh-issue-120831.i3eIjg.rst new file mode 100644 index 00000000000000..3784cc66c41219 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-02-12-42-25.gh-issue-120831.i3eIjg.rst @@ -0,0 +1 @@ +The default minimum iOS version was increased to 13.0. diff --git a/Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst b/Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst new file mode 100644 index 00000000000000..4bc8c6de0b7733 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-02-20-16-09.gh-issue-121103.TMef9j.rst @@ -0,0 +1,3 @@ +On POSIX systems, excluding macOS framework installs, the lib directory +for the free-threaded build now includes a "t" suffix to avoid conflicts +with a co-located default build installation. diff --git a/Misc/NEWS.d/next/Build/2024-07-08-01-11-54.gh-issue-121467.3qWRQj.rst b/Misc/NEWS.d/next/Build/2024-07-08-01-11-54.gh-issue-121467.3qWRQj.rst new file mode 100644 index 00000000000000..a2238475546eaa --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-08-01-11-54.gh-issue-121467.3qWRQj.rst @@ -0,0 +1 @@ +Fix a Makefile bug that prevented mimalloc header files from being installed. diff --git a/Misc/NEWS.d/next/Build/2024-07-08-14-01-17.gh-issue-121487.ekHmpR.rst b/Misc/NEWS.d/next/Build/2024-07-08-14-01-17.gh-issue-121487.ekHmpR.rst new file mode 100644 index 00000000000000..e30d4dcdbfe779 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-08-14-01-17.gh-issue-121487.ekHmpR.rst @@ -0,0 +1 @@ +Fix deprecation warning for ATOMIC_VAR_INIT in mimalloc. diff --git a/Misc/NEWS.d/next/Build/2024-07-14-01-29-47.gh-issue-121731.RMPGP3.rst b/Misc/NEWS.d/next/Build/2024-07-14-01-29-47.gh-issue-121731.RMPGP3.rst new file mode 100644 index 00000000000000..36e0f86a0ae455 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-14-01-29-47.gh-issue-121731.RMPGP3.rst @@ -0,0 +1 @@ +Fix mimalloc compile error on GNU/Hurd diff --git a/Misc/NEWS.d/next/Build/2024-07-16-12-29-54.gh-issue-120371.E7x858.rst b/Misc/NEWS.d/next/Build/2024-07-16-12-29-54.gh-issue-120371.E7x858.rst new file mode 100644 index 00000000000000..d57266dafd8d67 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-16-12-29-54.gh-issue-120371.E7x858.rst @@ -0,0 +1,2 @@ +Support WASI SDK 22 by explicitly skipping functions that are just stubs in +wasi-libc. diff --git a/Misc/NEWS.d/next/Build/2024-06-25-15-29-27.gh-issue-120522.5_n515.rst b/Misc/NEWS.d/next/Build/2024-07-18-07-53-07.gh-issue-120522.dg3o5A.rst similarity index 52% rename from Misc/NEWS.d/next/Build/2024-06-25-15-29-27.gh-issue-120522.5_n515.rst rename to Misc/NEWS.d/next/Build/2024-07-18-07-53-07.gh-issue-120522.dg3o5A.rst index 4a397a445ccc3b..e90c625a886b65 100644 --- a/Misc/NEWS.d/next/Build/2024-06-25-15-29-27.gh-issue-120522.5_n515.rst +++ b/Misc/NEWS.d/next/Build/2024-07-18-07-53-07.gh-issue-120522.dg3o5A.rst @@ -1,2 +1,2 @@ -Added a :option:`--with-app-store-compliance` option to patch out known issues -with macOS/iOS App Store review processes. +Added a :option:`--with-app-store-compliance` option to patch out known +issues with macOS/iOS App Store review processes. diff --git a/Misc/NEWS.d/next/Build/2024-07-19-10-14-31.gh-issue-121996.IEb2sz.rst b/Misc/NEWS.d/next/Build/2024-07-19-10-14-31.gh-issue-121996.IEb2sz.rst new file mode 100644 index 00000000000000..171efe8388bc0d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-07-19-10-14-31.gh-issue-121996.IEb2sz.rst @@ -0,0 +1,2 @@ +Introduce ./configure --disable-safety and --enable-slower-safety options. +Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/C API/2024-07-04-13-23-27.gh-issue-113601.K3RLqp.rst b/Misc/NEWS.d/next/C API/2024-07-04-13-23-27.gh-issue-113601.K3RLqp.rst new file mode 100644 index 00000000000000..009cc2bf017180 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-07-04-13-23-27.gh-issue-113601.K3RLqp.rst @@ -0,0 +1,2 @@ +Removed debug build assertions related to interning strings, which were +falsely triggered by stable ABI extensions. diff --git a/Misc/NEWS.d/next/C API/2024-07-04-15-41-10.gh-issue-113993.cLSiWV.rst b/Misc/NEWS.d/next/C API/2024-07-04-15-41-10.gh-issue-113993.cLSiWV.rst new file mode 100644 index 00000000000000..9b7f2082065eaa --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-07-04-15-41-10.gh-issue-113993.cLSiWV.rst @@ -0,0 +1,12 @@ +:c:func:`PyUnicode_InternInPlace` no longer prevents its argument from being +garbage collected. + +Several functions that take ``char *`` are now +documented as possibly preventing string objects from being garbage +collected; refer to their documentation for details: +:c:func:`PyUnicode_InternFromString`, +:c:func:`PyDict_SetItemString`, +:c:func:`PyObject_SetAttrString`, +:c:func:`PyObject_DelAttrString`, +:c:func:`PyUnicode_InternFromString`, +and ``PyModule_Add*`` convenience functions. diff --git a/Misc/NEWS.d/next/C API/2024-07-21-17-40-07.gh-issue-121489.SUMFCr.rst b/Misc/NEWS.d/next/C API/2024-07-21-17-40-07.gh-issue-121489.SUMFCr.rst new file mode 100644 index 00000000000000..8c18a49c05d547 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-07-21-17-40-07.gh-issue-121489.SUMFCr.rst @@ -0,0 +1 @@ +Export private :c:func:`!_PyBytes_Join` again. diff --git a/Misc/NEWS.d/next/C_API/2024-07-09-15-55-20.gh-issue-89364.yYYroI.rst b/Misc/NEWS.d/next/C_API/2024-07-09-15-55-20.gh-issue-89364.yYYroI.rst new file mode 100644 index 00000000000000..b82e78446e4e87 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-07-09-15-55-20.gh-issue-89364.yYYroI.rst @@ -0,0 +1,3 @@ +Export the :c:func:`PySignal_SetWakeupFd` function. Previously, the function +was documented but it couldn't be used in 3rd party code. Patch by Victor +Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-07-22-15-56-35.gh-issue-95144.FZYWX-.rst b/Misc/NEWS.d/next/Core and Builtins/2022-07-22-15-56-35.gh-issue-95144.FZYWX-.rst new file mode 100644 index 00000000000000..83b1126a8a455f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-07-22-15-56-35.gh-issue-95144.FZYWX-.rst @@ -0,0 +1,2 @@ +Improve the error message from ``a in b`` when ``b`` is not a container +to mention the term "container". diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-14-23-05-40.gh-issue-84978.Z0t6dg.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-14-23-05-40.gh-issue-84978.Z0t6dg.rst new file mode 100644 index 00000000000000..b1f08288f925da --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-14-23-05-40.gh-issue-84978.Z0t6dg.rst @@ -0,0 +1 @@ +Add class methods :meth:`float.from_number` and :meth:`complex.from_number`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-30-04-11-36.gh-issue-118934.fbDqve.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-30-04-11-36.gh-issue-118934.fbDqve.rst new file mode 100644 index 00000000000000..3087034fe458b8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-30-04-11-36.gh-issue-118934.fbDqve.rst @@ -0,0 +1 @@ +Make ``PyEval_GetLocals`` return borrowed reference diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-13-12-17-52.gh-issue-120384.w1UBGl.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-13-12-17-52.gh-issue-120384.w1UBGl.rst index 4a4db821ce29b8..65959ca2d28075 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-06-13-12-17-52.gh-issue-120384.w1UBGl.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-13-12-17-52.gh-issue-120384.w1UBGl.rst @@ -1,3 +1,3 @@ Fix an array out of bounds crash in ``list_ass_subscript``, which could be -invoked via some specificly tailored input: including concurrent modification +invoked via some specifically tailored input: including concurrent modification of a list object, where one thread assigns a slice and another clears it. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-29-10-46-14.gh-issue-121130.Rj66Xs.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-29-10-46-14.gh-issue-121130.Rj66Xs.rst new file mode 100644 index 00000000000000..7084f0cbebbb73 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-29-10-46-14.gh-issue-121130.Rj66Xs.rst @@ -0,0 +1,2 @@ +Fix f-strings with debug expressions in format specifiers. Patch by Pablo +Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-04-23-38-30.gh-issue-121368.m3EF9E.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-04-23-38-30.gh-issue-121368.m3EF9E.rst new file mode 100644 index 00000000000000..3df5b216cbc0af --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-04-23-38-30.gh-issue-121368.m3EF9E.rst @@ -0,0 +1,3 @@ +Fix race condition in ``_PyType_Lookup`` in the free-threaded build due to +a missing memory fence. This could lead to ``_PyType_Lookup`` returning +incorrect results on arm64. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-08-02-24-55.gh-issue-121439.jDHod3.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-02-24-55.gh-issue-121439.jDHod3.rst new file mode 100644 index 00000000000000..361f9fc71186c6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-02-24-55.gh-issue-121439.jDHod3.rst @@ -0,0 +1 @@ +Allow tuples of length 20 in the freelist to be reused. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-08-10-31-08.gh-issue-121012.M5hHk-.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-10-31-08.gh-issue-121012.M5hHk-.rst new file mode 100644 index 00000000000000..7b04eb68b03752 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-10-31-08.gh-issue-121012.M5hHk-.rst @@ -0,0 +1,2 @@ +Tier 2 execution now ensures that list iterators remain exhausted, once they +become exhausted. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-08-17-15-14.gh-issue-121497.I8hMDC.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-17-15-14.gh-issue-121497.I8hMDC.rst new file mode 100644 index 00000000000000..33de31abebe7c7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-08-17-15-14.gh-issue-121497.I8hMDC.rst @@ -0,0 +1,2 @@ +Fix a bug that was preventing the REPL to correctly respect the history when +an input hook was set. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-09-13-53-18.gh-issue-121499.rpp7il.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-09-13-53-18.gh-issue-121499.rpp7il.rst new file mode 100644 index 00000000000000..aec8ab9d4662b5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-09-13-53-18.gh-issue-121499.rpp7il.rst @@ -0,0 +1,2 @@ +Fix a bug affecting how multi-line history was being rendered in the new +REPL after interacting with the new screen cache. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-10-02-02-32.gh-issue-121562.8beIMi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-10-02-02-32.gh-issue-121562.8beIMi.rst new file mode 100644 index 00000000000000..940380971f407f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-10-02-02-32.gh-issue-121562.8beIMi.rst @@ -0,0 +1,2 @@ +Optimized performance of hex_from_char by replacing switch-case with a +lookup table diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-10-15-43-54.gh-issue-117482.5WYaXR.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-10-15-43-54.gh-issue-117482.5WYaXR.rst new file mode 100644 index 00000000000000..ec1e7327b77f19 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-10-15-43-54.gh-issue-117482.5WYaXR.rst @@ -0,0 +1,2 @@ +Unexpected slot wrappers are no longer created for builtin static types in +subinterpreters. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-12-18-18-44.gh-issue-121297.67VE7b.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-12-18-18-44.gh-issue-121297.67VE7b.rst new file mode 100644 index 00000000000000..25aae6c8c5cb10 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-12-18-18-44.gh-issue-121297.67VE7b.rst @@ -0,0 +1,4 @@ +Previously, incorrect usage of :keyword:`await` or asynchronous +comprehensions in code removed by the :option:`-O` option was not flagged by +the Python compiler. Now, such code raises :exc:`SyntaxError`. Patch by +Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-13-12-27-31.gh-issue-121657.wgOYLw.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-13-12-27-31.gh-issue-121657.wgOYLw.rst new file mode 100644 index 00000000000000..cb18629d79f5ce --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-13-12-27-31.gh-issue-121657.wgOYLw.rst @@ -0,0 +1,2 @@ +Improve the :exc:`SyntaxError` message if the user tries to use +:keyword:`yield from ` outside a function. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-15-16-26-32.gh-issue-121794.fhBtiQ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-15-16-26-32.gh-issue-121794.fhBtiQ.rst new file mode 100644 index 00000000000000..979efa0a0a1597 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-15-16-26-32.gh-issue-121794.fhBtiQ.rst @@ -0,0 +1,2 @@ +Fix bug in free-threaded Python where a resurrected object could lead to +a negative ref count assertion failure. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-15-20-41-06.gh-issue-121814.oR2ixR.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-15-20-41-06.gh-issue-121814.oR2ixR.rst new file mode 100644 index 00000000000000..14666de45f32e3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-15-20-41-06.gh-issue-121814.oR2ixR.rst @@ -0,0 +1 @@ +Fixed the SegFault when :c:func:`PyEval_SetTrace` is used with no Python frame on stack. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-16-15-11-51.gh-issue-121795.xkIHrI.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-16-15-11-51.gh-issue-121795.xkIHrI.rst new file mode 100644 index 00000000000000..b4102649c56758 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-16-15-11-51.gh-issue-121795.xkIHrI.rst @@ -0,0 +1 @@ +Improve performance of set membership testing, ``set.remove()`` and ``set.discard()`` when the argument is a set. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-16-18-23-22.gh-issue-121860.-FTauD.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-16-18-23-22.gh-issue-121860.-FTauD.rst new file mode 100644 index 00000000000000..a03ee83d6f8ec9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-16-18-23-22.gh-issue-121860.-FTauD.rst @@ -0,0 +1 @@ +Fix crash when rematerializing a managed dictionary after it was deleted. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-19-15-28-05.gh-issue-122026.sta2Ca.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-19-15-28-05.gh-issue-122026.sta2Ca.rst new file mode 100644 index 00000000000000..2721a405a50446 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-19-15-28-05.gh-issue-122026.sta2Ca.rst @@ -0,0 +1,2 @@ +Fix a bug that caused the tokenizer to not correctly identify mismatched +parentheses inside f-strings in some situations. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-21-01-23-54.gh-issue-122029.gKv-e2.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-21-01-23-54.gh-issue-122029.gKv-e2.rst new file mode 100644 index 00000000000000..bddee3a57fba80 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-21-01-23-54.gh-issue-122029.gKv-e2.rst @@ -0,0 +1 @@ +Emit ``c_call`` events in :func:`sys.setprofile` when a ``PyMethodObject`` pointing to a ``PyCFunction`` is called. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-23-11-57-36.gh-issue-122160.HSnrAP.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-23-11-57-36.gh-issue-122160.HSnrAP.rst new file mode 100644 index 00000000000000..78153fc1abdaeb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-23-11-57-36.gh-issue-122160.HSnrAP.rst @@ -0,0 +1 @@ +Remove the ``BUILD_CONST_KEY_MAP`` opcode. Use :opcode:`BUILD_MAP` instead. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-23-23-59-04.gh-issue-122208.z8KHsY.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-23-23-59-04.gh-issue-122208.z8KHsY.rst new file mode 100644 index 00000000000000..e4a89d137ede0e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-23-23-59-04.gh-issue-122208.z8KHsY.rst @@ -0,0 +1 @@ +Dictionary watchers now only deliver the PyDict_EVENT_ADDED event when the insertion is in a known good state to succeed. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-24-17-11-51.gh-issue-122234.VxsP_F.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-24-17-11-51.gh-issue-122234.VxsP_F.rst new file mode 100644 index 00000000000000..b86d6fbdfc648f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-24-17-11-51.gh-issue-122234.VxsP_F.rst @@ -0,0 +1,4 @@ +Specializations for sums with float and complex inputs in :func:`sum()` now +always use compensated summation. Also, for integer items in above +specializations: :c:func:`PyLong_AsDouble` is used, instead of +:c:func:`PyLong_AsLongAndOverflow`. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-24-22-39-07.gh-issue-122245.LVa9v8.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-24-22-39-07.gh-issue-122245.LVa9v8.rst new file mode 100644 index 00000000000000..fff99b4992e321 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-24-22-39-07.gh-issue-122245.LVa9v8.rst @@ -0,0 +1,4 @@ +Detection of writes to ``__debug__`` is moved from the compiler's codegen +stage to the symtable. This means that these errors are now detected even in +code that is optimized away before codegen (such as assertions with the +:option:`-O` command line option). diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-26-14-05-51.gh-issue-122300.SVIF-l.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-26-14-05-51.gh-issue-122300.SVIF-l.rst new file mode 100644 index 00000000000000..6b58f89247d1d4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-26-14-05-51.gh-issue-122300.SVIF-l.rst @@ -0,0 +1,2 @@ +Preserve AST nodes for f-string with single-element format specifiers. Patch +by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-07-13-09-51-44.gh-issue-121609.jWsE5t.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-07-13-09-51-44.gh-issue-121609.jWsE5t.rst new file mode 100644 index 00000000000000..72b5c071a5c67b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-07-13-09-51-44.gh-issue-121609.jWsE5t.rst @@ -0,0 +1 @@ +Fix pasting of characters containing unicode character joiners in the new REPL. Patch by Marta Gomez Macias diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-07-15-20-03-29.gh-issue-121295.w53ucI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-07-15-20-03-29.gh-issue-121295.w53ucI.rst new file mode 100644 index 00000000000000..7fca7d5461d39b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-07-15-20-03-29.gh-issue-121295.w53ucI.rst @@ -0,0 +1,2 @@ +Fix PyREPL console getting into a blocked state after interrupting a long +paste diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-07-18-21-19-04.gh-issue-121999.8IBbTK.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-07-18-21-19-04.gh-issue-121999.8IBbTK.rst new file mode 100644 index 00000000000000..e65aa993566446 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-07-18-21-19-04.gh-issue-121999.8IBbTK.rst @@ -0,0 +1,2 @@ +The default extraction filter for the :mod:`tarfile` module is now +set to :func:`'data' `. diff --git a/Misc/NEWS.d/next/Documentation/2024-07-14-11-48-10.gh-issue-121749.nxHoTk.rst b/Misc/NEWS.d/next/Documentation/2024-07-14-11-48-10.gh-issue-121749.nxHoTk.rst new file mode 100644 index 00000000000000..17dc60c11468f4 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-07-14-11-48-10.gh-issue-121749.nxHoTk.rst @@ -0,0 +1 @@ +Fix documentation for :c:func:`PyModule_AddObjectRef`. diff --git a/Misc/NEWS.d/next/Documentation/2024-07-14-12-25-53.gh-issue-117765.YFMOUv.rst b/Misc/NEWS.d/next/Documentation/2024-07-14-12-25-53.gh-issue-117765.YFMOUv.rst new file mode 100644 index 00000000000000..a727c1aa9a0571 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-07-14-12-25-53.gh-issue-117765.YFMOUv.rst @@ -0,0 +1 @@ +Improved documentation for :func:`unittest.mock.patch.dict` diff --git a/Misc/NEWS.d/next/IDLE/2024-07-16-16-57-03.gh-issue-78889.U7ghFD.rst b/Misc/NEWS.d/next/IDLE/2024-07-16-16-57-03.gh-issue-78889.U7ghFD.rst new file mode 100644 index 00000000000000..604194ebb2e859 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2024-07-16-16-57-03.gh-issue-78889.U7ghFD.rst @@ -0,0 +1,2 @@ +Stop Shell freezes by blocking user access to non-method sys.stdout.shell attributes, +which are all private. diff --git a/Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst b/Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst index 3f70168b81069e..bc9187309c6a53 100644 --- a/Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst +++ b/Misc/NEWS.d/next/Library/2023-04-24-05-34-23.gh-issue-103194.GwBwWL.rst @@ -1,4 +1,4 @@ Prepare Tkinter for C API changes in Tcl 8.7/9.0 to avoid -:class:`_tkinter.Tcl_Obj` being unexpectedly returned +:class:`!_tkinter.Tcl_Obj` being unexpectedly returned instead of :class:`bool`, :class:`str`, :class:`bytearray`, or :class:`int`. diff --git a/Misc/NEWS.d/next/Library/2024-05-15-01-21-44.gh-issue-73991.bNDqQN.rst b/Misc/NEWS.d/next/Library/2024-05-15-01-21-44.gh-issue-73991.bNDqQN.rst new file mode 100644 index 00000000000000..9aa7a7dba666af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-15-01-21-44.gh-issue-73991.bNDqQN.rst @@ -0,0 +1 @@ +Add :meth:`pathlib.Path.rmtree`, which recursively removes a directory. diff --git a/Misc/NEWS.d/next/Library/2024-05-21-23-39-22.gh-issue-118830.YTqvEo.rst b/Misc/NEWS.d/next/Library/2024-05-21-23-39-22.gh-issue-118830.YTqvEo.rst new file mode 100644 index 00000000000000..d06499831dc009 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-21-23-39-22.gh-issue-118830.YTqvEo.rst @@ -0,0 +1 @@ +Bump :mod:`pickle` default protocol to ``5``. diff --git a/Misc/NEWS.d/next/Library/2024-06-08-17-41-11.gh-issue-82017.WpSTGi.rst b/Misc/NEWS.d/next/Library/2024-06-08-17-41-11.gh-issue-82017.WpSTGi.rst new file mode 100644 index 00000000000000..7decee7ff3384e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-17-41-11.gh-issue-82017.WpSTGi.rst @@ -0,0 +1,2 @@ +Added support for converting any objects that have the +:meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. diff --git a/Misc/NEWS.d/next/Library/2024-06-09-19-53-11.gh-issue-120289.s4HXR0.rst b/Misc/NEWS.d/next/Library/2024-06-09-19-53-11.gh-issue-120289.s4HXR0.rst new file mode 100644 index 00000000000000..518f79dc446ae7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-09-19-53-11.gh-issue-120289.s4HXR0.rst @@ -0,0 +1,2 @@ +Fixed the use-after-free issue in :mod:`cProfile` by disallowing +``disable()`` and ``clear()`` in external timers. diff --git a/Misc/NEWS.d/next/Library/2024-06-11-07-17-25.gh-issue-119180.iH-2zy.rst b/Misc/NEWS.d/next/Library/2024-06-11-07-17-25.gh-issue-119180.iH-2zy.rst new file mode 100644 index 00000000000000..f24d7bd6b9d26c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-11-07-17-25.gh-issue-119180.iH-2zy.rst @@ -0,0 +1,4 @@ +As part of implementing :pep:`649` and :pep:`749`, add a new module +``annotationlib``. Add support for unresolved forward references in +annotations to :mod:`dataclasses`, :class:`typing.TypedDict`, and +:class:`typing.NamedTuple`. diff --git a/Misc/NEWS.d/next/Library/2024-06-22-17-01-56.gh-issue-120678.Ik8dCg.rst b/Misc/NEWS.d/next/Library/2024-06-22-17-01-56.gh-issue-120678.Ik8dCg.rst new file mode 100644 index 00000000000000..ef0d3e3299e2e9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-22-17-01-56.gh-issue-120678.Ik8dCg.rst @@ -0,0 +1,3 @@ +Fix regression in the new REPL that meant that globals from files passed +using the ``-i`` argument would not be included in the REPL's global +namespace. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst b/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst index eac5bab3e9fe6d..346a89879cad41 100644 --- a/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst +++ b/Misc/NEWS.d/next/Library/2024-06-26-03-04-24.gh-issue-121018.clVSc4.rst @@ -1,3 +1,3 @@ -Fixed issues where :meth:`!argparse.ArgumentParser.parses_args` did not honor +Fixed issues where :meth:`!argparse.ArgumentParser.parse_args` did not honor ``exit_on_error=False``. Based on patch by Ben Hsing. diff --git a/Misc/NEWS.d/next/Library/2024-06-29-19-30-15.gh-issue-121163.SJKDFq.rst b/Misc/NEWS.d/next/Library/2024-06-29-19-30-15.gh-issue-121163.SJKDFq.rst index 029838030278a6..50f945ab9f1436 100644 --- a/Misc/NEWS.d/next/Library/2024-06-29-19-30-15.gh-issue-121163.SJKDFq.rst +++ b/Misc/NEWS.d/next/Library/2024-06-29-19-30-15.gh-issue-121163.SJKDFq.rst @@ -1,3 +1,2 @@ Add support for ``all`` as an valid ``action`` for :func:`warnings.simplefilter` -and :func:`warnings.filterswarnings`. - +and :func:`warnings.filterwarnings`. diff --git a/Misc/NEWS.d/next/Library/2024-07-03-07-25-21.gh-issue-121332.Iz6FEq.rst b/Misc/NEWS.d/next/Library/2024-07-03-07-25-21.gh-issue-121332.Iz6FEq.rst new file mode 100644 index 00000000000000..480f27e05953a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-03-07-25-21.gh-issue-121332.Iz6FEq.rst @@ -0,0 +1,4 @@ +Fix constructor of :mod:`ast` nodes with custom ``_attributes``. Previously, +passing custom attributes would raise a :py:exc:`DeprecationWarning`. Passing +arguments to the constructor that are not in ``_fields`` or ``_attributes`` +remains deprecated. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2024-07-06-16-08-39.gh-issue-119169.o0YymL.rst b/Misc/NEWS.d/next/Library/2024-07-06-16-08-39.gh-issue-119169.o0YymL.rst new file mode 100644 index 00000000000000..5d9b50d452a9cd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-06-16-08-39.gh-issue-119169.o0YymL.rst @@ -0,0 +1 @@ +Slightly speed up :func:`os.walk` by simplifying exception handling. diff --git a/Misc/NEWS.d/next/Library/2024-07-06-23-39-38.gh-issue-121450.vGqb3c.rst b/Misc/NEWS.d/next/Library/2024-07-06-23-39-38.gh-issue-121450.vGqb3c.rst new file mode 100644 index 00000000000000..4a65fb737f025b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-06-23-39-38.gh-issue-121450.vGqb3c.rst @@ -0,0 +1,4 @@ +Hard-coded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace()`) now +reuse the most recent ``Pdb`` instance that calls ``Pdb.set_trace()``, +instead of creating a new one each time. As a result, all the instance specific +data like ``display`` and ``commands`` are preserved across Hard-coded breakpoints. diff --git a/Misc/NEWS.d/next/Library/2024-07-13-06-23-24.gh-issue-121245.RfOgf4.rst b/Misc/NEWS.d/next/Library/2024-07-13-06-23-24.gh-issue-121245.RfOgf4.rst new file mode 100644 index 00000000000000..1758f587157f36 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-13-06-23-24.gh-issue-121245.RfOgf4.rst @@ -0,0 +1,3 @@ +Simplify handling of the history file in ``site.register_readline()`` +helper. The ``CAN_USE_PYREPL`` variable now will be initialized, when +imported. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2024-07-14-06-24-02.gh-issue-57141.C3jhDh.rst b/Misc/NEWS.d/next/Library/2024-07-14-06-24-02.gh-issue-57141.C3jhDh.rst new file mode 100644 index 00000000000000..33e9ab94852e35 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-14-06-24-02.gh-issue-57141.C3jhDh.rst @@ -0,0 +1,2 @@ +The *shallow* argument to :class:`filecmp.dircmp` (new in Python 3.13) is +now keyword-only. diff --git a/Misc/NEWS.d/next/Library/2024-07-14-11-18-28.gh-issue-120930.Kuo4L0.rst b/Misc/NEWS.d/next/Library/2024-07-14-11-18-28.gh-issue-120930.Kuo4L0.rst new file mode 100644 index 00000000000000..9e11595cdb50b8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-14-11-18-28.gh-issue-120930.Kuo4L0.rst @@ -0,0 +1,2 @@ +Fixed a bug introduced by gh-92081 that added an incorrect extra +blank to encoded words occurring in wrapped headers. diff --git a/Misc/NEWS.d/next/Library/2024-07-17-09-44-35.gh-issue-119698.WlygzR.rst b/Misc/NEWS.d/next/Library/2024-07-17-09-44-35.gh-issue-119698.WlygzR.rst new file mode 100644 index 00000000000000..5134e609e7f1ca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-17-09-44-35.gh-issue-119698.WlygzR.rst @@ -0,0 +1,3 @@ +Due to the lack of interest for :meth:`symtable.Class.get_methods`, the +method is marked as deprecated and will be removed in Python 3.16. Patch by +Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-07-21-18-03-30.gh-issue-122088.vi2bP-.rst b/Misc/NEWS.d/next/Library/2024-07-21-18-03-30.gh-issue-122088.vi2bP-.rst new file mode 100644 index 00000000000000..9c173d8c462feb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-21-18-03-30.gh-issue-122088.vi2bP-.rst @@ -0,0 +1,3 @@ +:func:`@warnings.deprecated ` now copies the +coroutine status of functions and methods so that +:func:`inspect.iscoroutinefunction` returns the correct result. diff --git a/Misc/NEWS.d/next/Library/2024-07-22-08-14-04.gh-issue-113785.6B_KNB.rst b/Misc/NEWS.d/next/Library/2024-07-22-08-14-04.gh-issue-113785.6B_KNB.rst new file mode 100644 index 00000000000000..89d44a3f79c390 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-22-08-14-04.gh-issue-113785.6B_KNB.rst @@ -0,0 +1 @@ +:mod:`csv` now correctly parses numeric fields (when used with :const:`csv.QUOTE_NONNUMERIC` or :const:`csv.QUOTE_STRINGS`) which start with an escape character. diff --git a/Misc/NEWS.d/next/Library/2024-07-23-09-14-44.gh-issue-82951.-F5p5A.rst b/Misc/NEWS.d/next/Library/2024-07-23-09-14-44.gh-issue-82951.-F5p5A.rst new file mode 100644 index 00000000000000..b3f07889119c9f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-23-09-14-44.gh-issue-82951.-F5p5A.rst @@ -0,0 +1,3 @@ +Serializing objects with complex ``__qualname__`` (such as unbound methods +and nested classes) by name no longer involves serializing parent objects by +value in pickle protocols < 4. diff --git a/Misc/NEWS.d/next/Library/2024-07-23-13-07-12.gh-issue-122129.PwbC8q.rst b/Misc/NEWS.d/next/Library/2024-07-23-13-07-12.gh-issue-122129.PwbC8q.rst new file mode 100644 index 00000000000000..08beb45653d24b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-23-13-07-12.gh-issue-122129.PwbC8q.rst @@ -0,0 +1 @@ +Improve support of method descriptors and wrappers in the help title. diff --git a/Misc/NEWS.d/next/Library/2024-07-23-15-11-13.gh-issue-122163.4wRUuM.rst b/Misc/NEWS.d/next/Library/2024-07-23-15-11-13.gh-issue-122163.4wRUuM.rst new file mode 100644 index 00000000000000..a4625c2a0e50e3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-23-15-11-13.gh-issue-122163.4wRUuM.rst @@ -0,0 +1,2 @@ +Add notes for JSON serialization errors that allow to identify the source of +the error. diff --git a/Misc/NEWS.d/next/Library/2024-07-23-15-30-23.gh-issue-122170.Z9gi3Y.rst b/Misc/NEWS.d/next/Library/2024-07-23-15-30-23.gh-issue-122170.Z9gi3Y.rst new file mode 100644 index 00000000000000..7eeb9f67ad4b3a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-23-15-30-23.gh-issue-122170.Z9gi3Y.rst @@ -0,0 +1,2 @@ +Handle :exc:`ValueError`\s raised by :func:`os.stat` in :mod:`linecache`. +Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-07-23-17-13-10.gh-issue-119180.5PZELo.rst b/Misc/NEWS.d/next/Library/2024-07-23-17-13-10.gh-issue-119180.5PZELo.rst new file mode 100644 index 00000000000000..d65e89f7523b0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-23-17-13-10.gh-issue-119180.5PZELo.rst @@ -0,0 +1,2 @@ +Fix handling of classes with custom metaclasses in +``annotationlib.get_annotations``. diff --git a/Misc/NEWS.d/next/Library/2024-07-23-22-26-00.gh-issue-119180.B2IVT8.rst b/Misc/NEWS.d/next/Library/2024-07-23-22-26-00.gh-issue-119180.B2IVT8.rst new file mode 100644 index 00000000000000..13f51e4c42f4a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-23-22-26-00.gh-issue-119180.B2IVT8.rst @@ -0,0 +1,7 @@ +As part of :pep:`749`, add the following attributes for customizing +evaluation of annotation scopes: + +* ``evaluate_value`` on :class:`typing.TypeAliasType` +* ``evaluate_bound``, ``evaluate_constraints``, and ``evaluate_default`` on :class:`typing.TypeVar` +* ``evaluate_default`` on :class:`typing.ParamSpec` +* ``evaluate_default`` on :class:`typing.TypeVarTuple` diff --git a/Misc/NEWS.d/next/Library/2024-07-26-21-21-13.gh-issue-122332.fvw88r.rst b/Misc/NEWS.d/next/Library/2024-07-26-21-21-13.gh-issue-122332.fvw88r.rst new file mode 100644 index 00000000000000..55bb1dc44add1b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-26-21-21-13.gh-issue-122332.fvw88r.rst @@ -0,0 +1,2 @@ +Fixed segfault with :meth:`asyncio.Task.get_coro` when using an eager task +factory. diff --git a/Misc/NEWS.d/next/Library/2024-07-29-10-24-48.gh-issue-122311.xChV1b.rst b/Misc/NEWS.d/next/Library/2024-07-29-10-24-48.gh-issue-122311.xChV1b.rst new file mode 100644 index 00000000000000..8d70c610a8dad6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-29-10-24-48.gh-issue-122311.xChV1b.rst @@ -0,0 +1 @@ +Fix some error messages in :mod:`pickle`. diff --git a/Misc/NEWS.d/next/Security/2024-07-08-23-39-04.gh-issue-112301.TD8G01.rst b/Misc/NEWS.d/next/Security/2024-07-08-23-39-04.gh-issue-112301.TD8G01.rst new file mode 100644 index 00000000000000..d9b48993a2fb1a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-07-08-23-39-04.gh-issue-112301.TD8G01.rst @@ -0,0 +1,2 @@ +Enable runtime protections for glibc to abort execution when unsafe behavior is encountered, +for all platforms except Windows. diff --git a/Misc/NEWS.d/next/Security/2024-07-18-13-17-47.gh-issue-121957.QemKLU.rst b/Misc/NEWS.d/next/Security/2024-07-18-13-17-47.gh-issue-121957.QemKLU.rst new file mode 100644 index 00000000000000..49ccc5e14633cd --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-07-18-13-17-47.gh-issue-121957.QemKLU.rst @@ -0,0 +1,3 @@ +Fixed missing audit events around interactive use of Python, now also +properly firing for ``python -i``, as well as for ``python -m asyncio``. The +events in question are ``cpython.run_stdin`` and ``cpython.run_startup``. diff --git a/Misc/NEWS.d/next/Tests/2024-07-03-14-41-00.gh-issue-121160.LEtiTd.rst b/Misc/NEWS.d/next/Tests/2024-07-03-14-41-00.gh-issue-121160.LEtiTd.rst new file mode 100644 index 00000000000000..2c8c9ac7201836 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-07-03-14-41-00.gh-issue-121160.LEtiTd.rst @@ -0,0 +1,2 @@ +Add a test for :func:`readline.set_history_length`. Note that this test may +fail on readline libraries. diff --git a/Misc/NEWS.d/next/Tests/2024-07-13-11-04-44.gh-issue-99242.aGxnwz.rst b/Misc/NEWS.d/next/Tests/2024-07-13-11-04-44.gh-issue-99242.aGxnwz.rst new file mode 100644 index 00000000000000..7d904f26a36ff2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-07-13-11-04-44.gh-issue-99242.aGxnwz.rst @@ -0,0 +1,3 @@ +:func:`os.getloadavg` may throw :exc:`OSError` when running regression tests +under certain conditions (e.g. chroot). This error is now caught and +ignored, since reporting load average is optional. diff --git a/Misc/NEWS.d/next/Tests/2024-07-13-11-48-20.gh-issue-59022.fYNbQ8.rst b/Misc/NEWS.d/next/Tests/2024-07-13-11-48-20.gh-issue-59022.fYNbQ8.rst new file mode 100644 index 00000000000000..e1acebe922cfdf --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-07-13-11-48-20.gh-issue-59022.fYNbQ8.rst @@ -0,0 +1 @@ +Add tests for :func:`pkgutil.extend_path`. Patch by Andreas Stocker. diff --git a/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst b/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst new file mode 100644 index 00000000000000..d5718ed4be7606 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-07-13-21-55-58.gh-issue-112301.YJS1dl.rst @@ -0,0 +1,2 @@ +Add tooling to check for changes in compiler warnings. +Patch by Nate Ohlson. diff --git a/Misc/NEWS.d/next/Tests/2024-07-17-08-25-06.gh-issue-121921.HW8CIS.rst b/Misc/NEWS.d/next/Tests/2024-07-17-08-25-06.gh-issue-121921.HW8CIS.rst new file mode 100644 index 00000000000000..ef14fa9dfbd466 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-07-17-08-25-06.gh-issue-121921.HW8CIS.rst @@ -0,0 +1,2 @@ +Update ``Lib/test/crashers/bogus_code_obj.py`` so that it crashes properly +again. diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index aa4e60e272653b..4dcc0f55176d0e 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -30,6 +30,7 @@ _weakref _weakref.c _abc _abc.c _functools _functoolsmodule.c _locale _localemodule.c +_opcode _opcode.c _operator _operator.c _stat _stat.c _symtable symtablemodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 78b979698fcd75..dfc75077650df8 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -36,7 +36,6 @@ @MODULE__HEAPQ_TRUE@_heapq _heapqmodule.c @MODULE__JSON_TRUE@_json _json.c @MODULE__LSPROF_TRUE@_lsprof _lsprof.c rotatingtree.c -@MODULE__OPCODE_TRUE@_opcode _opcode.c @MODULE__PICKLE_TRUE@_pickle _pickle.c @MODULE__QUEUE_TRUE@_queue _queuemodule.c @MODULE__RANDOM_TRUE@_random _randommodule.c diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 87ad236cdbb39f..873c17cd78709d 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_dict.h" // _PyDict_GetItem_KnownHash() +#include "pycore_freelist.h" // _Py_FREELIST_POP() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _Py_SetImmortalUntracked @@ -75,12 +76,21 @@ typedef struct { #define Future_Check(state, obj) PyObject_TypeCheck(obj, state->FutureType) #define Task_Check(state, obj) PyObject_TypeCheck(obj, state->TaskType) -#define FI_FREELIST_MAXLEN 255 +#ifdef Py_GIL_DISABLED +# define ASYNCIO_STATE_LOCK(state) PyMutex_Lock(&state->mutex) +# define ASYNCIO_STATE_UNLOCK(state) PyMutex_Unlock(&state->mutex) +#else +# define ASYNCIO_STATE_LOCK(state) ((void)state) +# define ASYNCIO_STATE_UNLOCK(state) ((void)state) +#endif typedef struct futureiterobject futureiterobject; /* State of the _asyncio module */ typedef struct { +#ifdef Py_GIL_DISABLED + PyMutex mutex; +#endif PyTypeObject *FutureIterType; PyTypeObject *TaskStepMethWrapper_Type; PyTypeObject *FutureType; @@ -124,15 +134,9 @@ typedef struct { /* Imports from traceback. */ PyObject *traceback_extract_stack; - PyObject *cached_running_loop; // Borrowed reference - volatile uint64_t cached_running_loop_tsid; - /* Counter for autogenerated Task names */ uint64_t task_name_counter; - futureiterobject *fi_freelist; - Py_ssize_t fi_freelist_len; - /* Linked-list of all tasks which are instances of asyncio.Task or subclasses of it. Third party tasks implementations which don't inherit from asyncio.Task are tracked separately using the 'non_asyncio_tasks' WeakSet. @@ -310,96 +314,15 @@ get_future_loop(asyncio_state *state, PyObject *fut) return PyObject_GetAttr(fut, &_Py_ID(_loop)); } - -static int -get_running_loop(asyncio_state *state, PyObject **loop) -{ - PyObject *rl; - - PyThreadState *ts = _PyThreadState_GET(); - uint64_t ts_id = PyThreadState_GetID(ts); - if (state->cached_running_loop_tsid == ts_id && - state->cached_running_loop != NULL) - { - // Fast path, check the cache. - rl = state->cached_running_loop; - } - else { - PyObject *ts_dict = _PyThreadState_GetDict(ts); // borrowed - if (ts_dict == NULL) { - goto not_found; - } - - rl = PyDict_GetItemWithError( - ts_dict, &_Py_ID(__asyncio_running_event_loop__)); // borrowed - if (rl == NULL) { - if (PyErr_Occurred()) { - goto error; - } - else { - goto not_found; - } - } - - state->cached_running_loop = rl; - state->cached_running_loop_tsid = ts_id; - } - - - if (rl == Py_None) { - goto not_found; - } - - *loop = Py_NewRef(rl); - return 0; - -not_found: - *loop = NULL; - return 0; - -error: - *loop = NULL; - return -1; -} - - -static int -set_running_loop(asyncio_state *state, PyObject *loop) -{ - PyObject *ts_dict = NULL; - - PyThreadState *tstate = _PyThreadState_GET(); - if (tstate != NULL) { - ts_dict = _PyThreadState_GetDict(tstate); // borrowed - } - - if (ts_dict == NULL) { - PyErr_SetString( - PyExc_RuntimeError, "thread-local storage is not available"); - return -1; - } - if (PyDict_SetItem( - ts_dict, &_Py_ID(__asyncio_running_event_loop__), loop) < 0) - { - return -1; - } - - state->cached_running_loop = loop; // borrowed, kept alive by ts_dict - state->cached_running_loop_tsid = PyThreadState_GetID(tstate); - - return 0; -} - - static PyObject * get_event_loop(asyncio_state *state) { PyObject *loop; PyObject *policy; - if (get_running_loop(state, &loop)) { - return NULL; - } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + loop = Py_XNewRef(ts->asyncio_running_loop); + if (loop != NULL) { return loop; } @@ -1650,29 +1573,12 @@ FutureIter_dealloc(futureiterobject *it) { PyTypeObject *tp = Py_TYPE(it); - // FutureIter is a heap type so any subclass must also be a heap type. assert(_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); - PyObject *module = ((PyHeapTypeObject*)tp)->ht_module; - asyncio_state *state = NULL; - PyObject_GC_UnTrack(it); tp->tp_clear((PyObject *)it); - // GH-115874: We can't use PyType_GetModuleByDef here as the type might have - // already been cleared, which is also why we must check if ht_module != NULL. - // Due to this restriction, subclasses that belong to a different module - // will not be able to use the free list. - if (module && _PyModule_GetDef(module) == &_asynciomodule) { - state = get_asyncio_state(module); - } - - if (state && state->fi_freelist_len < FI_FREELIST_MAXLEN) { - state->fi_freelist_len++; - it->future = (FutureObj*) state->fi_freelist; - state->fi_freelist = it; - } - else { + if (!_Py_FREELIST_PUSH(futureiters, it, Py_futureiters_MAXFREELIST)) { PyObject_GC_Del(it); Py_DECREF(tp); } @@ -1876,14 +1782,8 @@ future_new_iter(PyObject *fut) asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); ENSURE_FUTURE_ALIVE(state, fut) - if (state->fi_freelist_len) { - state->fi_freelist_len--; - it = state->fi_freelist; - state->fi_freelist = (futureiterobject*) it->future; - it->future = NULL; - _Py_NewReference((PyObject*) it); - } - else { + it = _Py_FREELIST_POP(futureiterobject, futureiters); + if (it == NULL) { it = PyObject_GC_New(futureiterobject, state->FutureIterType); if (it == NULL) { return NULL; @@ -2018,10 +1918,12 @@ static PyMethodDef TaskWakeupDef = { static void register_task(asyncio_state *state, TaskObj *task) { + ASYNCIO_STATE_LOCK(state); assert(Task_Check(state, task)); assert(task != &state->asyncio_tasks.tail); if (task->next != NULL) { // already registered + ASYNCIO_STATE_UNLOCK(state); return; } assert(task->prev == NULL); @@ -2030,6 +1932,7 @@ register_task(asyncio_state *state, TaskObj *task) task->next = state->asyncio_tasks.head; state->asyncio_tasks.head->prev = task; state->asyncio_tasks.head = task; + ASYNCIO_STATE_UNLOCK(state); } static int @@ -2041,12 +1944,14 @@ register_eager_task(asyncio_state *state, PyObject *task) static void unregister_task(asyncio_state *state, TaskObj *task) { + ASYNCIO_STATE_LOCK(state); assert(Task_Check(state, task)); assert(task != &state->asyncio_tasks.tail); if (task->next == NULL) { // not registered assert(task->prev == NULL); assert(state->asyncio_tasks.head != task); + ASYNCIO_STATE_UNLOCK(state); return; } task->next->prev = task->prev; @@ -2059,6 +1964,7 @@ unregister_task(asyncio_state *state, TaskObj *task) task->next = NULL; task->prev = NULL; assert(state->asyncio_tasks.head != task); + ASYNCIO_STATE_UNLOCK(state); } static int @@ -2071,14 +1977,11 @@ static int enter_task(asyncio_state *state, PyObject *loop, PyObject *task) { PyObject *item; - Py_hash_t hash; - hash = PyObject_Hash(loop); - if (hash == -1) { + int res = PyDict_SetDefaultRef(state->current_tasks, loop, task, &item); + if (res < 0) { return -1; } - item = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); - if (item != NULL) { - Py_INCREF(item); + else if (res == 1) { PyErr_Format( PyExc_RuntimeError, "Cannot enter into task %R while another " \ @@ -2087,36 +1990,40 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) Py_DECREF(item); return -1; } - if (PyErr_Occurred()) { - return -1; - } - return _PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash); + Py_DECREF(item); + return 0; } +static int +err_leave_task(PyObject *item, PyObject *task) +{ + PyErr_Format( + PyExc_RuntimeError, + "Leaving task %R does not match the current task %R.", + task, item); + return -1; +} + +static int +leave_task_predicate(PyObject *item, void *task) +{ + if (item != task) { + return err_leave_task(item, (PyObject *)task); + } + return 1; +} static int leave_task(asyncio_state *state, PyObject *loop, PyObject *task) /*[clinic end generated code: output=0ebf6db4b858fb41 input=51296a46313d1ad8]*/ { - PyObject *item; - Py_hash_t hash; - hash = PyObject_Hash(loop); - if (hash == -1) { - return -1; - } - item = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); - if (item != task) { - if (item == NULL) { - /* Not entered, replace with None */ - item = Py_None; - } - PyErr_Format( - PyExc_RuntimeError, - "Leaving task %R does not match the current task %R.", - task, item, NULL); - return -1; + int res = _PyDict_DelItemIf(state->current_tasks, loop, + leave_task_predicate, task); + if (res == 0) { + // task was not found + return err_leave_task(Py_None, task); } - return _PyDict_DelItem_KnownHash(state->current_tasks, loop, hash); + return res; } static PyObject * @@ -2213,7 +2120,12 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, // optimization: defer task name formatting // store the task counter as PyLong in the name // for deferred formatting in get_name - name = PyLong_FromUnsignedLongLong(++state->task_name_counter); +#ifdef Py_GIL_DISABLED + unsigned long long counter = _Py_atomic_add_uint64(&state->task_name_counter, 1) + 1; +#else + unsigned long long counter = ++state->task_name_counter; +#endif + name = PyLong_FromUnsignedLongLong(counter); } else if (!PyUnicode_CheckExact(name)) { name = PyObject_Str(name); } else { @@ -2588,7 +2500,11 @@ static PyObject * _asyncio_Task_get_coro_impl(TaskObj *self) /*[clinic end generated code: output=bcac27c8cc6c8073 input=d2e8606c42a7b403]*/ { - return Py_NewRef(self->task_coro); + if (self->task_coro) { + return Py_NewRef(self->task_coro); + } + + Py_RETURN_NONE; } /*[clinic input] @@ -2867,7 +2783,7 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) if (task->task_state != STATE_PENDING) { PyErr_Format(state->asyncio_InvalidStateError, - "_step(): already done: %R %R", + "__step(): already done: %R %R", task, exc ? exc : Py_None); goto fail; @@ -3342,11 +3258,8 @@ static PyObject * _asyncio__get_running_loop_impl(PyObject *module) /*[clinic end generated code: output=b4390af721411a0a input=0a21627e25a4bd43]*/ { - PyObject *loop; - asyncio_state *state = get_asyncio_state(module); - if (get_running_loop(state, &loop)) { - return NULL; - } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + PyObject *loop = Py_XNewRef(ts->asyncio_running_loop); if (loop == NULL) { /* There's no currently running event loop */ Py_RETURN_NONE; @@ -3369,10 +3282,11 @@ static PyObject * _asyncio__set_running_loop(PyObject *module, PyObject *loop) /*[clinic end generated code: output=ae56bf7a28ca189a input=4c9720233d606604]*/ { - asyncio_state *state = get_asyncio_state(module); - if (set_running_loop(state, loop)) { - return NULL; + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (loop == Py_None) { + loop = NULL; } + Py_XSETREF(ts->asyncio_running_loop, Py_XNewRef(loop)); Py_RETURN_NONE; } @@ -3410,14 +3324,13 @@ _asyncio_get_running_loop_impl(PyObject *module) /*[clinic end generated code: output=c247b5f9e529530e input=2a3bf02ba39f173d]*/ { PyObject *loop; - asyncio_state *state = get_asyncio_state(module); - if (get_running_loop(state, &loop)) { - return NULL; - } + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + loop = Py_XNewRef(ts->asyncio_running_loop); if (loop == NULL) { /* There's no currently running event loop */ PyErr_SetString( PyExc_RuntimeError, "no running event loop"); + return NULL; } return loop; } @@ -3743,25 +3656,6 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) return tasks; } -static void -module_free_freelists(asyncio_state *state) -{ - PyObject *next; - PyObject *current; - - next = (PyObject*) state->fi_freelist; - while (next != NULL) { - assert(state->fi_freelist_len > 0); - state->fi_freelist_len--; - - current = next; - next = (PyObject*) ((futureiterobject*) current)->future; - PyObject_GC_Del(current); - } - assert(state->fi_freelist_len == 0); - state->fi_freelist = NULL; -} - static int module_traverse(PyObject *mod, visitproc visit, void *arg) { @@ -3790,13 +3684,6 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->context_kwname); - // Visit freelist. - PyObject *next = (PyObject*) state->fi_freelist; - while (next != NULL) { - PyObject *current = next; - Py_VISIT(current); - next = (PyObject*) ((futureiterobject*) current)->future; - } return 0; } @@ -3828,8 +3715,6 @@ module_clear(PyObject *mod) Py_CLEAR(state->context_kwname); - module_free_freelists(state); - return 0; } diff --git a/Modules/_csv.c b/Modules/_csv.c index 9964c383b8ad09..737b2c7468e13c 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -749,7 +749,6 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c) } else if (c == dialect->escapechar) { /* possible escaped character */ - self->unquoted_field = false; self->state = ESCAPED_CHAR; } else if (c == ' ' && dialect->skipinitialspace) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index db58f33511c166..b55102639c6786 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2303,9 +2303,14 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) if (!meth) { return -1; } - x = PyDict_SetItemString(((PyTypeObject*)self)->tp_dict, - ml->ml_name, - meth); + PyObject *name = PyUnicode_FromString(ml->ml_name); + if (name == NULL) { + Py_DECREF(meth); + return -1; + } + PyUnicode_InternInPlace(&name); + x = PyDict_SetItem(((PyTypeObject*)self)->tp_dict, name, meth); + Py_DECREF(name); Py_DECREF(meth); if (x == -1) { return -1; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index b8e613fd669d1b..0719a1b921f040 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -186,7 +186,7 @@ _testfunc_array_in_struct3B_set_defaults(void) /* * Test3C struct tests the MAX_STRUCT_SIZE 32. Structs containing arrays of up - * to four floating point types are passed in registers on Arm platforms. + * to four floating-point types are passed in registers on Arm platforms. * This struct is used for within bounds test on Arm platfroms and for an * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 32. * See gh-110190. @@ -210,7 +210,7 @@ _testfunc_array_in_struct3C_set_defaults(void) /* * Test3D struct tests the MAX_STRUCT_SIZE 64. Structs containing arrays of up - * to eight floating point types are passed in registers on PPC64LE platforms. + * to eight floating-point types are passed in registers on PPC64LE platforms. * This struct is used for within bounds test on PPC64LE platfroms and for an * out-of-bounds tests for platfroms where MAX_STRUCT_SIZE is less than 64. * See gh-110190. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index f20efd3d36d275..67b49aa6ac2301 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7256,49 +7256,51 @@ _datetime_exec(PyObject *module) Py_DECREF(value); \ } while(0) - /* timedelta values */ - PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); - DATETIME_ADD_MACRO(d, "max", - new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); - - /* date values */ - d = _PyType_GetDict(&PyDateTime_DateType); - DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); - DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); - - /* time values */ - d = _PyType_GetDict(&PyDateTime_TimeType); - DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* datetime values */ - d = _PyType_GetDict(&PyDateTime_DateTimeType); - DATETIME_ADD_MACRO(d, "min", - new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, - 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* timezone values */ - d = _PyType_GetDict(&PyDateTime_TimeZoneType); - if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { - goto error; - } + if (!reloading) { + /* timedelta values */ + PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); + DATETIME_ADD_MACRO(d, "max", + new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); + + /* date values */ + d = _PyType_GetDict(&PyDateTime_DateType); + DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); + DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); + + /* time values */ + d = _PyType_GetDict(&PyDateTime_TimeType); + DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* datetime values */ + d = _PyType_GetDict(&PyDateTime_DateTimeType); + DATETIME_ADD_MACRO(d, "min", + new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, + 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* timezone values */ + d = _PyType_GetDict(&PyDateTime_TimeZoneType); + if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { + goto error; + } - /* bpo-37642: These attributes are rounded to the nearest minute for backwards - * compatibility, even though the constructor will accept a wider range of - * values. This may change in the future.*/ + /* bpo-37642: These attributes are rounded to the nearest minute for backwards + * compatibility, even though the constructor will accept a wider range of + * values. This may change in the future.*/ - /* -23:59 */ - DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); + /* -23:59 */ + DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); - /* +23:59 */ - DATETIME_ADD_MACRO( - d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); + /* +23:59 */ + DATETIME_ADD_MACRO( + d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); + } #undef DATETIME_ADD_MACRO @@ -7342,12 +7344,6 @@ _datetime_exec(PyObject *module) static_assert(DI100Y == 25 * DI4Y - 1, "DI100Y"); assert(DI100Y == days_before_year(100+1)); - if (reloading) { - for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { - PyType_Modified(capi_types[i]); - } - } - if (set_current_module(interp, module) < 0) { goto error; } diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index ff8dacf5bd1ad0..a8b4a8d76b0eaa 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -18,7 +18,9 @@ #endif #define REGISTERS_HEAP_TYPES +#define HAS_UNBOUND_ITEMS #include "_interpreters_common.h" +#undef HAS_UNBOUND_ITEMS #undef REGISTERS_HEAP_TYPES @@ -511,8 +513,14 @@ _waiting_finish_releasing(_waiting_t *waiting) struct _channelitem; typedef struct _channelitem { + /* The interpreter that added the item to the queue. + The actual bound interpid is found in item->data. + This is necessary because item->data might be NULL, + meaning the interpreter has been destroyed. */ + int64_t interpid; _PyCrossInterpreterData *data; _waiting_t *waiting; + int unboundop; struct _channelitem *next; } _channelitem; @@ -524,11 +532,22 @@ _channelitem_ID(_channelitem *item) static void _channelitem_init(_channelitem *item, - _PyCrossInterpreterData *data, _waiting_t *waiting) + int64_t interpid, _PyCrossInterpreterData *data, + _waiting_t *waiting, int unboundop) { + if (interpid < 0) { + interpid = _get_interpid(data); + } + else { + assert(data == NULL + || _PyCrossInterpreterData_INTERPID(data) < 0 + || interpid == _PyCrossInterpreterData_INTERPID(data)); + } *item = (_channelitem){ + .interpid = interpid, .data = data, .waiting = waiting, + .unboundop = unboundop, }; if (waiting != NULL) { waiting->itemid = _channelitem_ID(item); @@ -536,17 +555,15 @@ _channelitem_init(_channelitem *item, } static void -_channelitem_clear(_channelitem *item) +_channelitem_clear_data(_channelitem *item, int removed) { - item->next = NULL; - if (item->data != NULL) { // It was allocated in channel_send(). (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); item->data = NULL; } - if (item->waiting != NULL) { + if (item->waiting != NULL && removed) { if (item->waiting->status == WAITING_ACQUIRED) { _waiting_release(item->waiting, 0); } @@ -554,15 +571,23 @@ _channelitem_clear(_channelitem *item) } } +static void +_channelitem_clear(_channelitem *item) +{ + item->next = NULL; + _channelitem_clear_data(item, 1); +} + static _channelitem * -_channelitem_new(_PyCrossInterpreterData *data, _waiting_t *waiting) +_channelitem_new(int64_t interpid, _PyCrossInterpreterData *data, + _waiting_t *waiting, int unboundop) { _channelitem *item = GLOBAL_MALLOC(_channelitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _channelitem_init(item, data, waiting); + _channelitem_init(item, interpid, data, waiting, unboundop); return item; } @@ -585,17 +610,48 @@ _channelitem_free_all(_channelitem *item) static void _channelitem_popped(_channelitem *item, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyCrossInterpreterData **p_data, _waiting_t **p_waiting, + int *p_unboundop) { assert(item->waiting == NULL || item->waiting->status == WAITING_ACQUIRED); *p_data = item->data; *p_waiting = item->waiting; + *p_unboundop = item->unboundop; // We clear them here, so they won't be released in _channelitem_clear(). item->data = NULL; item->waiting = NULL; _channelitem_free(item); } +static int +_channelitem_clear_interpreter(_channelitem *item) +{ + assert(item->interpid >= 0); + if (item->data == NULL) { + // Its interpreter was already cleared (or it was never bound). + // For UNBOUND_REMOVE it should have been freed at that time. + assert(item->unboundop != UNBOUND_REMOVE); + return 0; + } + assert(_PyCrossInterpreterData_INTERPID(item->data) == item->interpid); + + switch (item->unboundop) { + case UNBOUND_REMOVE: + // The caller must free/clear it. + return 1; + case UNBOUND_ERROR: + case UNBOUND_REPLACE: + // We won't need the cross-interpreter data later + // so we completely throw it away. + _channelitem_clear_data(item, 0); + return 0; + default: + Py_FatalError("not reachable"); + return -1; + } +} + + typedef struct _channelqueue { int64_t count; _channelitem *first; @@ -634,9 +690,10 @@ _channelqueue_free(_channelqueue *queue) static int _channelqueue_put(_channelqueue *queue, - _PyCrossInterpreterData *data, _waiting_t *waiting) + int64_t interpid, _PyCrossInterpreterData *data, + _waiting_t *waiting, int unboundop) { - _channelitem *item = _channelitem_new(data, waiting); + _channelitem *item = _channelitem_new(interpid, data, waiting, unboundop); if (item == NULL) { return -1; } @@ -659,7 +716,8 @@ _channelqueue_put(_channelqueue *queue, static int _channelqueue_get(_channelqueue *queue, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyCrossInterpreterData **p_data, _waiting_t **p_waiting, + int *p_unboundop) { _channelitem *item = queue->first; if (item == NULL) { @@ -671,7 +729,7 @@ _channelqueue_get(_channelqueue *queue, } queue->count -= 1; - _channelitem_popped(item, p_data, p_waiting); + _channelitem_popped(item, p_data, p_waiting, p_unboundop); return 0; } @@ -737,7 +795,8 @@ _channelqueue_remove(_channelqueue *queue, _channelitem_id_t itemid, } queue->count -= 1; - _channelitem_popped(item, p_data, p_waiting); + int unboundop; + _channelitem_popped(item, p_data, p_waiting, &unboundop); } static void @@ -748,14 +807,17 @@ _channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid) while (next != NULL) { _channelitem *item = next; next = item->next; - if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) { + int remove = (item->interpid == interpid) + ? _channelitem_clear_interpreter(item) + : 0; + if (remove) { + _channelitem_free(item); if (prev == NULL) { - queue->first = item->next; + queue->first = next; } else { - prev->next = item->next; + prev->next = next; } - _channelitem_free(item); queue->count -= 1; } else { @@ -1018,12 +1080,15 @@ typedef struct _channel { PyThread_type_lock mutex; _channelqueue *queue; _channelends *ends; + struct { + int unboundop; + } defaults; int open; struct _channel_closing *closing; } _channel_state; static _channel_state * -_channel_new(PyThread_type_lock mutex) +_channel_new(PyThread_type_lock mutex, int unboundop) { _channel_state *chan = GLOBAL_MALLOC(_channel_state); if (chan == NULL) { @@ -1041,6 +1106,7 @@ _channel_new(PyThread_type_lock mutex) GLOBAL_FREE(chan); return NULL; } + chan->defaults.unboundop = unboundop; chan->open = 1; chan->closing = NULL; return chan; @@ -1061,7 +1127,8 @@ _channel_free(_channel_state *chan) static int _channel_add(_channel_state *chan, int64_t interpid, - _PyCrossInterpreterData *data, _waiting_t *waiting) + _PyCrossInterpreterData *data, _waiting_t *waiting, + int unboundop) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1075,7 +1142,7 @@ _channel_add(_channel_state *chan, int64_t interpid, goto done; } - if (_channelqueue_put(chan->queue, data, waiting) != 0) { + if (_channelqueue_put(chan->queue, interpid, data, waiting, unboundop) != 0) { goto done; } // Any errors past this point must cause a _waiting_release() call. @@ -1088,7 +1155,8 @@ _channel_add(_channel_state *chan, int64_t interpid, static int _channel_next(_channel_state *chan, int64_t interpid, - _PyCrossInterpreterData **p_data, _waiting_t **p_waiting) + _PyCrossInterpreterData **p_data, _waiting_t **p_waiting, + int *p_unboundop) { int err = 0; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1102,11 +1170,15 @@ _channel_next(_channel_state *chan, int64_t interpid, goto done; } - int empty = _channelqueue_get(chan->queue, p_data, p_waiting); - assert(empty == 0 || empty == ERR_CHANNEL_EMPTY); + int empty = _channelqueue_get(chan->queue, p_data, p_waiting, p_unboundop); assert(!PyErr_Occurred()); - if (empty && chan->closing != NULL) { - chan->open = 0; + if (empty) { + assert(empty == ERR_CHANNEL_EMPTY); + if (chan->closing != NULL) { + chan->open = 0; + } + err = ERR_CHANNEL_EMPTY; + goto done; } done: @@ -1528,18 +1600,27 @@ _channels_release_cid_object(_channels *channels, int64_t cid) PyThread_release_lock(channels->mutex); } -static int64_t * +struct channel_id_and_info { + int64_t id; + int unboundop; +}; + +static struct channel_id_and_info * _channels_list_all(_channels *channels, int64_t *count) { - int64_t *cids = NULL; + struct channel_id_and_info *cids = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(channels->numopen)); + struct channel_id_and_info *ids = + PyMem_NEW(struct channel_id_and_info, (Py_ssize_t)(channels->numopen)); if (ids == NULL) { goto done; } _channelref *ref = channels->head; for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = ref->cid; + ids[i] = (struct channel_id_and_info){ + .id = ref->cid, + .unboundop = ref->chan->defaults.unboundop, + }; } *count = channels->numopen; @@ -1624,13 +1705,13 @@ _channel_finish_closing(_channel_state *chan) { // Create a new channel. static int64_t -channel_create(_channels *channels) +channel_create(_channels *channels, int unboundop) { PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_CHANNEL_MUTEX_INIT; } - _channel_state *chan = _channel_new(mutex); + _channel_state *chan = _channel_new(mutex, unboundop); if (chan == NULL) { PyThread_free_lock(mutex); return -1; @@ -1662,7 +1743,7 @@ channel_destroy(_channels *channels, int64_t cid) // Optionally request to be notified when it is received. static int channel_send(_channels *channels, int64_t cid, PyObject *obj, - _waiting_t *waiting) + _waiting_t *waiting, int unboundop) { PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { @@ -1698,7 +1779,7 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj, } // Add the data to the channel. - int res = _channel_add(chan, interpid, data, waiting); + int res = _channel_add(chan, interpid, data, waiting, unboundop); PyThread_release_lock(mutex); if (res != 0) { // We may chain an exception here: @@ -1735,7 +1816,7 @@ channel_clear_sent(_channels *channels, int64_t cid, _waiting_t *waiting) // Like channel_send(), but strictly wait for the object to be received. static int channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, - PY_TIMEOUT_T timeout) + int unboundop, PY_TIMEOUT_T timeout) { // We use a stack variable here, so we must ensure that &waiting // is not held by any channel item at the point this function exits. @@ -1746,7 +1827,7 @@ channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, } /* Queue up the object. */ - int res = channel_send(channels, cid, obj, &waiting); + int res = channel_send(channels, cid, obj, &waiting, unboundop); if (res < 0) { assert(waiting.status == WAITING_NO_STATUS); goto finally; @@ -1788,7 +1869,7 @@ channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, // The current interpreter gets associated with the recv end of the channel. // XXX Support a "wait" mutex? static int -channel_recv(_channels *channels, int64_t cid, PyObject **res) +channel_recv(_channels *channels, int64_t cid, PyObject **res, int *p_unboundop) { int err; *res = NULL; @@ -1816,13 +1897,15 @@ channel_recv(_channels *channels, int64_t cid, PyObject **res) // Pop off the next item from the channel. _PyCrossInterpreterData *data = NULL; _waiting_t *waiting = NULL; - err = _channel_next(chan, interpid, &data, &waiting); + err = _channel_next(chan, interpid, &data, &waiting, p_unboundop); PyThread_release_lock(mutex); if (err != 0) { return err; } else if (data == NULL) { + // The item was unbound. assert(!PyErr_Occurred()); + *res = NULL; return 0; } @@ -1915,6 +1998,23 @@ channel_is_associated(_channels *channels, int64_t cid, int64_t interpid, return (end != NULL && end->open); } +static int +_channel_get_count(_channels *channels, int64_t cid, Py_ssize_t *p_count) +{ + PyThread_type_lock mutex = NULL; + _channel_state *chan = NULL; + int err = _channels_lookup(channels, cid, &mutex, &chan); + if (err != 0) { + return err; + } + assert(chan != NULL); + int64_t count = chan->queue->count; + PyThread_release_lock(mutex); + + *p_count = (Py_ssize_t)count; + return 0; +} + /* channel info */ @@ -2615,10 +2715,10 @@ _get_current_channelend_type(int end) } if (cls == NULL) { // Force the module to be loaded, to register the type. - PyObject *highlevel = PyImport_ImportModule("interpreters.channel"); + PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); if (highlevel == NULL) { PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.channel"); + highlevel = PyImport_ImportModule("test.support.interpreters.channels"); if (highlevel == NULL) { return NULL; } @@ -2767,9 +2867,22 @@ clear_interpreter(void *data) static PyObject * -channelsmod_create(PyObject *self, PyObject *Py_UNUSED(ignored)) +channelsmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - int64_t cid = channel_create(&_globals.channels); + static char *kwlist[] = {"unboundop", NULL}; + int unboundop; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "i:create", kwlist, + &unboundop)) + { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); + return NULL; + } + + int64_t cid = channel_create(&_globals.channels, unboundop); if (cid < 0) { (void)handle_channel_error(-1, self, cid); return NULL; @@ -2796,7 +2909,7 @@ channelsmod_create(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(channelsmod_create_doc, -"channel_create() -> cid\n\ +"channel_create(unboundop) -> cid\n\ \n\ Create a new cross-interpreter channel and return a unique generated ID."); @@ -2831,7 +2944,8 @@ static PyObject * channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t count = 0; - int64_t *cids = _channels_list_all(&_globals.channels, &count); + struct channel_id_and_info *cids = + _channels_list_all(&_globals.channels, &count); if (cids == NULL) { if (count == 0) { return PyList_New(0); @@ -2848,19 +2962,26 @@ channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) ids = NULL; goto finally; } - int64_t *cur = cids; + struct channel_id_and_info *cur = cids; for (int64_t i=0; i < count; cur++, i++) { PyObject *cidobj = NULL; - int err = newchannelid(state->ChannelIDType, *cur, 0, + int err = newchannelid(state->ChannelIDType, cur->id, 0, &_globals.channels, 0, 0, (channelid **)&cidobj); - if (handle_channel_error(err, self, *cur)) { + if (handle_channel_error(err, self, cur->id)) { assert(cidobj == NULL); Py_SETREF(ids, NULL); break; } assert(cidobj != NULL); - PyList_SET_ITEM(ids, (Py_ssize_t)i, cidobj); + + PyObject *item = Py_BuildValue("Oi", cidobj, cur->unboundop); + Py_DECREF(cidobj); + if (item == NULL) { + Py_SETREF(ids, NULL); + break; + } + PyList_SET_ITEM(ids, (Py_ssize_t)i, item); } finally: @@ -2942,16 +3063,24 @@ receive end."); static PyObject * channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "blocking", "timeout", NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", + NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; + int unboundop = UNBOUND_REPLACE; int blocking = 1; PyObject *timeout_obj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|$pO:channel_send", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|i$pO:channel_send", kwlist, channel_id_converter, &cid_data, &obj, - &blocking, &timeout_obj)) { + &unboundop, &blocking, &timeout_obj)) + { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); return NULL; } @@ -2964,10 +3093,10 @@ channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) /* Queue up the object. */ int err = 0; if (blocking) { - err = channel_send_wait(&_globals.channels, cid, obj, timeout); + err = channel_send_wait(&_globals.channels, cid, obj, unboundop, timeout); } else { - err = channel_send(&_globals.channels, cid, obj, NULL); + err = channel_send(&_globals.channels, cid, obj, NULL, unboundop); } if (handle_channel_error(err, self, cid)) { return NULL; @@ -2977,7 +3106,7 @@ channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(channelsmod_send_doc, -"channel_send(cid, obj, blocking=True)\n\ +"channel_send(cid, obj, *, blocking=True, timeout=None)\n\ \n\ Add the object's data to the channel's queue.\n\ By default this waits for the object to be received."); @@ -2985,17 +3114,24 @@ By default this waits for the object to be received."); static PyObject * channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "blocking", "timeout", NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", + NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; + int unboundop = UNBOUND_REPLACE; int blocking = 1; PyObject *timeout_obj = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&O|$pO:channel_send_buffer", kwlist, + "O&O|i$pO:channel_send_buffer", kwlist, channel_id_converter, &cid_data, &obj, - &blocking, &timeout_obj)) { + &unboundop, &blocking, &timeout_obj)) { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); return NULL; } @@ -3013,10 +3149,11 @@ channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) /* Queue up the object. */ int err = 0; if (blocking) { - err = channel_send_wait(&_globals.channels, cid, tempobj, timeout); + err = channel_send_wait( + &_globals.channels, cid, tempobj, unboundop, timeout); } else { - err = channel_send(&_globals.channels, cid, tempobj, NULL); + err = channel_send(&_globals.channels, cid, tempobj, NULL, unboundop); } Py_DECREF(tempobj); if (handle_channel_error(err, self, cid)) { @@ -3027,7 +3164,7 @@ channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(channelsmod_send_buffer_doc, -"channel_send_buffer(cid, obj, blocking=True)\n\ +"channel_send_buffer(cid, obj, *, blocking=True, timeout=None)\n\ \n\ Add the object's buffer to the channel's queue.\n\ By default this waits for the object to be received."); @@ -3048,25 +3185,28 @@ channelsmod_recv(PyObject *self, PyObject *args, PyObject *kwds) cid = cid_data.cid; PyObject *obj = NULL; - int err = channel_recv(&_globals.channels, cid, &obj); - if (handle_channel_error(err, self, cid)) { - return NULL; - } - Py_XINCREF(dflt); - if (obj == NULL) { + int unboundop = 0; + int err = channel_recv(&_globals.channels, cid, &obj, &unboundop); + if (err == ERR_CHANNEL_EMPTY && dflt != NULL) { // Use the default. - if (dflt == NULL) { - (void)handle_channel_error(ERR_CHANNEL_EMPTY, self, cid); - return NULL; - } obj = Py_NewRef(dflt); + err = 0; } - Py_XDECREF(dflt); - return obj; + else if (handle_channel_error(err, self, cid)) { + return NULL; + } + else if (obj == NULL) { + // The item was unbound. + return Py_BuildValue("Oi", Py_None, unboundop); + } + + PyObject *res = Py_BuildValue("OO", obj, Py_None); + Py_DECREF(obj); + return res; } PyDoc_STRVAR(channelsmod_recv_doc, -"channel_recv(cid, [default]) -> obj\n\ +"channel_recv(cid, [default]) -> (obj, unboundop)\n\ \n\ Return a new object from the data at the front of the channel's queue.\n\ \n\ @@ -3167,6 +3307,34 @@ Close the channel for the current interpreter. 'send' and 'recv'\n\ (bool) may be used to indicate the ends to close. By default both\n\ ends are closed. Closing an already closed end is a noop."); +static PyObject * +channelsmod_get_count(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", NULL}; + struct channel_id_converter_data cid_data = { + .module = self, + }; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_count", kwlist, + channel_id_converter, &cid_data)) { + return NULL; + } + int64_t cid = cid_data.cid; + + Py_ssize_t count = -1; + int err = _channel_get_count(&_globals.channels, cid, &count); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + assert(count >= 0); + return PyLong_FromSsize_t(count); +} + +PyDoc_STRVAR(channelsmod_get_count_doc, +"get_count(cid)\n\ +\n\ +Return the number of items in the channel."); + static PyObject * channelsmod_get_info(PyObject *self, PyObject *args, PyObject *kwds) { @@ -3194,6 +3362,38 @@ PyDoc_STRVAR(channelsmod_get_info_doc, \n\ Return details about the channel."); +static PyObject * +channelsmod_get_channel_defaults(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", NULL}; + struct channel_id_converter_data cid_data = { + .module = self, + }; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:get_channel_defaults", kwlist, + channel_id_converter, &cid_data)) { + return NULL; + } + int64_t cid = cid_data.cid; + + PyThread_type_lock mutex = NULL; + _channel_state *channel = NULL; + int err = _channels_lookup(&_globals.channels, cid, &mutex, &channel); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + int unboundop = channel->defaults.unboundop; + PyThread_release_lock(mutex); + + PyObject *defaults = Py_BuildValue("i", unboundop); + return defaults; +} + +PyDoc_STRVAR(channelsmod_get_channel_defaults_doc, +"get_channel_defaults(cid)\n\ +\n\ +Return the channel's default values, set when it was created."); + static PyObject * channelsmod__channel_id(PyObject *self, PyObject *args, PyObject *kwds) { @@ -3240,8 +3440,8 @@ channelsmod__register_end_types(PyObject *self, PyObject *args, PyObject *kwds) } static PyMethodDef module_functions[] = { - {"create", channelsmod_create, - METH_NOARGS, channelsmod_create_doc}, + {"create", _PyCFunction_CAST(channelsmod_create), + METH_VARARGS | METH_KEYWORDS, channelsmod_create_doc}, {"destroy", _PyCFunction_CAST(channelsmod_destroy), METH_VARARGS | METH_KEYWORDS, channelsmod_destroy_doc}, {"list_all", channelsmod_list_all, @@ -3258,8 +3458,12 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, channelsmod_close_doc}, {"release", _PyCFunction_CAST(channelsmod_release), METH_VARARGS | METH_KEYWORDS, channelsmod_release_doc}, + {"get_count", _PyCFunction_CAST(channelsmod_get_count), + METH_VARARGS | METH_KEYWORDS, channelsmod_get_count_doc}, {"get_info", _PyCFunction_CAST(channelsmod_get_info), METH_VARARGS | METH_KEYWORDS, channelsmod_get_info_doc}, + {"get_channel_defaults", _PyCFunction_CAST(channelsmod_get_channel_defaults), + METH_VARARGS | METH_KEYWORDS, channelsmod_get_channel_defaults_doc}, {"_channel_id", _PyCFunction_CAST(channelsmod__channel_id), METH_VARARGS | METH_KEYWORDS, NULL}, {"_register_end_types", _PyCFunction_CAST(channelsmod__register_end_types), diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index c99111b861cc32..5dec240f02c4db 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -9,7 +9,9 @@ #include "pycore_crossinterp.h" // struct _xid #define REGISTERS_HEAP_TYPES +#define HAS_UNBOUND_ITEMS #include "_interpreters_common.h" +#undef HAS_UNBOUND_ITEMS #undef REGISTERS_HEAP_TYPES @@ -58,7 +60,6 @@ _release_xid_data(_PyCrossInterpreterData *data, int flags) return res; } - static PyInterpreterState * _get_current_interp(void) { @@ -394,42 +395,67 @@ handle_queue_error(int err, PyObject *mod, int64_t qid) struct _queueitem; typedef struct _queueitem { + /* The interpreter that added the item to the queue. + The actual bound interpid is found in item->data. + This is necessary because item->data might be NULL, + meaning the interpreter has been destroyed. */ + int64_t interpid; _PyCrossInterpreterData *data; int fmt; + int unboundop; struct _queueitem *next; } _queueitem; static void _queueitem_init(_queueitem *item, - _PyCrossInterpreterData *data, int fmt) + int64_t interpid, _PyCrossInterpreterData *data, + int fmt, int unboundop) { + if (interpid < 0) { + interpid = _get_interpid(data); + } + else { + assert(data == NULL + || _PyCrossInterpreterData_INTERPID(data) < 0 + || interpid == _PyCrossInterpreterData_INTERPID(data)); + } + assert(check_unbound(unboundop)); *item = (_queueitem){ + .interpid = interpid, .data = data, .fmt = fmt, + .unboundop = unboundop, }; } +static void +_queueitem_clear_data(_queueitem *item) +{ + if (item->data == NULL) { + return; + } + // It was allocated in queue_put(). + (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); + item->data = NULL; +} + static void _queueitem_clear(_queueitem *item) { item->next = NULL; - - if (item->data != NULL) { - // It was allocated in queue_put(). - (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); - item->data = NULL; - } + _queueitem_clear_data(item); } static _queueitem * -_queueitem_new(_PyCrossInterpreterData *data, int fmt) +_queueitem_new(int64_t interpid, _PyCrossInterpreterData *data, + int fmt, int unboundop) { _queueitem *item = GLOBAL_MALLOC(_queueitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _queueitem_init(item, data, fmt); + _queueitem_init(item, interpid, data, fmt, unboundop); return item; } @@ -452,15 +478,44 @@ _queueitem_free_all(_queueitem *item) static void _queueitem_popped(_queueitem *item, - _PyCrossInterpreterData **p_data, int *p_fmt) + _PyCrossInterpreterData **p_data, int *p_fmt, int *p_unboundop) { *p_data = item->data; *p_fmt = item->fmt; + *p_unboundop = item->unboundop; // We clear them here, so they won't be released in _queueitem_clear(). item->data = NULL; _queueitem_free(item); } +static int +_queueitem_clear_interpreter(_queueitem *item) +{ + assert(item->interpid >= 0); + if (item->data == NULL) { + // Its interpreter was already cleared (or it was never bound). + // For UNBOUND_REMOVE it should have been freed at that time. + assert(item->unboundop != UNBOUND_REMOVE); + return 0; + } + assert(_PyCrossInterpreterData_INTERPID(item->data) == item->interpid); + + switch (item->unboundop) { + case UNBOUND_REMOVE: + // The caller must free/clear it. + return 1; + case UNBOUND_ERROR: + case UNBOUND_REPLACE: + // We won't need the cross-interpreter data later + // so we completely throw it away. + _queueitem_clear_data(item); + return 0; + default: + Py_FatalError("not reachable"); + return -1; + } +} + /* the queue */ @@ -474,12 +529,16 @@ typedef struct _queue { _queueitem *first; _queueitem *last; } items; - int fmt; + struct { + int fmt; + int unboundop; + } defaults; } _queue; static int -_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt) +_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop) { + assert(check_unbound(unboundop)); PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_QUEUE_ALLOC; @@ -490,7 +549,10 @@ _queue_init(_queue *queue, Py_ssize_t maxsize, int fmt) .items = { .maxsize = maxsize, }, - .fmt = fmt, + .defaults = { + .fmt = fmt, + .unboundop = unboundop, + }, }; return 0; } @@ -571,7 +633,8 @@ _queue_unlock(_queue *queue) } static int -_queue_add(_queue *queue, _PyCrossInterpreterData *data, int fmt) +_queue_add(_queue *queue, int64_t interpid, _PyCrossInterpreterData *data, + int fmt, int unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -587,7 +650,7 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data, int fmt) return ERR_QUEUE_FULL; } - _queueitem *item = _queueitem_new(data, fmt); + _queueitem *item = _queueitem_new(interpid, data, fmt, unboundop); if (item == NULL) { _queue_unlock(queue); return -1; @@ -608,7 +671,7 @@ _queue_add(_queue *queue, _PyCrossInterpreterData *data, int fmt) static int _queue_next(_queue *queue, - _PyCrossInterpreterData **p_data, int *p_fmt) + _PyCrossInterpreterData **p_data, int *p_fmt, int *p_unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -627,7 +690,7 @@ _queue_next(_queue *queue, } queue->items.count -= 1; - _queueitem_popped(item, p_data, p_fmt); + _queueitem_popped(item, p_data, p_fmt, p_unboundop); _queue_unlock(queue); return 0; @@ -692,14 +755,17 @@ _queue_clear_interpreter(_queue *queue, int64_t interpid) while (next != NULL) { _queueitem *item = next; next = item->next; - if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) { + int remove = (item->interpid == interpid) + ? _queueitem_clear_interpreter(item) + : 0; + if (remove) { + _queueitem_free(item); if (prev == NULL) { - queue->items.first = item->next; + queue->items.first = next; } else { - prev->next = item->next; + prev->next = next; } - _queueitem_free(item); queue->items.count -= 1; } else { @@ -966,18 +1032,19 @@ _queues_decref(_queues *queues, int64_t qid) return res; } -struct queue_id_and_fmt { +struct queue_id_and_info { int64_t id; int fmt; + int unboundop; }; -static struct queue_id_and_fmt * -_queues_list_all(_queues *queues, int64_t *count) +static struct queue_id_and_info * +_queues_list_all(_queues *queues, int64_t *p_count) { - struct queue_id_and_fmt *qids = NULL; + struct queue_id_and_info *qids = NULL; PyThread_acquire_lock(queues->mutex, WAIT_LOCK); - struct queue_id_and_fmt *ids = PyMem_NEW(struct queue_id_and_fmt, - (Py_ssize_t)(queues->count)); + struct queue_id_and_info *ids = PyMem_NEW(struct queue_id_and_info, + (Py_ssize_t)(queues->count)); if (ids == NULL) { goto done; } @@ -985,9 +1052,10 @@ _queues_list_all(_queues *queues, int64_t *count) for (int64_t i=0; ref != NULL; ref = ref->next, i++) { ids[i].id = ref->qid; assert(ref->queue != NULL); - ids[i].fmt = ref->queue->fmt; + ids[i].fmt = ref->queue->defaults.fmt; + ids[i].unboundop = ref->queue->defaults.unboundop; } - *count = queues->count; + *p_count = queues->count; qids = ids; done: @@ -1021,13 +1089,13 @@ _queue_free(_queue *queue) // Create a new queue. static int64_t -queue_create(_queues *queues, Py_ssize_t maxsize, int fmt) +queue_create(_queues *queues, Py_ssize_t maxsize, int fmt, int unboundop) { _queue *queue = GLOBAL_MALLOC(_queue); if (queue == NULL) { return ERR_QUEUE_ALLOC; } - int err = _queue_init(queue, maxsize, fmt); + int err = _queue_init(queue, maxsize, fmt, unboundop); if (err < 0) { GLOBAL_FREE(queue); return (int64_t)err; @@ -1056,7 +1124,7 @@ queue_destroy(_queues *queues, int64_t qid) // Push an object onto the queue. static int -queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt) +queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) { // Look up the queue. _queue *queue = NULL; @@ -1077,9 +1145,12 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt) GLOBAL_FREE(data); return -1; } + assert(_PyCrossInterpreterData_INTERPID(data) == \ + PyInterpreterState_GetID(PyInterpreterState_Get())); // Add the data to the queue. - int res = _queue_add(queue, data, fmt); + int64_t interpid = -1; // _queueitem_init() will set it. + int res = _queue_add(queue, interpid, data, fmt, unboundop); _queue_unmark_waiter(queue, queues->mutex); if (res != 0) { // We may chain an exception here: @@ -1094,7 +1165,8 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt) // Pop the next object off the queue. Fail if empty. // XXX Support a "wait" mutex? static int -queue_get(_queues *queues, int64_t qid, PyObject **res, int *p_fmt) +queue_get(_queues *queues, int64_t qid, + PyObject **res, int *p_fmt, int *p_unboundop) { int err; *res = NULL; @@ -1110,7 +1182,7 @@ queue_get(_queues *queues, int64_t qid, PyObject **res, int *p_fmt) // Pop off the next item from the queue. _PyCrossInterpreterData *data = NULL; - err = _queue_next(queue, &data, p_fmt); + err = _queue_next(queue, &data, p_fmt, p_unboundop); _queue_unmark_waiter(queue, queues->mutex); if (err != 0) { return err; @@ -1397,15 +1469,22 @@ qidarg_converter(PyObject *arg, void *ptr) static PyObject * queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"maxsize", "fmt", NULL}; + static char *kwlist[] = {"maxsize", "fmt", "unboundop", NULL}; Py_ssize_t maxsize; int fmt; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "ni:create", kwlist, - &maxsize, &fmt)) { + int unboundop; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "nii:create", kwlist, + &maxsize, &fmt, &unboundop)) + { + return NULL; + } + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); return NULL; } - int64_t qid = queue_create(&_globals.queues, maxsize, fmt); + int64_t qid = queue_create(&_globals.queues, maxsize, fmt, unboundop); if (qid < 0) { (void)handle_queue_error((int)qid, self, qid); return NULL; @@ -1427,7 +1506,7 @@ queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_create_doc, -"create(maxsize, fmt) -> qid\n\ +"create(maxsize, fmt, unboundop) -> qid\n\ \n\ Create a new cross-interpreter queue and return its unique generated ID.\n\ It is a new reference as though bind() had been called on the queue.\n\ @@ -1463,9 +1542,9 @@ static PyObject * queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t count = 0; - struct queue_id_and_fmt *qids = _queues_list_all(&_globals.queues, &count); + struct queue_id_and_info *qids = _queues_list_all(&_globals.queues, &count); if (qids == NULL) { - if (count == 0) { + if (!PyErr_Occurred() && count == 0) { return PyList_New(0); } return NULL; @@ -1474,9 +1553,10 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) if (ids == NULL) { goto finally; } - struct queue_id_and_fmt *cur = qids; + struct queue_id_and_info *cur = qids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *item = Py_BuildValue("Li", cur->id, cur->fmt); + PyObject *item = Py_BuildValue("Lii", cur->id, cur->fmt, + cur->unboundop); if (item == NULL) { Py_SETREF(ids, NULL); break; @@ -1498,18 +1578,26 @@ Each corresponding default format is also included."); static PyObject * queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "obj", "fmt", NULL}; + static char *kwlist[] = {"qid", "obj", "fmt", "unboundop", NULL}; qidarg_converter_data qidarg; PyObject *obj; int fmt; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oi:put", kwlist, - qidarg_converter, &qidarg, &obj, &fmt)) { + int unboundop; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oii:put", kwlist, + qidarg_converter, &qidarg, &obj, &fmt, + &unboundop)) + { return NULL; } int64_t qid = qidarg.id; + if (!check_unbound(unboundop)) { + PyErr_Format(PyExc_ValueError, + "unsupported unboundop %d", unboundop); + return NULL; + } /* Queue up the object. */ - int err = queue_put(&_globals.queues, qid, obj, fmt); + int err = queue_put(&_globals.queues, qid, obj, fmt, unboundop); // This is the only place that raises QueueFull. if (handle_queue_error(err, self, qid)) { return NULL; @@ -1536,13 +1624,17 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) PyObject *obj = NULL; int fmt = 0; - int err = queue_get(&_globals.queues, qid, &obj, &fmt); + int unboundop = 0; + int err = queue_get(&_globals.queues, qid, &obj, &fmt, &unboundop); // This is the only place that raises QueueEmpty. if (handle_queue_error(err, self, qid)) { return NULL; } - PyObject *res = Py_BuildValue("Oi", obj, fmt); + if (obj == NULL) { + return Py_BuildValue("Oii", Py_None, fmt, unboundop); + } + PyObject *res = Py_BuildValue("OiO", obj, fmt, Py_None); Py_DECREF(obj); return res; } @@ -1656,17 +1748,12 @@ queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds) if (handle_queue_error(err, self, qid)) { return NULL; } - int fmt = queue->fmt; + int fmt = queue->defaults.fmt; + int unboundop = queue->defaults.unboundop; _queue_unmark_waiter(queue, _globals.queues.mutex); - PyObject *fmt_obj = PyLong_FromLong(fmt); - if (fmt_obj == NULL) { - return NULL; - } - // For now queues only have one default. - PyObject *res = PyTuple_Pack(1, fmt_obj); - Py_DECREF(fmt_obj); - return res; + PyObject *defaults = Py_BuildValue("ii", fmt, unboundop); + return defaults; } PyDoc_STRVAR(queuesmod_get_queue_defaults_doc, diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h index 07120f6ccc7207..0d2e0c9efd3837 100644 --- a/Modules/_interpreters_common.h +++ b/Modules/_interpreters_common.h @@ -19,3 +19,48 @@ clear_xid_class(PyTypeObject *cls) return _PyCrossInterpreterData_UnregisterClass(cls); } #endif + + +static inline int64_t +_get_interpid(_PyCrossInterpreterData *data) +{ + int64_t interpid; + if (data != NULL) { + interpid = _PyCrossInterpreterData_INTERPID(data); + assert(!PyErr_Occurred()); + } + else { + interpid = PyInterpreterState_GetID(PyInterpreterState_Get()); + } + return interpid; +} + + +/* unbound items ************************************************************/ + +#ifdef HAS_UNBOUND_ITEMS + +#define UNBOUND_REMOVE 1 +#define UNBOUND_ERROR 2 +#define UNBOUND_REPLACE 3 + +// It would also be possible to add UNBOUND_REPLACE where the replacement +// value is user-provided. There would be some limitations there, though. +// Another possibility would be something like UNBOUND_COPY, where the +// object is released but the underlying data is copied (with the "raw" +// allocator) and used when the item is popped off the queue. + +static int +check_unbound(int unboundop) +{ + switch (unboundop) { + case UNBOUND_REMOVE: + case UNBOUND_ERROR: + case UNBOUND_REPLACE: + return 1; + default: + return 0; + } +} + +#endif diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index aa52711941d374..e45323c93a17ef 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -8,7 +8,6 @@ */ #include "Python.h" -#include "pycore_bytesobject.h" // _PyBytes_Join() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() diff --git a/Modules/_json.c b/Modules/_json.c index c7fe1561bb1018..9e29de0f22465f 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -11,6 +11,7 @@ #include "Python.h" #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_runtime.h" // _PyRuntime +#include "pycore_pyerrors.h" // _PyErr_FormatNote #include "pycore_global_strings.h" // _Py_ID() #include // bool @@ -1461,6 +1462,7 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_DECREF(newobj); if (rv) { + _PyErr_FormatNote("when serializing %T object", obj); Py_XDECREF(ident); return -1; } @@ -1477,7 +1479,7 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, static int encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *first, - PyObject *key, PyObject *value, + PyObject *dct, PyObject *key, PyObject *value, PyObject *newline_indent, PyObject *item_separator) { @@ -1535,6 +1537,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir return -1; } if (encoder_listencode_obj(s, writer, value, newline_indent) < 0) { + _PyErr_FormatNote("when serializing %T item %R", dct, key); return -1; } return 0; @@ -1606,7 +1609,7 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, key = PyTuple_GET_ITEM(item, 0); value = PyTuple_GET_ITEM(item, 1); - if (encoder_encode_key_value(s, writer, &first, key, value, + if (encoder_encode_key_value(s, writer, &first, dct, key, value, new_newline_indent, current_item_separator) < 0) goto bail; @@ -1616,7 +1619,7 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } else { Py_ssize_t pos = 0; while (PyDict_Next(dct, &pos, &key, &value)) { - if (encoder_encode_key_value(s, writer, &first, key, value, + if (encoder_encode_key_value(s, writer, &first, dct, key, value, new_newline_indent, current_item_separator) < 0) goto bail; @@ -1710,8 +1713,10 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, if (_PyUnicodeWriter_WriteStr(writer, separator) < 0) goto bail; } - if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) + if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) { + _PyErr_FormatNote("when serializing %T item %zd", seq, i); goto bail; + } } if (ident != NULL) { if (PyDict_DelItem(s->markers, ident)) diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index d4923442478b3e..de7395b610e133 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -52,7 +52,7 @@ module _locale [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=ed98569b726feada]*/ -/* support functions for formatting floating point numbers */ +/* support functions for formatting floating-point numbers */ /* the grouping is terminated by either 0 or CHAR_MAX */ static PyObject* diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 5cf9eba243bd20..8b6906234bdc25 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -59,6 +59,7 @@ typedef struct { #define POF_ENABLED 0x001 #define POF_SUBCALLS 0x002 #define POF_BUILTINS 0x004 +#define POF_EXT_TIMER 0x008 #define POF_NOMEMORY 0x100 /*[clinic input] @@ -87,7 +88,14 @@ _lsprof_get_state(PyObject *module) static PyTime_t CallExternalTimer(ProfilerObject *pObj) { - PyObject *o = _PyObject_CallNoArgs(pObj->externalTimer); + PyObject *o = NULL; + + // External timer can do arbitrary things so we need a flag to prevent + // horrible things to happen + pObj->flags |= POF_EXT_TIMER; + o = _PyObject_CallNoArgs(pObj->externalTimer); + pObj->flags &= ~POF_EXT_TIMER; + if (o == NULL) { PyErr_WriteUnraisable(pObj->externalTimer); return 0; @@ -777,6 +785,11 @@ Stop collecting profiling information.\n\ static PyObject* profiler_disable(ProfilerObject *self, PyObject* noarg) { + if (self->flags & POF_EXT_TIMER) { + PyErr_SetString(PyExc_RuntimeError, + "cannot disable profiler in external timer"); + return NULL; + } if (self->flags & POF_ENABLED) { PyObject* result = NULL; PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); @@ -830,6 +843,11 @@ Clear all profiling information collected so far.\n\ static PyObject* profiler_clear(ProfilerObject *pObj, PyObject* noarg) { + if (pObj->flags & POF_EXT_TIMER) { + PyErr_SetString(PyExc_RuntimeError, + "cannot clear profiler in external timer"); + return NULL; + } clearEntries(pObj); Py_RETURN_NONE; } @@ -838,6 +856,7 @@ static int profiler_traverse(ProfilerObject *op, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(op)); + Py_VISIT(op->externalTimer); return 0; } diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 67643641bea861..dc93063aee7e54 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -10,6 +10,8 @@ #include "pycore_compile.h" #include "pycore_intrinsics.h" #include "pycore_optimizer.h" // _Py_GetExecutor() +#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, OPCODE_HAS_*, etc +#include "pycore_opcode_utils.h" /*[clinic input] module _opcode @@ -81,7 +83,7 @@ static int _opcode_is_valid_impl(PyObject *module, int opcode) /*[clinic end generated code: output=b0d918ea1d073f65 input=fe23e0aa194ddae0]*/ { - return _PyCompile_OpcodeIsValid(opcode); + return IS_VALID_OPCODE(opcode); } /*[clinic input] @@ -97,8 +99,7 @@ static int _opcode_has_arg_impl(PyObject *module, int opcode) /*[clinic end generated code: output=7a062d3b2dcc0815 input=93d878ba6361db5f]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasArg(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_ARG(opcode); } /*[clinic input] @@ -114,8 +115,7 @@ static int _opcode_has_const_impl(PyObject *module, int opcode) /*[clinic end generated code: output=c646d5027c634120 input=a6999e4cf13f9410]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasConst(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_CONST(opcode); } /*[clinic input] @@ -131,8 +131,7 @@ static int _opcode_has_name_impl(PyObject *module, int opcode) /*[clinic end generated code: output=b49a83555c2fa517 input=448aa5e4bcc947ba]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasName(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_NAME(opcode); } /*[clinic input] @@ -148,9 +147,7 @@ static int _opcode_has_jump_impl(PyObject *module, int opcode) /*[clinic end generated code: output=e9c583c669f1c46a input=35f711274357a0c3]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasJump(opcode); - + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_JUMP(opcode); } /*[clinic input] @@ -171,9 +168,7 @@ static int _opcode_has_free_impl(PyObject *module, int opcode) /*[clinic end generated code: output=d81ae4d79af0ee26 input=117dcd5c19c1139b]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasFree(opcode); - + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_FREE(opcode); } /*[clinic input] @@ -189,8 +184,7 @@ static int _opcode_has_local_impl(PyObject *module, int opcode) /*[clinic end generated code: output=da5a8616b7a5097b input=9a798ee24aaef49d]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasLocal(opcode); + return IS_VALID_OPCODE(opcode) && OPCODE_HAS_LOCAL(opcode); } /*[clinic input] @@ -206,8 +200,7 @@ static int _opcode_has_exc_impl(PyObject *module, int opcode) /*[clinic end generated code: output=41b68dff0ec82a52 input=db0e4bdb9bf13fa5]*/ { - return _PyCompile_OpcodeIsValid(opcode) && - _PyCompile_OpcodeHasExc(opcode); + return IS_VALID_OPCODE(opcode) && IS_BLOCK_PUSH_OPCODE(opcode); } /*[clinic input] @@ -424,7 +417,7 @@ opcode_functions[] = { {NULL, NULL, 0, NULL} }; -int +static int _opcode_exec(PyObject *m) { if (PyModule_AddIntMacro(m, ENABLE_SPECIALIZATION) < 0) { return -1; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 4a2191db0cf984..452b4aff0237ca 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -40,7 +40,7 @@ class _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "" already includes it. */ enum { HIGHEST_PROTOCOL = 5, - DEFAULT_PROTOCOL = 4 + DEFAULT_PROTOCOL = 5 }; #ifdef MS_WINDOWS @@ -1817,10 +1817,10 @@ get_dotted_path(PyObject *obj, PyObject *name) if (_PyUnicode_EqualToASCIIString(subpath, "")) { if (obj == NULL) PyErr_Format(PyExc_AttributeError, - "Can't pickle local object %R", name); + "Can't get local object %R", name); else PyErr_Format(PyExc_AttributeError, - "Can't pickle local attribute %R on %R", name, obj); + "Can't get local attribute %R on %R", name, obj); Py_DECREF(dotted_path); return NULL; } @@ -2507,7 +2507,7 @@ save_picklebuffer(PickleState *st, PicklerObject *self, PyObject *obj) { if (self->proto < 5) { PyErr_SetString(st->PicklingError, - "PickleBuffer can only pickled with protocol >= 5"); + "PickleBuffer can only be pickled with protocol >= 5"); return -1; } const Py_buffer* view = PyPickleBuffer_GetBuffer(obj); @@ -3592,7 +3592,6 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, PyObject *module = NULL; PyObject *parent = NULL; PyObject *dotted_path = NULL; - PyObject *lastname = NULL; PyObject *cls; int status = 0; @@ -3633,10 +3632,7 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, obj, module_name); goto error; } - lastname = Py_NewRef(PyList_GET_ITEM(dotted_path, - PyList_GET_SIZE(dotted_path) - 1)); cls = get_deep_attribute(module, dotted_path, &parent); - Py_CLEAR(dotted_path); if (cls == NULL) { PyErr_Format(st->PicklingError, "Can't pickle %R: attribute lookup %S on %S failed", @@ -3724,7 +3720,10 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, else { gen_global: if (parent == module) { - Py_SETREF(global_name, Py_NewRef(lastname)); + Py_SETREF(global_name, + Py_NewRef(PyList_GET_ITEM(dotted_path, + PyList_GET_SIZE(dotted_path) - 1))); + Py_CLEAR(dotted_path); } if (self->proto >= 4) { const char stack_global_op = STACK_GLOBAL; @@ -3737,20 +3736,30 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, if (_Pickler_Write(self, &stack_global_op, 1) < 0) goto error; } - else if (parent != module) { - PyObject *reduce_value = Py_BuildValue("(O(OO))", - st->getattr, parent, lastname); - if (reduce_value == NULL) - goto error; - status = save_reduce(st, self, reduce_value, NULL); - Py_DECREF(reduce_value); - if (status < 0) - goto error; - } else { /* Generate a normal global opcode if we are using a pickle protocol < 4, or if the object is not registered in the - extension registry. */ + extension registry. + + Objects with multi-part __qualname__ are represented as + getattr(getattr(..., attrname1), attrname2). */ + const char mark_op = MARK; + const char tupletwo_op = (self->proto < 2) ? TUPLE : TUPLE2; + const char reduce_op = REDUCE; + Py_ssize_t i; + if (dotted_path) { + if (PyList_GET_SIZE(dotted_path) > 1) { + Py_SETREF(global_name, Py_NewRef(PyList_GET_ITEM(dotted_path, 0))); + } + for (i = 1; i < PyList_GET_SIZE(dotted_path); i++) { + if (save(st, self, st->getattr, 0) < 0 || + (self->proto < 2 && _Pickler_Write(self, &mark_op, 1) < 0)) + { + goto error; + } + } + } + PyObject *encoded; PyObject *(*unicode_encoder)(PyObject *); @@ -3812,6 +3821,17 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, Py_DECREF(encoded); if (_Pickler_Write(self, "\n", 1) < 0) goto error; + + if (dotted_path) { + for (i = 1; i < PyList_GET_SIZE(dotted_path); i++) { + if (save(st, self, PyList_GET_ITEM(dotted_path, i), 0) < 0 || + _Pickler_Write(self, &tupletwo_op, 1) < 0 || + _Pickler_Write(self, &reduce_op, 1) < 0) + { + goto error; + } + } + } } /* Memoize the object. */ if (memo_put(st, self, obj) < 0) @@ -3827,7 +3847,6 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj, Py_XDECREF(module); Py_XDECREF(parent); Py_XDECREF(dotted_path); - Py_XDECREF(lastname); return status; } @@ -4692,7 +4711,7 @@ This takes a binary file for writing a pickle data stream. The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default -protocol is 4. It was introduced in Python 3.4, and is incompatible +protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest protocol @@ -4725,7 +4744,7 @@ static int _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, PyObject *protocol, int fix_imports, PyObject *buffer_callback) -/*[clinic end generated code: output=0abedc50590d259b input=a7c969699bf5dad3]*/ +/*[clinic end generated code: output=0abedc50590d259b input=cddc50f66b770002]*/ { /* In case of multiple __init__() calls, clear previous content. */ if (self->write != NULL) @@ -7507,7 +7526,7 @@ be more efficient. The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default -protocol is 4. It was introduced in Python 3.4, and is incompatible +protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest protocol @@ -7533,7 +7552,7 @@ static PyObject * _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, PyObject *protocol, int fix_imports, PyObject *buffer_callback) -/*[clinic end generated code: output=706186dba996490c input=5ed6653da99cd97c]*/ +/*[clinic end generated code: output=706186dba996490c input=b89ce8d0e911fd46]*/ { PickleState *state = _Pickle_GetState(module); PicklerObject *pickler = _Pickler_New(state); @@ -7578,7 +7597,7 @@ Return the pickled representation of the object as a bytes object. The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default -protocol is 4. It was introduced in Python 3.4, and is incompatible +protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest protocol @@ -7598,7 +7617,7 @@ into *file* as part of the pickle stream. It is an error if static PyObject * _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, int fix_imports, PyObject *buffer_callback) -/*[clinic end generated code: output=fbab0093a5580fdf input=e543272436c6f987]*/ +/*[clinic end generated code: output=fbab0093a5580fdf input=139fc546886c63ac]*/ { PyObject *result; PickleState *state = _Pickle_GetState(module); diff --git a/Modules/_struct.c b/Modules/_struct.c index 6a68478dd45d36..f744193469e2dc 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -281,7 +281,7 @@ get_size_t(_structmodulestate *state, PyObject *v, size_t *p) #define RANGE_ERROR(state, f, flag) return _range_error(state, f, flag) -/* Floating point helpers */ +/* Floating-point helpers */ static PyObject * unpack_halffloat(const char *p, /* start of 2-byte string */ diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index 4f72844535ebd6..850de6f9c3366b 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -125,6 +125,7 @@ test_atomic_fences(PyObject *self, PyObject *obj) { // Just make sure that the fences compile. We are not // testing any synchronizing ordering. _Py_atomic_fence_seq_cst(); + _Py_atomic_fence_acquire(); _Py_atomic_fence_release(); Py_RETURN_NONE; } diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index b30c5e8704c8af..03aaacb328e0b6 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -348,6 +348,9 @@ static PyObject * MethodDescriptor2_new(PyTypeObject* type, PyObject* args, PyObject *kw) { MethodDescriptor2Object *op = PyObject_New(MethodDescriptor2Object, type); + if (op == NULL) { + return NULL; + } op->base.vectorcall = NULL; op->vectorcall = MethodDescriptor_vectorcall; return (PyObject *)op; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 39d309729d88b8..d21a37d8866a5f 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1350,33 +1350,44 @@ newlockobject(PyObject *module) Our implementation uses small "localdummy" objects in order to break the reference chain. These trivial objects are hashable (using the default scheme of identity hashing) and weakrefable. - Each thread-state holds a separate localdummy for each local object - (as a /strong reference/), - and each thread-local object holds a dict mapping /weak references/ - of localdummies to local dicts. + + Each thread-state holds two separate localdummy objects: + + - `threading_local_key` is used as a key to retrieve the locals dictionary + for the thread in any `threading.local` object. + - `threading_local_sentinel` is used to signal when a thread is being + destroyed. Consequently, the associated thread-state must hold the only + reference. + + Each `threading.local` object contains a dict mapping localdummy keys to + locals dicts and a set containing weak references to localdummy + sentinels. Each sentinel weak reference has a callback that removes itself + and the locals dict for the key from the `threading.local` object when + called. Therefore: - - only the thread-state dict holds a strong reference to the dummies - - only the thread-local object holds a strong reference to the local dicts - - only outside objects (application- or library-level) hold strong - references to the thread-local objects - - as soon as a thread-state dict is destroyed, the weakref callbacks of all - dummies attached to that thread are called, and destroy the corresponding - local dicts from thread-local objects - - as soon as a thread-local object is destroyed, its local dicts are - destroyed and its dummies are manually removed from all thread states - - the GC can do its work correctly when a thread-local object is dangling, - without any interference from the thread-state dicts - - As an additional optimization, each localdummy holds a borrowed reference - to the corresponding localdict. This borrowed reference is only used - by the thread-local object which has created the localdummy, which should - guarantee that the localdict still exists when accessed. + - The thread-state only holds strong references to localdummy objects, which + cannot participate in cycles. + - Only outside objects (application- or library-level) hold strong + references to the thread-local objects. + - As soon as thread-state's sentinel dummy is destroyed the callbacks for + all weakrefs attached to the sentinel are called, and destroy the + corresponding local dicts from thread-local objects. + - As soon as a thread-local object is destroyed, its local dicts are + destroyed. + - The GC can do its work correctly when a thread-local object is dangling, + without any interference from the thread-state dicts. + + This dual key arrangement is necessary to ensure that `threading.local` + values can be retrieved from finalizers. If we were to only keep a mapping + of localdummy weakrefs to locals dicts it's possible that the weakrefs would + be cleared before finalizers were called (GC currently clears weakrefs that + are garbage before invoking finalizers), causing lookups in finalizers to + fail. */ typedef struct { PyObject_HEAD - PyObject *localdict; /* Borrowed reference! */ PyObject *weakreflist; /* List of weak references to self */ } localdummyobject; @@ -1413,80 +1424,60 @@ static PyType_Spec local_dummy_type_spec = { typedef struct { PyObject_HEAD - PyObject *key; PyObject *args; PyObject *kw; PyObject *weakreflist; /* List of weak references to self */ - /* A {localdummy weakref -> localdict} dict */ - PyObject *dummies; - /* The callback for weakrefs to localdummies */ - PyObject *wr_callback; + /* A {localdummy -> localdict} dict */ + PyObject *localdicts; + /* A set of weakrefs to thread sentinels localdummies*/ + PyObject *thread_watchdogs; } localobject; /* Forward declaration */ -static PyObject *_ldict(localobject *self, thread_module_state *state); -static PyObject *_localdummy_destroyed(PyObject *meth_self, PyObject *dummyweakref); +static int create_localsdict(localobject *self, thread_module_state *state, + PyObject **localsdict, PyObject **sentinel_wr); +static PyObject *clear_locals(PyObject *meth_self, PyObject *dummyweakref); -/* Create and register the dummy for the current thread. - Returns a borrowed reference of the corresponding local dict */ +/* Create a weakref to the sentinel localdummy for the current thread */ static PyObject * -_local_create_dummy(localobject *self, thread_module_state *state) +create_sentinel_wr(localobject *self) { - PyObject *ldict = NULL, *wr = NULL; - localdummyobject *dummy = NULL; - PyTypeObject *type = state->local_dummy_type; + static PyMethodDef wr_callback_def = { + "clear_locals", (PyCFunction) clear_locals, METH_O + }; - PyObject *tdict = PyThreadState_GetDict(); - if (tdict == NULL) { - PyErr_SetString(PyExc_SystemError, - "Couldn't get thread-state dictionary"); - goto err; - } + PyThreadState *tstate = PyThreadState_Get(); - ldict = PyDict_New(); - if (ldict == NULL) { - goto err; - } - dummy = (localdummyobject *) type->tp_alloc(type, 0); - if (dummy == NULL) { - goto err; - } - dummy->localdict = ldict; - wr = PyWeakref_NewRef((PyObject *) dummy, self->wr_callback); - if (wr == NULL) { - goto err; + /* We use a weak reference to self in the callback closure + in order to avoid spurious reference cycles */ + PyObject *self_wr = PyWeakref_NewRef((PyObject *) self, NULL); + if (self_wr == NULL) { + return NULL; } - /* As a side-effect, this will cache the weakref's hash before the - dummy gets deleted */ - int r = PyDict_SetItem(self->dummies, wr, ldict); - if (r < 0) { - goto err; + PyObject *args = PyTuple_New(2); + if (args == NULL) { + Py_DECREF(self_wr); + return NULL; } - Py_CLEAR(wr); - r = PyDict_SetItem(tdict, self->key, (PyObject *) dummy); - if (r < 0) { - goto err; + PyTuple_SET_ITEM(args, 0, self_wr); + PyTuple_SET_ITEM(args, 1, Py_NewRef(tstate->threading_local_key)); + + PyObject *cb = PyCFunction_New(&wr_callback_def, args); + Py_DECREF(args); + if (cb == NULL) { + return NULL; } - Py_CLEAR(dummy); - Py_DECREF(ldict); - return ldict; + PyObject *wr = PyWeakref_NewRef(tstate->threading_local_sentinel, cb); + Py_DECREF(cb); -err: - Py_XDECREF(ldict); - Py_XDECREF(wr); - Py_XDECREF(dummy); - return NULL; + return wr; } static PyObject * local_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - static PyMethodDef wr_callback_def = { - "_localdummy_destroyed", (PyCFunction) _localdummy_destroyed, METH_O - }; - if (type->tp_init == PyBaseObject_Type.tp_init) { int rc = 0; if (args != NULL) @@ -1513,30 +1504,25 @@ local_new(PyTypeObject *type, PyObject *args, PyObject *kw) self->args = Py_XNewRef(args); self->kw = Py_XNewRef(kw); - self->key = PyUnicode_FromFormat("thread.local.%p", self); - if (self->key == NULL) { - goto err; - } - self->dummies = PyDict_New(); - if (self->dummies == NULL) { + self->localdicts = PyDict_New(); + if (self->localdicts == NULL) { goto err; } - /* We use a weak reference to self in the callback closure - in order to avoid spurious reference cycles */ - PyObject *wr = PyWeakref_NewRef((PyObject *) self, NULL); - if (wr == NULL) { - goto err; - } - self->wr_callback = PyCFunction_NewEx(&wr_callback_def, wr, NULL); - Py_DECREF(wr); - if (self->wr_callback == NULL) { + self->thread_watchdogs = PySet_New(NULL); + if (self->thread_watchdogs == NULL) { goto err; } - if (_local_create_dummy(self, state) == NULL) { + + PyObject *localsdict = NULL; + PyObject *sentinel_wr = NULL; + if (create_localsdict(self, state, &localsdict, &sentinel_wr) < 0) { goto err; } + Py_DECREF(localsdict); + Py_DECREF(sentinel_wr); + return (PyObject *)self; err: @@ -1550,7 +1536,8 @@ local_traverse(localobject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); Py_VISIT(self->args); Py_VISIT(self->kw); - Py_VISIT(self->dummies); + Py_VISIT(self->localdicts); + Py_VISIT(self->thread_watchdogs); return 0; } @@ -1559,27 +1546,8 @@ local_clear(localobject *self) { Py_CLEAR(self->args); Py_CLEAR(self->kw); - Py_CLEAR(self->dummies); - Py_CLEAR(self->wr_callback); - /* Remove all strong references to dummies from the thread states */ - if (self->key) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - while (tstate) { - if (tstate->dict) { - if (PyDict_Pop(tstate->dict, self->key, NULL) < 0) { - // Silently ignore error - PyErr_Clear(); - } - } - HEAD_LOCK(runtime); - tstate = PyThreadState_Next(tstate); - HEAD_UNLOCK(runtime); - } - } + Py_CLEAR(self->localdicts); + Py_CLEAR(self->thread_watchdogs); return 0; } @@ -1595,48 +1563,142 @@ local_dealloc(localobject *self) PyObject_GC_UnTrack(self); local_clear(self); - Py_XDECREF(self->key); PyTypeObject *tp = Py_TYPE(self); tp->tp_free((PyObject*)self); Py_DECREF(tp); } -/* Returns a borrowed reference to the local dict, creating it if necessary */ +/* Create the TLS key and sentinel if they don't exist */ +static int +create_localdummies(thread_module_state *state) +{ + PyThreadState *tstate = _PyThreadState_GET(); + + if (tstate->threading_local_key != NULL) { + return 0; + } + + PyTypeObject *ld_type = state->local_dummy_type; + tstate->threading_local_key = ld_type->tp_alloc(ld_type, 0); + if (tstate->threading_local_key == NULL) { + return -1; + } + + tstate->threading_local_sentinel = ld_type->tp_alloc(ld_type, 0); + if (tstate->threading_local_sentinel == NULL) { + Py_CLEAR(tstate->threading_local_key); + return -1; + } + + return 0; +} + +/* Insert a localsdict and sentinel weakref for the current thread, placing + strong references in localsdict and sentinel_wr, respectively. +*/ +static int +create_localsdict(localobject *self, thread_module_state *state, + PyObject **localsdict, PyObject **sentinel_wr) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *ldict = NULL; + PyObject *wr = NULL; + + if (create_localdummies(state) < 0) { + goto err; + } + + /* Create and insert the locals dict and sentinel weakref */ + ldict = PyDict_New(); + if (ldict == NULL) { + goto err; + } + + if (PyDict_SetItem(self->localdicts, tstate->threading_local_key, ldict) < + 0) { + goto err; + } + + wr = create_sentinel_wr(self); + if (wr == NULL) { + PyObject *exc = PyErr_GetRaisedException(); + if (PyDict_DelItem(self->localdicts, tstate->threading_local_key) < + 0) { + PyErr_WriteUnraisable((PyObject *)self); + } + PyErr_SetRaisedException(exc); + goto err; + } + + if (PySet_Add(self->thread_watchdogs, wr) < 0) { + PyObject *exc = PyErr_GetRaisedException(); + if (PyDict_DelItem(self->localdicts, tstate->threading_local_key) < + 0) { + PyErr_WriteUnraisable((PyObject *)self); + } + PyErr_SetRaisedException(exc); + goto err; + } + + *localsdict = ldict; + *sentinel_wr = wr; + return 0; + +err: + Py_XDECREF(ldict); + Py_XDECREF(wr); + return -1; +} + +/* Return a strong reference to the locals dict for the current thread, + creating it if necessary. +*/ static PyObject * _ldict(localobject *self, thread_module_state *state) { - PyObject *tdict = PyThreadState_GetDict(); - if (tdict == NULL) { - PyErr_SetString(PyExc_SystemError, - "Couldn't get thread-state dictionary"); + if (create_localdummies(state) < 0) { return NULL; } + /* Check if a localsdict already exists */ PyObject *ldict; - PyObject *dummy = PyDict_GetItemWithError(tdict, self->key); - if (dummy == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - ldict = _local_create_dummy(self, state); - if (ldict == NULL) - return NULL; + PyThreadState *tstate = _PyThreadState_GET(); + if (PyDict_GetItemRef(self->localdicts, tstate->threading_local_key, + &ldict) < 0) { + return NULL; + } + if (ldict != NULL) { + return ldict; + } - if (Py_TYPE(self)->tp_init != PyBaseObject_Type.tp_init && - Py_TYPE(self)->tp_init((PyObject*)self, - self->args, self->kw) < 0) { - /* we need to get rid of ldict from thread so - we create a new one the next time we do an attr - access */ - PyDict_DelItem(tdict, self->key); - return NULL; - } + /* threading.local hasn't been instantiated for this thread */ + PyObject *wr; + if (create_localsdict(self, state, &ldict, &wr) < 0) { + return NULL; } - else { - assert(Py_IS_TYPE(dummy, state->local_dummy_type)); - ldict = ((localdummyobject *) dummy)->localdict; + + /* run __init__ if we're a subtype of `threading.local` */ + if (Py_TYPE(self)->tp_init != PyBaseObject_Type.tp_init && + Py_TYPE(self)->tp_init((PyObject *)self, self->args, self->kw) < 0) { + /* we need to get rid of ldict from thread so + we create a new one the next time we do an attr + access */ + PyObject *exc = PyErr_GetRaisedException(); + if (PyDict_DelItem(self->localdicts, tstate->threading_local_key) < + 0) { + PyErr_WriteUnraisable((PyObject *)self); + PyErr_Clear(); + } + if (PySet_Discard(self->thread_watchdogs, wr) < 0) { + PyErr_WriteUnraisable((PyObject *)self); + } + PyErr_SetRaisedException(exc); + Py_DECREF(ldict); + Py_DECREF(wr); + return NULL; } + Py_DECREF(wr); return ldict; } @@ -1650,21 +1712,28 @@ local_setattro(localobject *self, PyObject *name, PyObject *v) PyObject *ldict = _ldict(self, state); if (ldict == NULL) { - return -1; + goto err; } int r = PyObject_RichCompareBool(name, &_Py_ID(__dict__), Py_EQ); if (r == -1) { - return -1; + goto err; } if (r == 1) { PyErr_Format(PyExc_AttributeError, "'%.100s' object attribute '%U' is read-only", Py_TYPE(self)->tp_name, name); - return -1; + goto err; } - return _PyObject_GenericSetAttrWithDict((PyObject *)self, name, v, ldict); + int st = + _PyObject_GenericSetAttrWithDict((PyObject *)self, name, v, ldict); + Py_DECREF(ldict); + return st; + +err: + Py_XDECREF(ldict); + return -1; } static PyObject *local_getattro(localobject *, PyObject *); @@ -1707,34 +1776,42 @@ local_getattro(localobject *self, PyObject *name) int r = PyObject_RichCompareBool(name, &_Py_ID(__dict__), Py_EQ); if (r == 1) { - return Py_NewRef(ldict); + return ldict; } if (r == -1) { + Py_DECREF(ldict); return NULL; } if (!Py_IS_TYPE(self, state->local_type)) { /* use generic lookup for subtypes */ - return _PyObject_GenericGetAttrWithDict((PyObject *)self, name, - ldict, 0); + PyObject *res = + _PyObject_GenericGetAttrWithDict((PyObject *)self, name, ldict, 0); + Py_DECREF(ldict); + return res; } /* Optimization: just look in dict ourselves */ PyObject *value; if (PyDict_GetItemRef(ldict, name, &value) != 0) { // found or error + Py_DECREF(ldict); return value; } /* Fall back on generic to get __class__ and __dict__ */ - return _PyObject_GenericGetAttrWithDict( - (PyObject *)self, name, ldict, 0); + PyObject *res = + _PyObject_GenericGetAttrWithDict((PyObject *)self, name, ldict, 0); + Py_DECREF(ldict); + return res; } -/* Called when a dummy is destroyed. */ +/* Called when a dummy is destroyed, indicating that the owning thread is being + * cleared. */ static PyObject * -_localdummy_destroyed(PyObject *localweakref, PyObject *dummyweakref) +clear_locals(PyObject *locals_and_key, PyObject *dummyweakref) { + PyObject *localweakref = PyTuple_GetItem(locals_and_key, 0); localobject *self = (localobject *)_PyWeakref_GET_REF(localweakref); if (self == NULL) { Py_RETURN_NONE; @@ -1742,11 +1819,18 @@ _localdummy_destroyed(PyObject *localweakref, PyObject *dummyweakref) /* If the thread-local object is still alive and not being cleared, remove the corresponding local dict */ - if (self->dummies != NULL) { - if (PyDict_Pop(self->dummies, dummyweakref, NULL) < 0) { + if (self->localdicts != NULL) { + PyObject *key = PyTuple_GetItem(locals_and_key, 1); + if (PyDict_Pop(self->localdicts, key, NULL) < 0) { PyErr_WriteUnraisable((PyObject*)self); } } + if (self->thread_watchdogs != NULL) { + if (PySet_Discard(self->thread_watchdogs, dummyweakref) < 0) { + PyErr_WriteUnraisable((PyObject *)self); + } + } + Py_DECREF(self); Py_RETURN_NONE; } diff --git a/Modules/_weakref.c b/Modules/_weakref.c index a5c15c0f10b930..ecaa08ff60f203 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -31,7 +31,7 @@ _weakref_getweakrefcount_impl(PyObject *module, PyObject *object) static int -is_dead_weakref(PyObject *value) +is_dead_weakref(PyObject *value, void *unused) { if (!PyWeakref_Check(value)) { PyErr_SetString(PyExc_TypeError, "not a weakref"); @@ -56,15 +56,8 @@ _weakref__remove_dead_weakref_impl(PyObject *module, PyObject *dct, PyObject *key) /*[clinic end generated code: output=d9ff53061fcb875c input=19fc91f257f96a1d]*/ { - if (_PyDict_DelItemIf(dct, key, is_dead_weakref) < 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) - /* This function is meant to allow safe weak-value dicts - with GC in another thread (see issue #28427), so it's - ok if the key doesn't exist anymore. - */ - PyErr_Clear(); - else - return NULL; + if (_PyDict_DelItemIf(dct, key, is_dead_weakref, NULL) < 0) { + return NULL; } Py_RETURN_NONE; } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index e6c84d588be98b..b80c964f20d65e 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -2863,7 +2863,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ -an array of basic values: characters, integers, floating point\n\ +an array of basic values: characters, integers, floating-point\n\ numbers. Arrays are sequence types and behave very much like lists,\n\ except that the type of objects stored in them is constrained.\n"); @@ -2891,8 +2891,8 @@ The following type codes are defined:\n\ 'L' unsigned integer 4\n\ 'q' signed integer 8 (see note)\n\ 'Q' unsigned integer 8 (see note)\n\ - 'f' floating point 4\n\ - 'd' floating point 8\n\ + 'f' floating-point 4\n\ + 'd' floating-point 8\n\ \n\ NOTE: The 'u' typecode corresponds to Python's unicode character. On\n\ narrow builds this is 2-bytes on wide builds this is 4-bytes.\n\ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index 693c7d59e9d7a6..40f1309b6aa03c 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -111,7 +111,7 @@ PyDoc_STRVAR(_pickle_Pickler___init____doc__, "\n" "The optional *protocol* argument tells the pickler to use the given\n" "protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\n" -"protocol is 4. It was introduced in Python 3.4, and is incompatible\n" +"protocol is 5. It was introduced in Python 3.8, and is incompatible\n" "with previous versions.\n" "\n" "Specifying a negative protocol version selects the highest protocol\n" @@ -614,7 +614,7 @@ PyDoc_STRVAR(_pickle_dump__doc__, "\n" "The optional *protocol* argument tells the pickler to use the given\n" "protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\n" -"protocol is 4. It was introduced in Python 3.4, and is incompatible\n" +"protocol is 5. It was introduced in Python 3.8, and is incompatible\n" "with previous versions.\n" "\n" "Specifying a negative protocol version selects the highest protocol\n" @@ -724,7 +724,7 @@ PyDoc_STRVAR(_pickle_dumps__doc__, "\n" "The optional *protocol* argument tells the pickler to use the given\n" "protocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\n" -"protocol is 4. It was introduced in Python 3.4, and is incompatible\n" +"protocol is 5. It was introduced in Python 3.8, and is incompatible\n" "with previous versions.\n" "\n" "Specifying a negative protocol version selects the highest protocol\n" @@ -1077,4 +1077,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=c7dd60d20ee4895f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a9452cf1219f2e7a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index d16db722a74bc4..81eec310ddb21d 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -34,9 +34,9 @@ PyDoc_STRVAR(math_fsum__doc__, "fsum($module, seq, /)\n" "--\n" "\n" -"Return an accurate floating point sum of values in the iterable seq.\n" +"Return an accurate floating-point sum of values in the iterable seq.\n" "\n" -"Assumes IEEE-754 floating point arithmetic."); +"Assumes IEEE-754 floating-point arithmetic."); #define MATH_FSUM_METHODDEF \ {"fsum", (PyCFunction)math_fsum, METH_O, math_fsum__doc__}, @@ -610,7 +610,7 @@ PyDoc_STRVAR(math_isclose__doc__, "isclose($module, /, a, b, *, rel_tol=1e-09, abs_tol=0.0)\n" "--\n" "\n" -"Determine whether two floating point numbers are close in value.\n" +"Determine whether two floating-point numbers are close in value.\n" "\n" " rel_tol\n" " maximum difference for being considered \"close\", relative to the\n" @@ -1011,4 +1011,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=7d03f84f77342496 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=755da3b1dbd9e45f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index ca40bec345bda9..4b9dbac9af031f 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -6348,7 +6348,7 @@ PyDoc_STRVAR(os_times__doc__, "\n" "The object returned behaves like a named tuple with these fields:\n" " (utime, stime, cutime, cstime, elapsed_time)\n" -"All fields are floating point numbers."); +"All fields are floating-point numbers."); #define OS_TIMES_METHODDEF \ {"times", (PyCFunction)os_times, METH_NOARGS, os_times__doc__}, @@ -12837,4 +12837,4 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=31dc3bb3cba924d0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2fafa0d2814948f8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index dc7d3fb814396d..49c0e48d2e0eac 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() @@ -25,7 +26,7 @@ PyDoc_STRVAR(select_select__doc__, "gotten from a fileno() method call on one of those.\n" "\n" "The optional 4th argument specifies a timeout in seconds; it may be\n" -"a floating point number to specify fractions of seconds. If it is absent\n" +"a floating-point number to specify fractions of seconds. If it is absent\n" "or None, the call will never time out.\n" "\n" "The return value is a tuple of three lists corresponding to the first three\n" @@ -110,7 +111,9 @@ select_poll_register(pollObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_register_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -155,7 +158,9 @@ select_poll_modify(pollObject *self, PyObject *const *args, Py_ssize_t nargs) if (!_PyLong_UnsignedShort_Converter(args[1], &eventmask)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_modify_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -187,7 +192,9 @@ select_poll_unregister(pollObject *self, PyObject *arg) if (fd < 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_unregister_impl(self, fd); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -230,7 +237,9 @@ select_poll_poll(pollObject *self, PyObject *const *args, Py_ssize_t nargs) } timeout_obj = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_poll_poll_impl(self, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -281,7 +290,9 @@ select_devpoll_register(devpollObject *self, PyObject *const *args, Py_ssize_t n goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_register_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -332,7 +343,9 @@ select_devpoll_modify(devpollObject *self, PyObject *const *args, Py_ssize_t nar goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_modify_impl(self, fd, eventmask); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -364,7 +377,9 @@ select_devpoll_unregister(devpollObject *self, PyObject *arg) if (fd < 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_unregister_impl(self, fd); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -407,7 +422,9 @@ select_devpoll_poll(devpollObject *self, PyObject *const *args, Py_ssize_t nargs } timeout_obj = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = select_devpoll_poll_impl(self, timeout_obj); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -434,7 +451,13 @@ select_devpoll_close_impl(devpollObject *self); static PyObject * select_devpoll_close(devpollObject *self, PyObject *Py_UNUSED(ignored)) { - return select_devpoll_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) && defined(HAVE_SYS_DEVPOLL_H) */ @@ -456,7 +479,13 @@ select_devpoll_fileno_impl(devpollObject *self); static PyObject * select_devpoll_fileno(devpollObject *self, PyObject *Py_UNUSED(ignored)) { - return select_devpoll_fileno_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_devpoll_fileno_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) && defined(HAVE_SYS_DEVPOLL_H) */ @@ -615,7 +644,13 @@ select_epoll_close_impl(pyEpoll_Object *self); static PyObject * select_epoll_close(pyEpoll_Object *self, PyObject *Py_UNUSED(ignored)) { - return select_epoll_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_epoll_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_EPOLL) */ @@ -1108,7 +1143,13 @@ select_kqueue_close_impl(kqueue_queue_Object *self); static PyObject * select_kqueue_close(kqueue_queue_Object *self, PyObject *Py_UNUSED(ignored)) { - return select_kqueue_close_impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = select_kqueue_close_impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_KQUEUE) */ @@ -1319,4 +1360,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=4fc17ae9b6cfdc86 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f99427b75cbe6d44 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index d074cc30d1e746..1d3a143dfd8d39 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -597,7 +597,7 @@ PyDoc_STRVAR(signal_sigtimedwait__doc__, "\n" "Like sigwaitinfo(), but with a timeout.\n" "\n" -"The timeout is specified in seconds, with floating point numbers allowed."); +"The timeout is specified in seconds, with floating-point numbers allowed."); #define SIGNAL_SIGTIMEDWAIT_METHODDEF \ {"sigtimedwait", _PyCFunction_CAST(signal_sigtimedwait), METH_FASTCALL, signal_sigtimedwait__doc__}, @@ -776,4 +776,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=1c11c1b6f12f26be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6d8e17a32cef668f input=a9049054013a1b77]*/ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index cfa3cbdc34bc86..b62362f277797e 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -75,7 +75,7 @@ static fault_handler_t faulthandler_handlers[] = { #ifdef SIGILL {SIGILL, 0, "Illegal instruction", }, #endif - {SIGFPE, 0, "Floating point exception", }, + {SIGFPE, 0, "Floating-point exception", }, {SIGABRT, 0, "Aborted", }, /* define SIGSEGV at the end to make it the default choice if searching the handler fails in faulthandler_fatal_error() */ diff --git a/Modules/getpath.c b/Modules/getpath.c index abed139028244a..d0128b20faeeae 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -951,6 +951,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !wchar_to_dict(dict, "executable_dir", NULL) || !wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) || !funcs_to_dict(dict, config->pathconfig_warnings) || +#ifdef Py_GIL_DISABLED + !decode_to_dict(dict, "ABI_THREAD", "t") || +#else + !decode_to_dict(dict, "ABI_THREAD", "") || +#endif #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || #endif diff --git a/Modules/getpath.py b/Modules/getpath.py index bc7053224aaf16..1f1bfcb4f64dd4 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -40,6 +40,7 @@ # EXE_SUFFIX -- [in, opt] '.exe' on Windows/Cygwin/similar # VERSION_MAJOR -- [in] sys.version_info.major # VERSION_MINOR -- [in] sys.version_info.minor +# ABI_THREAD -- [in] either 't' for free-threaded builds or '' # PYWINVER -- [in] the Windows platform-specific version (e.g. 3.8-32) # ** Values read from the environment ** @@ -172,17 +173,18 @@ # ****************************************************************************** platlibdir = config.get('platlibdir') or PLATLIBDIR +ABI_THREAD = ABI_THREAD or '' if os_name == 'posix' or os_name == 'darwin': BUILDDIR_TXT = 'pybuilddir.txt' BUILD_LANDMARK = 'Modules/Setup.local' DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' - STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}' STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] - PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}/lib-dynload' BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] VENV_LANDMARK = 'pyvenv.cfg' - ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}{ABI_THREAD}.zip' DELIM = ':' SEP = '/' diff --git a/Modules/main.c b/Modules/main.c index 1a70b300b6ad17..15ea49a1bad19e 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -4,6 +4,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_initconfig.h" // _PyArgv #include "pycore_interp.h" // _PyInterpreterState.sysdict +#include "pycore_long.h" // _PyLong_GetOne() #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -259,6 +260,57 @@ pymain_run_command(wchar_t *command) } +static int +pymain_start_pyrepl_no_main(void) +{ + int res = 0; + PyObject *console = NULL; + PyObject *empty_tuple = NULL; + PyObject *kwargs = NULL; + PyObject *console_result = NULL; + + PyObject *pyrepl = PyImport_ImportModule("_pyrepl.main"); + if (pyrepl == NULL) { + fprintf(stderr, "Could not import _pyrepl.main\n"); + res = pymain_exit_err_print(); + goto done; + } + console = PyObject_GetAttrString(pyrepl, "interactive_console"); + if (console == NULL) { + fprintf(stderr, "Could not access _pyrepl.main.interactive_console\n"); + res = pymain_exit_err_print(); + goto done; + } + empty_tuple = PyTuple_New(0); + if (empty_tuple == NULL) { + res = pymain_exit_err_print(); + goto done; + } + kwargs = PyDict_New(); + if (kwargs == NULL) { + res = pymain_exit_err_print(); + goto done; + } + if (!PyDict_SetItemString(kwargs, "pythonstartup", _PyLong_GetOne())) { + _PyRuntime.signals.unhandled_keyboard_interrupt = 0; + console_result = PyObject_Call(console, empty_tuple, kwargs); + if (!console_result && PyErr_Occurred() == PyExc_KeyboardInterrupt) { + _PyRuntime.signals.unhandled_keyboard_interrupt = 1; + } + if (console_result == NULL) { + res = pymain_exit_err_print(); + } + } +done: + Py_XDECREF(console_result); + Py_XDECREF(kwargs); + Py_XDECREF(empty_tuple); + Py_XDECREF(console); + Py_XDECREF(pyrepl); + return res; +} + + static int pymain_run_module(const wchar_t *modname, int set_argv0) { @@ -542,6 +594,10 @@ pymain_repl(PyConfig *config, int *exitcode) return; } + if (PySys_Audit("cpython.run_stdin", NULL) < 0) { + return; + } + if (!isatty(fileno(stdin)) || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { PyCompilerFlags cf = _PyCompilerFlags_INIT; @@ -549,7 +605,7 @@ pymain_repl(PyConfig *config, int *exitcode) *exitcode = (run != 0); return; } - int run = pymain_run_module(L"_pyrepl", 0); + int run = pymain_start_pyrepl_no_main(); *exitcode = (run != 0); return; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 6defa973da0952..64dfceac8bfea3 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -107,7 +107,7 @@ typedef struct{ double hi; double lo; } DoubleLength; static DoubleLength dl_fast_sum(double a, double b) { - /* Algorithm 1.1. Compensated summation of two floating point numbers. */ + /* Algorithm 1.1. Compensated summation of two floating-point numbers. */ assert(fabs(a) >= fabs(b)); double x = a + b; double y = (a - x) + b; @@ -1354,14 +1354,14 @@ math.fsum seq: object / -Return an accurate floating point sum of values in the iterable seq. +Return an accurate floating-point sum of values in the iterable seq. -Assumes IEEE-754 floating point arithmetic. +Assumes IEEE-754 floating-point arithmetic. [clinic start generated code]*/ static PyObject * math_fsum(PyObject *module, PyObject *seq) -/*[clinic end generated code: output=ba5c672b87fe34fc input=c51b7d8caf6f6e82]*/ +/*[clinic end generated code: output=ba5c672b87fe34fc input=4506244ded6057dc]*/ { PyObject *item, *iter, *sum = NULL; Py_ssize_t i, j, n = 0, m = NUM_PARTIALS; @@ -2453,7 +2453,7 @@ Since lo**2 is less than 1/2 ulp(csum), we have csum+lo*lo == csum. To minimize loss of information during the accumulation of fractional values, each term has a separate accumulator. This also breaks up sequential dependencies in the inner loop so the CPU can maximize -floating point throughput. [4] On an Apple M1 Max, hypot(*vec) +floating-point throughput. [4] On an Apple M1 Max, hypot(*vec) takes only 3.33 µsec when len(vec) == 1000. The square root differential correction is needed because a @@ -3136,7 +3136,7 @@ math.isclose -> bool maximum difference for being considered "close", regardless of the magnitude of the input values -Determine whether two floating point numbers are close in value. +Determine whether two floating-point numbers are close in value. Return True if a is close in value to b, and False otherwise. @@ -3151,7 +3151,7 @@ only close to themselves. static int math_isclose_impl(PyObject *module, double a, double b, double rel_tol, double abs_tol) -/*[clinic end generated code: output=b73070207511952d input=f28671871ea5bfba]*/ +/*[clinic end generated code: output=b73070207511952d input=12d41764468bfdb8]*/ { double diff = 0.0; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index fc218383d5ff95..f02b6d1779827f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10585,12 +10585,12 @@ Return a collection containing process timing information. The object returned behaves like a named tuple with these fields: (utime, stime, cutime, cstime, elapsed_time) -All fields are floating point numbers. +All fields are floating-point numbers. [clinic start generated code]*/ static PyObject * os_times_impl(PyObject *module) -/*[clinic end generated code: output=35f640503557d32a input=2bf9df3d6ab2e48b]*/ +/*[clinic end generated code: output=35f640503557d32a input=8dbfe33a2dcc3df3]*/ { #ifdef MS_WINDOWS FILETIME create, exit, kernel, user; diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 3eaee22c652c28..5bd9b7732a44a4 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -262,7 +262,7 @@ A file descriptor is either a socket or file object, or a small integer gotten from a fileno() method call on one of those. The optional 4th argument specifies a timeout in seconds; it may be -a floating point number to specify fractions of seconds. If it is absent +a floating-point number to specify fractions of seconds. If it is absent or None, the call will never time out. The return value is a tuple of three lists corresponding to the first three @@ -277,7 +277,7 @@ descriptors can be used. static PyObject * select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, PyObject *xlist, PyObject *timeout_obj) -/*[clinic end generated code: output=2b3cfa824f7ae4cf input=e467f5d68033de00]*/ +/*[clinic end generated code: output=2b3cfa824f7ae4cf input=1199d5e101abca4a]*/ { #ifdef SELECT_USES_HEAP pylist *rfd2obj, *wfd2obj, *efd2obj; @@ -473,6 +473,7 @@ update_ufd_array(pollObject *self) } /*[clinic input] +@critical_section select.poll.register fd: fildes @@ -486,7 +487,7 @@ Register a file descriptor with the polling object. static PyObject * select_poll_register_impl(pollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=0dc7173c800a4a65 input=34e16cfb28d3c900]*/ +/*[clinic end generated code: output=0dc7173c800a4a65 input=c475e029ce6c2830]*/ { PyObject *key, *value; int err; @@ -514,6 +515,7 @@ select_poll_register_impl(pollObject *self, int fd, unsigned short eventmask) /*[clinic input] +@critical_section select.poll.modify fd: fildes @@ -528,7 +530,7 @@ Modify an already registered file descriptor. static PyObject * select_poll_modify_impl(pollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=1a7b88bf079eff17 input=a8e383df075c32cf]*/ +/*[clinic end generated code: output=1a7b88bf079eff17 input=38c9db5346711872]*/ { PyObject *key, *value; int err; @@ -566,6 +568,7 @@ select_poll_modify_impl(pollObject *self, int fd, unsigned short eventmask) /*[clinic input] +@critical_section select.poll.unregister fd: fildes @@ -576,7 +579,7 @@ Remove a file descriptor being tracked by the polling object. static PyObject * select_poll_unregister_impl(pollObject *self, int fd) -/*[clinic end generated code: output=8c9f42e75e7d291b input=4b4fccc1040e79cb]*/ +/*[clinic end generated code: output=8c9f42e75e7d291b input=ae6315d7f5243704]*/ { PyObject *key; @@ -599,6 +602,7 @@ select_poll_unregister_impl(pollObject *self, int fd) } /*[clinic input] +@critical_section select.poll.poll timeout as timeout_obj: object = None @@ -614,7 +618,7 @@ report, as a list of (fd, event) 2-tuples. static PyObject * select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=876e837d193ed7e4 input=c2f6953ec45e5622]*/ +/*[clinic end generated code: output=876e837d193ed7e4 input=54310631457efdec]*/ { PyObject *result_list = NULL; int poll_result, i, j; @@ -857,6 +861,7 @@ internal_devpoll_register(devpollObject *self, int fd, } /*[clinic input] +@critical_section select.devpoll.register fd: fildes @@ -872,12 +877,13 @@ Register a file descriptor with the polling object. static PyObject * select_devpoll_register_impl(devpollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=6e07fe8b74abba0c input=22006fabe9567522]*/ +/*[clinic end generated code: output=6e07fe8b74abba0c input=8d48bd2653a61c42]*/ { return internal_devpoll_register(self, fd, eventmask, 0); } /*[clinic input] +@critical_section select.devpoll.modify fd: fildes @@ -893,12 +899,13 @@ Modify a possible already registered file descriptor. static PyObject * select_devpoll_modify_impl(devpollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=bc2e6d23aaff98b4 input=09fa335db7cdc09e]*/ +/*[clinic end generated code: output=bc2e6d23aaff98b4 input=773b37e9abca2460]*/ { return internal_devpoll_register(self, fd, eventmask, 1); } /*[clinic input] +@critical_section select.devpoll.unregister fd: fildes @@ -909,7 +916,7 @@ Remove a file descriptor being tracked by the polling object. static PyObject * select_devpoll_unregister_impl(devpollObject *self, int fd) -/*[clinic end generated code: output=95519ffa0c7d43fe input=b4ea42a4442fd467]*/ +/*[clinic end generated code: output=95519ffa0c7d43fe input=6052d368368d4d05]*/ { if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -926,6 +933,7 @@ select_devpoll_unregister_impl(devpollObject *self, int fd) } /*[clinic input] +@critical_section select.devpoll.poll timeout as timeout_obj: object = None The maximum time to wait in milliseconds, or else None (or a negative @@ -940,7 +948,7 @@ report, as a list of (fd, event) 2-tuples. static PyObject * select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=2654e5457cca0b3c input=3c3f0a355ec2bedb]*/ +/*[clinic end generated code: output=2654e5457cca0b3c input=fe7a3f6dcbc118c5]*/ { struct dvpoll dvp; PyObject *result_list = NULL; @@ -1059,6 +1067,7 @@ devpoll_internal_close(devpollObject *self) } /*[clinic input] +@critical_section select.devpoll.close Close the devpoll file descriptor. @@ -1068,7 +1077,7 @@ Further operations on the devpoll object will raise an exception. static PyObject * select_devpoll_close_impl(devpollObject *self) -/*[clinic end generated code: output=26b355bd6429f21b input=6273c30f5560a99b]*/ +/*[clinic end generated code: output=26b355bd6429f21b input=408fde21a377ccfb]*/ { errno = devpoll_internal_close(self); if (errno < 0) { @@ -1088,6 +1097,7 @@ devpoll_get_closed(devpollObject *self, void *Py_UNUSED(ignored)) } /*[clinic input] +@critical_section select.devpoll.fileno Return the file descriptor. @@ -1095,7 +1105,7 @@ Return the file descriptor. static PyObject * select_devpoll_fileno_impl(devpollObject *self) -/*[clinic end generated code: output=26920929f8d292f4 input=ef15331ebde6c368]*/ +/*[clinic end generated code: output=26920929f8d292f4 input=8c9db2efa1ade538]*/ { if (self->fd_devpoll < 0) return devpoll_err_closed(); @@ -1378,6 +1388,7 @@ pyepoll_dealloc(pyEpoll_Object *self) } /*[clinic input] +@critical_section select.epoll.close Close the epoll control file descriptor. @@ -1387,7 +1398,7 @@ Further operations on the epoll object will raise an exception. static PyObject * select_epoll_close_impl(pyEpoll_Object *self) -/*[clinic end generated code: output=ee2144c446a1a435 input=ca6c66ba5a736bfd]*/ +/*[clinic end generated code: output=ee2144c446a1a435 input=f626a769192e1dbe]*/ { errno = pyepoll_internal_close(self); if (errno < 0) { @@ -2023,10 +2034,8 @@ kqueue_tracking_init(PyObject *module) { } static int -kqueue_tracking_add(_selectstate *state, kqueue_queue_Object *self) { - if (!state->kqueue_tracking_initialized) { - kqueue_tracking_init(PyType_GetModule(Py_TYPE(self))); - } +kqueue_tracking_add_lock_held(_selectstate *state, kqueue_queue_Object *self) +{ assert(self->kqfd >= 0); _kqueue_list_item *item = PyMem_New(_kqueue_list_item, 1); if (item == NULL) { @@ -2039,8 +2048,23 @@ kqueue_tracking_add(_selectstate *state, kqueue_queue_Object *self) { return 0; } +static int +kqueue_tracking_add(_selectstate *state, kqueue_queue_Object *self) +{ + int ret; + PyObject *module = PyType_GetModule(Py_TYPE(self)); + Py_BEGIN_CRITICAL_SECTION(module); + if (!state->kqueue_tracking_initialized) { + kqueue_tracking_init(module); + } + ret = kqueue_tracking_add_lock_held(state, self); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void -kqueue_tracking_remove(_selectstate *state, kqueue_queue_Object *self) { +kqueue_tracking_remove_lock_held(_selectstate *state, kqueue_queue_Object *self) +{ _kqueue_list *listptr = &state->kqueue_open_list; while (*listptr != NULL) { _kqueue_list_item *item = *listptr; @@ -2056,6 +2080,14 @@ kqueue_tracking_remove(_selectstate *state, kqueue_queue_Object *self) { assert(0); } +static void +kqueue_tracking_remove(_selectstate *state, kqueue_queue_Object *self) +{ + Py_BEGIN_CRITICAL_SECTION(PyType_GetModule(Py_TYPE(self))); + kqueue_tracking_remove_lock_held(state, self); + Py_END_CRITICAL_SECTION(); +} + static int kqueue_queue_internal_close(kqueue_queue_Object *self) { @@ -2150,6 +2182,7 @@ kqueue_queue_finalize(kqueue_queue_Object *self) } /*[clinic input] +@critical_section select.kqueue.close Close the kqueue control file descriptor. @@ -2159,7 +2192,7 @@ Further operations on the kqueue object will raise an exception. static PyObject * select_kqueue_close_impl(kqueue_queue_Object *self) -/*[clinic end generated code: output=d1c7df0b407a4bc1 input=0b12d95430e0634c]*/ +/*[clinic end generated code: output=d1c7df0b407a4bc1 input=6d763c858b17b690]*/ { errno = kqueue_queue_internal_close(self); if (errno < 0) { diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 7de5ebe0899b35..73bfcb756657b8 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -638,7 +638,7 @@ signal_strsignal_impl(PyObject *module, int signalnum) res = "Aborted"; break; case SIGFPE: - res = "Floating point exception"; + res = "Floating-point exception"; break; case SIGSEGV: res = "Segmentation fault"; @@ -1199,13 +1199,13 @@ signal.sigtimedwait Like sigwaitinfo(), but with a timeout. -The timeout is specified in seconds, with floating point numbers allowed. +The timeout is specified in seconds, with floating-point numbers allowed. [clinic start generated code]*/ static PyObject * signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, PyObject *timeout_obj) -/*[clinic end generated code: output=59c8971e8ae18a64 input=87fd39237cf0b7ba]*/ +/*[clinic end generated code: output=59c8971e8ae18a64 input=955773219c1596cd]*/ { PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 4b10730436c1aa..46f85bc9c30f9c 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -116,7 +116,7 @@ time_time(PyObject *self, PyObject *unused) PyDoc_STRVAR(time_doc, -"time() -> floating point number\n\ +"time() -> floating-point number\n\ \n\ Return the current time in seconds since the Epoch.\n\ Fractions of a second may be present if the system clock provides them."); @@ -350,7 +350,7 @@ time_clock_getres(PyObject *self, PyObject *args) } PyDoc_STRVAR(clock_getres_doc, -"clock_getres(clk_id) -> floating point number\n\ +"clock_getres(clk_id) -> floating-point number\n\ \n\ Return the resolution (precision) of the specified clock clk_id."); @@ -413,7 +413,7 @@ PyDoc_STRVAR(sleep_doc, "sleep(seconds)\n\ \n\ Delay execution for a given number of seconds. The argument may be\n\ -a floating point number for subsecond precision."); +a floating-point number for subsecond precision."); static PyStructSequence_Field struct_time_type_fields[] = { {"tm_year", "year, for example, 1993"}, @@ -1104,7 +1104,7 @@ time_mktime(PyObject *module, PyObject *tm_tuple) } PyDoc_STRVAR(mktime_doc, -"mktime(tuple) -> floating point number\n\ +"mktime(tuple) -> floating-point number\n\ \n\ Convert a time tuple in local time to seconds since the Epoch.\n\ Note that mktime(gmtime(0)) will not generally return zero for most\n\ @@ -1902,7 +1902,7 @@ PyDoc_STRVAR(module_doc, \n\ There are two standard representations of time. One is the number\n\ of seconds since the Epoch, in UTC (a.k.a. GMT). It may be an integer\n\ -or a floating point number (to represent fractions of seconds).\n\ +or a floating-point number (to represent fractions of seconds).\n\ The epoch is the point where the time starts, the return value of time.gmtime(0).\n\ It is January 1, 1970, 00:00:00 (UTC) on all platforms.\n\ \n\ diff --git a/Objects/abstract.c b/Objects/abstract.c index 200817064e3cda..afb068718bb010 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2141,7 +2141,7 @@ PySequence_Fast(PyObject *v, const char *m) PY_ITERSEARCH_COUNT: -1 if error, else # of times obj appears in seq. PY_ITERSEARCH_INDEX: 0-based index of first occurrence of obj in seq; set ValueError and return -1 if none found; also return -1 on error. - Py_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on error. + PY_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on error. */ Py_ssize_t _PySequence_IterSearch(PyObject *seq, PyObject *obj, int operation) @@ -2158,7 +2158,15 @@ _PySequence_IterSearch(PyObject *seq, PyObject *obj, int operation) it = PyObject_GetIter(seq); if (it == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - type_error("argument of type '%.200s' is not iterable", seq); + if (operation == PY_ITERSEARCH_CONTAINS) { + type_error( + "argument of type '%.200s' is not a container or iterable", + seq + ); + } + else { + type_error("argument of type '%.200s' is not iterable", seq); + } } return -1; } diff --git a/Objects/clinic/complexobject.c.h b/Objects/clinic/complexobject.c.h index 46c3b352562445..58fd4e26871b4d 100644 --- a/Objects/clinic/complexobject.c.h +++ b/Objects/clinic/complexobject.c.h @@ -160,4 +160,13 @@ complex_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=295ecfd71389d7fe input=a9049054013a1b77]*/ + +PyDoc_STRVAR(complex_from_number__doc__, +"from_number($type, number, /)\n" +"--\n" +"\n" +"Convert number to a complex floating-point number."); + +#define COMPLEX_FROM_NUMBER_METHODDEF \ + {"from_number", (PyCFunction)complex_from_number, METH_O|METH_CLASS, complex_from_number__doc__}, +/*[clinic end generated code: output=188438cc9ae167f7 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index 10f6149cc88c22..dd29135590a6a6 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -197,7 +197,7 @@ PyDoc_STRVAR(float_new__doc__, "float(x=0, /)\n" "--\n" "\n" -"Convert a string or number to a floating point number, if possible."); +"Convert a string or number to a floating-point number, if possible."); static PyObject * float_new_impl(PyTypeObject *type, PyObject *x); @@ -227,6 +227,15 @@ float_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(float_from_number__doc__, +"from_number($type, number, /)\n" +"--\n" +"\n" +"Convert real number to a floating-point number."); + +#define FLOAT_FROM_NUMBER_METHODDEF \ + {"from_number", (PyCFunction)float_from_number, METH_O|METH_CLASS, float_from_number__doc__}, + PyDoc_STRVAR(float___getnewargs____doc__, "__getnewargs__($self, /)\n" "--\n" @@ -256,7 +265,7 @@ PyDoc_STRVAR(float___getformat____doc__, "It exists mainly to be used in Python\'s test suite.\n" "\n" "This function returns whichever of \'unknown\', \'IEEE, big-endian\' or \'IEEE,\n" -"little-endian\' best describes the format of floating point numbers used by the\n" +"little-endian\' best describes the format of floating-point numbers used by the\n" "C type named by typestr."); #define FLOAT___GETFORMAT___METHODDEF \ @@ -318,4 +327,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=c79743c8551c30d9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=366cea9463cc5bf6 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 7493280c898750..d45ba5ed4a9c06 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -147,7 +147,7 @@ intern_strings(PyObject *tuple) "non-string found in code slot"); return -1; } - _PyUnicode_InternMortal(interp, &_PyTuple_ITEMS(tuple)[i]); + _PyUnicode_InternImmortal(interp, &_PyTuple_ITEMS(tuple)[i]); } return 0; } diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 31897463dbe689..7c8a6bd9dfcd3f 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -757,22 +757,6 @@ complex___complex___impl(PyComplexObject *self) } -static PyMethodDef complex_methods[] = { - COMPLEX_CONJUGATE_METHODDEF - COMPLEX___COMPLEX___METHODDEF - COMPLEX___GETNEWARGS___METHODDEF - COMPLEX___FORMAT___METHODDEF - {NULL, NULL} /* sentinel */ -}; - -static PyMemberDef complex_members[] = { - {"real", Py_T_DOUBLE, offsetof(PyComplexObject, cval.real), Py_READONLY, - "the real part of a complex number"}, - {"imag", Py_T_DOUBLE, offsetof(PyComplexObject, cval.imag), Py_READONLY, - "the imaginary part of a complex number"}, - {0}, -}; - static PyObject * complex_from_string_inner(const char *s, Py_ssize_t len, void *type) { @@ -1142,6 +1126,52 @@ complex_new_impl(PyTypeObject *type, PyObject *r, PyObject *i) return complex_subtype_from_doubles(type, cr.real, ci.real); } +/*[clinic input] +@classmethod +complex.from_number + + number: object + / + +Convert number to a complex floating-point number. +[clinic start generated code]*/ + +static PyObject * +complex_from_number(PyTypeObject *type, PyObject *number) +/*[clinic end generated code: output=658a7a5fb0de074d input=3f8bdd3a2bc3facd]*/ +{ + if (PyComplex_CheckExact(number) && type == &PyComplex_Type) { + Py_INCREF(number); + return number; + } + Py_complex cv = PyComplex_AsCComplex(number); + if (cv.real == -1.0 && PyErr_Occurred()) { + return NULL; + } + PyObject *result = PyComplex_FromCComplex(cv); + if (type != &PyComplex_Type && result != NULL) { + Py_SETREF(result, PyObject_CallOneArg((PyObject *)type, result)); + } + return result; +} + +static PyMethodDef complex_methods[] = { + COMPLEX_FROM_NUMBER_METHODDEF + COMPLEX_CONJUGATE_METHODDEF + COMPLEX___COMPLEX___METHODDEF + COMPLEX___GETNEWARGS___METHODDEF + COMPLEX___FORMAT___METHODDEF + {NULL, NULL} /* sentinel */ +}; + +static PyMemberDef complex_members[] = { + {"real", Py_T_DOUBLE, offsetof(PyComplexObject, cval.real), Py_READONLY, + "the real part of a complex number"}, + {"imag", Py_T_DOUBLE, offsetof(PyComplexObject, cval.imag), Py_READONLY, + "the imaginary part of a complex number"}, + {0}, +}; + static PyNumberMethods complex_as_number = { (binaryfunc)complex_add, /* nb_add */ (binaryfunc)complex_sub, /* nb_subtract */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 2b11a01595b0bc..6a16a04102a6c0 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -158,6 +158,10 @@ ASSERT_DICT_LOCKED(PyObject *op) if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ ASSERT_DICT_LOCKED(op); \ } +#define ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(op) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); \ + } #define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp) #define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp) @@ -165,16 +169,15 @@ ASSERT_DICT_LOCKED(PyObject *op) #define STORE_INDEX(keys, size, idx, value) _Py_atomic_store_int##size##_relaxed(&((int##size##_t*)keys->dk_indices)[idx], (int##size##_t)value); #define ASSERT_OWNED_OR_SHARED(mp) \ assert(_Py_IsOwnedByCurrentThread((PyObject *)mp) || IS_DICT_SHARED(mp)); -#define LOAD_KEYS_NENTRIES(d) #define LOCK_KEYS_IF_SPLIT(keys, kind) \ if (kind == DICT_KEYS_SPLIT) { \ - LOCK_KEYS(dk); \ + LOCK_KEYS(keys); \ } #define UNLOCK_KEYS_IF_SPLIT(keys, kind) \ if (kind == DICT_KEYS_SPLIT) { \ - UNLOCK_KEYS(dk); \ + UNLOCK_KEYS(keys); \ } static inline Py_ssize_t @@ -208,7 +211,7 @@ set_values(PyDictObject *mp, PyDictValues *values) #define INCREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, 1) // Dec refs the keys object, giving the previous value #define DECREF_KEYS(dk) _Py_atomic_add_ssize(&dk->dk_refcnt, -1) -#define LOAD_KEYS_NENTIRES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) +#define LOAD_KEYS_NENTRIES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) #define INCREF_KEYS_FT(dk) dictkeys_incref(dk) #define DECREF_KEYS_FT(dk, shared) dictkeys_decref(_PyInterpreterState_GET(), dk, shared) @@ -227,6 +230,7 @@ static inline void split_keys_entry_added(PyDictKeysObject *keys) #define ASSERT_DICT_LOCKED(op) #define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) +#define ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(op) #define LOCK_KEYS(keys) #define UNLOCK_KEYS(keys) #define ASSERT_KEYS_LOCKED(keys) @@ -234,7 +238,7 @@ static inline void split_keys_entry_added(PyDictKeysObject *keys) #define STORE_SHARED_KEY(key, value) key = value #define INCREF_KEYS(dk) dk->dk_refcnt++ #define DECREF_KEYS(dk) dk->dk_refcnt-- -#define LOAD_KEYS_NENTIRES(keys) keys->dk_nentries +#define LOAD_KEYS_NENTRIES(keys) keys->dk_nentries #define INCREF_KEYS_FT(dk) #define DECREF_KEYS_FT(dk, shared) #define LOCK_KEYS_IF_SPLIT(keys, kind) @@ -391,44 +395,6 @@ static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); #include "clinic/dictobject.c.h" -#ifdef WITH_FREELISTS -static struct _Py_dict_freelist * -get_dict_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->dicts; -} - -static struct _Py_dictkeys_freelist * -get_dictkeys_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->dictkeys; -} -#endif - - -void -_PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *freelist = &freelists->dicts; - while (freelist->numfree > 0) { - PyDictObject *op = freelist->items[--freelist->numfree]; - assert(PyDict_CheckExact(op)); - PyObject_GC_Del(op); - } - struct _Py_dictkeys_freelist *keys_freelist = &freelists->dictkeys; - while (keys_freelist->numfree > 0) { - PyMem_Free(keys_freelist->items[--keys_freelist->numfree]); - } - if (is_finalization) { - freelist->numfree = -1; - keys_freelist->numfree = -1; - } -#endif -} - static inline Py_hash_t unicode_get_hash(PyObject *o) { @@ -441,12 +407,12 @@ void _PyDict_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - struct _Py_dict_freelist *dict_freelist = get_dict_freelist(); _PyDebugAllocatorStats(out, "free PyDictObject", - dict_freelist->numfree, sizeof(PyDictObject)); - struct _Py_dictkeys_freelist *dictkeys_freelist = get_dictkeys_freelist(); + _Py_FREELIST_SIZE(dicts), + sizeof(PyDictObject)); _PyDebugAllocatorStats(out, "free PyDictKeysObject", - dictkeys_freelist->numfree, sizeof(PyDictKeysObject)); + _Py_FREELIST_SIZE(dictkeys), + sizeof(PyDictKeysObject)); #endif } @@ -689,10 +655,15 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) int splitted = _PyDict_HasSplitTable(mp); Py_ssize_t usable = USABLE_FRACTION(DK_SIZE(keys)); + // In the free-threaded build, shared keys may be concurrently modified, + // so use atomic loads. + Py_ssize_t dk_usable = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_usable); + Py_ssize_t dk_nentries = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(keys->dk_nentries); + CHECK(0 <= mp->ma_used && mp->ma_used <= usable); - CHECK(0 <= keys->dk_usable && keys->dk_usable <= usable); - CHECK(0 <= keys->dk_nentries && keys->dk_nentries <= usable); - CHECK(keys->dk_usable + keys->dk_nentries <= usable); + CHECK(0 <= dk_usable && dk_usable <= usable); + CHECK(0 <= dk_nentries && dk_nentries <= usable); + CHECK(dk_usable + dk_nentries <= usable); if (!splitted) { /* combined table */ @@ -709,6 +680,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) } if (check_content) { + LOCK_KEYS_IF_SPLIT(keys, keys->dk_kind); for (Py_ssize_t i=0; i < DK_SIZE(keys); i++) { Py_ssize_t ix = dictkeys_get_index(keys, i); CHECK(DKIX_DUMMY <= ix && ix <= usable); @@ -764,6 +736,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) CHECK(mp->ma_values->values[index] != NULL); } } + UNLOCK_KEYS_IF_SPLIT(keys, keys->dk_kind); } return 1; @@ -774,7 +747,6 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) { - PyDictKeysObject *dk; Py_ssize_t usable; int log2_bytes; size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry); @@ -797,15 +769,11 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) log2_bytes = log2_size + 2; } -#ifdef WITH_FREELISTS - struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); - if (log2_size == PyDict_LOG_MINSIZE && unicode && freelist->numfree > 0) { - dk = freelist->items[--freelist->numfree]; - OBJECT_STAT_INC(from_freelist); + PyDictKeysObject *dk = NULL; + if (log2_size == PyDict_LOG_MINSIZE && unicode) { + dk = _Py_FREELIST_POP_MEM(dictkeys); } - else -#endif - { + if (dk == NULL) { dk = PyMem_Malloc(sizeof(PyDictKeysObject) + ((size_t)1 << log2_bytes) + entry_size * usable); @@ -841,18 +809,12 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) return; } #endif -#ifdef WITH_FREELISTS - struct _Py_dictkeys_freelist *freelist = get_dictkeys_freelist(); - if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE - && freelist->numfree < PyDict_MAXFREELIST - && freelist->numfree >= 0 - && DK_IS_UNICODE(keys)) { - freelist->items[freelist->numfree++] = keys; - OBJECT_STAT_INC(to_freelist); - return; + if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE && keys->dk_kind == DICT_KEYS_UNICODE) { + _Py_FREELIST_FREE(dictkeys, keys, PyMem_Free); + } + else { + PyMem_Free(keys); } -#endif - PyMem_Free(keys); } static size_t @@ -901,20 +863,9 @@ new_dict(PyInterpreterState *interp, PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { - PyDictObject *mp; assert(keys != NULL); -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *freelist = get_dict_freelist(); - if (freelist->numfree > 0) { - mp = freelist->items[--freelist->numfree]; - assert (mp != NULL); - assert (Py_IS_TYPE(mp, &PyDict_Type)); - OBJECT_STAT_INC(from_freelist); - _Py_NewReference((PyObject *)mp); - } - else -#endif - { + PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts); + if (mp == NULL) { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { dictkeys_decref(interp, keys, false); @@ -924,6 +875,7 @@ new_dict(PyInterpreterState *interp, return NULL; } } + assert(Py_IS_TYPE(mp, &PyDict_Type)); mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = used; @@ -1039,7 +991,7 @@ lookdict_index(PyDictKeysObject *k, Py_hash_t hash, Py_ssize_t index) static inline Py_ALWAYS_INLINE Py_ssize_t do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, - Py_ssize_t (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t)) + int (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t)) { void *ep0 = _DK_ENTRIES(dk); size_t mask = DK_MASK(dk); @@ -1049,7 +1001,7 @@ do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, for (;;) { ix = dictkeys_get_index(dk, i); if (ix >= 0) { - Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + int cmp = check_lookup(mp, dk, ep0, ix, key, hash); if (cmp < 0) { return cmp; } else if (cmp) { @@ -1065,7 +1017,7 @@ do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, // Manual loop unrolling ix = dictkeys_get_index(dk, i); if (ix >= 0) { - Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash); + int cmp = check_lookup(mp, dk, ep0, ix, key, hash); if (cmp < 0) { return cmp; } else if (cmp) { @@ -1081,7 +1033,7 @@ do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, Py_UNREACHABLE(); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline int compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { @@ -1116,7 +1068,7 @@ unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key return do_lookup(mp, dk, key, hash, compare_unicode_generic); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline int compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { @@ -1137,7 +1089,7 @@ unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) return do_lookup(NULL, dk, key, hash, compare_unicode_unicode); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline int compare_generic(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { @@ -1332,8 +1284,8 @@ ensure_shared_on_resize(PyDictObject *mp) #ifdef Py_GIL_DISABLED -static inline Py_ALWAYS_INLINE -Py_ssize_t compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, +static inline Py_ALWAYS_INLINE int +compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; @@ -1375,7 +1327,7 @@ unicodekeys_lookup_generic_threadsafe(PyDictObject *mp, PyDictKeysObject* dk, Py return do_lookup(mp, dk, key, hash, compare_unicode_generic_threadsafe); } -static inline Py_ALWAYS_INLINE Py_ssize_t +static inline Py_ALWAYS_INLINE int compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { @@ -1409,8 +1361,8 @@ unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, Py_ha return do_lookup(NULL, dk, key, hash, compare_unicode_unicode_threadsafe); } -static inline Py_ALWAYS_INLINE -Py_ssize_t compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, +static inline Py_ALWAYS_INLINE int +compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; @@ -1675,6 +1627,10 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, } } + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, value); + mp->ma_keys->dk_version = 0; + Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash); dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); @@ -1691,6 +1647,7 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, STORE_VALUE(ep, value); STORE_HASH(ep, hash); } + mp->ma_version_tag = new_version; STORE_KEYS_USABLE(mp->ma_keys, mp->ma_keys->dk_usable - 1); STORE_KEYS_NENTRIES(mp->ma_keys, mp->ma_keys->dk_nentries + 1); assert(mp->ma_keys->dk_usable >= 0); @@ -1794,15 +1751,11 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, if (ix == DKIX_EMPTY) { assert(!_PyDict_HasSplitTable(mp)); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, value); /* Insert into new slot. */ - mp->ma_keys->dk_version = 0; assert(old_value == NULL); if (insert_combined_dict(interp, mp, hash, key, value) < 0) { goto Fail; } - mp->ma_version_tag = new_version; STORE_USED(mp, mp->ma_used + 1); ASSERT_CONSISTENT(mp); return 0; @@ -1843,9 +1796,6 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, assert(mp->ma_keys == Py_EMPTY_KEYS); ASSERT_DICT_LOCKED(mp); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, value); - int unicode = PyUnicode_CheckExact(key); PyDictKeysObject *newkeys = new_keys_object( interp, PyDict_LOG_MINSIZE, unicode); @@ -1854,6 +1804,9 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, Py_DECREF(value); return -1; } + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, value); + /* We don't decref Py_EMPTY_KEYS here because it is immortal. */ assert(mp->ma_values == NULL); @@ -2556,7 +2509,7 @@ delete_index_from_values(PyDictValues *values, Py_ssize_t ix) values->size = size; } -static int +static void delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, PyObject *old_value, uint64_t new_version) { @@ -2598,7 +2551,6 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, Py_DECREF(old_value); ASSERT_CONSISTENT(mp); - return 0; } int @@ -2641,7 +2593,8 @@ delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) PyInterpreterState *interp = _PyInterpreterState_GET(); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_DELETED, mp, key, NULL); - return delitem_common(mp, hash, ix, old_value, new_version); + delitem_common(mp, hash, ix, old_value, new_version); + return 0; } int @@ -2656,7 +2609,8 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) static int delitemif_lock_held(PyObject *op, PyObject *key, - int (*predicate)(PyObject *value)) + int (*predicate)(PyObject *value, void *arg), + void *arg) { Py_ssize_t ix; PyDictObject *mp; @@ -2666,24 +2620,20 @@ delitemif_lock_held(PyObject *op, PyObject *key, ASSERT_DICT_LOCKED(op); - if (!PyDict_Check(op)) { - PyErr_BadInternalCall(); - return -1; - } assert(key); hash = PyObject_Hash(key); if (hash == -1) return -1; mp = (PyDictObject *)op; ix = _Py_dict_lookup(mp, key, hash, &old_value); - if (ix == DKIX_ERROR) + if (ix == DKIX_ERROR) { return -1; + } if (ix == DKIX_EMPTY || old_value == NULL) { - _PyErr_SetKeyError(key); - return -1; + return 0; } - res = predicate(old_value); + res = predicate(old_value, arg); if (res == -1) return -1; @@ -2691,7 +2641,8 @@ delitemif_lock_held(PyObject *op, PyObject *key, PyInterpreterState *interp = _PyInterpreterState_GET(); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_DELETED, mp, key, NULL); - return delitem_common(mp, hash, ix, old_value, new_version); + delitem_common(mp, hash, ix, old_value, new_version); + return 1; } else { return 0; } @@ -2703,11 +2654,13 @@ delitemif_lock_held(PyObject *op, PyObject *key, */ int _PyDict_DelItemIf(PyObject *op, PyObject *key, - int (*predicate)(PyObject *value)) + int (*predicate)(PyObject *value, void *arg), + void *arg) { + assert(PyDict_Check(op)); int res; Py_BEGIN_CRITICAL_SECTION(op); - res = delitemif_lock_held(op, key, predicate); + res = delitemif_lock_held(op, key, predicate, arg); Py_END_CRITICAL_SECTION(); return res; } @@ -3142,16 +3095,10 @@ dict_dealloc(PyObject *self) assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); dictkeys_decref(interp, keys, false); } -#ifdef WITH_FREELISTS - struct _Py_dict_freelist *freelist = get_dict_freelist(); - if (freelist->numfree < PyDict_MAXFREELIST && freelist->numfree >=0 && - Py_IS_TYPE(mp, &PyDict_Type)) { - freelist->items[freelist->numfree++] = mp; - OBJECT_STAT_INC(to_freelist); + if (Py_IS_TYPE(mp, &PyDict_Type)) { + _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free); } - else -#endif - { + else { Py_TYPE(mp)->tp_free((PyObject *)mp); } Py_TRASHCAN_END @@ -4032,7 +3979,7 @@ dict_equal_lock_held(PyDictObject *a, PyDictObject *b) /* can't be equal if # of entries differ */ return 0; /* Same # of entries -- check all of 'em. Exit early on any diff. */ - for (i = 0; i < LOAD_KEYS_NENTIRES(a->ma_keys); i++) { + for (i = 0; i < LOAD_KEYS_NENTRIES(a->ma_keys); i++) { PyObject *key, *aval; Py_hash_t hash; if (DK_IS_UNICODE(a->ma_keys)) { @@ -4253,9 +4200,6 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu if (ix == DKIX_EMPTY) { assert(!_PyDict_HasSplitTable(mp)); - uint64_t new_version = _PyDict_NotifyEvent( - interp, PyDict_EVENT_ADDED, mp, key, default_value); - mp->ma_keys->dk_version = 0; value = default_value; if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { @@ -4268,7 +4212,6 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu MAINTAIN_TRACKING(mp, key, value); STORE_USED(mp, mp->ma_used + 1); - mp->ma_version_tag = new_version; assert(mp->ma_keys->dk_usable >= 0); ASSERT_CONSISTENT(mp); if (result) { @@ -6667,18 +6610,25 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, return res; } -static PyDictObject * -materialize_managed_dict_lock_held(PyObject *obj) +PyDictObject * +_PyObject_MaterializeManagedDict_LockHeld(PyObject *obj) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(obj); - PyDictValues *values = _PyObject_InlineValues(obj); - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); OBJECT_STAT_INC(dict_materialized_on_request); - PyDictObject *dict = make_dict_from_instance_attributes(interp, keys, values); + + PyDictValues *values = _PyObject_InlineValues(obj); + PyDictObject *dict; + if (values->valid) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); + dict = make_dict_from_instance_attributes(interp, keys, values); + } + else { + dict = (PyDictObject *)PyDict_New(); + } FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, - (PyDictObject *)dict); + dict); return dict; } @@ -6699,7 +6649,7 @@ _PyObject_MaterializeManagedDict(PyObject *obj) goto exit; } #endif - dict = materialize_managed_dict_lock_held(obj); + dict = _PyObject_MaterializeManagedDict_LockHeld(obj); #ifdef Py_GIL_DISABLED exit: @@ -7132,7 +7082,7 @@ PyObject_ClearManagedDict(PyObject *obj) int _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(obj); assert(_PyObject_ManagedDictPointer(obj)->dict == mp); assert(_PyObject_InlineValuesConsistencyCheck(obj)); diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 6376f2f012a7d6..fda62f159c1540 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3287,7 +3287,7 @@ SimpleExtendsException(PyExc_Exception, ArithmeticError, * FloatingPointError extends ArithmeticError */ SimpleExtendsException(PyExc_ArithmeticError, FloatingPointError, - "Floating point operation failed."); + "Floating-point operation failed."); /* diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 2627ba80eed8ca..82f39de421f245 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -7,8 +7,8 @@ #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_dtoa.h" // _Py_dg_dtoa() #include "pycore_floatobject.h" // _PyFloat_FormatAdvancedWriter() +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_interp.h" // _Py_float_freelist #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats() @@ -26,16 +26,6 @@ class float "PyObject *" "&PyFloat_Type" #include "clinic/floatobject.c.h" -#ifdef WITH_FREELISTS -static struct _Py_float_freelist * -get_float_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - assert(freelists != NULL); - return &freelists->floats; -} -#endif - double PyFloat_GetMax(void) @@ -132,24 +122,14 @@ PyFloat_GetInfo(void) PyObject * PyFloat_FromDouble(double fval) { - PyFloatObject *op; -#ifdef WITH_FREELISTS - struct _Py_float_freelist *float_freelist = get_float_freelist(); - op = float_freelist->items; - if (op != NULL) { - float_freelist->items = (PyFloatObject *) Py_TYPE(op); - float_freelist->numfree--; - OBJECT_STAT_INC(from_freelist); - } - else -#endif - { + PyFloatObject *op = _Py_FREELIST_POP(PyFloatObject, floats); + if (op == NULL) { op = PyObject_Malloc(sizeof(PyFloatObject)); if (!op) { return PyErr_NoMemory(); } + _PyObject_Init((PyObject*)op, &PyFloat_Type); } - _PyObject_Init((PyObject*)op, &PyFloat_Type); op->ob_fval = fval; return (PyObject *) op; } @@ -248,20 +228,7 @@ void _PyFloat_ExactDealloc(PyObject *obj) { assert(PyFloat_CheckExact(obj)); - PyFloatObject *op = (PyFloatObject *)obj; -#ifdef WITH_FREELISTS - struct _Py_float_freelist *float_freelist = get_float_freelist(); - if (float_freelist->numfree >= PyFloat_MAXFREELIST || float_freelist->numfree < 0) { - PyObject_Free(op); - return; - } - float_freelist->numfree++; - Py_SET_TYPE(op, (PyTypeObject *)float_freelist->items); - float_freelist->items = op; - OBJECT_STAT_INC(to_freelist); -#else - PyObject_Free(op); -#endif + _Py_FREELIST_FREE(floats, obj, PyObject_Free); } static void @@ -1142,69 +1109,39 @@ char_from_hex(int x) return Py_hexdigits[x]; } +/* This table maps characters to their hexadecimal values, only + * works with encodings whose lower half is ASCII (like UTF-8). + * '0' maps to 0, ..., '9' maps to 9. + * 'a' and 'A' map to 10, ..., 'f' and 'F' map to 15. + * All other indices map to -1. + */ +static const int +_CHAR_TO_HEX[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +}; + +/* Convert a character to its hexadecimal value, or -1 if it's not a + * valid hexadecimal character, only works with encodings whose lower + * half is ASCII (like UTF-8). + */ static int -hex_from_char(char c) { - int x; - switch(c) { - case '0': - x = 0; - break; - case '1': - x = 1; - break; - case '2': - x = 2; - break; - case '3': - x = 3; - break; - case '4': - x = 4; - break; - case '5': - x = 5; - break; - case '6': - x = 6; - break; - case '7': - x = 7; - break; - case '8': - x = 8; - break; - case '9': - x = 9; - break; - case 'a': - case 'A': - x = 10; - break; - case 'b': - case 'B': - x = 11; - break; - case 'c': - case 'C': - x = 12; - break; - case 'd': - case 'D': - x = 13; - break; - case 'e': - case 'E': - x = 14; - break; - case 'f': - case 'F': - x = 15; - break; - default: - x = -1; - break; - } - return x; +hex_from_char(unsigned char c) { + return _CHAR_TO_HEX[c]; } /* convert a float to a hexadecimal string */ @@ -1632,12 +1569,12 @@ float.__new__ as float_new x: object(c_default="NULL") = 0 / -Convert a string or number to a floating point number, if possible. +Convert a string or number to a floating-point number, if possible. [clinic start generated code]*/ static PyObject * float_new_impl(PyTypeObject *type, PyObject *x) -/*[clinic end generated code: output=ccf1e8dc460ba6ba input=f43661b7de03e9d8]*/ +/*[clinic end generated code: output=ccf1e8dc460ba6ba input=55909f888aa0c8a6]*/ { if (type != &PyFloat_Type) { if (x == NULL) { @@ -1699,6 +1636,36 @@ float_vectorcall(PyObject *type, PyObject * const*args, } +/*[clinic input] +@classmethod +float.from_number + + number: object + / + +Convert real number to a floating-point number. +[clinic start generated code]*/ + +static PyObject * +float_from_number(PyTypeObject *type, PyObject *number) +/*[clinic end generated code: output=bbcf05529fe907a3 input=1f8424d9bc11866a]*/ +{ + if (PyFloat_CheckExact(number) && type == &PyFloat_Type) { + Py_INCREF(number); + return number; + } + double x = PyFloat_AsDouble(number); + if (x == -1.0 && PyErr_Occurred()) { + return NULL; + } + PyObject *result = PyFloat_FromDouble(x); + if (type != &PyFloat_Type && result != NULL) { + Py_SETREF(result, PyObject_CallOneArg((PyObject *)type, result)); + } + return result; +} + + /*[clinic input] float.__getnewargs__ [clinic start generated code]*/ @@ -1733,13 +1700,13 @@ You probably don't want to use this function. It exists mainly to be used in Python's test suite. This function returns whichever of 'unknown', 'IEEE, big-endian' or 'IEEE, -little-endian' best describes the format of floating point numbers used by the +little-endian' best describes the format of floating-point numbers used by the C type named by typestr. [clinic start generated code]*/ static PyObject * float___getformat___impl(PyTypeObject *type, const char *typestr) -/*[clinic end generated code: output=2bfb987228cc9628 input=d5a52600f835ad67]*/ +/*[clinic end generated code: output=2bfb987228cc9628 input=90d5e246409a246e]*/ { float_format_type r; @@ -1812,6 +1779,7 @@ float___format___impl(PyObject *self, PyObject *format_spec) } static PyMethodDef float_methods[] = { + FLOAT_FROM_NUMBER_METHODDEF FLOAT_CONJUGATE_METHODDEF FLOAT___TRUNC___METHODDEF FLOAT___FLOOR___METHODDEF @@ -1925,7 +1893,7 @@ _init_global_state(void) float_format_type detected_double_format, detected_float_format; /* We attempt to determine if this machine is using IEEE - floating point formats by peering at the bits of some + floating-point formats by peering at the bits of some carefully chosen values. If it looks like we are on an IEEE platform, the float packing/unpacking routines can just copy bits, if not they resort to arithmetic & shifts @@ -1993,27 +1961,6 @@ _PyFloat_InitTypes(PyInterpreterState *interp) return _PyStatus_OK(); } -void -_PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_float_freelist *state = &freelists->floats; - PyFloatObject *f = state->items; - while (f != NULL) { - PyFloatObject *next = (PyFloatObject*) Py_TYPE(f); - PyObject_Free(f); - f = next; - } - state->items = NULL; - if (is_finalization) { - state->numfree = -1; - } - else { - state->numfree = 0; - } -#endif -} - void _PyFloat_FiniType(PyInterpreterState *interp) { @@ -2025,10 +1972,10 @@ void _PyFloat_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - struct _Py_float_freelist *float_freelist = get_float_freelist(); _PyDebugAllocatorStats(out, "free PyFloatObject", - float_freelist->numfree, sizeof(PyFloatObject)); + _Py_FREELIST_SIZE(floats), + sizeof(PyFloatObject)); #endif } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 3dc9ff058a5c9e..88093eb9071ae4 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1620,13 +1620,16 @@ frame_dealloc(PyFrameObject *f) Py_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); - for (int i = 0; i < frame->stacktop; i++) { - PyStackRef_CLEAR(locals[i]); + _PyStackRef *sp = frame->stackpointer; + while (sp > locals) { + sp--; + PyStackRef_CLEAR(*sp); } } Py_CLEAR(f->f_back); Py_CLEAR(f->f_trace); Py_CLEAR(f->f_extra_locals); + Py_CLEAR(f->f_locals_cache); PyObject_GC_Del(f); Py_XDECREF(co); Py_TRASHCAN_END; @@ -1638,6 +1641,7 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) Py_VISIT(f->f_back); Py_VISIT(f->f_trace); Py_VISIT(f->f_extra_locals); + Py_VISIT(f->f_locals_cache); if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) { return 0; } @@ -1650,14 +1654,17 @@ frame_tp_clear(PyFrameObject *f) { Py_CLEAR(f->f_trace); Py_CLEAR(f->f_extra_locals); + Py_CLEAR(f->f_locals_cache); /* locals and stack */ _PyStackRef *locals = _PyFrame_GetLocalsArray(f->f_frame); - assert(f->f_frame->stacktop >= 0); - for (int i = 0; i < f->f_frame->stacktop; i++) { - PyStackRef_CLEAR(locals[i]); + _PyStackRef *sp = f->f_frame->stackpointer; + assert(sp >= locals); + while (sp > locals) { + sp--; + PyStackRef_CLEAR(*sp); } - f->f_frame->stacktop = 0; + f->f_frame->stackpointer = locals; Py_CLEAR(f->f_frame->f_locals); return 0; } @@ -1787,6 +1794,7 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace_opcodes = 0; f->f_lineno = 0; f->f_extra_locals = NULL; + f->f_locals_cache = NULL; return f; } @@ -1874,8 +1882,9 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, return 0; } - PyObject *value = PyStackRef_AsPyObjectBorrow(frame->localsplus[i]); - if (frame->stacktop) { + PyObject *value = NULL; + if (frame->stackpointer == NULL || frame->stackpointer > frame->localsplus + i) { + value = PyStackRef_AsPyObjectBorrow(frame->localsplus[i]); if (kind & CO_FAST_FREE) { // The cell was set by COPY_FREE_VARS. assert(value != NULL && PyCell_Check(value)); @@ -1893,9 +1902,6 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, } } } - else { - assert(value == NULL); - } *pvalue = value; return 1; } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 40211297be20c0..8df0da800980a9 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -547,6 +547,7 @@ func_get_annotation_dict(PyFunctionObject *op) PyTuple_GET_ITEM(ann_tuple, i + 1)); if (err < 0) { + Py_DECREF(ann_dict); return NULL; } } diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index f5fefd656539c8..64b4e2645cbaee 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -4,6 +4,7 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" +#include "pycore_typevarobject.h" // _Py_typing_type_repr #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check @@ -50,69 +51,6 @@ ga_traverse(PyObject *self, visitproc visit, void *arg) return 0; } -static int -ga_repr_item(PyUnicodeWriter *writer, PyObject *p) -{ - PyObject *qualname = NULL; - PyObject *module = NULL; - int rc; - - if (p == Py_Ellipsis) { - // The Ellipsis object - rc = PyUnicodeWriter_WriteUTF8(writer, "...", 3); - goto done; - } - - if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && - (rc = PyObject_HasAttrWithError(p, &_Py_ID(__args__))) > 0) - { - // It looks like a GenericAlias - goto use_repr; - } - if (rc < 0) { - goto error; - } - - if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { - goto error; - } - if (qualname == NULL) { - goto use_repr; - } - if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { - goto error; - } - if (module == NULL || module == Py_None) { - goto use_repr; - } - - // Looks like a class - if (PyUnicode_Check(module) && - _PyUnicode_EqualToASCIIString(module, "builtins")) - { - // builtins don't need a module name - rc = PyUnicodeWriter_WriteStr(writer, qualname); - goto done; - } - else { - rc = PyUnicodeWriter_Format(writer, "%S.%S", module, qualname); - goto done; - } - -error: - rc = -1; - goto done; - -use_repr: - rc = PyUnicodeWriter_WriteRepr(writer, p); - goto done; - -done: - Py_XDECREF(qualname); - Py_XDECREF(module); - return rc; -} - static int ga_repr_items_list(PyUnicodeWriter *writer, PyObject *p) { @@ -131,7 +69,7 @@ ga_repr_items_list(PyUnicodeWriter *writer, PyObject *p) } } PyObject *item = PyList_GET_ITEM(p, i); - if (ga_repr_item(writer, item) < 0) { + if (_Py_typing_type_repr(writer, item) < 0) { return -1; } } @@ -162,7 +100,7 @@ ga_repr(PyObject *self) goto error; } } - if (ga_repr_item(writer, alias->origin) < 0) { + if (_Py_typing_type_repr(writer, alias->origin) < 0) { goto error; } if (PyUnicodeWriter_WriteChar(writer, '[') < 0) { @@ -181,7 +119,7 @@ ga_repr(PyObject *self) goto error; } } - else if (ga_repr_item(writer, p) < 0) { + else if (_Py_typing_type_repr(writer, p) < 0) { goto error; } } @@ -563,6 +501,10 @@ ga_getitem(PyObject *self, PyObject *item) } PyObject *res = Py_GenericAlias(alias->origin, newargs); + if (res == NULL) { + Py_DECREF(newargs); + return NULL; + } ((gaobject *)res)->starred = alias->starred; Py_DECREF(newargs); diff --git a/Objects/genobject.c b/Objects/genobject.c index 37b40530589d57..c204ac04b480a4 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -6,7 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_frame.h" // _PyInterpreterFrame -#include "pycore_freelist.h" // struct _Py_async_gen_freelist +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() @@ -1629,23 +1629,6 @@ PyTypeObject PyAsyncGen_Type = { }; -#ifdef WITH_FREELISTS -static struct _Py_async_gen_freelist * -get_async_gen_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->async_gens; -} - -static struct _Py_async_gen_asend_freelist * -get_async_gen_asend_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->async_gen_asends; -} -#endif - - PyObject * PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) { @@ -1662,36 +1645,6 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname) return (PyObject*)o; } - -void -_PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelist_state, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *freelist = &freelist_state->async_gens; - - while (freelist->numfree > 0) { - _PyAsyncGenWrappedValue *o; - o = freelist->items[--freelist->numfree]; - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - PyObject_GC_Del(o); - } - - struct _Py_async_gen_asend_freelist *asend_freelist = &freelist_state->async_gen_asends; - - while (asend_freelist->numfree > 0) { - PyAsyncGenASend *o; - o = asend_freelist->items[--asend_freelist->numfree]; - assert(Py_IS_TYPE(o, &_PyAsyncGenASend_Type)); - PyObject_GC_Del(o); - } - - if (is_finalization) { - freelist->numfree = -1; - asend_freelist->numfree = -1; - } -#endif -} - static PyObject * async_gen_unwrap_value(PyAsyncGenObject *gen, PyObject *result) { @@ -1735,18 +1688,11 @@ async_gen_asend_dealloc(PyAsyncGenASend *o) _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_sendval); -#ifdef WITH_FREELISTS - struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); - if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { - assert(PyAsyncGenASend_CheckExact(o)); - _PyGC_CLEAR_FINALIZED((PyObject *)o); - freelist->items[freelist->numfree++] = o; - } - else -#endif - { - PyObject_GC_Del(o); - } + + assert(PyAsyncGenASend_CheckExact(o)); + _PyGC_CLEAR_FINALIZED((PyObject *)o); + + _Py_FREELIST_FREE(async_gen_asends, o, PyObject_GC_Del); } static int @@ -1936,17 +1882,8 @@ PyTypeObject _PyAsyncGenASend_Type = { static PyObject * async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval) { - PyAsyncGenASend *o; -#ifdef WITH_FREELISTS - struct _Py_async_gen_asend_freelist *freelist = get_async_gen_asend_freelist(); - if (freelist->numfree > 0) { - freelist->numfree--; - o = freelist->items[freelist->numfree]; - _Py_NewReference((PyObject *)o); - } - else -#endif - { + PyAsyncGenASend *o = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends); + if (o == NULL) { o = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type); if (o == NULL) { return NULL; @@ -1972,18 +1909,7 @@ async_gen_wrapped_val_dealloc(_PyAsyncGenWrappedValue *o) { _PyObject_GC_UNTRACK((PyObject *)o); Py_CLEAR(o->agw_val); -#ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); - if (freelist->numfree >= 0 && freelist->numfree < _PyAsyncGen_MAXFREELIST) { - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - freelist->items[freelist->numfree++] = o; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - PyObject_GC_Del(o); - } + _Py_FREELIST_FREE(async_gens, o, PyObject_GC_Del); } @@ -2042,27 +1968,17 @@ PyTypeObject _PyAsyncGenWrappedValue_Type = { PyObject * _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val) { - _PyAsyncGenWrappedValue *o; assert(val); -#ifdef WITH_FREELISTS - struct _Py_async_gen_freelist *freelist = get_async_gen_freelist(); - if (freelist->numfree > 0) { - freelist->numfree--; - o = freelist->items[freelist->numfree]; - OBJECT_STAT_INC(from_freelist); - assert(_PyAsyncGenWrappedValue_CheckExact(o)); - _Py_NewReference((PyObject*)o); - } - else -#endif - { + _PyAsyncGenWrappedValue *o = _Py_FREELIST_POP(_PyAsyncGenWrappedValue, async_gens); + if (o == NULL) { o = PyObject_GC_New(_PyAsyncGenWrappedValue, &_PyAsyncGenWrappedValue_Type); if (o == NULL) { return NULL; } } + assert(_PyAsyncGenWrappedValue_CheckExact(o)); o->agw_val = Py_NewRef(val); _PyObject_GC_TRACK((PyObject*)o); return (PyObject*)o; diff --git a/Objects/listobject.c b/Objects/listobject.c index f29f58dc25be04..4d654c2ed33177 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -4,6 +4,7 @@ #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_dict.h" // _PyDictViewObject +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_interp.h" // PyInterpreterState.list #include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject @@ -23,16 +24,6 @@ class list "PyListObject *" "&PyList_Type" _Py_DECLARE_STR(list_err, "list index out of range"); -#ifdef WITH_FREELISTS -static struct _Py_list_freelist * -get_list_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - assert(freelists != NULL); - return &freelists->lists; -} -#endif - #ifdef Py_GIL_DISABLED typedef struct { Py_ssize_t allocated; @@ -205,55 +196,28 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) return 0; } -void -_PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_list_freelist *state = &freelists->lists; - while (state->numfree > 0) { - PyListObject *op = state->items[--state->numfree]; - assert(PyList_CheckExact(op)); - PyObject_GC_Del(op); - } - if (is_finalization) { - state->numfree = -1; - } -#endif -} - /* Print summary info about the state of the optimized allocator */ void _PyList_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - struct _Py_list_freelist *list_freelist = get_list_freelist(); _PyDebugAllocatorStats(out, "free PyListObject", - list_freelist->numfree, sizeof(PyListObject)); + _Py_FREELIST_SIZE(lists), + sizeof(PyListObject)); #endif } PyObject * PyList_New(Py_ssize_t size) { - PyListObject *op; - if (size < 0) { PyErr_BadInternalCall(); return NULL; } -#ifdef WITH_FREELISTS - struct _Py_list_freelist *list_freelist = get_list_freelist(); - if (PyList_MAXFREELIST && list_freelist->numfree > 0) { - list_freelist->numfree--; - op = list_freelist->items[list_freelist->numfree]; - OBJECT_STAT_INC(from_freelist); - _Py_NewReference((PyObject *)op); - } - else -#endif - { + PyListObject *op = _Py_FREELIST_POP(PyListObject, lists); + if (op == NULL) { op = PyObject_GC_New(PyListObject, &PyList_Type); if (op == NULL) { return NULL; @@ -548,16 +512,11 @@ list_dealloc(PyObject *self) } free_list_items(op->ob_item, false); } -#ifdef WITH_FREELISTS - struct _Py_list_freelist *list_freelist = get_list_freelist(); - if (list_freelist->numfree < PyList_MAXFREELIST && list_freelist->numfree >= 0 && PyList_CheckExact(op)) { - list_freelist->items[list_freelist->numfree++] = op; - OBJECT_STAT_INC(to_freelist); + if (PyList_CheckExact(op)) { + _Py_FREELIST_FREE(lists, op, PyObject_GC_Del); } - else -#endif - { - Py_TYPE(op)->tp_free((PyObject *)op); + else { + PyObject_GC_Del(op); } Py_TRASHCAN_END } diff --git a/Objects/longobject.c b/Objects/longobject.c index 4ca259fb08e8c7..050ce1a7303842 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -483,11 +483,18 @@ PyLong_AsLongAndOverflow(PyObject *vv, int *overflow) do_decref = 1; } if (_PyLong_IsCompact(v)) { -#if SIZEOF_LONG < SIZEOF_VOID_P - intptr_t tmp = _PyLong_CompactValue(v); - res = (long)tmp; - if (res != tmp) { - *overflow = tmp < 0 ? -1 : 1; +#if SIZEOF_LONG < SIZEOF_SIZE_T + Py_ssize_t tmp = _PyLong_CompactValue(v); + if (tmp < LONG_MIN) { + *overflow = -1; + res = -1; + } + else if (tmp > LONG_MAX) { + *overflow = 1; + res = -1; + } + else { + res = (long)tmp; } #else res = _PyLong_CompactValue(v); @@ -632,14 +639,15 @@ PyLong_AsUnsignedLong(PyObject *vv) v = (PyLongObject *)vv; if (_PyLong_IsNonNegativeCompact(v)) { -#if SIZEOF_LONG < SIZEOF_VOID_P - intptr_t tmp = _PyLong_CompactValue(v); +#if SIZEOF_LONG < SIZEOF_SIZE_T + size_t tmp = (size_t)_PyLong_CompactValue(v); unsigned long res = (unsigned long)tmp; if (res != tmp) { goto overflow; } + return res; #else - return _PyLong_CompactValue(v); + return (unsigned long)(size_t)_PyLong_CompactValue(v); #endif } if (_PyLong_IsNegative(v)) { @@ -685,7 +693,7 @@ PyLong_AsSize_t(PyObject *vv) v = (PyLongObject *)vv; if (_PyLong_IsNonNegativeCompact(v)) { - return _PyLong_CompactValue(v); + return (size_t)_PyLong_CompactValue(v); } if (_PyLong_IsNegative(v)) { PyErr_SetString(PyExc_OverflowError, @@ -722,7 +730,11 @@ _PyLong_AsUnsignedLongMask(PyObject *vv) } v = (PyLongObject *)vv; if (_PyLong_IsCompact(v)) { - return (unsigned long)_PyLong_CompactValue(v); +#if SIZEOF_LONG < SIZEOF_SIZE_T + return (unsigned long)(size_t)_PyLong_CompactValue(v); +#else + return (unsigned long)(long)_PyLong_CompactValue(v); +#endif } i = _PyLong_DigitCount(v); int sign = _PyLong_NonCompactSign(v); @@ -1540,7 +1552,18 @@ PyLong_AsUnsignedLongLong(PyObject *vv) v = (PyLongObject*)vv; if (_PyLong_IsNonNegativeCompact(v)) { res = 0; - bytes = _PyLong_CompactValue(v); +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + size_t tmp = (size_t)_PyLong_CompactValue(v); + bytes = (unsigned long long)tmp; + if (bytes != tmp) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert " + "to C unsigned long long"); + res = -1; + } +#else + bytes = (unsigned long long)(size_t)_PyLong_CompactValue(v); +#endif } else { res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes, @@ -1571,7 +1594,11 @@ _PyLong_AsUnsignedLongLongMask(PyObject *vv) } v = (PyLongObject *)vv; if (_PyLong_IsCompact(v)) { - return (unsigned long long)(signed long long)_PyLong_CompactValue(v); +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + return (unsigned long long)(size_t)_PyLong_CompactValue(v); +#else + return (unsigned long long)(long long)_PyLong_CompactValue(v); +#endif } i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); @@ -1643,7 +1670,22 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow) do_decref = 1; } if (_PyLong_IsCompact(v)) { +#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T + Py_ssize_t tmp = _PyLong_CompactValue(v); + if (tmp < LLONG_MIN) { + *overflow = -1; + res = -1; + } + else if (tmp > LLONG_MAX) { + *overflow = 1; + res = -1; + } + else { + res = (long long)tmp; + } +#else res = _PyLong_CompactValue(v); +#endif } else { i = _PyLong_DigitCount(v); @@ -3579,7 +3621,7 @@ long_hash(PyLongObject *v) int sign; if (_PyLong_IsCompact(v)) { - x = _PyLong_CompactValue(v); + x = (Py_uhash_t)_PyLong_CompactValue(v); if (x == (Py_uhash_t)-1) { x = (Py_uhash_t)-2; } @@ -6503,7 +6545,7 @@ PyDoc_STRVAR(long_doc, int(x, base=10) -> integer\n\ \n\ Convert a number or string to an integer, or return 0 if no arguments\n\ -are given. If x is a number, return x.__int__(). For floating point\n\ +are given. If x is a number, return x.__int__(). For floating-point\n\ numbers, this truncates towards zero.\n\ \n\ If x is not a number or if base is given, then x must be a string,\n\ diff --git a/Objects/mimalloc/prim/unix/prim.c b/Objects/mimalloc/prim/unix/prim.c index c6ea05bbe7a2ac..c4816af1a0d6e5 100644 --- a/Objects/mimalloc/prim/unix/prim.c +++ b/Objects/mimalloc/prim/unix/prim.c @@ -27,6 +27,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // mmap #include // sysconf +#include // open, close, read, access #if defined(__linux__) #include diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 73ad9711b6b0fc..efc74dafb5fc73 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -251,7 +251,7 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) } } m->md_def = module; -#ifdef Py_GIL_DISABLE +#ifdef Py_GIL_DISABLED m->md_gil = Py_MOD_GIL_USED; #endif return (PyObject*)m; diff --git a/Objects/object.c b/Objects/object.c index c4622359bb1035..8a648a46487910 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -10,6 +10,7 @@ #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type #include "pycore_hashtable.h" // _Py_hashtable_new() @@ -375,7 +376,6 @@ _Py_MergeZeroLocalRefcount(PyObject *op) { assert(op->ob_ref_local == 0); - _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); if (shared == 0) { // Fast-path: shared refcount is zero (including flags) @@ -383,6 +383,11 @@ _Py_MergeZeroLocalRefcount(PyObject *op) return; } + // gh-121794: This must be before the store to `ob_ref_shared` (gh-119999), + // but should outside the fast-path to maintain the invariant that + // a zero `ob_tid` implies a merged refcount. + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + // Slow-path: atomically set the flags (low two bits) to _Py_REF_MERGED. Py_ssize_t new_shared; do { @@ -804,20 +809,57 @@ PyObject_Bytes(PyObject *v) return PyBytes_FromObject(v); } +#ifdef WITH_FREELISTS +static void +clear_freelist(struct _Py_freelist *freelist, int is_finalization, + freefunc dofree) +{ + void *ptr; + while ((ptr = _PyFreeList_PopNoStats(freelist)) != NULL) { + dofree(ptr); + } + assert(freelist->size == 0 || freelist->size == -1); + assert(freelist->freelist == NULL); + if (is_finalization) { + freelist->size = -1; + } +} + +static void +free_object(void *obj) +{ + PyObject *op = (PyObject *)obj; + PyTypeObject *tp = Py_TYPE(op); + tp->tp_free(op); + Py_DECREF(tp); +} + +#endif + void -_PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization) +_PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) { +#ifdef WITH_FREELISTS // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() - _PyFloat_ClearFreeList(freelists, is_finalization); - _PyTuple_ClearFreeList(freelists, is_finalization); - _PyList_ClearFreeList(freelists, is_finalization); - _PyDict_ClearFreeList(freelists, is_finalization); - _PyContext_ClearFreeList(freelists, is_finalization); - _PyAsyncGen_ClearFreeLists(freelists, is_finalization); - // Only be cleared if is_finalization is true. - _PyObjectStackChunk_ClearFreeList(freelists, is_finalization); - _PySlice_ClearFreeList(freelists, is_finalization); + clear_freelist(&freelists->floats, is_finalization, free_object); + for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) { + clear_freelist(&freelists->tuples[i], is_finalization, free_object); + } + clear_freelist(&freelists->lists, is_finalization, free_object); + clear_freelist(&freelists->dicts, is_finalization, free_object); + clear_freelist(&freelists->dictkeys, is_finalization, PyMem_Free); + clear_freelist(&freelists->slices, is_finalization, free_object); + clear_freelist(&freelists->contexts, is_finalization, free_object); + clear_freelist(&freelists->async_gens, is_finalization, free_object); + clear_freelist(&freelists->async_gen_asends, is_finalization, free_object); + clear_freelist(&freelists->futureiters, is_finalization, free_object); + if (is_finalization) { + // Only clear object stack chunks during finalization. We use object + // stacks during GC, so emptying the free-list is counterproductive. + clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree); + } +#endif } /* @@ -2724,7 +2766,6 @@ _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op) _PyObject_ASSERT(op, !_PyObject_GC_IS_TRACKED(op)); _PyObject_ASSERT(op, Py_REFCNT(op) == 0); #ifdef Py_GIL_DISABLED - _PyObject_ASSERT(op, op->ob_tid == 0); op->ob_tid = (uintptr_t)tstate->delete_later; #else _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)tstate->delete_later); @@ -2757,6 +2798,7 @@ _PyTrash_thread_destroy_chain(PyThreadState *tstate) #ifdef Py_GIL_DISABLED tstate->delete_later = (PyObject*) op->ob_tid; op->ob_tid = 0; + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, _Py_REF_MERGED); #else tstate->delete_later = (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); #endif diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index d033e2bad1891a..a6a71802ef8e01 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -386,8 +386,16 @@ _PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr, ) { #ifdef MS_WINDOWS + /* Unlike free(), VirtualFree() does not special-case NULL to noop. */ + if (ptr == NULL) { + return; + } VirtualFree(ptr, 0, MEM_RELEASE); #elif defined(ARENAS_USE_MMAP) + /* Unlike free(), munmap() does not special-case NULL to noop. */ + if (ptr == NULL) { + return; + } munmap(ptr, size); #else free(ptr); diff --git a/Objects/setobject.c b/Objects/setobject.c index eb0c404bf6b8e0..9c17e3eef2fe00 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -184,14 +184,14 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) found_unused_or_dummy: if (freeslot == NULL) goto found_unused; - so->used++; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used + 1); freeslot->key = key; freeslot->hash = hash; return 0; found_unused: so->fill++; - so->used++; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used + 1); entry->key = key; entry->hash = hash; if ((size_t)so->fill*5 < mask*3) @@ -357,7 +357,7 @@ set_discard_entry(PySetObject *so, PyObject *key, Py_hash_t hash) old_key = entry->key; entry->key = dummy; entry->hash = -1; - so->used--; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used - 1); Py_DECREF(old_key); return DISCARD_FOUND; } @@ -397,7 +397,7 @@ set_empty_to_minsize(PySetObject *so) { memset(so->smalltable, 0, sizeof(so->smalltable)); so->fill = 0; - so->used = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, 0); so->mask = PySet_MINSIZE - 1; so->table = so->smalltable; so->hash = -1; @@ -615,7 +615,7 @@ set_merge_lock_held(PySetObject *so, PyObject *otherset) } } so->fill = other->fill; - so->used = other->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, other->used); return 0; } @@ -624,7 +624,7 @@ set_merge_lock_held(PySetObject *so, PyObject *otherset) setentry *newtable = so->table; size_t newmask = (size_t)so->mask; so->fill = other->used; - so->used = other->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, other->used); for (i = other->mask + 1; i > 0 ; i--, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { @@ -678,7 +678,7 @@ set_pop_impl(PySetObject *so) key = entry->key; entry->key = dummy; entry->hash = -1; - so->used--; + FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used - 1); so->finger = entry - so->table + 1; /* next place to start */ return key; } @@ -709,18 +709,20 @@ _shuffle_bits(Py_uhash_t h) large primes with "interesting bit patterns" and that passed tests for good collision statistics on a variety of problematic datasets including powersets and graph structures (such as David Eppstein's - graph recipes in Lib/test/test_set.py) */ + graph recipes in Lib/test/test_set.py). + + This hash algorithm can be used on either a frozenset or a set. + When it is used on a set, it computes the hash value of the equivalent + frozenset without creating a new frozenset object. */ static Py_hash_t -frozenset_hash(PyObject *self) +frozenset_hash_impl(PyObject *self) { + assert(PyAnySet_Check(self)); PySetObject *so = (PySetObject *)self; Py_uhash_t hash = 0; setentry *entry; - if (so->hash != -1) - return so->hash; - /* Xor-in shuffled bits from every entry's hash field because xor is commutative and a frozenset hash should be independent of order. @@ -753,6 +755,20 @@ frozenset_hash(PyObject *self) if (hash == (Py_uhash_t)-1) hash = 590923713UL; + return (Py_hash_t)hash; +} + +static Py_hash_t +frozenset_hash(PyObject *self) +{ + PySetObject *so = (PySetObject *)self; + Py_uhash_t hash; + + if (so->hash != -1) { + return so->hash; + } + + hash = frozenset_hash_impl(self); so->hash = hash; return hash; } @@ -1173,7 +1189,9 @@ set_swap_bodies(PySetObject *a, PySetObject *b) Py_hash_t h; t = a->fill; a->fill = b->fill; b->fill = t; - t = a->used; a->used = b->used; b->used = t; + t = a->used; + FT_ATOMIC_STORE_SSIZE_RELAXED(a->used, b->used); + FT_ATOMIC_STORE_SSIZE_RELAXED(b->used, t); t = a->mask; a->mask = b->mask; b->mask = t; u = a->table; @@ -2135,7 +2153,6 @@ set_add_impl(PySetObject *so, PyObject *key) static int set_contains_lock_held(PySetObject *so, PyObject *key) { - PyObject *tmpkey; int rv; rv = set_contains_key(so, key); @@ -2143,11 +2160,11 @@ set_contains_lock_held(PySetObject *so, PyObject *key) if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return -1; PyErr_Clear(); - tmpkey = make_new_set(&PyFrozenSet_Type, key); - if (tmpkey == NULL) - return -1; - rv = set_contains_key(so, tmpkey); - Py_DECREF(tmpkey); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(key); + hash = frozenset_hash_impl(key); + Py_END_CRITICAL_SECTION(); + rv = set_contains_entry(so, key, hash); } return rv; } @@ -2201,7 +2218,6 @@ static PyObject * set_remove_impl(PySetObject *so, PyObject *key) /*[clinic end generated code: output=0b9134a2a2200363 input=893e1cb1df98227a]*/ { - PyObject *tmpkey; int rv; rv = set_discard_key(so, key); @@ -2209,11 +2225,11 @@ set_remove_impl(PySetObject *so, PyObject *key) if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return NULL; PyErr_Clear(); - tmpkey = make_new_set(&PyFrozenSet_Type, key); - if (tmpkey == NULL) - return NULL; - rv = set_discard_key(so, tmpkey); - Py_DECREF(tmpkey); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(key); + hash = frozenset_hash_impl(key); + Py_END_CRITICAL_SECTION(); + rv = set_discard_entry(so, key, hash); if (rv < 0) return NULL; } @@ -2242,7 +2258,6 @@ static PyObject * set_discard_impl(PySetObject *so, PyObject *key) /*[clinic end generated code: output=eec3b687bf32759e input=861cb7fb69b4def0]*/ { - PyObject *tmpkey; int rv; rv = set_discard_key(so, key); @@ -2250,11 +2265,11 @@ set_discard_impl(PySetObject *so, PyObject *key) if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) return NULL; PyErr_Clear(); - tmpkey = make_new_set(&PyFrozenSet_Type, key); - if (tmpkey == NULL) - return NULL; - rv = set_discard_key(so, tmpkey); - Py_DECREF(tmpkey); + Py_hash_t hash; + Py_BEGIN_CRITICAL_SECTION(key); + hash = frozenset_hash_impl(key); + Py_END_CRITICAL_SECTION(); + rv = set_discard_entry(so, key, hash); if (rv < 0) return NULL; } diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 245bea98d58509..1b6d35998c2b69 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -15,6 +15,7 @@ this type and there is exactly one in existence. #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -108,20 +109,6 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type); /* Slice object implementation */ -void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ - if (!is_finalization) { - return; - } -#ifdef WITH_FREELISTS - PySliceObject *obj = freelists->slices.slice_cache; - if (obj != NULL) { - freelists->slices.slice_cache = NULL; - PyObject_GC_Del(obj); - } -#endif -} - /* start, stop, and step are python objects with None indicating no index is present. */ @@ -130,17 +117,8 @@ static PySliceObject * _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) { assert(start != NULL && stop != NULL && step != NULL); - PySliceObject *obj; -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (freelists->slices.slice_cache != NULL) { - obj = freelists->slices.slice_cache; - freelists->slices.slice_cache = NULL; - _Py_NewReference((PyObject *)obj); - } - else -#endif - { + PySliceObject *obj = _Py_FREELIST_POP(PySliceObject, slices); + if (obj == NULL) { obj = PyObject_GC_New(PySliceObject, &PySlice_Type); if (obj == NULL) { goto error; @@ -369,16 +347,7 @@ slice_dealloc(PySliceObject *r) Py_DECREF(r->step); Py_DECREF(r->start); Py_DECREF(r->stop); -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (freelists->slices.slice_cache == NULL) { - freelists->slices.slice_cache = r; - } - else -#endif - { - PyObject_GC_Del(r); - } + _Py_FREELIST_FREE(slices, r, PyObject_GC_Del); } static PyObject * diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 994258f20b495d..bd6e568191167a 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_freelist.h" // _Py_FREELIST_PUSH(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_modsupport.h" // _PyArg_NoKwnames() @@ -17,7 +18,6 @@ class tuple "PyTupleObject *" "&PyTuple_Type" #include "clinic/tupleobject.c.h" -static inline PyTupleObject * maybe_freelist_pop(Py_ssize_t); static inline int maybe_freelist_push(PyTupleObject *); @@ -38,22 +38,20 @@ tuple_alloc(Py_ssize_t size) PyErr_BadInternalCall(); return NULL; } -#ifdef Py_DEBUG assert(size != 0); // The empty tuple is statically allocated. -#endif - - PyTupleObject *op = maybe_freelist_pop(size); - if (op == NULL) { - /* Check for overflow */ - if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) - - sizeof(PyObject *))) / sizeof(PyObject *)) { - return (PyTupleObject *)PyErr_NoMemory(); + Py_ssize_t index = size - 1; + if (index < PyTuple_MAXSAVESIZE) { + PyTupleObject *op = _Py_FREELIST_POP(PyTupleObject, tuples[index]); + if (op != NULL) { + return op; } - op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size); - if (op == NULL) - return NULL; } - return op; + /* Check for overflow */ + if ((size_t)size > ((size_t)PY_SSIZE_T_MAX - (sizeof(PyTupleObject) - + sizeof(PyObject *))) / sizeof(PyObject *)) { + return (PyTupleObject *)PyErr_NoMemory(); + } + return PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size); } // The empty tuple singleton is not tracked by the GC. @@ -982,16 +980,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) return 0; } - -static void maybe_freelist_clear(struct _Py_object_freelists *, int); - - -void -_PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ - maybe_freelist_clear(freelists, is_finalization); -} - /*********************** Tuple Iterator **************************/ @@ -1141,102 +1129,31 @@ tuple_iter(PyObject *seq) * freelists * *************/ -#define TUPLE_FREELIST (freelists->tuples) -#define FREELIST_FINALIZED (TUPLE_FREELIST.numfree[0] < 0) - -static inline PyTupleObject * -maybe_freelist_pop(Py_ssize_t size) -{ -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (size == 0) { - return NULL; - } - assert(size > 0); - if (size < PyTuple_MAXSAVESIZE) { - Py_ssize_t index = size - 1; - PyTupleObject *op = TUPLE_FREELIST.items[index]; - if (op != NULL) { - /* op is the head of a linked list, with the first item - pointing to the next node. Here we pop off the old head. */ - TUPLE_FREELIST.items[index] = (PyTupleObject *) op->ob_item[0]; - TUPLE_FREELIST.numfree[index]--; - /* Inlined _PyObject_InitVar() without _PyType_HasFeature() test */ -#ifdef Py_TRACE_REFS - /* maybe_freelist_push() ensures these were already set. */ - // XXX Can we drop these? See commit 68055ce6fe01 (GvR, Dec 1998). - Py_SET_SIZE(op, size); - Py_SET_TYPE(op, &PyTuple_Type); -#endif - _Py_NewReference((PyObject *)op); - /* END inlined _PyObject_InitVar() */ - OBJECT_STAT_INC(from_freelist); - return op; - } - } -#endif - return NULL; -} - static inline int maybe_freelist_push(PyTupleObject *op) { -#ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - if (Py_SIZE(op) == 0) { + if (!Py_IS_TYPE(op, &PyTuple_Type)) { return 0; } Py_ssize_t index = Py_SIZE(op) - 1; - if (index < PyTuple_NFREELISTS - && TUPLE_FREELIST.numfree[index] < PyTuple_MAXFREELIST - && TUPLE_FREELIST.numfree[index] >= 0 - && Py_IS_TYPE(op, &PyTuple_Type)) - { - /* op is the head of a linked list, with the first item - pointing to the next node. Here we set op as the new head. */ - op->ob_item[0] = (PyObject *) TUPLE_FREELIST.items[index]; - TUPLE_FREELIST.items[index] = op; - TUPLE_FREELIST.numfree[index]++; - OBJECT_STAT_INC(to_freelist); - return 1; + if (index < PyTuple_MAXSAVESIZE) { + return _Py_FREELIST_PUSH(tuples[index], op, Py_tuple_MAXFREELIST); } -#endif return 0; } -static void -maybe_freelist_clear(struct _Py_object_freelists *freelists, int fini) -{ -#ifdef WITH_FREELISTS - for (Py_ssize_t i = 0; i < PyTuple_NFREELISTS; i++) { - PyTupleObject *p = TUPLE_FREELIST.items[i]; - TUPLE_FREELIST.items[i] = NULL; - TUPLE_FREELIST.numfree[i] = fini ? -1 : 0; - while (p) { - PyTupleObject *q = p; - p = (PyTupleObject *)(p->ob_item[0]); - PyObject_GC_Del(q); - } - } -#endif -} - /* Print summary info about the state of the optimized allocator */ void _PyTuple_DebugMallocStats(FILE *out) { #ifdef WITH_FREELISTS - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - for (int i = 0; i < PyTuple_NFREELISTS; i++) { + for (int i = 0; i < PyTuple_MAXSAVESIZE; i++) { int len = i + 1; char buf[128]; PyOS_snprintf(buf, sizeof(buf), "free %d-sized PyTupleObject", len); - _PyDebugAllocatorStats(out, buf, TUPLE_FREELIST.numfree[i], + _PyDebugAllocatorStats(out, buf, _Py_FREELIST_SIZE(tuples[i]), _PyObject_VAR_SIZE(&PyTuple_Type, len)); } #endif } - -#undef STATE -#undef FREELIST_FINALIZED diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 447e561c0d4440..5b0a466f913495 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -314,6 +314,16 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, } } +static PyTypeObject * +managed_static_type_get_def(PyTypeObject *self, int isbuiltin) +{ + size_t index = managed_static_type_index_get(self); + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + return &_PyRuntime.types.managed_static.types[full_index].def; +} + // Also see _PyStaticType_InitBuiltin() and _PyStaticType_FiniBuiltin(). /* end static builtin helpers */ @@ -1016,6 +1026,8 @@ type_modified_unlocked(PyTypeObject *type) if (type->tp_version_tag == 0) { return; } + // Cannot modify static builtin types. + assert((type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) == 0); PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses != NULL) { @@ -5387,7 +5399,7 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name) #ifdef Py_GIL_DISABLED // synchronize-with other writing threads by doing an acquire load on the sequence while (1) { - int sequence = _PySeqLock_BeginRead(&entry->sequence); + uint32_t sequence = _PySeqLock_BeginRead(&entry->sequence); uint32_t entry_version = _Py_atomic_load_uint32_relaxed(&entry->version); uint32_t type_version = _Py_atomic_load_uint32_acquire(&type->tp_version_tag); if (entry_version == type_version && @@ -5840,7 +5852,6 @@ fini_static_type(PyInterpreterState *interp, PyTypeObject *type, _PyStaticType_ClearWeakRefs(interp, type); managed_static_type_state_clear(interp, type, isbuiltin, final); - /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ } void @@ -6540,28 +6551,11 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* return 0; } -static int -object_set_class(PyObject *self, PyObject *value, void *closure) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "can't delete __class__ attribute"); - return -1; - } - if (!PyType_Check(value)) { - PyErr_Format(PyExc_TypeError, - "__class__ must be set to a class, not '%s' object", - Py_TYPE(value)->tp_name); - return -1; - } - PyTypeObject *newto = (PyTypeObject *)value; - - if (PySys_Audit("object.__setattr__", "OsO", - self, "__class__", value) < 0) { - return -1; - } +static int +object_set_class_world_stopped(PyObject *self, PyTypeObject *newto) +{ PyTypeObject *oldto = Py_TYPE(self); /* In versions of CPython prior to 3.5, the code in @@ -6627,39 +6621,66 @@ object_set_class(PyObject *self, PyObject *value, void *closure) /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) { - PyDictObject *dict = _PyObject_MaterializeManagedDict(self); + PyDictObject *dict = _PyObject_GetManagedDict(self); if (dict == NULL) { - return -1; + dict = _PyObject_MaterializeManagedDict_LockHeld(self); + if (dict == NULL) { + return -1; + } } - bool error = false; - - Py_BEGIN_CRITICAL_SECTION2(self, dict); - - // If we raced after materialization and replaced the dict - // then the materialized dict should no longer have the - // inline values in which case detach is a nop. - assert(_PyObject_GetManagedDict(self) == dict || - dict->ma_values != _PyObject_InlineValues(self)); + assert(_PyObject_GetManagedDict(self) == dict); if (_PyDict_DetachFromObject(dict, self) < 0) { - error = true; - } - - Py_END_CRITICAL_SECTION2(); - if (error) { return -1; } + } if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_INCREF(newto); } - Py_BEGIN_CRITICAL_SECTION(self); - // The real Py_TYPE(self) (`oldto`) may have changed from - // underneath us in another thread, so we re-fetch it here. - oldto = Py_TYPE(self); + Py_SET_TYPE(self, newto); - Py_END_CRITICAL_SECTION(); + + return 0; + } + else { + return -1; + } +} + +static int +object_set_class(PyObject *self, PyObject *value, void *closure) +{ + + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "can't delete __class__ attribute"); + return -1; + } + if (!PyType_Check(value)) { + PyErr_Format(PyExc_TypeError, + "__class__ must be set to a class, not '%s' object", + Py_TYPE(value)->tp_name); + return -1; + } + PyTypeObject *newto = (PyTypeObject *)value; + + if (PySys_Audit("object.__setattr__", "OsO", + self, "__class__", value) < 0) { + return -1; + } + +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); +#endif + PyTypeObject *oldto = Py_TYPE(self); + int res = object_set_class_world_stopped(self, newto); +#ifdef Py_GIL_DISABLED + _PyEval_StartTheWorld(interp); +#endif + if (res == 0) { if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_DECREF(oldto); } @@ -6667,9 +6688,7 @@ object_set_class(PyObject *self, PyObject *value, void *closure) RARE_EVENT_INC(set_class); return 0; } - else { - return -1; - } + return res; } static PyGetSetDef object_getsets[] = { @@ -7842,7 +7861,7 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) return 0; } -static int add_operators(PyTypeObject *); +static int add_operators(PyTypeObject *, PyTypeObject *); static int add_tp_new_wrapper(PyTypeObject *type); #define COLLECTION_FLAGS (Py_TPFLAGS_SEQUENCE | Py_TPFLAGS_MAPPING) @@ -8007,10 +8026,10 @@ type_dict_set_doc(PyTypeObject *type) static int -type_ready_fill_dict(PyTypeObject *type) +type_ready_fill_dict(PyTypeObject *type, PyTypeObject *def) { /* Add type-specific descriptors to tp_dict */ - if (add_operators(type) < 0) { + if (add_operators(type, def) < 0) { return -1; } if (type_add_methods(type) < 0) { @@ -8329,7 +8348,7 @@ type_ready_post_checks(PyTypeObject *type) static int -type_ready(PyTypeObject *type, int initial) +type_ready(PyTypeObject *type, PyTypeObject *def, int initial) { ASSERT_TYPE_LOCK_HELD(); @@ -8368,7 +8387,7 @@ type_ready(PyTypeObject *type, int initial) if (type_ready_set_new(type, initial) < 0) { goto error; } - if (type_ready_fill_dict(type) < 0) { + if (type_ready_fill_dict(type, def) < 0) { goto error; } if (initial) { @@ -8425,7 +8444,7 @@ PyType_Ready(PyTypeObject *type) int res; BEGIN_TYPE_LOCK(); if (!(type->tp_flags & Py_TPFLAGS_READY)) { - res = type_ready(type, 1); + res = type_ready(type, NULL, 1); } else { res = 0; assert(_PyType_CheckConsistency(type)); @@ -8461,14 +8480,20 @@ init_static_type(PyInterpreterState *interp, PyTypeObject *self, managed_static_type_state_init(interp, self, isbuiltin, initial); + PyTypeObject *def = managed_static_type_get_def(self, isbuiltin); + if (initial) { + memcpy(def, self, sizeof(PyTypeObject)); + } + int res; BEGIN_TYPE_LOCK(); - res = type_ready(self, initial); + res = type_ready(self, def, initial); END_TYPE_LOCK(); if (res < 0) { _PyStaticType_ClearWeakRefs(interp, self); managed_static_type_state_clear(interp, self, isbuiltin, initial); } + return res; } @@ -11056,17 +11081,22 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, infinite recursion here.) */ static int -add_operators(PyTypeObject *type) +add_operators(PyTypeObject *type, PyTypeObject *def) { PyObject *dict = lookup_tp_dict(type); pytype_slotdef *p; PyObject *descr; void **ptr; + assert(def == NULL || (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + if (def == NULL) { + def = type; + } + for (p = slotdefs; p->name; p++) { if (p->wrapper == NULL) continue; - ptr = slotptr(type, p->offset); + ptr = slotptr(def, p->offset); if (!ptr || !*ptr) continue; int r = PyDict_Contains(dict, p->name_strobj); diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index c8ab14053de418..3c96850589d378 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -116,6 +116,203 @@ PyTypeObject _PyNoDefault_Type = { PyObject _Py_NoDefaultStruct = _PyObject_HEAD_INIT(&_PyNoDefault_Type); +typedef struct { + PyObject_HEAD + PyObject *value; +} constevaluatorobject; + +static void +constevaluator_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + constevaluatorobject *ce = (constevaluatorobject *)self; + + _PyObject_GC_UNTRACK(self); + + Py_XDECREF(ce->value); + + Py_TYPE(self)->tp_free(self); + Py_DECREF(tp); +} + +static int +constevaluator_traverse(PyObject *self, visitproc visit, void *arg) +{ + constevaluatorobject *ce = (constevaluatorobject *)self; + Py_VISIT(ce->value); + return 0; +} + +static int +constevaluator_clear(PyObject *self) +{ + Py_CLEAR(((constevaluatorobject *)self)->value); + return 0; +} + +static PyObject * +constevaluator_repr(PyObject *self, PyObject *repr) +{ + PyObject *value = ((constevaluatorobject *)self)->value; + return PyUnicode_FromFormat("", value); +} + +static PyObject * +constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (!_PyArg_NoKeywords("constevaluator.__call__", kwargs)) { + return NULL; + } + int format; + if (!PyArg_ParseTuple(args, "i:constevaluator.__call__", &format)) { + return NULL; + } + PyObject *value = ((constevaluatorobject *)self)->value; + if (format == 3) { // SOURCE + PyUnicodeWriter *writer = PyUnicodeWriter_Create(5); // cannot be <5 + if (writer == NULL) { + return NULL; + } + if (PyTuple_Check(value)) { + if (PyUnicodeWriter_WriteChar(writer, '(') < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(value); i++) { + PyObject *item = PyTuple_GET_ITEM(value, i); + if (i > 0) { + if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + if (_Py_typing_type_repr(writer, item) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + if (PyUnicodeWriter_WriteChar(writer, ')') < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + else { + if (_Py_typing_type_repr(writer, value) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + } + return PyUnicodeWriter_Finish(writer); + } + return Py_NewRef(value); +} + +static PyObject * +constevaluator_alloc(PyObject *value) +{ + PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.constevaluator_type; + assert(tp != NULL); + constevaluatorobject *ce = PyObject_GC_New(constevaluatorobject, tp); + if (ce == NULL) { + return NULL; + } + ce->value = Py_NewRef(value); + _PyObject_GC_TRACK(ce); + return (PyObject *)ce; + +} + +PyDoc_STRVAR(constevaluator_doc, +"_ConstEvaluator()\n" +"--\n\n" +"Internal type for implementing evaluation functions."); + +static PyType_Slot constevaluator_slots[] = { + {Py_tp_doc, (void *)constevaluator_doc}, + {Py_tp_dealloc, constevaluator_dealloc}, + {Py_tp_traverse, constevaluator_traverse}, + {Py_tp_clear, constevaluator_clear}, + {Py_tp_repr, constevaluator_repr}, + {Py_tp_call, constevaluator_call}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +PyType_Spec constevaluator_spec = { + .name = "_typing._ConstEvaluator", + .basicsize = sizeof(constevaluatorobject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, + .slots = constevaluator_slots, +}; + +int +_Py_typing_type_repr(PyUnicodeWriter *writer, PyObject *p) +{ + PyObject *qualname = NULL; + PyObject *module = NULL; + PyObject *r = NULL; + int rc; + + if (p == Py_Ellipsis) { + // The Ellipsis object + r = PyUnicode_FromString("..."); + goto exit; + } + + if (p == (PyObject *)&_PyNone_Type) { + return PyUnicodeWriter_WriteUTF8(writer, "None", 4); + } + + if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && + (rc = PyObject_HasAttrWithError(p, &_Py_ID(__args__))) > 0) + { + // It looks like a GenericAlias + goto use_repr; + } + if (rc < 0) { + goto exit; + } + + if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { + goto exit; + } + if (qualname == NULL) { + goto use_repr; + } + if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { + goto exit; + } + if (module == NULL || module == Py_None) { + goto use_repr; + } + + // Looks like a class + if (PyUnicode_Check(module) && + _PyUnicode_EqualToASCIIString(module, "builtins")) + { + // builtins don't need a module name + r = PyObject_Str(qualname); + goto exit; + } + else { + r = PyUnicode_FromFormat("%S.%S", module, qualname); + goto exit; + } + +use_repr: + r = PyObject_Repr(p); +exit: + Py_XDECREF(qualname); + Py_XDECREF(module); + if (r == NULL) { + return -1; + } + rc = PyUnicodeWriter_WriteStr(writer, r); + Py_DECREF(r); + return rc; +} + static PyObject * call_typing_func_object(const char *name, PyObject **args, size_t nargs) @@ -364,10 +561,49 @@ typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) return constraints; } +static PyObject * +typevar_evaluate_bound(typevarobject *self, void *Py_UNUSED(ignored)) +{ + if (self->evaluate_bound != NULL) { + return Py_NewRef(self->evaluate_bound); + } + if (self->bound != NULL) { + return constevaluator_alloc(self->bound); + } + Py_RETURN_NONE; +} + +static PyObject * +typevar_evaluate_constraints(typevarobject *self, void *Py_UNUSED(ignored)) +{ + if (self->evaluate_constraints != NULL) { + return Py_NewRef(self->evaluate_constraints); + } + if (self->constraints != NULL) { + return constevaluator_alloc(self->constraints); + } + Py_RETURN_NONE; +} + +static PyObject * +typevar_evaluate_default(typevarobject *self, void *Py_UNUSED(ignored)) +{ + if (self->evaluate_default != NULL) { + return Py_NewRef(self->evaluate_default); + } + if (self->default_value != NULL) { + return constevaluator_alloc(self->default_value); + } + Py_RETURN_NONE; +} + static PyGetSetDef typevar_getset[] = { {"__bound__", (getter)typevar_bound, NULL, NULL, NULL}, {"__constraints__", (getter)typevar_constraints, NULL, NULL, NULL}, {"__default__", (getter)typevar_default, NULL, NULL, NULL}, + {"evaluate_bound", (getter)typevar_evaluate_bound, NULL, NULL, NULL}, + {"evaluate_constraints", (getter)typevar_evaluate_constraints, NULL, NULL, NULL}, + {"evaluate_default", (getter)typevar_evaluate_default, NULL, NULL, NULL}, {0} }; @@ -995,10 +1231,23 @@ paramspec_default(paramspecobject *self, void *unused) return default_value; } +static PyObject * +paramspec_evaluate_default(paramspecobject *self, void *unused) +{ + if (self->evaluate_default != NULL) { + return Py_NewRef(self->evaluate_default); + } + if (self->default_value != NULL) { + return constevaluator_alloc(self->default_value); + } + Py_RETURN_NONE; +} + static PyGetSetDef paramspec_getset[] = { {"args", (getter)paramspec_args, NULL, PyDoc_STR("Represents positional arguments."), NULL}, {"kwargs", (getter)paramspec_kwargs, NULL, PyDoc_STR("Represents keyword arguments."), NULL}, {"__default__", (getter)paramspec_default, NULL, "The default value for this ParamSpec.", NULL}, + {"evaluate_default", (getter)paramspec_evaluate_default, NULL, NULL, NULL}, {0}, }; @@ -1437,8 +1686,21 @@ typevartuple_default(typevartupleobject *self, void *unused) return default_value; } +static PyObject * +typevartuple_evaluate_default(typevartupleobject *self, void *unused) +{ + if (self->evaluate_default != NULL) { + return Py_NewRef(self->evaluate_default); + } + if (self->default_value != NULL) { + return constevaluator_alloc(self->default_value); + } + Py_RETURN_NONE; +} + static PyGetSetDef typevartuple_getset[] = { {"__default__", (getter)typevartuple_default, NULL, "The default value for this TypeVarTuple.", NULL}, + {"evaluate_default", (getter)typevartuple_evaluate_default, NULL, NULL, NULL}, {0}, }; @@ -1584,6 +1846,17 @@ typealias_value(PyObject *self, void *unused) return typealias_get_value(ta); } +static PyObject * +typealias_evaluate_value(PyObject *self, void *unused) +{ + typealiasobject *ta = (typealiasobject *)self; + if (ta->compute_value != NULL) { + return Py_NewRef(ta->compute_value); + } + assert(ta->value != NULL); + return constevaluator_alloc(ta->value); +} + static PyObject * typealias_parameters(PyObject *self, void *unused) { @@ -1627,6 +1900,7 @@ static PyGetSetDef typealias_getset[] = { {"__parameters__", typealias_parameters, (setter)NULL, NULL, NULL}, {"__type_params__", typealias_type_params, (setter)NULL, NULL, NULL}, {"__value__", typealias_value, (setter)NULL, NULL, NULL}, + {"evaluate_value", typealias_evaluate_value, (setter)NULL, NULL, NULL}, {"__module__", typealias_module, (setter)NULL, NULL, NULL}, {0} }; @@ -1952,6 +2226,7 @@ int _Py_initialize_generic(PyInterpreterState *interp) MAKE_TYPE(paramspec); MAKE_TYPE(paramspecargs); MAKE_TYPE(paramspeckwargs); + MAKE_TYPE(constevaluator); #undef MAKE_TYPE return 0; } @@ -1964,6 +2239,7 @@ void _Py_clear_generic_types(PyInterpreterState *interp) Py_CLEAR(interp->cached_objects.paramspec_type); Py_CLEAR(interp->cached_objects.paramspecargs_type); Py_CLEAR(interp->cached_objects.paramspeckwargs_type); + Py_CLEAR(interp->cached_objects.constevaluator_type); } PyObject * diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 394ea888fc9231..ffb879a68745b1 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -252,7 +252,8 @@ _PyUnicode_InternedSize_Immortal(void) // value, to help detect bugs in optimizations. while (PyDict_Next(dict, &pos, &key, &value)) { - if (_Py_IsImmortal(key)) { + assert(PyUnicode_CHECK_INTERNED(key) != SSTATE_INTERNED_IMMORTAL_STATIC); + if (PyUnicode_CHECK_INTERNED(key) == SSTATE_INTERNED_IMMORTAL) { count++; } } @@ -324,7 +325,8 @@ init_global_interned_strings(PyInterpreterState *interp) return _PyStatus_ERR("failed to create global interned dict"); } - /* Intern statically allocated string identifiers and deepfreeze strings. + /* Intern statically allocated string identifiers, deepfreeze strings, + * and one-byte latin-1 strings. * This must be done before any module initialization so that statically * allocated string identifiers are used instead of heap allocated strings. * Deepfreeze uses the interned identifiers if present to save space @@ -332,14 +334,11 @@ init_global_interned_strings(PyInterpreterState *interp) */ _PyUnicode_InitStaticStrings(interp); -#ifdef Py_GIL_DISABLED -// In the free-threaded build, intern the 1-byte strings as well for (int i = 0; i < 256; i++) { PyObject *s = LATIN1(i); _PyUnicode_InternStatic(interp, &s); assert(s == LATIN1(i)); } -#endif #ifdef Py_DEBUG assert(_PyUnicode_CheckConsistency(&_Py_STR(empty), 1)); @@ -688,10 +687,14 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) /* Check interning state */ #ifdef Py_DEBUG + // Note that we do not check `_Py_IsImmortal(op)`, since stable ABI + // extensions can make immortal strings mortal (but with a high enough + // refcount). + // The other way is extremely unlikely (worth a potential failed assertion + // in a debug build), so we do check `!_Py_IsImmortal(op)`. switch (PyUnicode_CHECK_INTERNED(op)) { case SSTATE_NOT_INTERNED: if (ascii->state.statically_allocated) { - CHECK(_Py_IsImmortal(op)); // This state is for two exceptions: // - strings are currently checked before they're interned // - the 256 one-latin1-character strings @@ -707,11 +710,9 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) break; case SSTATE_INTERNED_IMMORTAL: CHECK(!ascii->state.statically_allocated); - CHECK(_Py_IsImmortal(op)); break; case SSTATE_INTERNED_IMMORTAL_STATIC: CHECK(ascii->state.statically_allocated); - CHECK(_Py_IsImmortal(op)); break; default: Py_UNREACHABLE(); @@ -1867,7 +1868,6 @@ static PyObject* get_latin1_char(Py_UCS1 ch) { PyObject *o = LATIN1(ch); - assert(_Py_IsImmortal(o)); return o; } @@ -13635,27 +13635,28 @@ int _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) { - Py_UCS4 maxchar; - Py_ssize_t len; - assert(0 <= start); assert(end <= PyUnicode_GET_LENGTH(str)); assert(start <= end); - if (end == 0) - return 0; - if (start == 0 && end == PyUnicode_GET_LENGTH(str)) return _PyUnicodeWriter_WriteStr(writer, str); - if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar) + Py_ssize_t len = end - start; + if (len == 0) { + return 0; + } + + Py_UCS4 maxchar; + if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar) { maxchar = _PyUnicode_FindMaxChar(str, start, end); - else + } + else { maxchar = writer->maxchar; - len = end - start; - - if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) + } + if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) { return -1; + } _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, str, start, len); @@ -15352,27 +15353,14 @@ intern_static(PyInterpreterState *interp, PyObject *s /* stolen */) assert(s != NULL); assert(_PyUnicode_CHECK(s)); assert(_PyUnicode_STATE(s).statically_allocated); - assert(_Py_IsImmortal(s)); - - switch (PyUnicode_CHECK_INTERNED(s)) { - case SSTATE_NOT_INTERNED: - break; - case SSTATE_INTERNED_IMMORTAL_STATIC: - return s; - default: - Py_FatalError("_PyUnicode_InternStatic called on wrong string"); - } + assert(!PyUnicode_CHECK_INTERNED(s)); #ifdef Py_DEBUG /* We must not add process-global interned string if there's already a * per-interpreter interned_dict, which might contain duplicates. - * Except "short string" singletons: those are special-cased. */ + */ PyObject *interned = get_interned_dict(interp); - assert(interned == NULL || unicode_is_singleton(s)); -#ifdef Py_GIL_DISABLED - // In the free-threaded build, don't allow even the short strings. assert(interned == NULL); -#endif #endif /* Look in the global cache first. */ @@ -15444,11 +15432,6 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, return s; } - /* Handle statically allocated strings. */ - if (_PyUnicode_STATE(s).statically_allocated) { - return intern_static(interp, s); - } - /* Is it already interned? */ switch (PyUnicode_CHECK_INTERNED(s)) { case SSTATE_NOT_INTERNED: @@ -15465,6 +15448,9 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, return s; } + /* Statically allocated strings must be already interned. */ + assert(!_PyUnicode_STATE(s).statically_allocated); + #if Py_GIL_DISABLED /* In the free-threaded build, all interned strings are immortal */ immortalize = 1; @@ -15475,13 +15461,11 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, immortalize = 1; } - /* if it's a short string, get the singleton -- and intern it */ + /* if it's a short string, get the singleton */ if (PyUnicode_GET_LENGTH(s) == 1 && PyUnicode_KIND(s) == PyUnicode_1BYTE_KIND) { PyObject *r = LATIN1(*(unsigned char*)PyUnicode_DATA(s)); - if (!PyUnicode_CHECK_INTERNED(r)) { - r = intern_static(interp, r); - } + assert(PyUnicode_CHECK_INTERNED(r)); Py_DECREF(s); return r; } @@ -15493,7 +15477,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, { PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s); if (r != NULL) { - assert(_Py_IsImmortal(r)); + assert(_PyUnicode_STATE(r).statically_allocated); assert(r != s); // r must be statically_allocated; s is not Py_DECREF(s); return Py_NewRef(r); @@ -15583,7 +15567,7 @@ void PyUnicode_InternInPlace(PyObject **p) { PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUnicode_InternImmortal(interp, p); + _PyUnicode_InternMortal(interp, p); } // Public-looking name kept for the stable ABI; user should not call this: diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 7931f4345f7fdd..6e65a653a95c46 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -1,11 +1,10 @@ // types.UnionType -- used to represent e.g. Union[int, str], int | str #include "Python.h" #include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK -#include "pycore_typevarobject.h" // _PyTypeAlias_Type +#include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_typing_type_repr #include "pycore_unionobject.h" - static PyObject *make_union(PyObject *); @@ -181,67 +180,6 @@ _Py_union_type_or(PyObject* self, PyObject* other) return new_union; } -static int -union_repr_item(PyUnicodeWriter *writer, PyObject *p) -{ - PyObject *qualname = NULL; - PyObject *module = NULL; - int rc; - - if (p == (PyObject *)&_PyNone_Type) { - return PyUnicodeWriter_WriteUTF8(writer, "None", 4); - } - - if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && - (rc = PyObject_HasAttrWithError(p, &_Py_ID(__args__))) > 0) - { - // It looks like a GenericAlias - goto use_repr; - } - if (rc < 0) { - goto error; - } - - if (PyObject_GetOptionalAttr(p, &_Py_ID(__qualname__), &qualname) < 0) { - goto error; - } - if (qualname == NULL) { - goto use_repr; - } - if (PyObject_GetOptionalAttr(p, &_Py_ID(__module__), &module) < 0) { - goto error; - } - if (module == NULL || module == Py_None) { - goto use_repr; - } - - // Looks like a class - if (PyUnicode_Check(module) && - _PyUnicode_EqualToASCIIString(module, "builtins")) - { - // builtins don't need a module name - rc = PyUnicodeWriter_WriteStr(writer, qualname); - goto done; - } - else { - rc = PyUnicodeWriter_Format(writer, "%S.%S", module, qualname); - goto done; - } - -error: - rc = -1; - goto done; - -use_repr: - rc = PyUnicodeWriter_WriteRepr(writer, p); - goto done; - -done: - Py_XDECREF(qualname); - Py_XDECREF(module); - return rc; -} - static PyObject * union_repr(PyObject *self) { @@ -260,7 +198,7 @@ union_repr(PyObject *self) goto error; } PyObject *p = PyTuple_GET_ITEM(alias->args, i); - if (union_repr_item(writer, p) < 0) { + if (_Py_typing_type_repr(writer, p) < 0) { goto error; } } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 0fcd37d949b34a..61f05514a48023 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -426,6 +426,10 @@ get_or_create_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) return basic_ref; } PyWeakReference *newref = allocate_weakref(type, obj, callback); + if (newref == NULL) { + UNLOCK_WEAKREFS(obj); + return NULL; + } insert_weakref(newref, list); UNLOCK_WEAKREFS(obj); return newref; @@ -433,6 +437,9 @@ get_or_create_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) else { // We may not be able to safely allocate inside the lock PyWeakReference *newref = allocate_weakref(type, obj, callback); + if (newref == NULL) { + return NULL; + } LOCK_WEAKREFS(obj); insert_weakref(newref, list); UNLOCK_WEAKREFS(obj); diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f36fcb8caece33..9e3af689f4a288 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -239,6 +239,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index a1b43addf9e36a..31f7971bda845d 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -639,6 +639,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 44bf87da8288a6..1972c606827cdb 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -969,6 +969,8 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv) return result_token_with_metadata(p, conv, conv_token->metadata); } +static asdl_expr_seq * +unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions); ResultTokenWithMetadata * _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) @@ -1007,8 +1009,16 @@ _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, in assert(j == non_empty_count); spec = resized_spec; } - expr_ty res = _PyAST_JoinedStr(spec, lineno, col_offset, end_lineno, - end_col_offset, p->arena); + expr_ty res; + Py_ssize_t n = asdl_seq_LEN(spec); + if (n == 0 || (n == 1 && asdl_seq_GET(spec, 0)->kind == Constant_kind)) { + res = _PyAST_JoinedStr(spec, lineno, col_offset, end_lineno, + end_col_offset, p->arena); + } else { + res = _PyPegen_concatenate_strings(p, spec, + lineno, col_offset, end_lineno, + end_col_offset, arena); + } if (!res) { return NULL; } @@ -1308,6 +1318,7 @@ unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions) expr_ty _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) { + asdl_expr_seq *expr = unpack_top_level_joined_strs(p, raw_expressions); Py_ssize_t n_items = asdl_seq_LEN(expr); @@ -1472,7 +1483,6 @@ expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, Re debug_end_offset = end_col_offset; debug_metadata = closing_brace->metadata; } - expr_ty debug_text = _PyAST_Constant(debug_metadata, NULL, lineno, col_offset + 1, debug_end_line, debug_end_offset - 1, p->arena); if (!debug_text) { @@ -1505,16 +1515,23 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, Py_ssize_t n_flattened_elements = 0; for (i = 0; i < len; i++) { expr_ty elem = asdl_seq_GET(strings, i); - if (elem->kind == Constant_kind) { - if (PyBytes_CheckExact(elem->v.Constant.value)) { - bytes_found = 1; - } else { - unicode_string_found = 1; - } - n_flattened_elements++; - } else { - n_flattened_elements += asdl_seq_LEN(elem->v.JoinedStr.values); - f_string_found = 1; + switch(elem->kind) { + case Constant_kind: + if (PyBytes_CheckExact(elem->v.Constant.value)) { + bytes_found = 1; + } else { + unicode_string_found = 1; + } + n_flattened_elements++; + break; + case JoinedStr_kind: + n_flattened_elements += asdl_seq_LEN(elem->v.JoinedStr.values); + f_string_found = 1; + break; + default: + n_flattened_elements++; + f_string_found = 1; + break; } } @@ -1556,16 +1573,19 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, Py_ssize_t j = 0; for (i = 0; i < len; i++) { expr_ty elem = asdl_seq_GET(strings, i); - if (elem->kind == Constant_kind) { - asdl_seq_SET(flattened, current_pos++, elem); - } else { - for (j = 0; j < asdl_seq_LEN(elem->v.JoinedStr.values); j++) { - expr_ty subvalue = asdl_seq_GET(elem->v.JoinedStr.values, j); - if (subvalue == NULL) { - return NULL; + switch(elem->kind) { + case JoinedStr_kind: + for (j = 0; j < asdl_seq_LEN(elem->v.JoinedStr.values); j++) { + expr_ty subvalue = asdl_seq_GET(elem->v.JoinedStr.values, j); + if (subvalue == NULL) { + return NULL; + } + asdl_seq_SET(flattened, current_pos++, subvalue); } - asdl_seq_SET(flattened, current_pos++, subvalue); - } + break; + default: + asdl_seq_SET(flattened, current_pos++, elem); + break; } } @@ -1575,7 +1595,7 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, for (i = 0; i < n_flattened_elements; i++) { expr_ty elem = asdl_seq_GET(flattened, i); - /* The concatenation of a FormattedValue and an empty Contant should + /* The concatenation of a FormattedValue and an empty Constant should lead to the FormattedValue itself. Thus, we will not take any empty constants into account, just as in `_PyPegen_joined_str` */ if (f_string_found && elem->kind == Constant_kind && diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index f3667801782f2b..9fed69b12479d6 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -131,7 +131,7 @@ def emit(self, s, depth, reflow=True): def metadata(self): if self._metadata is None: raise ValueError( - "%s was expecting to be annnotated with metadata" + "%s was expecting to be annotated with metadata" % type(self).__name__ ) return self._metadata @@ -880,7 +880,7 @@ def visitModule(self, mod): Py_ssize_t i, numfields = 0; int res = -1; - PyObject *key, *value, *fields, *remaining_fields = NULL; + PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL; if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -947,22 +947,32 @@ def visitModule(self, mod): goto cleanup; } } - else if ( - PyUnicode_CompareWithASCIIString(key, "lineno") != 0 && - PyUnicode_CompareWithASCIIString(key, "col_offset") != 0 && - PyUnicode_CompareWithASCIIString(key, "end_lineno") != 0 && - PyUnicode_CompareWithASCIIString(key, "end_col_offset") != 0 - ) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "%.400s.__init__ got an unexpected keyword argument '%U'. " - "Support for arbitrary keyword arguments is deprecated " - "and will be removed in Python 3.15.", - Py_TYPE(self)->tp_name, key - ) < 0) { + else { + // Lazily initialize "attributes" + if (attributes == NULL) { + attributes = PyObject_GetAttr((PyObject*)Py_TYPE(self), state->_attributes); + if (attributes == NULL) { + res = -1; + goto cleanup; + } + } + int contains = PySequence_Contains(attributes, key); + if (contains == -1) { res = -1; goto cleanup; } + else if (contains == 0) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ got an unexpected keyword argument '%U'. " + "Support for arbitrary keyword arguments is deprecated " + "and will be removed in Python 3.15.", + Py_TYPE(self)->tp_name, key + ) < 0) { + res = -1; + goto cleanup; + } + } } res = PyObject_SetAttr(self, key, value); if (res < 0) { @@ -1045,6 +1055,7 @@ def visitModule(self, mod): Py_DECREF(field_types); } cleanup: + Py_XDECREF(attributes); Py_XDECREF(fields); Py_XDECREF(remaining_fields); return res; diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 82b0e4ee352d62..8c868593f944c8 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -884,7 +884,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } { - /* Accept floating point numbers. */ + /* Accept floating-point numbers. */ if (c == '.') { c = tok_nextc(tok); fraction: @@ -989,6 +989,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t the_current_tok->last_expr_buffer = NULL; the_current_tok->last_expr_size = 0; the_current_tok->last_expr_end = -1; + the_current_tok->in_format_spec = 0; the_current_tok->f_string_debug = 0; switch (*tok->start) { @@ -1137,15 +1138,20 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t * by the `{` case, so for ensuring that we are on the 0th level, we need * to adjust it manually */ int cursor = current_tok->curly_bracket_depth - (c != '{'); - if (cursor == 0 && !_PyLexer_update_fstring_expr(tok, c)) { + int in_format_spec = current_tok->in_format_spec; + int cursor_in_format_with_debug = + cursor == 1 && (current_tok->f_string_debug || in_format_spec); + int cursor_valid = cursor == 0 || cursor_in_format_with_debug; + if ((cursor_valid) && !_PyLexer_update_fstring_expr(tok, c)) { return MAKE_TOKEN(ENDMARKER); } - if (cursor == 0 && c != '{' && set_fstring_expr(tok, token, c)) { + if ((cursor_valid) && c != '{' && set_fstring_expr(tok, token, c)) { return MAKE_TOKEN(ERRORTOKEN); } if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) { current_tok->kind = TOK_FSTRING_MODE; + current_tok->in_format_spec = 1; p_start = tok->start; p_end = tok->cur; return MAKE_TOKEN(_PyToken_OneChar(c)); @@ -1232,9 +1238,13 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if (INSIDE_FSTRING(tok)) { current_tok->curly_bracket_depth--; + if (current_tok->curly_bracket_depth < 0) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: unmatched '%c'", c)); + } if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) { current_tok->curly_bracket_expr_start_depth--; current_tok->kind = TOK_FSTRING_MODE; + current_tok->in_format_spec = 0; current_tok->f_string_debug = 0; } } @@ -1317,11 +1327,11 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct tok->multi_line_start = tok->line_start; while (end_quote_size != current_tok->f_string_quote_size) { int c = tok_nextc(tok); - if (tok->done == E_ERROR) { + if (tok->done == E_ERROR || tok->done == E_DECODE) { return MAKE_TOKEN(ERRORTOKEN); } int in_format_spec = ( - current_tok->last_expr_end != -1 + current_tok->in_format_spec && INSIDE_FSTRING_EXPR(current_tok) ); @@ -1337,6 +1347,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct if (in_format_spec && c == '\n') { tok_backup(tok, c); TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + current_tok->in_format_spec = 0; p_start = tok->start; p_end = tok->cur; return MAKE_TOKEN(FSTRING_MIDDLE); @@ -1378,6 +1389,9 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct } if (c == '{') { + if (!_PyLexer_update_fstring_expr(tok, c)) { + return MAKE_TOKEN(ENDMARKER); + } int peek = tok_nextc(tok); if (peek != '{' || in_format_spec) { tok_backup(tok, peek); @@ -1387,6 +1401,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: expressions nested too deeply")); } TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + current_tok->in_format_spec = 0; p_start = tok->start; p_end = tok->cur; } else { @@ -1406,13 +1421,15 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct // scanning (indicated by the end of the expression being set) and we are not at the top level // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double // brackets, we can bypass it here. - if (peek == '}' && !in_format_spec) { + int cursor = current_tok->curly_bracket_depth; + if (peek == '}' && !in_format_spec && cursor == 0) { p_start = tok->start; p_end = tok->cur - 1; } else { tok_backup(tok, peek); tok_backup(tok, c); TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + current_tok->in_format_spec = 0; p_start = tok->start; p_end = tok->cur; } diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 653ddafd411095..647f291911564c 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -74,6 +74,7 @@ free_fstring_expressions(struct tok_state *tok) mode->last_expr_buffer = NULL; mode->last_expr_size = 0; mode->last_expr_end = -1; + mode->in_format_spec = 0; } } } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 61d090d6d2fe21..9ed3babfdbfbf1 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -58,6 +58,7 @@ typedef struct _tokenizer_mode { Py_ssize_t last_expr_end; char* last_expr_buffer; int f_string_debug; + int in_format_spec; } tokenizer_mode; /* Tokenizer state */ diff --git a/Parser/pegen.c b/Parser/pegen.c index 6efb5477c7b80f..ac428be0958bdf 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -341,7 +341,7 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) for (Memo *m = t->memo; m != NULL; m = m->next) { if (m->type == type) { -#if defined(PY_DEBUG) +#if defined(Py_DEBUG) if (0 <= type && type < NSTATISTICS) { long count = m->mark - p->mark; // A memoized negative result counts for one. diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index e8f11a67e50fa0..e94a4923228d0f 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -155,7 +155,7 @@ _Pypegen_raise_decode_error(Parser *p) static int _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) { // Tokenize the whole input to see if there are any tokenization - // errors such as mistmatching parentheses. These will get priority + // errors such as mismatching parentheses. These will get priority // over generic syntax errors only if the line number of the error is // before the one that we had for the generic error. diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index b2a7196bd6081c..22354c9bbf8a35 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,17 +1,17 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,243,166,0,0,0,149,0,81,0,81,1, - 72,0,113,0,81,0,81,1,72,1,113,1,90,2,31,0, - 81,2,50,1,0,0,0,0,0,0,29,0,90,2,31,0, - 81,3,90,0,79,6,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,50,2,0,0,0,0,0,0, - 29,0,90,1,79,8,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,31,0,50,0,0,0,0,0, - 0,0,81,4,2,0,0,0,113,5,81,5,16,0,69,20, - 0,0,113,6,90,2,31,0,81,6,90,6,12,0,81,7, - 90,5,90,6,2,0,0,0,12,0,48,4,50,1,0,0, - 0,0,0,0,29,0,74,22,0,0,9,0,29,0,102,1, + 0,0,0,0,0,243,166,0,0,0,149,0,80,0,80,1, + 71,0,112,0,80,0,80,1,71,1,112,1,89,2,31,0, + 80,2,49,1,0,0,0,0,0,0,29,0,89,2,31,0, + 80,3,89,0,78,6,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,49,2,0,0,0,0,0,0, + 29,0,89,1,78,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,31,0,49,0,0,0,0,0, + 0,0,80,4,2,0,0,0,112,5,80,5,16,0,68,20, + 0,0,112,6,89,2,31,0,80,6,89,6,12,0,80,7, + 89,5,89,6,2,0,0,0,12,0,47,4,49,1,0,0, + 0,0,0,0,29,0,73,22,0,0,9,0,29,0,101,1, 41,8,233,0,0,0,0,78,122,18,70,114,111,122,101,110, 32,72,101,108,108,111,32,87,111,114,108,100,122,8,115,121, 115,46,97,114,103,118,218,6,99,111,110,102,105,103,41,5, diff --git a/Python/Python-ast.c b/Python/Python-ast.c index cca2ee409e7978..4d0db457a8b172 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5081,7 +5081,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) Py_ssize_t i, numfields = 0; int res = -1; - PyObject *key, *value, *fields, *remaining_fields = NULL; + PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL; if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) { goto cleanup; } @@ -5148,22 +5148,32 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } } - else if ( - PyUnicode_CompareWithASCIIString(key, "lineno") != 0 && - PyUnicode_CompareWithASCIIString(key, "col_offset") != 0 && - PyUnicode_CompareWithASCIIString(key, "end_lineno") != 0 && - PyUnicode_CompareWithASCIIString(key, "end_col_offset") != 0 - ) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "%.400s.__init__ got an unexpected keyword argument '%U'. " - "Support for arbitrary keyword arguments is deprecated " - "and will be removed in Python 3.15.", - Py_TYPE(self)->tp_name, key - ) < 0) { + else { + // Lazily initialize "attributes" + if (attributes == NULL) { + attributes = PyObject_GetAttr((PyObject*)Py_TYPE(self), state->_attributes); + if (attributes == NULL) { + res = -1; + goto cleanup; + } + } + int contains = PySequence_Contains(attributes, key); + if (contains == -1) { res = -1; goto cleanup; } + else if (contains == 0) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "%.400s.__init__ got an unexpected keyword argument '%U'. " + "Support for arbitrary keyword arguments is deprecated " + "and will be removed in Python 3.15.", + Py_TYPE(self)->tp_name, key + ) < 0) { + res = -1; + goto cleanup; + } + } } res = PyObject_SetAttr(self, key, value); if (res < 0) { @@ -5246,6 +5256,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) Py_DECREF(field_types); } cleanup: + Py_XDECREF(attributes); Py_XDECREF(fields); Py_XDECREF(remaining_fields); return res; diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 55c821754c2031..34b4445be27f62 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -1,9 +1,10 @@ #include "Python.h" #include "errcode.h" +#include "internal/pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION #include "../Parser/lexer/state.h" #include "../Parser/lexer/lexer.h" #include "../Parser/tokenizer/tokenizer.h" -#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() +#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() static struct PyModuleDef _tokenizemodule; @@ -84,14 +85,16 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, } static int -_tokenizer_error(struct tok_state *tok) +_tokenizer_error(tokenizeriterobject *it) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); if (PyErr_Occurred()) { return -1; } const char *msg = NULL; PyObject* errtype = PyExc_SyntaxError; + struct tok_state *tok = it->tok; switch (tok->done) { case E_TOKEN: msg = "invalid token"; @@ -177,17 +180,78 @@ _tokenizer_error(struct tok_state *tok) return result; } +static PyObject * +_get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t size, + int *line_changed) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); + PyObject *line; + if (it->tok->lineno != it->last_lineno) { + // Line has changed since last token, so we fetch the new line and cache it + // in the iter object. + Py_XDECREF(it->last_line); + line = PyUnicode_DecodeUTF8(line_start, size, "replace"); + it->last_line = line; + it->byte_col_offset_diff = 0; + } + else { + line = it->last_line; + *line_changed = 0; + } + return line; +} + +static void +_get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_start, + PyObject *line, int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno, + Py_ssize_t *col_offset, Py_ssize_t *end_col_offset) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); + Py_ssize_t byte_offset = -1; + if (token.start != NULL && token.start >= line_start) { + byte_offset = token.start - line_start; + if (line_changed) { + *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); + it->byte_col_offset_diff = byte_offset - *col_offset; + } + else { + *col_offset = byte_offset - it->byte_col_offset_diff; + } + } + + if (token.end != NULL && token.end >= it->tok->line_start) { + Py_ssize_t end_byte_offset = token.end - it->tok->line_start; + if (lineno == end_lineno) { + // If the whole token is at the same line, we can just use the token.start + // buffer for figuring out the new column offset, since using line is not + // performant for very long lines. + Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); + *end_col_offset = *col_offset + token_col_offset; + it->byte_col_offset_diff += token.end - token.start - token_col_offset; + } + else { + *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset); + it->byte_col_offset_diff += end_byte_offset - *end_col_offset; + } + } + it->last_lineno = lineno; + it->last_end_lineno = end_lineno; +} + static PyObject * tokenizeriter_next(tokenizeriterobject *it) { PyObject* result = NULL; + + Py_BEGIN_CRITICAL_SECTION(it); + struct token token; _PyToken_Init(&token); int type = _PyTokenizer_Get(it->tok, &token); if (type == ERRORTOKEN) { if(!PyErr_Occurred()) { - _tokenizer_error(it->tok); + _tokenizer_error(it); assert(PyErr_Occurred()); } goto exit; @@ -224,18 +288,7 @@ tokenizeriter_next(tokenizeriterobject *it) size -= 1; } - if (it->tok->lineno != it->last_lineno) { - // Line has changed since last token, so we fetch the new line and cache it - // in the iter object. - Py_XDECREF(it->last_line); - line = PyUnicode_DecodeUTF8(line_start, size, "replace"); - it->last_line = line; - it->byte_col_offset_diff = 0; - } else { - // Line hasn't changed so we reuse the cached one. - line = it->last_line; - line_changed = 0; - } + line = _get_current_line(it, line_start, size, &line_changed); } if (line == NULL) { Py_DECREF(str); @@ -244,36 +297,10 @@ tokenizeriter_next(tokenizeriterobject *it) Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno; Py_ssize_t end_lineno = it->tok->lineno; - it->last_lineno = lineno; - it->last_end_lineno = end_lineno; - Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - Py_ssize_t byte_offset = -1; - if (token.start != NULL && token.start >= line_start) { - byte_offset = token.start - line_start; - if (line_changed) { - col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); - it->byte_col_offset_diff = byte_offset - col_offset; - } - else { - col_offset = byte_offset - it->byte_col_offset_diff; - } - } - if (token.end != NULL && token.end >= it->tok->line_start) { - Py_ssize_t end_byte_offset = token.end - it->tok->line_start; - if (lineno == end_lineno) { - // If the whole token is at the same line, we can just use the token.start - // buffer for figuring out the new column offset, since using line is not - // performant for very long lines. - Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); - end_col_offset = col_offset + token_col_offset; - it->byte_col_offset_diff += token.end - token.start - token_col_offset; - } else { - end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset); - it->byte_col_offset_diff += end_byte_offset - end_col_offset; - } - } + _get_col_offsets(it, token, line_start, line, line_changed, + lineno, end_lineno, &col_offset, &end_col_offset); if (it->tok->tok_extra_tokens) { if (is_trailing_token) { @@ -315,6 +342,8 @@ tokenizeriter_next(tokenizeriterobject *it) if (type == ENDMARKER) { it->done = 1; } + + Py_END_CRITICAL_SECTION(); return result; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index a5b45e358d9efb..ae025e767ec838 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2644,8 +2644,8 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) b = PyLong_AsLongAndOverflow(item, &overflow); } if (overflow == 0 && - (i_result >= 0 ? (b <= LONG_MAX - i_result) - : (b >= LONG_MIN - i_result))) + (i_result >= 0 ? (b <= PY_SSIZE_T_MAX - i_result) + : (b >= PY_SSIZE_T_MIN - i_result))) { i_result += b; Py_DECREF(item); @@ -2687,14 +2687,15 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) continue; } if (PyLong_Check(item)) { - long value; - int overflow; - value = PyLong_AsLongAndOverflow(item, &overflow); - if (!overflow) { - re_sum.hi += (double)value; + double value = PyLong_AsDouble(item); + if (value != -1.0 || !PyErr_Occurred()) { + re_sum = cs_add(re_sum, value); Py_DECREF(item); continue; } + else { + return NULL; + } } result = PyFloat_FromDouble(cs_to_double(re_sum)); if (result == NULL) { @@ -2736,19 +2737,20 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) continue; } if (PyLong_Check(item)) { - long value; - int overflow; - value = PyLong_AsLongAndOverflow(item, &overflow); - if (!overflow) { - re_sum.hi += (double)value; + double value = PyLong_AsDouble(item); + if (value != -1.0 || !PyErr_Occurred()) { + re_sum = cs_add(re_sum, value); im_sum.hi += 0.0; Py_DECREF(item); continue; } + else { + return NULL; + } } if (PyFloat_Check(item)) { double value = PyFloat_AS_DOUBLE(item); - re_sum.hi += value; + re_sum = cs_add(re_sum, value); im_sum.hi += 0.0; _Py_DECREF_SPECIALIZED(item, _PyFloat_ExactDealloc); continue; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 76587a4f0dc695..d74f2aae0483ce 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -286,7 +286,7 @@ dummy_func( tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - if (PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver))) { + if (PyStackRef_GenCheck(receiver)) { if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { ERROR_NO_POP(); } @@ -317,7 +317,7 @@ dummy_func( } pure inst(UNARY_NOT, (value -- res)) { - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + assert(PyStackRef_BoolCheck(value)); res = PyStackRef_Is(value, PyStackRef_False) ? PyStackRef_True : PyStackRef_False; } @@ -353,7 +353,7 @@ dummy_func( macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL; inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { - EXIT_IF(!PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + EXIT_IF(!PyStackRef_BoolCheck(value)); STAT_INC(TO_BOOL, hit); } @@ -581,12 +581,18 @@ dummy_func( // So the inputs are the same as for all BINARY_OP // specializations, but there is no output. // At the end we just skip over the STORE_FAST. - tier1 op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { + op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int next_oparg; + #if TIER_ONE assert(next_instr->op.code == STORE_FAST); - _PyStackRef *target_local = &GETLOCAL(next_instr->op.arg); + next_oparg = next_instr->op.arg; + #else + next_oparg = CURRENT_OPERAND(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); DEOPT_IF(!PyStackRef_Is(*target_local, left)); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -607,9 +613,12 @@ dummy_func( *target_local = PyStackRef_FromPyObjectSteal(temp); _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); ERROR_IF(PyStackRef_IsNull(*target_local), error); - // The STORE_FAST is already done. + #if TIER_ONE + // The STORE_FAST is already done. This is done here in tier one, + // and during trace projection in tier two: assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); + #endif } macro(BINARY_OP_INPLACE_ADD_UNICODE) = @@ -936,48 +945,25 @@ dummy_func( LLTRACE_RESUME_FRAME(); } - inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { + tier1 op(_RETURN_VALUE_EVENT, (val -- val)) { int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); if (err) ERROR_NO_POP(); - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; } + macro(INSTRUMENTED_RETURN_VALUE) = + _RETURN_VALUE_EVENT + + RETURN_VALUE; + macro(RETURN_CONST) = LOAD_CONST + RETURN_VALUE; - inst(INSTRUMENTED_RETURN_CONST, (--)) { - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) ERROR_NO_POP(); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval)); - LOAD_IP(frame->return_offset); - goto resume_frame; - } + macro(INSTRUMENTED_RETURN_CONST) = + LOAD_CONST + + _RETURN_VALUE_EVENT + + RETURN_VALUE; inst(GET_AITER, (obj -- iter)) { unaryfunc getter = NULL; @@ -1154,47 +1140,25 @@ dummy_func( macro(SEND) = _SPECIALIZE_SEND + _SEND; - inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { - DEOPT_IF(tstate->interp->eval_frame); + op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame: _PyInterpreterFrame *)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); + gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); + assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); } - inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); - if (err) ERROR_NO_POP(); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; - } + macro(SEND_GEN) = + unused/1 + + _CHECK_PEP_523 + + _SEND_GEN_FRAME + + _PUSH_FRAME; inst(YIELD_VALUE, (retval -- value)) { // NOTE: It's important that YIELD_VALUE never raises an exception! @@ -1232,6 +1196,23 @@ dummy_func( LLTRACE_RESUME_FRAME(); } + tier1 op(_YIELD_VALUE_EVENT, (val -- val)) { + SAVE_SP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + LOAD_SP(); + if (err) ERROR_NO_POP(); + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); + } + } + + macro(INSTRUMENTED_YIELD_VALUE) = + _YIELD_VALUE_EVENT + + YIELD_VALUE; + inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, @@ -1547,22 +1528,16 @@ dummy_func( ERROR_IF(true, error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) { - ERROR_NO_POP(); - } + ERROR_IF(PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0, error); if (v_o == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) { - ERROR_NO_POP(); - } + ERROR_IF(PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0, error); if (v_o == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) { - ERROR_NO_POP(); - } + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0, error); if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - ERROR_NO_POP(); + ERROR_IF(true, error); } } } @@ -1828,7 +1803,8 @@ dummy_func( inst(BUILD_SET, (values[oparg] -- set)) { PyObject *set_o = PySet_New(NULL); if (set_o == NULL) { - ERROR_NO_POP(); + DECREF_INPUTS(); + ERROR_IF(true, error); } int err = 0; for (int i = 0; i < oparg; i++) { @@ -1884,25 +1860,6 @@ dummy_func( } } - inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) { - PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); - - assert(PyTuple_CheckExact(keys_o)); - assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); - STACKREFS_TO_PYOBJECTS(values, oparg, values_o); - if (CONVERSION_FAILED(values_o)) { - DECREF_INPUTS(); - ERROR_IF(true, error); - } - PyObject *map_o = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys_o, 0), 1, - values_o, 1, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - DECREF_INPUTS(); - ERROR_IF(map_o == NULL, error); - map = PyStackRef_FromPyObjectSteal(map_o); - } - inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) { PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); @@ -2264,32 +2221,30 @@ dummy_func( unused/2 + _LOAD_ATTR_CLASS; - inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _PyInterpreterFrame *)) { assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame); - - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version); PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 1); + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED); + DEOPT_IF(code->co_kwonlyargcount); + DEOPT_IF(code->co_argcount != 1); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1); new_frame->localsplus[0] = owner; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); } + macro(LOAD_ATTR_PROPERTY) = + unused/1 + + _CHECK_PEP_523 + + _GUARD_TYPE_VERSION + + unused/2 + + _LOAD_ATTR_PROPERTY_FRAME + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -2616,9 +2571,9 @@ dummy_func( b = res ? PyStackRef_True : PyStackRef_False; } - tier1 inst(IMPORT_NAME, (level, fromlist -- res)) { + inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = import_name(tstate, frame, name, + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); DECREF_INPUTS(); @@ -2626,9 +2581,9 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - tier1 inst(IMPORT_FROM, (from -- from, res)) { + inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2709,7 +2664,7 @@ dummy_func( } replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -2718,7 +2673,7 @@ dummy_func( } replaced op(_POP_JUMP_IF_TRUE, (cond -- )) { - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -2898,7 +2853,7 @@ dummy_func( } /* iterator ended normally */ /* The translator sets the deopt target just past the matching END_FOR */ - DEOPT_IF(true); + EXIT_IF(true); } next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator @@ -2967,7 +2922,10 @@ dummy_func( assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; EXIT_IF(seq == NULL); - EXIT_IF((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)); + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + it->it_index = -1; + EXIT_IF(1); + } } op(_ITER_NEXT_LIST, (iter -- iter, next)) { @@ -3139,7 +3097,7 @@ dummy_func( else { Py_DECREF(tb); } - assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti))); + assert(PyStackRef_LongCheck(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); @@ -3181,7 +3139,7 @@ dummy_func( else { prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc))); + assert(PyStackRef_ExceptionInstanceCheck(new_exc)); exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); } @@ -3283,20 +3241,6 @@ dummy_func( unused/1 + _LOAD_ATTR_METHOD_LAZY_DICT; - inst(INSTRUMENTED_CALL, (unused/3 -- )) { - int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - ERROR_IF(err, error); - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL); - } - // Cache layout: counter/1, func_version/2 // CALL_INTRINSIC_1/2, CALL_KW, and CALL_FUNCTION_EX aren't members! family(CALL, INLINE_CACHE_ENTRIES_CALL) = { @@ -3334,28 +3278,34 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + } + // When calling Python, inline the call using DISPATCH_INLINED(). - op(_CALL, (callable, self_or_null, args[oparg] -- res)) { + op(_DO_CALL, (callable, self_or_null, args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - else if (Py_TYPE(callable_o) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; - } // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && @@ -3418,7 +3368,28 @@ dummy_func( CHECK_EVAL_BREAKER(); } - macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC; + op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) { + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + ERROR_IF(err, error); + } + + macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC; + macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC; op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -3445,7 +3416,7 @@ dummy_func( } } - op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; @@ -3476,9 +3447,8 @@ dummy_func( assert(PyStackRef_IsNull(null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); + assert(PyStackRef_FunctionCheck(method)); PyStackRef_CLOSE(callable); } @@ -3487,6 +3457,7 @@ dummy_func( _CHECK_PEP_523 + _CHECK_METHOD_VERSION + _EXPAND_METHOD + + flush + // so that self is in the argument array _PY_FRAME_GENERAL + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -3541,16 +3512,12 @@ dummy_func( EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type); } - op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- func, self, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization - self = stack_pointer[-1 - oparg]; - func = stack_pointer[-2 - oparg]; + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); PyStackRef_CLOSE(callable); - // self may be unused in tier 1, so silence warnings. - (void)self; } op(_CHECK_PEP_523, (--)) { @@ -3565,7 +3532,7 @@ dummy_func( EXIT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))); } - op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_STACK_SPACE, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -3606,6 +3573,7 @@ dummy_func( _CHECK_PEP_523 + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + _INIT_CALL_BOUND_METHOD_EXACT_ARGS + + flush + // In case the following deopt _CHECK_FUNCTION_VERSION + _CHECK_FUNCTION_EXACT_ARGS + _CHECK_STACK_SPACE + @@ -3942,7 +3910,7 @@ dummy_func( } // This is secretly a super-instruction - tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- unused)) { + inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- )) { assert(oparg == 1); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); @@ -3952,16 +3920,16 @@ dummy_func( assert(self_o != NULL); DEOPT_IF(!PyList_Check(self_o)); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); - STACK_SHRINK(3); - // Skip POP_TOP + ERROR_IF(err, error); + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); - DISPATCH(); + #endif } op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res)) { @@ -4464,6 +4432,36 @@ dummy_func( assert(oparg >= 2); } + inst(INSTRUMENTED_LINE, ( -- )) { + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyCode_CODE(code))].original_opcode; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + inst(INSTRUMENTED_INSTRUCTION, ( -- )) { int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); @@ -4488,7 +4486,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { _PyStackRef cond = POP(); - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -4499,7 +4497,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { _PyStackRef cond = POP(); - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -4611,8 +4609,8 @@ dummy_func( #endif } - tier2 op(_EXIT_TRACE, (--)) { - _PyExitData *exit = ¤t_executor->exits[oparg]; + tier2 op(_EXIT_TRACE, (exit_p/4 --)) { + _PyExitData *exit = (_PyExitData *)exit_p; PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) @@ -4621,11 +4619,15 @@ dummy_func( printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - oparg, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); } #endif + if (exit->executor && !exit->executor->vm_data.valid) { + exit->temperature = initial_temperature_backoff_counter(); + Py_CLEAR(exit->executor); + } if (exit->executor == NULL) { _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { @@ -4696,9 +4698,9 @@ dummy_func( exe->count++; } - tier2 op(_DYNAMIC_EXIT, (--)) { + tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { tstate->previous_executor = (PyObject *)current_executor; - _PyExitData *exit = (_PyExitData *)¤t_executor->exits[oparg]; + _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); @@ -4706,7 +4708,7 @@ dummy_func( printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - oparg, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); } @@ -4745,7 +4747,7 @@ dummy_func( #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif - DEOPT_IF(!((_PyExecutorObject *)executor)->vm_data.valid); + assert(((_PyExecutorObject *)executor)->vm_data.valid); } tier2 op(_FATAL_ERROR, (--)) { diff --git a/Python/ceval.c b/Python/ceval.c index a240ed4321f7ee..c0074c45b27111 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -245,9 +245,6 @@ static void monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, - PyObject *, PyObject *, PyObject *); -static PyObject * import_from(PyThreadState *, PyObject *, PyObject *); static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg); static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); static _PyInterpreterFrame * @@ -730,15 +727,6 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) * so consume 3 units of C stack */ #define PY_EVAL_C_STACK_UNITS 2 -#if defined(_MSC_VER) && defined(_Py_USING_PGO) -/* gh-111786: _PyEval_EvalFrameDefault is too large to optimize for speed with - PGO on MSVC. Disable that optimization temporarily. If this is fixed - upstream, we should gate this on the version of MSVC. - */ -# pragma optimize("t", off) -/* This setting is reversed below following _PyEval_EvalFrameDefault */ -#endif - PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { @@ -773,7 +761,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif entry_frame.f_executable = Py_None; entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; - entry_frame.stacktop = 0; + entry_frame.stackpointer = entry_frame.localsplus; entry_frame.owner = FRAME_OWNED_BY_CSTACK; entry_frame.return_offset = 0; /* Push frame */ @@ -847,46 +835,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #include "generated_cases.c.h" - /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, - * because it needs to capture frame->instr_ptr before it is updated, - * as happens in the standard instruction prologue. - */ -#if USE_COMPUTED_GOTOS - TARGET_INSTRUMENTED_LINE: -#else - case INSTRUMENTED_LINE: -#endif - { - _Py_CODEUNIT *prev = frame->instr_ptr; - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - original_opcode = code->_co_monitoring->lines[(int)(here - _PyCode_CODE(code))].original_opcode; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, here, prev); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->instr_ptr; - if (next_instr != here) { - DISPATCH(); - } - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - /* Prevent the underlying instruction from specializing - * and overwriting the instrumentation. */ - PAUSE_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - } - #if USE_COMPUTED_GOTOS _unknown_opcode: @@ -1158,7 +1106,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int # pragma GCC diagnostic pop #elif defined(_MSC_VER) /* MS_WINDOWS */ # pragma warning(pop) -# pragma optimize("", on) #endif static void @@ -2535,6 +2482,7 @@ _PyEval_GetBuiltinId(_Py_Identifier *name) PyObject * PyEval_GetLocals(void) { + // We need to return a borrowed reference here, so some tricks are needed PyThreadState *tstate = _PyThreadState_GET(); _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); if (current_frame == NULL) { @@ -2542,7 +2490,37 @@ PyEval_GetLocals(void) return NULL; } - PyObject *locals = _PyEval_GetFrameLocals(); + // Be aware that this returns a new reference + PyObject *locals = _PyFrame_GetLocals(current_frame); + + if (locals == NULL) { + return NULL; + } + + if (PyFrameLocalsProxy_Check(locals)) { + PyFrameObject *f = _PyFrame_GetFrameObject(current_frame); + PyObject *ret = f->f_locals_cache; + if (ret == NULL) { + PyObject *ret = PyDict_New(); + if (ret == NULL) { + Py_DECREF(locals); + return NULL; + } + f->f_locals_cache = ret; + } + if (PyDict_Update(ret, locals) < 0) { + // At this point, if the cache dict is broken, it will stay broken, as + // trying to clean it up or replace it will just cause other problems + ret = NULL; + } + Py_DECREF(locals); + return ret; + } + + assert(PyMapping_Check(locals)); + assert(Py_REFCNT(locals) > 1); + Py_DECREF(locals); + return locals; } @@ -2706,8 +2684,8 @@ _PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) return 1; } -static PyObject * -import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, +PyObject * +_PyEval_ImportName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name, PyObject *fromlist, PyObject *level) { PyObject *import_func; @@ -2745,8 +2723,8 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, return res; } -static PyObject * -import_from(PyThreadState *tstate, PyObject *v, PyObject *name) +PyObject * +_PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) { PyObject *x; PyObject *fullmodname, *pkgname, *pkgpath, *pkgname_or_unknown, *errmsg; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 5617504a495686..0b45caba0d49ff 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -901,6 +901,18 @@ unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) #endif } +static void +clear_pending_handling_thread(struct _pending_calls *pending) +{ +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&pending->mutex); + pending->handling_thread = NULL; + PyMutex_Unlock(&pending->mutex); +#else + pending->handling_thread = NULL; +#endif +} + static int make_pending_calls(PyThreadState *tstate) { @@ -933,7 +945,7 @@ make_pending_calls(PyThreadState *tstate) int32_t npending; if (_make_pending_calls(pending, &npending) != 0) { - pending->handling_thread = NULL; + clear_pending_handling_thread(pending); /* There might not be more calls to make, but we play it safe. */ signal_pending_calls(tstate, interp); return -1; @@ -945,7 +957,7 @@ make_pending_calls(PyThreadState *tstate) if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) { if (_make_pending_calls(pending_main, &npending) != 0) { - pending->handling_thread = NULL; + clear_pending_handling_thread(pending); /* There might not be more calls to make, but we play it safe. */ signal_pending_calls(tstate, interp); return -1; @@ -956,7 +968,7 @@ make_pending_calls(PyThreadState *tstate) } } - pending->handling_thread = NULL; + clear_pending_handling_thread(pending); return 0; } @@ -991,12 +1003,34 @@ _Py_FinishPendingCalls(PyThreadState *tstate) assert(PyGILState_Check()); assert(_PyThreadState_CheckConsistency(tstate)); - if (make_pending_calls(tstate) < 0) { - PyObject *exc = _PyErr_GetRaisedException(tstate); - PyErr_BadInternalCall(); - _PyErr_ChainExceptions1(exc); - _PyErr_Print(tstate); - } + struct _pending_calls *pending = &tstate->interp->ceval.pending; + struct _pending_calls *pending_main = + _Py_IsMainThread() && _Py_IsMainInterpreter(tstate->interp) + ? &_PyRuntime.ceval.pending_mainthread + : NULL; + /* make_pending_calls() may return early without making all pending + calls, so we keep trying until we're actually done. */ + int32_t npending; +#ifndef NDEBUG + int32_t npending_prev = INT32_MAX; +#endif + do { + if (make_pending_calls(tstate) < 0) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyErr_BadInternalCall(); + _PyErr_ChainExceptions1(exc); + _PyErr_Print(tstate); + } + + npending = _Py_atomic_load_int32_relaxed(&pending->npending); + if (pending_main != NULL) { + npending += _Py_atomic_load_int32_relaxed(&pending_main->npending); + } +#ifndef NDEBUG + assert(npending_prev > npending); + npending_prev = npending; +#endif + } while (npending > 0); } int diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 595b72bfaf9613..60efe3d78ff22c 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -402,7 +402,10 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { /* There's no STORE_IP(), it's inlined by the code generator. */ #define LOAD_SP() \ -stack_pointer = _PyFrame_GetStackPointer(frame); +stack_pointer = _PyFrame_GetStackPointer(frame) + +#define SAVE_SP() \ +_PyFrame_SetStackPointer(frame, stack_pointer) /* Tier-switching macros. */ diff --git a/Python/compile.c b/Python/compile.c index 1d6b54d411daf1..02b5345cedd0a3 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -71,9 +71,26 @@ ((C)->c_flags.cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \ && ((C)->u->u_ste->ste_type == ModuleBlock)) +struct compiler; + +typedef _PyInstruction instruction; +typedef _PyInstructionSequence instr_sequence; + +static instr_sequence *compiler_instr_sequence(struct compiler *c); +static int compiler_future_features(struct compiler *c); +static struct symtable *compiler_symtable(struct compiler *c); +static PySTEntryObject *compiler_symtable_entry(struct compiler *c); + +#define INSTR_SEQUENCE(C) compiler_instr_sequence(C) +#define FUTURE_FEATURES(C) compiler_future_features(C) +#define SYMTABLE(C) compiler_symtable(C) +#define SYMTABLE_ENTRY(C) compiler_symtable_entry(C) + typedef _Py_SourceLocation location; typedef struct _PyCfgBuilder cfg_builder; +static PyObject *compiler_maybe_mangle(struct compiler *c, PyObject *name); + #define LOCATION(LNO, END_LNO, COL, END_COL) \ ((const _Py_SourceLocation){(LNO), (END_LNO), (COL), (END_COL)}) @@ -129,12 +146,6 @@ enum { }; -typedef _PyInstruction instruction; -typedef _PyInstructionSequence instr_sequence; - -#define INITIAL_INSTR_SEQUENCE_SIZE 100 -#define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 - static const int compare_masks[] = { [Py_LT] = COMPARISON_LESS_THAN, [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, @@ -254,8 +265,6 @@ struct compiler { */ }; -#define INSTR_SEQUENCE(C) ((C)->u->u_instr_sequence) - typedef struct { // A list of strings corresponding to name captures. It is used to track: @@ -313,7 +322,6 @@ static int compiler_call_helper(struct compiler *c, location loc, asdl_keyword_seq *keywords); static int compiler_try_except(struct compiler *, stmt_ty); static int compiler_try_star_except(struct compiler *, stmt_ty); -static int compiler_set_qualname(struct compiler *); static int compiler_sync_comprehension_generator( struct compiler *c, location loc, @@ -554,8 +562,8 @@ compiler_unit_free(struct compiler_unit *u) PyMem_Free(u); } -static struct compiler_unit * -get_class_compiler_unit(struct compiler *c) +static int +compiler_add_static_attribute_to_class(struct compiler *c, PyObject *attr) { Py_ssize_t stack_size = PyList_GET_SIZE(c->c_stack); for (Py_ssize_t i = stack_size - 1; i >= 0; i--) { @@ -564,10 +572,12 @@ get_class_compiler_unit(struct compiler *c) capsule, CAPSULE_NAME); assert(u); if (u->u_scope_type == COMPILER_SCOPE_CLASS) { - return u; + assert(u->u_static_attributes); + RETURN_IF_ERROR(PySet_Add(u->u_static_attributes, attr)); + break; } } - return NULL; + return SUCCESS; } static int @@ -655,54 +665,6 @@ compiler_set_qualname(struct compiler *c) return SUCCESS; } -int -_PyCompile_OpcodeIsValid(int opcode) -{ - return IS_VALID_OPCODE(opcode); -} - -int -_PyCompile_OpcodeHasArg(int opcode) -{ - return OPCODE_HAS_ARG(opcode); -} - -int -_PyCompile_OpcodeHasConst(int opcode) -{ - return OPCODE_HAS_CONST(opcode); -} - -int -_PyCompile_OpcodeHasName(int opcode) -{ - return OPCODE_HAS_NAME(opcode); -} - -int -_PyCompile_OpcodeHasJump(int opcode) -{ - return OPCODE_HAS_JUMP(opcode); -} - -int -_PyCompile_OpcodeHasFree(int opcode) -{ - return OPCODE_HAS_FREE(opcode); -} - -int -_PyCompile_OpcodeHasLocal(int opcode) -{ - return OPCODE_HAS_LOCAL(opcode); -} - -int -_PyCompile_OpcodeHasExc(int opcode) -{ - return IS_BLOCK_PUSH_OPCODE(opcode); -} - static int codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { @@ -737,9 +699,11 @@ dict_add_o(PyObject *dict, PyObject *o) return arg; } -// Merge const *o* recursively and return constant key object. +/* Merge const *o* and return constant key object. + * If recursive, insert all elements if o is a tuple or frozen set. + */ static PyObject* -merge_consts_recursive(PyObject *const_cache, PyObject *o) +const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive) { assert(PyDict_CheckExact(const_cache)); // None and Ellipsis are immortal objects, and key is the singleton. @@ -763,6 +727,10 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) } Py_DECREF(t); + if (!recursive) { + return key; + } + // We registered o in const_cache. // When o is a tuple or frozenset, we want to merge its // items too. @@ -770,7 +738,7 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) Py_ssize_t len = PyTuple_GET_SIZE(o); for (Py_ssize_t i = 0; i < len; i++) { PyObject *item = PyTuple_GET_ITEM(o, i); - PyObject *u = merge_consts_recursive(const_cache, item); + PyObject *u = const_cache_insert(const_cache, item, recursive); if (u == NULL) { Py_DECREF(key); return NULL; @@ -812,7 +780,7 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) PyObject *item; Py_hash_t hash; while (_PySet_NextEntry(o, &pos, &item, &hash)) { - PyObject *k = merge_consts_recursive(const_cache, item); + PyObject *k = const_cache_insert(const_cache, item, recursive); if (k == NULL) { Py_DECREF(tuple); Py_DECREF(key); @@ -846,39 +814,44 @@ merge_consts_recursive(PyObject *const_cache, PyObject *o) return key; } +static PyObject* +merge_consts_recursive(PyObject *const_cache, PyObject *o) +{ + return const_cache_insert(const_cache, o, true); +} + static Py_ssize_t -compiler_add_const(PyObject *const_cache, struct compiler_unit *u, PyObject *o) +compiler_add_const(struct compiler *c, PyObject *o) { - assert(PyDict_CheckExact(const_cache)); - PyObject *key = merge_consts_recursive(const_cache, o); + PyObject *key = merge_consts_recursive(c->c_const_cache, o); if (key == NULL) { return ERROR; } - Py_ssize_t arg = dict_add_o(u->u_metadata.u_consts, key); + Py_ssize_t arg = dict_add_o(c->u->u_metadata.u_consts, key); Py_DECREF(key); return arg; } static int -compiler_addop_load_const(PyObject *const_cache, struct compiler_unit *u, location loc, PyObject *o) +compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) { - Py_ssize_t arg = compiler_add_const(const_cache, u, o); + Py_ssize_t arg = compiler_add_const(c, o); if (arg < 0) { return ERROR; } - return codegen_addop_i(u->u_instr_sequence, LOAD_CONST, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), LOAD_CONST, arg, loc); } static int -compiler_addop_o(struct compiler_unit *u, location loc, +compiler_addop_o(struct compiler *c, location loc, int opcode, PyObject *dict, PyObject *o) { Py_ssize_t arg = dict_add_o(dict, o); if (arg < 0) { return ERROR; } - return codegen_addop_i(u->u_instr_sequence, opcode, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); } #define LOAD_METHOD -1 @@ -887,10 +860,10 @@ compiler_addop_o(struct compiler_unit *u, location loc, #define LOAD_ZERO_SUPER_METHOD -4 static int -compiler_addop_name(struct compiler_unit *u, location loc, +compiler_addop_name(struct compiler *c, location loc, int opcode, PyObject *dict, PyObject *o) { - PyObject *mangled = _Py_MaybeMangle(u->u_private, u->u_ste, o); + PyObject *mangled = compiler_maybe_mangle(c, o); if (!mangled) { return ERROR; } @@ -925,7 +898,7 @@ compiler_addop_name(struct compiler_unit *u, location loc, arg <<= 2; arg |= 1; } - return codegen_addop_i(u->u_instr_sequence, opcode, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); } /* Add an opcode with an integer argument */ @@ -968,7 +941,7 @@ codegen_addop_j(instr_sequence *seq, location loc, #define ADDOP_IN_SCOPE(C, LOC, OP) RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) #define ADDOP_LOAD_CONST(C, LOC, O) \ - RETURN_IF_ERROR(compiler_addop_load_const((C)->c_const_cache, (C)->u, (LOC), (O))) + RETURN_IF_ERROR(compiler_addop_load_const((C), (LOC), (O))) /* Same as ADDOP_LOAD_CONST, but steals a reference. */ #define ADDOP_LOAD_CONST_NEW(C, LOC, O) { \ @@ -976,7 +949,7 @@ codegen_addop_j(instr_sequence *seq, location loc, if (__new_const == NULL) { \ return ERROR; \ } \ - if (compiler_addop_load_const((C)->c_const_cache, (C)->u, (LOC), __new_const) < 0) { \ + if (compiler_addop_load_const((C), (LOC), __new_const) < 0) { \ Py_DECREF(__new_const); \ return ERROR; \ } \ @@ -985,7 +958,7 @@ codegen_addop_j(instr_sequence *seq, location loc, #define ADDOP_N(C, LOC, OP, O, TYPE) { \ assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \ - if (compiler_addop_o((C)->u, (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O)) < 0) { \ + if (compiler_addop_o((C), (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O)) < 0) { \ Py_DECREF((O)); \ return ERROR; \ } \ @@ -993,7 +966,7 @@ codegen_addop_j(instr_sequence *seq, location loc, } #define ADDOP_NAME(C, LOC, OP, O, TYPE) \ - RETURN_IF_ERROR(compiler_addop_name((C)->u, (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O))) + RETURN_IF_ERROR(compiler_addop_name((C), (LOC), (OP), (C)->u->u_metadata.u_ ## TYPE, (O))) #define ADDOP_I(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) @@ -1052,8 +1025,8 @@ codegen_addop_j(instr_sequence *seq, location loc, static int -compiler_enter_scope(struct compiler *c, identifier name, - int scope_type, void *key, int lineno) +compiler_enter_scope(struct compiler *c, identifier name, int scope_type, + void *key, int lineno, PyObject *private) { location loc = LOCATION(lineno, lineno, 0, 0); @@ -1132,7 +1105,6 @@ compiler_enter_scope(struct compiler *c, identifier name, return ERROR; } - u->u_private = NULL; u->u_deferred_annotations = NULL; if (scope_type == COMPILER_SCOPE_CLASS) { u->u_static_attributes = PySet_New(0); @@ -1146,6 +1118,10 @@ compiler_enter_scope(struct compiler *c, identifier name, } u->u_instr_sequence = (instr_sequence*)_PyInstructionSequence_New(); + if (!u->u_instr_sequence) { + compiler_unit_free(u); + return ERROR; + } /* Push the old compiler_unit on the stack. */ if (c->u) { @@ -1156,8 +1132,13 @@ compiler_enter_scope(struct compiler *c, identifier name, return ERROR; } Py_DECREF(capsule); - u->u_private = Py_XNewRef(c->u->u_private); + if (private == NULL) { + private = c->u->u_private; + } } + + u->u_private = Py_XNewRef(private); + c->u = u; c->c_nestlevel++; @@ -1170,9 +1151,6 @@ compiler_enter_scope(struct compiler *c, identifier name, } ADDOP_I(c, loc, RESUME, RESUME_AT_FUNC_START); - if (u->u_scope_type == COMPILER_SCOPE_MODULE) { - loc.lineno = -1; - } return SUCCESS; } @@ -1436,7 +1414,7 @@ compiler_setup_annotations_scope(struct compiler *c, location loc, void *key, PyObject *name) { if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, - key, loc.lineno) == -1) { + key, loc.lineno, NULL) == -1) { return ERROR; } c->u->u_metadata.u_posonlyargcount = 1; @@ -1478,19 +1456,10 @@ compiler_leave_annotations_scope(struct compiler *c, location loc, static int compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) { - - /* Set current line number to the line number of first statement. - This way line number for SETUP_ANNOTATIONS will always - coincide with the line number of first "real" statement in module. - If body is empty, then lineno will be set later in optimize_and_assemble. */ - if (c->u->u_scope_type == COMPILER_SCOPE_MODULE && asdl_seq_LEN(stmts)) { - stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); - loc = LOC(st); - } /* If from __future__ import annotations is active, * every annotated class and module should have __annotations__. * Else __annotate__ is created when necessary. */ - if ((c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && c->u->u_ste->ste_annotations_used) { + if ((FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && SYMTABLE_ENTRY(c)->ste_annotations_used) { ADDOP(c, loc, SETUP_ANNOTATIONS); } if (!asdl_seq_LEN(stmts)) { @@ -1522,7 +1491,7 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) // If there are annotations and the future import is not on, we // collect the annotations in a separate pass and generate an // __annotate__ function. See PEP 649. - if (!(c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && + if (!(FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && c->u->u_deferred_annotations != NULL) { // It's possible that ste_annotations_block is set but @@ -1530,11 +1499,12 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) // set if there are only non-simple annotations (i.e., annotations // for attributes, subscripts, or parenthesized names). However, the // reverse should not be possible. - assert(c->u->u_ste->ste_annotation_block != NULL); + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + assert(ste->ste_annotation_block != NULL); PyObject *deferred_anno = Py_NewRef(c->u->u_deferred_annotations); - void *key = (void *)((uintptr_t)c->u->u_ste->ste_id + 1); + void *key = (void *)((uintptr_t)ste->ste_id + 1); if (compiler_setup_annotations_scope(c, loc, key, - c->u->u_ste->ste_annotation_block->ste_name) == -1) { + ste->ste_annotation_block->ste_name) == -1) { Py_DECREF(deferred_anno); return ERROR; } @@ -1563,31 +1533,51 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) return SUCCESS; } +static location +start_location(asdl_stmt_seq *stmts) +{ + if (asdl_seq_LEN(stmts) > 0) { + /* Set current line number to the line number of first statement. + * This way line number for SETUP_ANNOTATIONS will always + * coincide with the line number of first "real" statement in module. + * If body is empty, then lineno will be set later in optimize_and_assemble. + */ + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); + return LOC(st); + } + return LOCATION(1, 1, 0, 0); +} + static int compiler_codegen(struct compiler *c, mod_ty mod) { - location loc = LOCATION(1, 1, 0, 0); + assert(c->u->u_scope_type == COMPILER_SCOPE_MODULE); switch (mod->kind) { - case Module_kind: - if (compiler_body(c, loc, mod->v.Module.body) < 0) { + case Module_kind: { + asdl_stmt_seq *stmts = mod->v.Module.body; + if (compiler_body(c, start_location(stmts), stmts) < 0) { return ERROR; } break; - case Interactive_kind: + } + case Interactive_kind: { c->c_interactive = 1; - if (compiler_body(c, loc, mod->v.Interactive.body) < 0) { + asdl_stmt_seq *stmts = mod->v.Interactive.body; + if (compiler_body(c, start_location(stmts), stmts) < 0) { return ERROR; } break; - case Expression_kind: + } + case Expression_kind: { VISIT(c, expr, mod->v.Expression.body); break; - default: + } + default: { PyErr_Format(PyExc_SystemError, "module kind %d should not be possible", mod->kind); return ERROR; - } + }} return SUCCESS; } @@ -1597,7 +1587,7 @@ compiler_enter_anonymous_scope(struct compiler* c, mod_ty mod) _Py_DECLARE_STR(anon_module, ""); RETURN_IF_ERROR( compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1)); + mod, 1, NULL)); return SUCCESS; } @@ -1618,36 +1608,33 @@ compiler_mod(struct compiler *c, mod_ty mod) return co; } -/* The test for LOCAL must come before the test for FREE in order to - handle classes where name is both local and free. The local var is - a method and the free var is a free var referenced within a method. -*/ - static int -get_ref_type(struct compiler *c, PyObject *name) +compiler_get_ref_type(struct compiler *c, PyObject *name) { - int scope; if (c->u->u_scope_type == COMPILER_SCOPE_CLASS && (_PyUnicode_EqualToASCIIString(name, "__class__") || _PyUnicode_EqualToASCIIString(name, "__classdict__"))) { return CELL; } - scope = _PyST_GetScope(c->u->u_ste, name); + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + int scope = _PyST_GetScope(ste, name); if (scope == 0) { PyErr_Format(PyExc_SystemError, "_PyST_GetScope(name=%R) failed: " "unknown scope in unit %S (%R); " - "symbols: %R; locals: %R; globals: %R", + "symbols: %R; locals: %R; " + "globals: %R", name, - c->u->u_metadata.u_name, c->u->u_ste->ste_id, - c->u->u_ste->ste_symbols, c->u->u_metadata.u_varnames, c->u->u_metadata.u_names); + c->u->u_metadata.u_name, ste->ste_id, + ste->ste_symbols, c->u->u_metadata.u_varnames, + c->u->u_metadata.u_names); return ERROR; } return scope; } static int -compiler_lookup_arg(PyObject *dict, PyObject *name) +dict_lookup_arg(PyObject *dict, PyObject *name) { PyObject *v = PyDict_GetItemWithError(dict, name); if (v == NULL) { @@ -1656,6 +1643,45 @@ compiler_lookup_arg(PyObject *dict, PyObject *name) return PyLong_AS_LONG(v); } +static int +compiler_lookup_arg(struct compiler *c, PyCodeObject *co, PyObject *name) +{ + /* Special case: If a class contains a method with a + * free variable that has the same name as a method, + * the name will be considered free *and* local in the + * class. It should be handled by the closure, as + * well as by the normal name lookup logic. + */ + int reftype = compiler_get_ref_type(c, name); + if (reftype == -1) { + return ERROR; + } + int arg; + if (reftype == CELL) { + arg = dict_lookup_arg(c->u->u_metadata.u_cellvars, name); + } + else { + arg = dict_lookup_arg(c->u->u_metadata.u_freevars, name); + } + if (arg == -1) { + PyObject *freevars = _PyCode_GetFreevars(co); + if (freevars == NULL) { + PyErr_Clear(); + } + PyErr_Format(PyExc_SystemError, + "compiler_lookup_arg(name=%R) with reftype=%d failed in %S; " + "freevars of code %S: %R", + name, + reftype, + c->u->u_metadata.u_name, + co->co_name, + freevars); + Py_DECREF(freevars); + return ERROR; + } + return arg; +} + static int compiler_make_closure(struct compiler *c, location loc, PyCodeObject *co, Py_ssize_t flags) @@ -1667,40 +1693,8 @@ compiler_make_closure(struct compiler *c, location loc, LOAD_DEREF but LOAD_CLOSURE is needed. */ PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - - /* Special case: If a class contains a method with a - free variable that has the same name as a method, - the name will be considered free *and* local in the - class. It should be handled by the closure, as - well as by the normal name lookup logic. - */ - int reftype = get_ref_type(c, name); - if (reftype == -1) { - return ERROR; - } - int arg; - if (reftype == CELL) { - arg = compiler_lookup_arg(c->u->u_metadata.u_cellvars, name); - } - else { - arg = compiler_lookup_arg(c->u->u_metadata.u_freevars, name); - } - if (arg == -1) { - PyObject *freevars = _PyCode_GetFreevars(co); - if (freevars == NULL) { - PyErr_Clear(); - } - PyErr_Format(PyExc_SystemError, - "compiler_lookup_arg(name=%R) with reftype=%d failed in %S; " - "freevars of code %S: %R", - name, - reftype, - c->u->u_metadata.u_name, - co->co_name, - freevars); - Py_DECREF(freevars); - return ERROR; - } + int arg = compiler_lookup_arg(c, co, name); + RETURN_IF_ERROR(arg); ADDOP_I(c, loc, LOAD_CLOSURE, arg); } flags |= MAKE_FUNCTION_CLOSURE; @@ -1765,42 +1759,24 @@ compiler_kwonlydefaults(struct compiler *c, location loc, */ int i; PyObject *keys = NULL; - + int default_count = 0; for (i = 0; i < asdl_seq_LEN(kwonlyargs); i++) { arg_ty arg = asdl_seq_GET(kwonlyargs, i); expr_ty default_ = asdl_seq_GET(kw_defaults, i); if (default_) { - PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, arg->arg); + default_count++; + PyObject *mangled = compiler_maybe_mangle(c, arg->arg); if (!mangled) { goto error; } - if (keys == NULL) { - keys = PyList_New(1); - if (keys == NULL) { - Py_DECREF(mangled); - return ERROR; - } - PyList_SET_ITEM(keys, 0, mangled); - } - else { - int res = PyList_Append(keys, mangled); - Py_DECREF(mangled); - if (res == -1) { - goto error; - } - } + ADDOP_LOAD_CONST_NEW(c, loc, mangled); if (compiler_visit_expr(c, default_) < 0) { goto error; } } } - if (keys != NULL) { - Py_ssize_t default_count = PyList_GET_SIZE(keys); - PyObject *keys_tuple = PyList_AsTuple(keys); - Py_DECREF(keys); - ADDOP_LOAD_CONST_NEW(c, loc, keys_tuple); - ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, default_count); - assert(default_count > 0); + if (default_count) { + ADDOP_I(c, loc, BUILD_MAP, default_count); return 1; } else { @@ -1827,14 +1803,14 @@ compiler_argannotation(struct compiler *c, identifier id, if (!annotation) { return SUCCESS; } - PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, id); + PyObject *mangled = compiler_maybe_mangle(c, id); if (!mangled) { return ERROR; } ADDOP_LOAD_CONST(c, loc, mangled); Py_DECREF(mangled); - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { + if (FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) { VISIT(c, annexpr, annotation); } else { @@ -1916,7 +1892,7 @@ compiler_annotations(struct compiler *c, location loc, Py_ssize_t annotations_len = 0; PySTEntryObject *ste; - if (_PySymtable_LookupOptional(c->c_st, args, &ste) < 0) { + if (_PySymtable_LookupOptional(SYMTABLE(c), args, &ste) < 0) { return ERROR; } assert(ste != NULL); @@ -1978,55 +1954,6 @@ compiler_default_arguments(struct compiler *c, location loc, return funcflags; } -static bool -forbidden_name(struct compiler *c, location loc, identifier name, - expr_context_ty ctx) -{ - if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) { - compiler_error(c, loc, "cannot assign to __debug__"); - return true; - } - if (ctx == Del && _PyUnicode_EqualToASCIIString(name, "__debug__")) { - compiler_error(c, loc, "cannot delete __debug__"); - return true; - } - return false; -} - -static int -compiler_check_debug_one_arg(struct compiler *c, arg_ty arg) -{ - if (arg != NULL) { - if (forbidden_name(c, LOC(arg), arg->arg, Store)) { - return ERROR; - } - } - return SUCCESS; -} - -static int -compiler_check_debug_args_seq(struct compiler *c, asdl_arg_seq *args) -{ - if (args != NULL) { - for (Py_ssize_t i = 0, n = asdl_seq_LEN(args); i < n; i++) { - RETURN_IF_ERROR( - compiler_check_debug_one_arg(c, asdl_seq_GET(args, i))); - } - } - return SUCCESS; -} - -static int -compiler_check_debug_args(struct compiler *c, arguments_ty args) -{ - RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->posonlyargs)); - RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->args)); - RETURN_IF_ERROR(compiler_check_debug_one_arg(c, args->vararg)); - RETURN_IF_ERROR(compiler_check_debug_args_seq(c, args->kwonlyargs)); - RETURN_IF_ERROR(compiler_check_debug_one_arg(c, args->kwarg)); - return SUCCESS; -} - static int wrap_in_stopiteration_handler(struct compiler *c) { @@ -2051,8 +1978,9 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, identifier name, void *key, bool allow_starred) { - if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, - key, e->lineno) == -1) { + PyObject *defaults = PyTuple_Pack(1, _PyLong_GetOne()); + ADDOP_LOAD_CONST_NEW(c, LOC(e), defaults); + if (compiler_setup_annotations_scope(c, LOC(e), key, name) == -1) { return ERROR; } if (allow_starred && e->kind == Starred_kind) { @@ -2068,7 +1996,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, LOC(e), co, 0) < 0) { + if (compiler_make_closure(c, LOC(e), co, MAKE_FUNCTION_DEFAULTS) < 0) { Py_DECREF(co); return ERROR; } @@ -2197,7 +2125,7 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f } RETURN_IF_ERROR( - compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno)); + compiler_enter_scope(c, name, scope_type, (void *)s, firstlineno, NULL)); Py_ssize_t first_instr = 0; PyObject *docstring = _PyAST_GetDocString(body); @@ -2215,7 +2143,7 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f docstring = NULL; } } - if (compiler_add_const(c->c_const_cache, c->u, docstring ? docstring : Py_None) < 0) { + if (compiler_add_const(c, docstring ? docstring : Py_None) < 0) { Py_XDECREF(docstring); compiler_exit_scope(c); return ERROR; @@ -2228,7 +2156,8 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f NEW_JUMP_TARGET_LABEL(c, start); USE_LABEL(c, start); - bool add_stopiteration_handler = c->u->u_ste->ste_coroutine || c->u->u_ste->ste_generator; + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + bool add_stopiteration_handler = ste->ste_coroutine || ste->ste_generator; if (add_stopiteration_handler) { /* wrap_in_stopiteration_handler will push a block, so we need to account for that */ RETURN_IF_ERROR( @@ -2290,7 +2219,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) type_params = s->v.FunctionDef.type_params; } - RETURN_IF_ERROR(compiler_check_debug_args(c, args)); RETURN_IF_ERROR(compiler_decorators(c, decos)); firstlineno = s->lineno; @@ -2324,7 +2252,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) return ERROR; } if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, - (void *)type_params, firstlineno) == -1) { + (void *)type_params, firstlineno, NULL) == -1) { Py_DECREF(type_params_name); return ERROR; } @@ -2407,12 +2335,10 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) /* 1. compile the class body into a code object */ RETURN_IF_ERROR( - compiler_enter_scope(c, s->v.ClassDef.name, - COMPILER_SCOPE_CLASS, (void *)s, firstlineno)); + compiler_enter_scope(c, s->v.ClassDef.name, COMPILER_SCOPE_CLASS, + (void *)s, firstlineno, s->v.ClassDef.name)); location loc = LOCATION(firstlineno, firstlineno, 0, 0); - /* use the class name for name mangling */ - Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); /* load (global) __name__ ... */ if (compiler_nameop(c, loc, &_Py_ID(__name__), Load) < 0) { compiler_exit_scope(c); @@ -2441,14 +2367,14 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) return ERROR; } } - if (c->u->u_ste->ste_needs_classdict) { + if (SYMTABLE_ENTRY(c)->ste_needs_classdict) { ADDOP(c, loc, LOAD_LOCALS); // We can't use compiler_nameop here because we need to generate a // STORE_DEREF in a class namespace, and compiler_nameop() won't do // that by default. PyObject *cellvars = c->u->u_metadata.u_cellvars; - if (compiler_addop_o(c->u, loc, STORE_DEREF, cellvars, + if (compiler_addop_o(c, loc, STORE_DEREF, cellvars, &_Py_ID(__classdict__)) < 0) { compiler_exit_scope(c); return ERROR; @@ -2473,9 +2399,9 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) } /* The following code is artificial */ /* Set __classdictcell__ if necessary */ - if (c->u->u_ste->ste_needs_classdict) { + if (SYMTABLE_ENTRY(c)->ste_needs_classdict) { /* Store __classdictcell__ into class namespace */ - int i = compiler_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__classdict__)); + int i = dict_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__classdict__)); if (i < 0) { compiler_exit_scope(c); return ERROR; @@ -2487,9 +2413,9 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) } } /* Return __classcell__ if it is referenced, otherwise return None */ - if (c->u->u_ste->ste_needs_class_closure) { + if (SYMTABLE_ENTRY(c)->ste_needs_class_closure) { /* Store __classcell__ into class namespace & return it */ - int i = compiler_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__class__)); + int i = dict_lookup_arg(c->u->u_metadata.u_cellvars, &_Py_ID(__class__)); if (i < 0) { compiler_exit_scope(c); return ERROR; @@ -2558,12 +2484,11 @@ compiler_class(struct compiler *c, stmt_ty s) return ERROR; } if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, - (void *)type_params, firstlineno) == -1) { + (void *)type_params, firstlineno, s->v.ClassDef.name) == -1) { Py_DECREF(type_params_name); return ERROR; } Py_DECREF(type_params_name); - Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); _Py_DECLARE_STR(type_params, ".type_params"); RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store)); @@ -2642,11 +2567,13 @@ compiler_typealias_body(struct compiler *c, stmt_ty s) { location loc = LOC(s); PyObject *name = s->v.TypeAlias.name->v.Name.id; + PyObject *defaults = PyTuple_Pack(1, _PyLong_GetOne()); + ADDOP_LOAD_CONST_NEW(c, loc, defaults); RETURN_IF_ERROR( - compiler_enter_scope(c, name, COMPILER_SCOPE_FUNCTION, s, loc.lineno)); + compiler_setup_annotations_scope(c, LOC(s), s, name)); /* Make None the first constant, so the evaluate function can't have a docstring. */ - RETURN_IF_ERROR(compiler_add_const(c->c_const_cache, c->u, Py_None)); + RETURN_IF_ERROR(compiler_add_const(c, Py_None)); VISIT_IN_SCOPE(c, expr, s->v.TypeAlias.value); ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); PyCodeObject *co = optimize_and_assemble(c, 0); @@ -2654,7 +2581,7 @@ compiler_typealias_body(struct compiler *c, stmt_ty s) if (co == NULL) { return ERROR; } - if (compiler_make_closure(c, loc, co, 0) < 0) { + if (compiler_make_closure(c, loc, co, MAKE_FUNCTION_DEFAULTS) < 0) { Py_DECREF(co); return ERROR; } @@ -2678,13 +2605,13 @@ compiler_typealias(struct compiler *c, stmt_ty s) return ERROR; } if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, - (void *)type_params, loc.lineno) == -1) { + (void *)type_params, loc.lineno, NULL) == -1) { Py_DECREF(type_params_name); return ERROR; } Py_DECREF(type_params_name); RETURN_IF_ERROR_IN_SCOPE( - c, compiler_addop_load_const(c->c_const_cache, c->u, loc, name) + c, compiler_addop_load_const(c, loc, name) ); RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params)); } @@ -2936,8 +2863,6 @@ compiler_lambda(struct compiler *c, expr_ty e) arguments_ty args = e->v.Lambda.args; assert(e->kind == Lambda_kind); - RETURN_IF_ERROR(compiler_check_debug_args(c, args)); - location loc = LOC(e); funcflags = compiler_default_arguments(c, loc, args); if (funcflags == -1) { @@ -2947,17 +2872,17 @@ compiler_lambda(struct compiler *c, expr_ty e) _Py_DECLARE_STR(anon_lambda, ""); RETURN_IF_ERROR( compiler_enter_scope(c, &_Py_STR(anon_lambda), COMPILER_SCOPE_LAMBDA, - (void *)e, e->lineno)); + (void *)e, e->lineno, NULL)); /* Make None the first constant, so the lambda can't have a docstring. */ - RETURN_IF_ERROR(compiler_add_const(c->c_const_cache, c->u, Py_None)); + RETURN_IF_ERROR(compiler_add_const(c, Py_None)); c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); - if (c->u->u_ste->ste_generator) { + if (SYMTABLE_ENTRY(c)->ste_generator) { co = optimize_and_assemble(c, 0); } else { @@ -3133,12 +3058,12 @@ compiler_return(struct compiler *c, stmt_ty s) location loc = LOC(s); int preserve_tos = ((s->v.Return.value != NULL) && (s->v.Return.value->kind != Constant_kind)); - if (!_PyST_IsFunctionLike(c->u->u_ste)) { + + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + if (!_PyST_IsFunctionLike(ste)) { return compiler_error(c, loc, "'return' outside function"); } - if (s->v.Return.value != NULL && - c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator) - { + if (s->v.Return.value != NULL && ste->ste_coroutine && ste->ste_generator) { return compiler_error(c, loc, "'return' with value in async generator"); } @@ -4088,7 +4013,8 @@ addop_binary(struct compiler *c, location loc, operator_ty binop, static int addop_yield(struct compiler *c, location loc) { - if (c->u->u_ste->ste_generator && c->u->u_ste->ste_coroutine) { + PySTEntryObject *ste = SYMTABLE_ENTRY(c); + if (ste->ste_generator && ste->ste_coroutine) { ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_ASYNC_GEN_WRAP); } ADDOP_I(c, loc, YIELD_VALUE, 0); @@ -4111,18 +4037,14 @@ compiler_nameop(struct compiler *c, location loc, !_PyUnicode_EqualToASCIIString(name, "True") && !_PyUnicode_EqualToASCIIString(name, "False")); - if (forbidden_name(c, loc, name, ctx)) { - return ERROR; - } - - mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, name); + mangled = compiler_maybe_mangle(c, name); if (!mangled) { return ERROR; } op = 0; optype = OP_NAME; - scope = _PyST_GetScope(c->u->u_ste, mangled); + scope = _PyST_GetScope(SYMTABLE_ENTRY(c), mangled); switch (scope) { case FREE: dict = c->u->u_metadata.u_freevars; @@ -4133,7 +4055,7 @@ compiler_nameop(struct compiler *c, location loc, optype = OP_DEREF; break; case LOCAL: - if (_PyST_IsFunctionLike(c->u->u_ste)) { + if (_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { optype = OP_FAST; } else { @@ -4149,7 +4071,7 @@ compiler_nameop(struct compiler *c, location loc, } break; case GLOBAL_IMPLICIT: - if (_PyST_IsFunctionLike(c->u->u_ste)) + if (_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) optype = OP_GLOBAL; break; case GLOBAL_EXPLICIT: @@ -4167,17 +4089,17 @@ compiler_nameop(struct compiler *c, location loc, case OP_DEREF: switch (ctx) { case Load: - if (c->u->u_ste->ste_type == ClassBlock && !c->u->u_in_inlined_comp) { + if (SYMTABLE_ENTRY(c)->ste_type == ClassBlock && !c->u->u_in_inlined_comp) { op = LOAD_FROM_DICT_OR_DEREF; // First load the locals if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { goto error; } } - else if (c->u->u_ste->ste_can_see_class_scope) { + else if (SYMTABLE_ENTRY(c)->ste_can_see_class_scope) { op = LOAD_FROM_DICT_OR_DEREF; // First load the classdict - if (compiler_addop_o(c->u, loc, LOAD_DEREF, + if (compiler_addop_o(c, loc, LOAD_DEREF, c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { goto error; } @@ -4201,10 +4123,10 @@ compiler_nameop(struct compiler *c, location loc, case OP_GLOBAL: switch (ctx) { case Load: - if (c->u->u_ste->ste_can_see_class_scope && scope == GLOBAL_IMPLICIT) { + if (SYMTABLE_ENTRY(c)->ste_can_see_class_scope && scope == GLOBAL_IMPLICIT) { op = LOAD_FROM_DICT_OR_GLOBALS; // First load the classdict - if (compiler_addop_o(c->u, loc, LOAD_DEREF, + if (compiler_addop_o(c, loc, LOAD_DEREF, c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { goto error; } @@ -4219,7 +4141,7 @@ compiler_nameop(struct compiler *c, location loc, case OP_NAME: switch (ctx) { case Load: - op = (c->u->u_ste->ste_type == ClassBlock + op = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock && c->u->u_in_inlined_comp) ? LOAD_GLOBAL : LOAD_NAME; @@ -4461,25 +4383,8 @@ static int compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end) { Py_ssize_t i, n = end - begin; - PyObject *keys, *key; int big = n*2 > STACK_USE_GUIDELINE; location loc = LOC(e); - if (n > 1 && !big && are_all_items_const(e->v.Dict.keys, begin, end)) { - for (i = begin; i < end; i++) { - VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i)); - } - keys = PyTuple_New(n); - if (keys == NULL) { - return SUCCESS; - } - for (i = begin; i < end; i++) { - key = ((expr_ty)asdl_seq_GET(e->v.Dict.keys, i))->v.Constant.value; - PyTuple_SET_ITEM(keys, i - begin, Py_NewRef(key)); - } - ADDOP_LOAD_CONST_NEW(c, loc, keys); - ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, n); - return SUCCESS; - } if (big) { ADDOP_I(c, loc, BUILD_MAP, 0); } @@ -4724,7 +4629,7 @@ is_import_originated(struct compiler *c, expr_ty e) return 0; } - long flags = _PyST_GetSymbol(c->c_st->st_top, e->v.Name.id); + long flags = _PyST_GetSymbol(SYMTABLE(c)->st_top, e->v.Name.id); return flags & DEF_IMPORT; } @@ -4743,11 +4648,11 @@ can_optimize_super_call(struct compiler *c, expr_ty attr) PyObject *super_name = e->v.Call.func->v.Name.id; // detect statically-visible shadowing of 'super' name - int scope = _PyST_GetScope(c->u->u_ste, super_name); + int scope = _PyST_GetScope(SYMTABLE_ENTRY(c), super_name); if (scope != GLOBAL_IMPLICIT) { return 0; } - scope = _PyST_GetScope(c->c_st->st_top, super_name); + scope = _PyST_GetScope(SYMTABLE(c)->st_top, super_name); if (scope != 0) { return 0; } @@ -4775,7 +4680,7 @@ can_optimize_super_call(struct compiler *c, expr_ty attr) return 0; } // __class__ cell should be available - if (get_ref_type(c, &_Py_ID(__class__)) == FREE) { + if (compiler_get_ref_type(c, &_Py_ID(__class__)) == FREE) { return 1; } return 0; @@ -4797,7 +4702,7 @@ load_args_for_super(struct compiler *c, expr_ty e) { // load __class__ cell PyObject *name = &_Py_ID(__class__); - assert(get_ref_type(c, name) == FREE); + assert(compiler_get_ref_type(c, name) == FREE); RETURN_IF_ERROR(compiler_nameop(c, loc, name, Load)); // load self (first argument) @@ -4920,10 +4825,6 @@ validate_keywords(struct compiler *c, asdl_keyword_seq *keywords) if (key->arg == NULL) { continue; } - location loc = LOC(key); - if (forbidden_name(c, loc, key->arg, Store)) { - return ERROR; - } for (Py_ssize_t j = i + 1; j < nkeywords; j++) { keyword_ty other = ((keyword_ty)asdl_seq_GET(keywords, j)); if (other->arg && !PyUnicode_Compare(key->arg, other->arg)) { @@ -5039,26 +4940,8 @@ compiler_subkwargs(struct compiler *c, location loc, { Py_ssize_t i, n = end - begin; keyword_ty kw; - PyObject *keys, *key; assert(n > 0); int big = n*2 > STACK_USE_GUIDELINE; - if (n > 1 && !big) { - for (i = begin; i < end; i++) { - kw = asdl_seq_GET(keywords, i); - VISIT(c, expr, kw->value); - } - keys = PyTuple_New(n); - if (keys == NULL) { - return ERROR; - } - for (i = begin; i < end; i++) { - key = ((keyword_ty) asdl_seq_GET(keywords, i))->arg; - PyTuple_SET_ITEM(keys, i - begin, Py_NewRef(key)); - } - ADDOP_LOAD_CONST_NEW(c, loc, keys); - ADDOP_I(c, loc, BUILD_CONST_KEY_MAP, n); - return SUCCESS; - } if (big) { ADDOP_I(c, NO_LOCATION, BUILD_MAP, 0); } @@ -5469,7 +5352,7 @@ push_inlined_comprehension_state(struct compiler *c, location loc, PySTEntryObject *entry, inlined_comprehension_state *state) { - int in_class_block = (c->u->u_ste->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; + int in_class_block = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; c->u->u_in_inlined_comp++; // iterate over names bound in the comprehension and ensure we isolate // them from the outer scope as needed @@ -5479,7 +5362,7 @@ push_inlined_comprehension_state(struct compiler *c, location loc, assert(PyLong_Check(v)); long symbol = PyLong_AS_LONG(v); long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; - PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); + PyObject *outv = PyDict_GetItemWithError(SYMTABLE_ENTRY(c)->ste_symbols, k); if (outv == NULL) { if (PyErr_Occurred()) { return ERROR; @@ -5508,7 +5391,7 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // the outer version; we'll restore it after running the // comprehension Py_INCREF(outv); - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { + if (PyDict_SetItem(SYMTABLE_ENTRY(c)->ste_symbols, k, v) < 0) { Py_DECREF(outv); return ERROR; } @@ -5521,7 +5404,7 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // locals handling for names bound in comprehension (DEF_LOCAL | // DEF_NONLOCAL occurs in assignment expression to nonlocal) if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { - if (!_PyST_IsFunctionLike(c->u->u_ste)) { + if (!_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { // non-function scope: override this name to use fast locals PyObject *orig; if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { @@ -5623,7 +5506,7 @@ pop_inlined_comprehension_state(struct compiler *c, location loc, Py_ssize_t pos = 0; if (state.temp_symbols) { while (PyDict_Next(state.temp_symbols, &pos, &k, &v)) { - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v)) { + if (PyDict_SetItem(SYMTABLE_ENTRY(c)->ste_symbols, k, v)) { return ERROR; } } @@ -5690,14 +5573,16 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, PyCodeObject *co = NULL; inlined_comprehension_state inline_state = {NULL, NULL, NULL, NO_LABEL, NO_LABEL}; comprehension_ty outermost; +#ifndef NDEBUG int scope_type = c->u->u_scope_type; int is_top_level_await = IS_TOP_LEVEL_AWAIT(c); - PySTEntryObject *entry = _PySymtable_Lookup(c->c_st, (void *)e); +#endif + PySTEntryObject *entry = _PySymtable_Lookup(SYMTABLE(c), (void *)e); if (entry == NULL) { goto error; } int is_inlined = entry->ste_comp_inlined; - int is_async_generator = entry->ste_coroutine; + int is_async_comprehension = entry->ste_coroutine; location loc = LOC(e); @@ -5712,22 +5597,17 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, } else { if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION, - (void *)e, e->lineno) < 0) - { + (void *)e, e->lineno, NULL) < 0) { goto error; } } Py_CLEAR(entry); - if (is_async_generator && type != COMP_GENEXP && - scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && - scope_type != COMPILER_SCOPE_COMPREHENSION && - !is_top_level_await) - { - compiler_error(c, loc, "asynchronous comprehension outside of " - "an asynchronous function"); - goto error_in_scope; - } + assert (!is_async_comprehension || + type == COMP_GENEXP || + scope_type == COMPILER_SCOPE_ASYNC_FUNCTION || + scope_type == COMPILER_SCOPE_COMPREHENSION || + is_top_level_await); if (type != COMP_GENEXP) { int op; @@ -5792,7 +5672,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADDOP_I(c, loc, CALL, 0); - if (is_async_generator && type != COMP_GENEXP) { + if (is_async_comprehension && type != COMP_GENEXP) { ADDOP_I(c, loc, GET_AWAITABLE, 0); ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); @@ -6129,7 +6009,7 @@ compiler_visit_expr(struct compiler *c, expr_ty e) case DictComp_kind: return compiler_dictcomp(c, e); case Yield_kind: - if (!_PyST_IsFunctionLike(c->u->u_ste)) { + if (!_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { return compiler_error(c, loc, "'yield' outside function"); } if (e->v.Yield.value) { @@ -6141,8 +6021,8 @@ compiler_visit_expr(struct compiler *c, expr_ty e) ADDOP_YIELD(c, loc); break; case YieldFrom_kind: - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - return compiler_error(c, loc, "'yield' outside function"); + if (!_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) { + return compiler_error(c, loc, "'yield from' outside function"); } if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION) { return compiler_error(c, loc, "'yield from' inside async function"); @@ -6153,16 +6033,10 @@ compiler_visit_expr(struct compiler *c, expr_ty e) ADD_YIELD_FROM(c, loc, 0); break; case Await_kind: - if (!IS_TOP_LEVEL_AWAIT(c)){ - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - return compiler_error(c, loc, "'await' outside function"); - } - - if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && - c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION) { - return compiler_error(c, loc, "'await' outside async function"); - } - } + assert(IS_TOP_LEVEL_AWAIT(c) || (_PyST_IsFunctionLike(SYMTABLE_ENTRY(c)) && ( + c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION || + c->u->u_scope_type == COMPILER_SCOPE_COMPREHENSION + ))); VISIT(c, expr, e->v.Await.value); ADDOP_I(c, loc, GET_AWAITABLE, 0); @@ -6194,13 +6068,7 @@ compiler_visit_expr(struct compiler *c, expr_ty e) if (e->v.Attribute.value->kind == Name_kind && _PyUnicode_EqualToASCIIString(e->v.Attribute.value->v.Name.id, "self")) { - struct compiler_unit *class_u = get_class_compiler_unit(c); - if (class_u != NULL) { - assert(class_u->u_scope_type == COMPILER_SCOPE_CLASS); - assert(class_u->u_static_attributes); - RETURN_IF_ERROR( - PySet_Add(class_u->u_static_attributes, e->v.Attribute.attr)); - } + RETURN_IF_ERROR(compiler_add_static_attribute_to_class(c, e->v.Attribute.attr)); } VISIT(c, expr, e->v.Attribute.value); loc = LOC(e); @@ -6210,9 +6078,6 @@ compiler_visit_expr(struct compiler *c, expr_ty e) ADDOP_NAME(c, loc, LOAD_ATTR, e->v.Attribute.attr, names); break; case Store: - if (forbidden_name(c, loc, e->v.Attribute.attr, e->v.Attribute.ctx)) { - return ERROR; - } ADDOP_NAME(c, loc, STORE_ATTR, e->v.Attribute.attr, names); break; case Del: @@ -6347,7 +6212,7 @@ check_annotation(struct compiler *c, stmt_ty s) { /* Annotations of complex targets does not produce anything under annotations future */ - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { + if (FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) { return SUCCESS; } @@ -6394,7 +6259,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) { location loc = LOC(s); expr_ty targ = s->v.AnnAssign.target; - bool future_annotations = c->c_future.ff_features & CO_FUTURE_ANNOTATIONS; + bool future_annotations = FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS; PyObject *mangled; assert(s->kind == AnnAssign_kind); @@ -6406,9 +6271,6 @@ compiler_annassign(struct compiler *c, stmt_ty s) } switch (targ->kind) { case Name_kind: - if (forbidden_name(c, loc, targ->v.Name.id, Store)) { - return ERROR; - } /* If we have a simple name in a module or class, store annotation. */ if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || @@ -6416,7 +6278,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) if (future_annotations) { VISIT(c, annexpr, s->v.AnnAssign.annotation); ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); - mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id); + mangled = compiler_maybe_mangle(c, targ->v.Name.id); ADDOP_LOAD_CONST_NEW(c, loc, mangled); ADDOP(c, loc, STORE_SUBSCR); } @@ -6440,9 +6302,6 @@ compiler_annassign(struct compiler *c, stmt_ty s) } break; case Attribute_kind: - if (forbidden_name(c, loc, targ->v.Attribute.attr, Store)) { - return ERROR; - } if (!s->v.AnnAssign.value && check_ann_expr(c, targ->v.Attribute.value) < 0) { return ERROR; @@ -6706,9 +6565,6 @@ pattern_helper_store_name(struct compiler *c, location loc, ADDOP(c, loc, POP_TOP); return SUCCESS; } - if (forbidden_name(c, loc, n, Store)) { - return ERROR; - } // Can't assign to the same name twice: int duplicate = PySequence_Contains(pc->stores, n); RETURN_IF_ERROR(duplicate); @@ -6866,10 +6722,6 @@ validate_kwd_attrs(struct compiler *c, asdl_identifier_seq *attrs, asdl_pattern_ Py_ssize_t nattrs = asdl_seq_LEN(attrs); for (Py_ssize_t i = 0; i < nattrs; i++) { identifier attr = ((identifier)asdl_seq_GET(attrs, i)); - location loc = LOC((pattern_ty) asdl_seq_GET(patterns, i)); - if (forbidden_name(c, loc, attr, Store)) { - return ERROR; - } for (Py_ssize_t j = i + 1; j < nattrs; j++) { identifier other = ((identifier)asdl_seq_GET(attrs, j)); if (!PyUnicode_Compare(attr, other)) { @@ -7458,19 +7310,47 @@ consts_dict_keys_inorder(PyObject *dict) return consts; } +static PyObject * +compiler_maybe_mangle(struct compiler *c, PyObject *name) +{ + return _Py_MaybeMangle(c->u->u_private, c->u->u_ste, name); +} + +static instr_sequence * +compiler_instr_sequence(struct compiler *c) +{ + return c->u->u_instr_sequence; +} + +static int +compiler_future_features(struct compiler *c) +{ + return c->c_future.ff_features; +} + +static struct symtable * +compiler_symtable(struct compiler *c) +{ + return c->c_st; +} + +static PySTEntryObject * +compiler_symtable_entry(struct compiler *c) +{ + return c->u->u_ste; +} + static int compute_code_flags(struct compiler *c) { - PySTEntryObject *ste = c->u->u_ste; + PySTEntryObject *ste = SYMTABLE_ENTRY(c); int flags = 0; - if (_PyST_IsFunctionLike(c->u->u_ste)) { + if (_PyST_IsFunctionLike(ste)) { flags |= CO_NEWLOCALS | CO_OPTIMIZED; if (ste->ste_nested) flags |= CO_NESTED; if (ste->ste_generator && !ste->ste_coroutine) flags |= CO_GENERATOR; - if (!ste->ste_generator && ste->ste_coroutine) - flags |= CO_COROUTINE; if (ste->ste_generator && ste->ste_coroutine) flags |= CO_ASYNC_GENERATOR; if (ste->ste_varargs) @@ -7479,49 +7359,33 @@ compute_code_flags(struct compiler *c) flags |= CO_VARKEYWORDS; } - /* (Only) inherit compilerflags in PyCF_MASK */ - flags |= (c->c_flags.cf_flags & PyCF_MASK); - - if ((IS_TOP_LEVEL_AWAIT(c)) && - ste->ste_coroutine && - !ste->ste_generator) { + if (ste->ste_coroutine && !ste->ste_generator) { + assert (IS_TOP_LEVEL_AWAIT(c) || _PyST_IsFunctionLike(ste)); flags |= CO_COROUTINE; } + /* (Only) inherit compilerflags in PyCF_MASK */ + flags |= (c->c_flags.cf_flags & PyCF_MASK); + return flags; } -// Merge *obj* with constant cache. -// Unlike merge_consts_recursive(), this function doesn't work recursively. +// Merge *obj* with constant cache, without recursion. int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) { - assert(PyDict_CheckExact(const_cache)); - PyObject *key = _PyCode_ConstantKey(*obj); + PyObject *key = const_cache_insert(const_cache, *obj, false); if (key == NULL) { return ERROR; } - - PyObject *t; - int res = PyDict_SetDefaultRef(const_cache, key, key, &t); - Py_DECREF(key); - if (res < 0) { - return ERROR; - } - if (res == 0) { // inserted: obj is new constant. - Py_DECREF(t); - return SUCCESS; - } - - if (PyTuple_CheckExact(t)) { - PyObject *item = PyTuple_GET_ITEM(t, 1); + if (PyTuple_CheckExact(key)) { + PyObject *item = PyTuple_GET_ITEM(key, 1); Py_SETREF(*obj, Py_NewRef(item)); - Py_DECREF(t); + Py_DECREF(key); } else { - Py_SETREF(*obj, t); + Py_SETREF(*obj, key); } - return SUCCESS; } diff --git a/Python/context.c b/Python/context.c index a3830be17908fe..5cafde4dab9336 100644 --- a/Python/context.c +++ b/Python/context.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_VectorcallTstate() #include "pycore_context.h" +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_gc.h" // _PyObject_GC_MAY_BE_TRACKED() #include "pycore_hamt.h" #include "pycore_initconfig.h" // _PyStatus_OK() @@ -64,16 +65,6 @@ static int contextvar_del(PyContextVar *var); -#ifdef WITH_FREELISTS -static struct _Py_context_freelist * -get_context_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->contexts; -} -#endif - - PyObject * _PyContext_NewHamtForTests(void) { @@ -203,6 +194,7 @@ PyContextVar_Get(PyObject *ovar, PyObject *def, PyObject **val) goto not_found; } +#ifndef Py_GIL_DISABLED if (var->var_cached != NULL && var->var_cached_tsid == ts->id && var->var_cached_tsver == ts->context_ver) @@ -210,6 +202,7 @@ PyContextVar_Get(PyObject *ovar, PyObject *def, PyObject **val) *val = var->var_cached; goto found; } +#endif assert(PyContext_CheckExact(ts->context)); PyHamtObject *vars = ((PyContext *)ts->context)->ctx_vars; @@ -221,9 +214,11 @@ PyContextVar_Get(PyObject *ovar, PyObject *def, PyObject **val) } if (res == 1) { assert(found != NULL); +#ifndef Py_GIL_DISABLED var->var_cached = found; /* borrow */ var->var_cached_tsid = ts->id; var->var_cached_tsver = ts->context_ver; +#endif *val = found; goto found; @@ -339,20 +334,8 @@ class _contextvars.Context "PyContext *" "&PyContext_Type" static inline PyContext * _context_alloc(void) { - PyContext *ctx; -#ifdef WITH_FREELISTS - struct _Py_context_freelist *context_freelist = get_context_freelist(); - if (context_freelist->numfree > 0) { - context_freelist->numfree--; - ctx = context_freelist->items; - context_freelist->items = (PyContext *)ctx->ctx_weakreflist; - OBJECT_STAT_INC(from_freelist); - ctx->ctx_weakreflist = NULL; - _Py_NewReference((PyObject *)ctx); - } - else -#endif - { + PyContext *ctx = _Py_FREELIST_POP(PyContext, contexts); + if (ctx == NULL) { ctx = PyObject_GC_New(PyContext, &PyContext_Type); if (ctx == NULL) { return NULL; @@ -467,19 +450,7 @@ context_tp_dealloc(PyContext *self) } (void)context_tp_clear(self); -#ifdef WITH_FREELISTS - struct _Py_context_freelist *context_freelist = get_context_freelist(); - if (context_freelist->numfree >= 0 && context_freelist->numfree < PyContext_MAXFREELIST) { - context_freelist->numfree++; - self->ctx_weakreflist = (PyObject *)context_freelist->items; - context_freelist->items = self; - OBJECT_STAT_INC(to_freelist); - } - else -#endif - { - Py_TYPE(self)->tp_free(self); - } + _Py_FREELIST_FREE(contexts, self, Py_TYPE(self)->tp_free); } static PyObject * @@ -723,8 +694,10 @@ PyTypeObject PyContext_Type = { static int contextvar_set(PyContextVar *var, PyObject *val) { +#ifndef Py_GIL_DISABLED var->var_cached = NULL; PyThreadState *ts = _PyThreadState_GET(); +#endif PyContext *ctx = context_get(); if (ctx == NULL) { @@ -739,16 +712,20 @@ contextvar_set(PyContextVar *var, PyObject *val) Py_SETREF(ctx->ctx_vars, new_vars); +#ifndef Py_GIL_DISABLED var->var_cached = val; /* borrow */ var->var_cached_tsid = ts->id; var->var_cached_tsver = ts->context_ver; +#endif return 0; } static int contextvar_del(PyContextVar *var) { +#ifndef Py_GIL_DISABLED var->var_cached = NULL; +#endif PyContext *ctx = context_get(); if (ctx == NULL) { @@ -823,9 +800,11 @@ contextvar_new(PyObject *name, PyObject *def) var->var_default = Py_XNewRef(def); +#ifndef Py_GIL_DISABLED var->var_cached = NULL; var->var_cached_tsid = 0; var->var_cached_tsver = 0; +#endif if (_PyObject_GC_MAY_BE_TRACKED(name) || (def != NULL && _PyObject_GC_MAY_BE_TRACKED(def))) @@ -863,9 +842,11 @@ contextvar_tp_clear(PyContextVar *self) { Py_CLEAR(self->var_name); Py_CLEAR(self->var_default); +#ifndef Py_GIL_DISABLED self->var_cached = NULL; self->var_cached_tsid = 0; self->var_cached_tsver = 0; +#endif return 0; } @@ -1250,24 +1231,6 @@ get_token_missing(void) /////////////////////////// -void -_PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ -#ifdef WITH_FREELISTS - struct _Py_context_freelist *state = &freelists->contexts; - for (; state->numfree > 0; state->numfree--) { - PyContext *ctx = state->items; - state->items = (PyContext *)ctx->ctx_weakreflist; - ctx->ctx_weakreflist = NULL; - PyObject_GC_Del(ctx); - } - if (is_finalization) { - state->numfree = -1; - } -#endif -} - - PyStatus _PyContext_Init(PyInterpreterState *interp) { diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index 2a80ec4f18d757..960c6b4a2ef995 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -10,7 +10,17 @@ * https://github.com/GoogleChromeLabs/wasm-feature-detect/blob/main/src/detectors/type-reflection/index.js */ EM_JS(int, _PyEM_detect_type_reflection, (), { - return "Function" in WebAssembly; + if (!("Function" in WebAssembly)) { + return false; + } + if (WebAssembly.Function.type) { + // Node v20 + Module.PyEM_CountArgs = (func) => WebAssembly.Function.type(wasmTable.get(func)).parameters.length; + } else { + // Node >= 22, v8-based browsers + Module.PyEM_CountArgs = (func) => wasmTable.get(func).type().parameters.length; + } + return true; }); void @@ -43,7 +53,7 @@ EM_JS(int, _PyEM_CountFuncParams, (PyCFunctionWithKeywords func), if (n !== undefined) { return n; } - n = WebAssembly.Function.type(wasmTable.get(func)).parameters.length; + n = Module.PyEM_CountArgs(func); _PyEM_CountFuncParams.cache.set(func, n); return n; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3b999465aac815..6e3f6cc62fe11f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -324,7 +324,7 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + assert(PyStackRef_BoolCheck(value)); res = PyStackRef_Is(value, PyStackRef_False) ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; @@ -346,7 +346,7 @@ case _TO_BOOL_BOOL: { _PyStackRef value; value = stack_pointer[-1]; - if (!PyBool_Check(PyStackRef_AsPyObjectBorrow(value))) { + if (!PyStackRef_BoolCheck(value)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -693,6 +693,55 @@ break; } + case _BINARY_OP_INPLACE_ADD_UNICODE: { + _PyStackRef right; + _PyStackRef left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = CURRENT_OPERAND(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + if (!PyStackRef_Is(*target_local, left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + /* Handle `left = left + right` or `left += right` for str. + * + * When possible, extend `left` in place rather than + * allocating a new PyUnicodeObject. This attempts to avoid + * quadratic behavior when one neglects to use str.join(). + * + * If `left` has only two references remaining (one from + * the stack, one in the locals), DECREFing `left` leaves + * only the locals reference, so PyUnicode_Append knows + * that the string is safe to mutate. + */ + assert(Py_REFCNT(left_o) >= 2); + _Py_DECREF_NO_DEALLOC(left_o); + PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + if (PyStackRef_IsNull(*target_local)) JUMP_TO_ERROR(); + #if TIER_ONE + // The STORE_FAST is already done. This is done here in tier one, + // and during trace projection in tier two: + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_SUBSCR: { _PyStackRef sub; _PyStackRef container; @@ -1104,10 +1153,6 @@ break; } - /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 because it is instrumented */ - - /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 because it is instrumented */ - case _GET_AITER: { _PyStackRef obj; _PyStackRef iter; @@ -1227,9 +1272,33 @@ /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _SEND_GEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */ + case _SEND_GEN_FRAME: { + _PyStackRef v; + _PyStackRef receiver; + _PyInterpreterFrame *gen_frame; + oparg = CURRENT_OPARG(); + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + stack_pointer[-1].bits = (uintptr_t)gen_frame; + break; + } case _YIELD_VALUE: { _PyStackRef retval; @@ -1551,7 +1620,36 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - /* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ + case _LOAD_NAME: { + _PyStackRef v; + oparg = CURRENT_OPARG(); + PyObject *v_o; + PyObject *mod_or_class_dict = LOCALS(); + if (mod_or_class_dict == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) JUMP_TO_ERROR(); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) JUMP_TO_ERROR(); + if (v_o == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) JUMP_TO_ERROR(); + if (v_o == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) JUMP_TO_ERROR(); + if (v_o == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) JUMP_TO_ERROR(); + } + } + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _LOAD_GLOBAL: { _PyStackRef res; @@ -1890,7 +1988,36 @@ break; } - /* _BUILD_SET is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ + case _BUILD_SET: { + _PyStackRef *values; + _PyStackRef set; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + PyObject *set_o = PySet_New(NULL); + if (set_o == NULL) { + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) JUMP_TO_ERROR(); + } + int err = 0; + for (int i = 0; i < oparg; i++) { + PyObject *item = PyStackRef_AsPyObjectSteal(values[i]); + if (err == 0) { + err = PySet_Add(set_o, item); + } + Py_DECREF(item); + } + if (err != 0) { + Py_DECREF(set_o); + if (true) JUMP_TO_ERROR(); + } + set = PyStackRef_FromPyObjectSteal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _BUILD_MAP: { _PyStackRef *values; @@ -1944,40 +2071,6 @@ break; } - case _BUILD_CONST_KEY_MAP: { - _PyStackRef keys; - _PyStackRef *values; - _PyStackRef map; - oparg = CURRENT_OPARG(); - keys = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); - assert(PyTuple_CheckExact(keys_o)); - assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); - STACKREFS_TO_PYOBJECTS(values, oparg, values_o); - if (CONVERSION_FAILED(values_o)) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - PyStackRef_CLOSE(keys); - if (true) JUMP_TO_ERROR(); - } - PyObject *map_o = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys_o, 0), 1, - values_o, 1, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - PyStackRef_CLOSE(keys); - if (map_o == NULL) JUMP_TO_ERROR(); - map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - break; - } - case _DICT_UPDATE: { _PyStackRef update; _PyStackRef dict; @@ -2457,7 +2550,39 @@ /* _LOAD_ATTR_CLASS is split on (oparg & 1) */ - /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _LOAD_ATTR_PROPERTY_FRAME: { + _PyStackRef owner; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + PyObject *fget = (PyObject *)CURRENT_OPERAND(); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (code->co_kwonlyargcount) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (code->co_argcount != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(fget); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame->localsplus[0] = owner; + stack_pointer[-1].bits = (uintptr_t)new_frame; + break; + } /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ @@ -2843,6 +2968,42 @@ break; } + case _IMPORT_NAME: { + _PyStackRef fromlist; + _PyStackRef level; + _PyStackRef res; + oparg = CURRENT_OPARG(); + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + PyStackRef_CLOSE(level); + PyStackRef_CLOSE(fromlist); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IMPORT_FROM: { + _PyStackRef from; + _PyStackRef res; + oparg = CURRENT_OPARG(); + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + if (res_o == NULL) JUMP_TO_ERROR(); + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ @@ -3055,8 +3216,11 @@ JUMP_TO_JUMP_TARGET(); } if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + it->it_index = -1; + if (1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } @@ -3255,7 +3419,7 @@ else { Py_DECREF(tb); } - assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti))); + assert(PyStackRef_LongCheck(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); @@ -3279,7 +3443,7 @@ else { prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc))); + assert(PyStackRef_ExceptionInstanceCheck(new_exc)); exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; @@ -3420,15 +3584,45 @@ break; } - /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 because it is instrumented */ + case _MAYBE_EXPAND_METHOD: { + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef func; + _PyStackRef maybe_self; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + break; + } - /* _CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + /* _DO_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _CHECK_PERIODIC: { CHECK_EVAL_BREAKER(); break; } + /* _MONITOR_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _PY_FRAME_GENERAL: { _PyStackRef *args; _PyStackRef self_or_null; @@ -3524,9 +3718,8 @@ assert(PyStackRef_IsNull(null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); + assert(PyStackRef_FunctionCheck(method)); PyStackRef_CLOSE(callable); stack_pointer[-2 - oparg] = method; stack_pointer[-1 - oparg] = self; @@ -3621,13 +3814,9 @@ callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization - self = stack_pointer[-1 - oparg]; - func = stack_pointer[-2 - oparg]; + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); PyStackRef_CLOSE(callable); - // self may be unused in tier 1, so silence warnings. - (void)self; stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = self; break; @@ -4253,6 +4442,43 @@ break; } + case _CALL_LIST_APPEND: { + _PyStackRef arg; + _PyStackRef self; + _PyStackRef callable; + oparg = CURRENT_OPARG(); + arg = stack_pointer[-1]; + self = stack_pointer[-2]; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + assert(self_o != NULL); + if (!PyList_Check(self_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); + if (err) JUMP_TO_ERROR(); + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _CALL_METHOD_DESCRIPTOR_O: { _PyStackRef *args; _PyStackRef self_or_null; @@ -4711,6 +4937,8 @@ break; } + /* _INSTRUMENTED_LINE is not a viable micro-op for tier 2 because it is instrumented */ + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 because it is instrumented */ /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */ @@ -4816,8 +5044,8 @@ } case _EXIT_TRACE: { - oparg = CURRENT_OPARG(); - _PyExitData *exit = ¤t_executor->exits[oparg]; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND(); + _PyExitData *exit = (_PyExitData *)exit_p; PyCodeObject *code = _PyFrame_GetCode(frame); _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) @@ -4826,11 +5054,15 @@ printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - oparg, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); } #endif + if (exit->executor && !exit->executor->vm_data.valid) { + exit->temperature = initial_temperature_backoff_counter(); + Py_CLEAR(exit->executor); + } if (exit->executor == NULL) { _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { @@ -4950,9 +5182,9 @@ } case _DYNAMIC_EXIT: { - oparg = CURRENT_OPARG(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND(); tstate->previous_executor = (PyObject *)current_executor; - _PyExitData *exit = (_PyExitData *)¤t_executor->exits[oparg]; + _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); @@ -4960,7 +5192,7 @@ printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", - oparg, exit->temperature.as_counter, + exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); } @@ -5001,10 +5233,7 @@ #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif - if (!((_PyExecutorObject *)executor)->vm_data.valid) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(((_PyExecutorObject *)executor)->vm_data.valid); break; } diff --git a/Python/frame.c b/Python/frame.c index 9c7e59601e6faf..25fa2824630f9b 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -17,10 +17,11 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) Py_VISIT(_PyFrame_GetCode(frame)); /* locals */ _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); - int i = 0; + _PyStackRef *sp = frame->stackpointer; /* locals and stack */ - for (; i stacktop; i++) { - Py_VISIT(PyStackRef_AsPyObjectBorrow(locals[i])); + while (sp > locals) { + sp--; + Py_VISIT(PyStackRef_AsPyObjectBorrow(*sp)); } return 0; } @@ -59,10 +60,11 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) assert(frame->owner != FRAME_OWNED_BY_CSTACK); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); assert(frame->owner != FRAME_CLEARED); - Py_ssize_t size = ((char*)&frame->localsplus[frame->stacktop]) - (char *)frame; + Py_ssize_t size = ((char*)frame->stackpointer) - (char *)frame; Py_INCREF(_PyFrame_GetCode(frame)); memcpy((_PyInterpreterFrame *)f->_f_frame_data, frame, size); frame = (_PyInterpreterFrame *)f->_f_frame_data; + frame->stackpointer = (_PyStackRef *)(((char *)frame) + size); f->f_frame = frame; frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; if (_PyFrame_IsIncomplete(frame)) { @@ -97,11 +99,13 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) void _PyFrame_ClearLocals(_PyInterpreterFrame *frame) { - assert(frame->stacktop >= 0); - int stacktop = frame->stacktop; - frame->stacktop = 0; - for (int i = 0; i < stacktop; i++) { - PyStackRef_XCLOSE(frame->localsplus[i]); + assert(frame->stackpointer != NULL); + _PyStackRef *sp = frame->stackpointer; + _PyStackRef *locals = frame->localsplus; + frame->stackpointer = locals; + while (sp > locals) { + sp--; + PyStackRef_XCLOSE(*sp); } Py_CLEAR(frame->f_locals); } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index f19362c9573812..53f04160c38841 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -4,6 +4,7 @@ #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_context.h" #include "pycore_dict.h" // _PyDict_MaybeUntrack() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_object.h" @@ -455,6 +456,30 @@ mark_reachable(PyObject *op) } #ifdef GC_DEBUG +static bool +validate_refcounts(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + + _PyObject_ASSERT_WITH_MSG(op, !gc_is_unreachable(op), + "object should not be marked as unreachable yet"); + + if (_Py_REF_IS_MERGED(op->ob_ref_shared)) { + _PyObject_ASSERT_WITH_MSG(op, op->ob_tid == 0, + "merged objects should have ob_tid == 0"); + } + else if (!_Py_IsImmortal(op)) { + _PyObject_ASSERT_WITH_MSG(op, op->ob_tid != 0, + "unmerged objects should have ob_tid != 0"); + } + + return true; +} + static bool validate_gc_objects(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *args) @@ -498,6 +523,19 @@ mark_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } +static bool +restore_refs(const mi_heap_t *heap, const mi_heap_area_t *area, + void *block, size_t block_size, void *args) +{ + PyObject *op = op_from_block(block, args, false); + if (op == NULL) { + return true; + } + gc_restore_tid(op); + gc_clear_unreachable(op); + return true; +} + /* Return true if object has a pre-PEP 442 finalization method. */ static int has_legacy_finalizer(PyObject *op) @@ -549,6 +587,13 @@ static int deduce_unreachable_heap(PyInterpreterState *interp, struct collection_state *state) { + +#ifdef GC_DEBUG + // Check that all objects are marked as unreachable and that the computed + // reference count difference (stored in `ob_tid`) is non-negative. + gc_visit_heaps(interp, &validate_refcounts, &state->base); +#endif + // Identify objects that are directly reachable from outside the GC heap // by computing the difference between the refcount and the number of // incoming references. @@ -563,6 +608,8 @@ deduce_unreachable_heap(PyInterpreterState *interp, // Transitively mark reachable objects by clearing the // _PyGC_BITS_UNREACHABLE flag. if (gc_visit_heaps(interp, &mark_heap_visitor, &state->base) < 0) { + // On out-of-memory, restore the refcounts and bail out. + gc_visit_heaps(interp, &restore_refs, &state->base); return -1; } @@ -1066,7 +1113,8 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, int err = deduce_unreachable_heap(interp, state); if (err < 0) { _PyEval_StartTheWorld(interp); - goto error; + PyErr_NoMemory(); + return; } // Print debugging information. @@ -1100,7 +1148,12 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, _PyEval_StartTheWorld(interp); if (err < 0) { - goto error; + cleanup_worklist(&state->unreachable); + cleanup_worklist(&state->legacy_finalizers); + cleanup_worklist(&state->wrcb_to_call); + cleanup_worklist(&state->objs_to_decref); + PyErr_NoMemory(); + return; } // Call tp_clear on objects in the unreachable set. This will cause @@ -1110,15 +1163,6 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Append objects with legacy finalizers to the "gc.garbage" list. handle_legacy_finalizers(state); - return; - -error: - cleanup_worklist(&state->unreachable); - cleanup_worklist(&state->legacy_finalizers); - cleanup_worklist(&state->wrcb_to_call); - cleanup_worklist(&state->objs_to_decref); - PyErr_NoMemory(); - PyErr_FormatUnraisable("Out of memory during garbage collection"); } /* This is the main function. Read this to understand how the diff --git a/Python/gc_gil.c b/Python/gc_gil.c index 48646c7af86b7f..7d62a9ed0b1ca8 100644 --- a/Python/gc_gil.c +++ b/Python/gc_gil.c @@ -1,5 +1,5 @@ #include "Python.h" -#include "pycore_pystate.h" // _Py_ClearFreeLists() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #ifndef Py_GIL_DISABLED diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 61057221291c0a..76d1cc7ad6cf95 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -16,8 +16,8 @@ PREDICTED(BINARY_OP); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - _PyStackRef rhs; _PyStackRef lhs; + _PyStackRef rhs; _PyStackRef res; // _SPECIALIZE_BINARY_OP rhs = stack_pointer[-1]; @@ -59,8 +59,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT right = stack_pointer[-1]; @@ -95,8 +95,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT right = stack_pointer[-1]; @@ -130,8 +130,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_UNICODE right = stack_pointer[-1]; @@ -165,8 +165,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; // _GUARD_BOTH_UNICODE right = stack_pointer[-1]; left = stack_pointer[-2]; @@ -181,8 +181,14 @@ { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int next_oparg; + #if TIER_ONE assert(next_instr->op.code == STORE_FAST); - _PyStackRef *target_local = &GETLOCAL(next_instr->op.arg); + next_oparg = next_instr->op.arg; + #else + next_oparg = CURRENT_OPERAND(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); DEOPT_IF(!PyStackRef_Is(*target_local, left), BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -203,9 +209,12 @@ *target_local = PyStackRef_FromPyObjectSteal(temp); _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); if (PyStackRef_IsNull(*target_local)) goto pop_2_error; - // The STORE_FAST is already done. + #if TIER_ONE + // The STORE_FAST is already done. This is done here in tier one, + // and during trace projection in tier two: assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); + #endif } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -217,8 +226,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT right = stack_pointer[-1]; @@ -253,8 +262,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT right = stack_pointer[-1]; @@ -288,8 +297,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT right = stack_pointer[-1]; @@ -324,8 +333,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT right = stack_pointer[-1]; @@ -358,9 +367,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); - _PyStackRef stop; - _PyStackRef start; _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; _PyStackRef res; stop = stack_pointer[-1]; start = stack_pointer[-2]; @@ -393,8 +402,8 @@ PREDICTED(BINARY_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - _PyStackRef sub; _PyStackRef container; + _PyStackRef sub; _PyStackRef res; // _SPECIALIZE_BINARY_SUBSCR sub = stack_pointer[-1]; @@ -433,8 +442,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef sub_st; _PyStackRef dict_st; + _PyStackRef sub_st; _PyStackRef res; /* Skip 1 cache entry */ sub_st = stack_pointer[-1]; @@ -464,8 +473,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef sub_st; _PyStackRef container_st; + _PyStackRef sub_st; /* Skip 1 cache entry */ sub_st = stack_pointer[-1]; container_st = stack_pointer[-2]; @@ -498,8 +507,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef sub_st; _PyStackRef list_st; + _PyStackRef sub_st; _PyStackRef res; /* Skip 1 cache entry */ sub_st = stack_pointer[-1]; @@ -530,8 +539,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef sub_st; _PyStackRef str_st; + _PyStackRef sub_st; _PyStackRef res; /* Skip 1 cache entry */ sub_st = stack_pointer[-1]; @@ -562,8 +571,8 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - _PyStackRef sub_st; _PyStackRef tuple_st; + _PyStackRef sub_st; _PyStackRef res; /* Skip 1 cache entry */ sub_st = stack_pointer[-1]; @@ -589,42 +598,6 @@ DISPATCH(); } - TARGET(BUILD_CONST_KEY_MAP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); - _PyStackRef keys; - _PyStackRef *values; - _PyStackRef map; - keys = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); - assert(PyTuple_CheckExact(keys_o)); - assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); - STACKREFS_TO_PYOBJECTS(values, oparg, values_o); - if (CONVERSION_FAILED(values_o)) { - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - PyStackRef_CLOSE(keys); - if (true) { stack_pointer += -1 - oparg; goto error; } - } - PyObject *map_o = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys_o, 0), 1, - values_o, 1, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); - } - PyStackRef_CLOSE(keys); - if (map_o == NULL) { stack_pointer += -1 - oparg; goto error; } - map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - TARGET(BUILD_LIST) { frame->instr_ptr = next_instr; next_instr += 1; @@ -688,7 +661,10 @@ values = &stack_pointer[-oparg]; PyObject *set_o = PySet_New(NULL); if (set_o == NULL) { - goto error; + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(values[_i]); + } + if (true) { stack_pointer += -oparg; goto error; } } int err = 0; for (int i = 0; i < oparg; i++) { @@ -713,9 +689,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef step = PyStackRef_NULL; - _PyStackRef stop; _PyStackRef start; + _PyStackRef stop; + _PyStackRef step = PyStackRef_NULL; _PyStackRef slice; if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; @@ -794,15 +770,17 @@ PREDICTED(CALL); _Py_CODEUNIT *this_instr = next_instr - 4; (void)this_instr; - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef func; + _PyStackRef maybe_self; _PyStackRef res; // _SPECIALIZE_CALL - args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { + args = &stack_pointer[-oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -816,26 +794,34 @@ #endif /* ENABLE_SPECIALIZATION */ } /* Skip 2 cache entries */ - // _CALL + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + } + // _DO_CALL + self_or_null = maybe_self; + callable = func; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null_o != NULL) { - args--; - total_args++; - } - else if (Py_TYPE(callable_o) == &PyMethod_Type) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - args[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - args[-1] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(callable); - callable_o = method; - callable = args[-1]; } // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && @@ -913,9 +899,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef null; _PyStackRef callable; + _PyStackRef null; + _PyStackRef *args; /* Skip 1 cache entry */ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; @@ -977,8 +963,8 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; _PyStackRef callable; + _PyStackRef null; _PyStackRef func; _PyStackRef self; _PyStackRef self_or_null; @@ -1000,16 +986,15 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization - self = stack_pointer[-1 - oparg]; - func = stack_pointer[-2 - oparg]; + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + func = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); PyStackRef_CLOSE(callable); - // self may be unused in tier 1, so silence warnings. - (void)self; } + // flush + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = self; // _CHECK_FUNCTION_VERSION - callable = func; + callable = stack_pointer[-2 - oparg]; { uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -1036,7 +1021,6 @@ } // _INIT_CALL_PY_EXACT_ARGS args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null); @@ -1082,12 +1066,12 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; _PyStackRef callable; + _PyStackRef null; _PyStackRef method; _PyStackRef self; - _PyStackRef *args; _PyStackRef self_or_null; + _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 @@ -1112,15 +1096,17 @@ assert(PyStackRef_IsNull(null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); + assert(PyStackRef_FunctionCheck(method)); PyStackRef_CLOSE(callable); } + // flush + stack_pointer[-2 - oparg] = method; + stack_pointer[-1 - oparg] = self; // _PY_FRAME_GENERAL args = &stack_pointer[-oparg]; - self_or_null = self; - callable = method; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); @@ -1176,9 +1162,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -1231,9 +1217,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -1292,9 +1278,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -1352,9 +1338,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -1404,9 +1390,9 @@ PREDICTED(CALL_FUNCTION_EX); _Py_CODEUNIT *this_instr = next_instr - 1; (void)this_instr; - _PyStackRef kwargs_st = PyStackRef_NULL; - _PyStackRef callargs_st; _PyStackRef func_st; + _PyStackRef callargs_st; + _PyStackRef kwargs_st = PyStackRef_NULL; _PyStackRef result; if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; } callargs_st = stack_pointer[-1 - (oparg & 1)]; @@ -1509,8 +1495,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); - _PyStackRef value1_st; _PyStackRef value2_st; + _PyStackRef value1_st; _PyStackRef res; value1_st = stack_pointer[-1]; value2_st = stack_pointer[-2]; @@ -1533,9 +1519,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -1577,10 +1563,10 @@ PREDICTED(CALL_KW); _Py_CODEUNIT *this_instr = next_instr - 1; (void)this_instr; - _PyStackRef kwnames; - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; _PyStackRef res; kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; @@ -1683,9 +1669,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -1728,9 +1714,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef arg; - _PyStackRef self; _PyStackRef callable; + _PyStackRef self; + _PyStackRef arg; /* Skip 1 cache entry */ /* Skip 2 cache entries */ arg = stack_pointer[-1]; @@ -1744,15 +1730,18 @@ assert(self_o != NULL); DEOPT_IF(!PyList_Check(self_o), CALL); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) { - goto pop_1_error; // Since arg is DECREF'ed already - } + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); - STACK_SHRINK(3); - // Skip POP_TOP + if (err) goto pop_3_error; + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); + #endif + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1761,9 +1750,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -1824,9 +1813,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -1887,9 +1876,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -1941,9 +1930,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *args; - _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -1999,8 +1988,8 @@ INSTRUCTION_STATS(CALL_NON_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef *args; _PyStackRef self_or_null; + _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -2100,7 +2089,6 @@ } // _INIT_CALL_PY_EXACT_ARGS args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null); @@ -2147,8 +2135,8 @@ INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; - _PyStackRef *args; _PyStackRef self_or_null; + _PyStackRef *args; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 @@ -2222,9 +2210,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef arg; - _PyStackRef null; _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -2258,9 +2246,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef arg; - _PyStackRef null; _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -2294,9 +2282,9 @@ next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef arg; - _PyStackRef null; _PyStackRef callable; + _PyStackRef null; + _PyStackRef arg; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -2321,8 +2309,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); - _PyStackRef match_type_st; _PyStackRef exc_value_st; + _PyStackRef match_type_st; _PyStackRef rest; _PyStackRef match; match_type_st = stack_pointer[-1]; @@ -2357,8 +2345,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef b; right = stack_pointer[-1]; left = stack_pointer[-2]; @@ -2381,9 +2369,9 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); - _PyStackRef exc_value_st; - _PyStackRef last_sent_val_st; _PyStackRef sub_iter_st; + _PyStackRef last_sent_val_st; + _PyStackRef exc_value_st; _PyStackRef none; _PyStackRef value; exc_value_st = stack_pointer[-1]; @@ -2418,8 +2406,8 @@ PREDICTED(COMPARE_OP); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _SPECIALIZE_COMPARE_OP right = stack_pointer[-1]; @@ -2467,8 +2455,8 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_FLOAT right = stack_pointer[-1]; @@ -2505,8 +2493,8 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_INT right = stack_pointer[-1]; @@ -2547,8 +2535,8 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef res; // _GUARD_BOTH_UNICODE right = stack_pointer[-1]; @@ -2588,12 +2576,11 @@ PREDICTED(CONTAINS_OP); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef b; // _SPECIALIZE_CONTAINS_OP right = stack_pointer[-1]; - left = stack_pointer[-2]; { uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; @@ -2608,6 +2595,7 @@ #endif /* ENABLE_SPECIALIZATION */ } // _CONTAINS_OP + left = stack_pointer[-2]; { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -2628,8 +2616,8 @@ next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef b; /* Skip 1 cache entry */ right = stack_pointer[-1]; @@ -2654,8 +2642,8 @@ next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef b; /* Skip 1 cache entry */ right = stack_pointer[-1]; @@ -2818,8 +2806,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); - _PyStackRef sub; _PyStackRef container; + _PyStackRef sub; sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ @@ -2837,9 +2825,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); - _PyStackRef update; - _PyStackRef dict; _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; @@ -2861,8 +2849,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); - _PyStackRef update; _PyStackRef dict; + _PyStackRef update; update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); @@ -2887,8 +2875,8 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); - _PyStackRef exc_st; _PyStackRef awaitable_st; + _PyStackRef exc_st; exc_st = stack_pointer[-1]; awaitable_st = stack_pointer[-2]; PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); @@ -2924,8 +2912,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); - _PyStackRef value; _PyStackRef receiver; + _PyStackRef value; value = stack_pointer[-1]; receiver = stack_pointer[-2]; (void)receiver; @@ -3024,8 +3012,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); - _PyStackRef fmt_spec; _PyStackRef value; + _PyStackRef fmt_spec; _PyStackRef res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; @@ -3489,7 +3477,7 @@ _PyStackRef res; from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); if (res_o == NULL) goto error; res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -3502,13 +3490,13 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); - _PyStackRef fromlist; _PyStackRef level; + _PyStackRef fromlist; _PyStackRef res; fromlist = stack_pointer[-1]; level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = import_name(tstate, frame, name, + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); PyStackRef_CLOSE(level); @@ -3526,34 +3514,150 @@ (void)this_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef res; /* Skip 3 cache entries */ - int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; - int total_args = oparg + is_meth; - PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - if (err) goto error; - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL); - } - - TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - } - - TARGET(INSTRUMENTED_CALL_KW) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); - int total_args = oparg + is_meth; + // _MAYBE_EXPAND_METHOD + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + func = PyStackRef_FromPyObjectNew(method); + /* Make sure that callable and all args are in memory */ + args[-2] = func; + args[-1] = maybe_self; + PyStackRef_CLOSE(callable); + } + else { + func = callable; + maybe_self = self_or_null; + } + } + // _MONITOR_CALL + { + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + if (err) goto error; + } + // _DO_CALL + self_or_null = maybe_self; + callable = func; + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = (uint16_t)(next_instr - this_instr); + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(self_or_null); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (true) { stack_pointer += -2 - oparg; goto error; } + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); + for (int i = 0; i < total_args; i++) { + PyStackRef_CLOSE(args[i]); + } + if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + + TARGET(INSTRUMENTED_CALL_KW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); + int total_args = oparg + is_meth; PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); @@ -3569,13 +3673,13 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - _PyStackRef value; _PyStackRef receiver; + _PyStackRef value; value = stack_pointer[-1]; receiver = stack_pointer[-2]; /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - if (PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver))) { + if (PyStackRef_GenCheck(receiver)) { if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { goto error; } @@ -3591,8 +3695,8 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - _PyStackRef value; _PyStackRef receiver; + _PyStackRef value; value = stack_pointer[-1]; receiver = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); @@ -3679,6 +3783,41 @@ DISPATCH(); } + TARGET(INSTRUMENTED_LINE) { + _Py_CODEUNIT *prev_instr = frame->instr_ptr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + original_opcode = code->_co_monitoring->lines[(int)(this_instr - _PyCode_CODE(code))].original_opcode; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; (void)this_instr; @@ -3698,7 +3837,7 @@ INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ _PyStackRef cond = POP(); - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -3761,7 +3900,7 @@ INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ _PyStackRef cond = POP(); - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -3807,23 +3946,44 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); - if (err) goto error; - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval)); - LOAD_IP(frame->return_offset); - goto resume_frame; + _PyStackRef value; + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _LOAD_CONST + { + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); + } + // _RETURN_VALUE_EVENT + val = value; + { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + if (err) goto error; + } + // _RETURN_VALUE + retval = val; + { + #if TIER_ONE + assert(frame != &entry_frame); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + LOAD_SP(); + LOAD_IP(frame->return_offset); + res = retval; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(INSTRUMENTED_RETURN_VALUE) { @@ -3831,24 +3991,41 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef val; _PyStackRef retval; - retval = stack_pointer[-1]; - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); - if (err) goto error; - STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); - LOAD_IP(frame->return_offset); - goto resume_frame; + _PyStackRef res; + // _RETURN_VALUE_EVENT + val = stack_pointer[-1]; + { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + if (err) goto error; + } + // _RETURN_VALUE + retval = val; + { + #if TIER_ONE + assert(frame != &entry_frame); + #endif + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + LOAD_SP(); + LOAD_IP(frame->return_offset); + res = retval; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(INSTRUMENTED_YIELD_VALUE) { @@ -3856,30 +4033,65 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + _PyStackRef val; _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame != &entry_frame); - frame->instr_ptr = next_instr; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); - if (err) goto error; - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); - /* We don't know which of these is relevant here, so keep them equal */ - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + _PyStackRef value; + // _YIELD_VALUE_EVENT + val = stack_pointer[-1]; + { + SAVE_SP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + LOAD_SP(); + if (err) goto error; + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + DISPATCH(); + } + } + // _YIELD_VALUE + retval = val; + { + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); } TARGET(INTERPRETER_EXIT) { @@ -3901,8 +4113,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); - _PyStackRef right; _PyStackRef left; + _PyStackRef right; _PyStackRef b; right = stack_pointer[-1]; left = stack_pointer[-2]; @@ -3986,8 +4198,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); - _PyStackRef v; _PyStackRef list; + _PyStackRef v; v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), @@ -4001,8 +4213,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); - _PyStackRef iterable_st; _PyStackRef list_st; + _PyStackRef iterable_st; iterable_st = stack_pointer[-1]; list_st = stack_pointer[-2 - (oparg-1)]; PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); @@ -4460,32 +4672,63 @@ INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + } + // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *fget = read_obj(&this_instr[6].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); - STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); - // Manipulate stack directly because we exit with DISPATCH_INLINED(). - STACK_SHRINK(1); - new_frame->localsplus[0] = owner; - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); + DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); + DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + STAT_INC(LOAD_ATTR, hit); + Py_INCREF(fget); + new_frame = _PyFrame_PushUnchecked(tstate, f, 1); + new_frame->localsplus[0] = owner; + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { @@ -4981,22 +5224,16 @@ if (true) goto error; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) { - goto error; - } + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) goto error; if (v_o == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) { - goto error; - } + if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) goto error; if (v_o == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) { - goto error; - } + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) goto error; if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; + if (true) goto error; } } } @@ -5043,8 +5280,8 @@ PREDICTED(LOAD_SUPER_ATTR); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - _PyStackRef class_st; _PyStackRef global_super_st; + _PyStackRef class_st; _PyStackRef self_st; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; @@ -5120,9 +5357,9 @@ next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef self_st; - _PyStackRef class_st; _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; _PyStackRef attr_st; /* Skip 1 cache entry */ self_st = stack_pointer[-1]; @@ -5153,9 +5390,9 @@ next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef self_st; - _PyStackRef class_st; _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; _PyStackRef attr; _PyStackRef self_or_null; /* Skip 1 cache entry */ @@ -5234,9 +5471,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); - _PyStackRef value; - _PyStackRef key; _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; value = stack_pointer[-1]; key = stack_pointer[-2]; dict_st = stack_pointer[-3 - (oparg - 1)]; @@ -5254,9 +5491,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); - _PyStackRef names; - _PyStackRef type; _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; _PyStackRef attrs; names = stack_pointer[-1]; type = stack_pointer[-2]; @@ -5290,8 +5527,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); - _PyStackRef keys; _PyStackRef subject; + _PyStackRef keys; _PyStackRef values_or_none; keys = stack_pointer[-1]; subject = stack_pointer[-2]; @@ -5366,7 +5603,7 @@ _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -5400,7 +5637,7 @@ // _POP_JUMP_IF_TRUE cond = b; { - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -5435,7 +5672,7 @@ // _POP_JUMP_IF_FALSE cond = b; { - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_False); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -5455,7 +5692,7 @@ _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_Is(cond, PyStackRef_True); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -5492,7 +5729,7 @@ else { prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc))); + assert(PyStackRef_ExceptionInstanceCheck(new_exc)); exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; @@ -5548,8 +5785,8 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); - _PyStackRef exc_st; _PyStackRef *values; + _PyStackRef exc_st; exc_st = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); @@ -5807,29 +6044,53 @@ } TARGET(SEND_GEN) { - _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - _PyStackRef v; _PyStackRef receiver; + _PyStackRef v; + _PyInterpreterFrame *gen_frame; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, SEND); + } + // _SEND_GEN_FRAME v = stack_pointer[-1]; receiver = stack_pointer[-2]; - DEOPT_IF(tstate->interp->eval_frame, SEND); - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); - STAT_INC(SEND, hit); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(next_instr - this_instr + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); - DISPATCH_INLINED(gen_frame); + { + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + STAT_INC(SEND, hit); + gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + } + // _PUSH_FRAME + new_frame = gen_frame; + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(SETUP_ANNOTATIONS) { @@ -5863,8 +6124,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); - _PyStackRef v; _PyStackRef set; + _PyStackRef v; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), @@ -5880,8 +6141,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - _PyStackRef func_st; _PyStackRef attr_st; + _PyStackRef func_st; func_st = stack_pointer[-1]; attr_st = stack_pointer[-2]; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); @@ -5925,8 +6186,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); - _PyStackRef iterable; _PyStackRef set; + _PyStackRef iterable; iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), @@ -6163,8 +6424,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - _PyStackRef value1; _PyStackRef value2; + _PyStackRef value1; value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; uint32_t oparg1 = oparg >> 4; @@ -6221,10 +6482,10 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); - _PyStackRef stop; - _PyStackRef start; - _PyStackRef container; _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; @@ -6254,8 +6515,8 @@ PREDICTED(STORE_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; (void)this_instr; - _PyStackRef sub; _PyStackRef container; + _PyStackRef sub; _PyStackRef v; // _SPECIALIZE_STORE_SUBSCR sub = stack_pointer[-1]; @@ -6293,9 +6554,9 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef sub_st; - _PyStackRef dict_st; _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub_st; /* Skip 1 cache entry */ sub_st = stack_pointer[-1]; dict_st = stack_pointer[-2]; @@ -6317,9 +6578,9 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef sub_st; - _PyStackRef list_st; _PyStackRef value; + _PyStackRef list_st; + _PyStackRef sub_st; /* Skip 1 cache entry */ sub_st = stack_pointer[-1]; list_st = stack_pointer[-2]; @@ -6349,8 +6610,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); - _PyStackRef top; _PyStackRef bottom; + _PyStackRef top; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); @@ -6431,7 +6692,7 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyBool_Check(PyStackRef_AsPyObjectBorrow(value)), TO_BOOL); + DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); STAT_INC(TO_BOOL, hit); DISPATCH(); } @@ -6561,7 +6822,7 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; - assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + assert(PyStackRef_BoolCheck(value)); res = PyStackRef_Is(value, PyStackRef_False) ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; @@ -6698,10 +6959,10 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); - _PyStackRef val; - _PyStackRef lasti; - _PyStackRef exit_self; _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; _PyStackRef res; val = stack_pointer[-1]; lasti = stack_pointer[-3]; @@ -6728,7 +6989,7 @@ else { Py_DECREF(tb); } - assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti))); + assert(PyStackRef_LongCheck(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); diff --git a/Python/import.c b/Python/import.c index 20ad10020044df..40b7feac001d6e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1532,6 +1532,35 @@ switch_to_main_interpreter(PyThreadState *tstate) return main_tstate; } +static void +switch_back_from_main_interpreter(PyThreadState *tstate, + PyThreadState *main_tstate, + PyObject *tempobj) +{ + assert(main_tstate == PyThreadState_GET()); + assert(_Py_IsMainInterpreter(main_tstate->interp)); + assert(tstate->interp != main_tstate->interp); + + /* Handle any exceptions, which we cannot propagate directly + * to the subinterpreter. */ + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_MemoryError)) { + /* We trust it will be caught again soon. */ + PyErr_Clear(); + } + else { + /* Printing the exception should be sufficient. */ + PyErr_PrintEx(0); + } + } + + Py_XDECREF(tempobj); + + PyThreadState_Clear(main_tstate); + (void)PyThreadState_Swap(tstate); + PyThreadState_Delete(main_tstate); +} + static PyObject * get_core_module_dict(PyInterpreterState *interp, PyObject *name, PyObject *path) @@ -2027,27 +2056,10 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, /* Switch back to the subinterpreter. */ if (switched) { assert(main_tstate != tstate); - - /* Handle any exceptions, which we cannot propagate directly - * to the subinterpreter. */ - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_MemoryError)) { - /* We trust it will be caught again soon. */ - PyErr_Clear(); - } - else { - /* Printing the exception should be sufficient. */ - PyErr_PrintEx(0); - } - } - + switch_back_from_main_interpreter(tstate, main_tstate, mod); /* Any module we got from the init function will have to be * reloaded in the subinterpreter. */ - Py_CLEAR(mod); - - PyThreadState_Clear(main_tstate); - (void)PyThreadState_Swap(tstate); - PyThreadState_Delete(main_tstate); + mod = NULL; } /*****************************************************************/ @@ -2141,9 +2153,21 @@ clear_singlephase_extension(PyInterpreterState *interp, } } + /* We must use the main interpreter to clean up the cache. + * See the note in import_run_extension(). */ + PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *main_tstate = switch_to_main_interpreter(tstate); + if (main_tstate == NULL) { + return -1; + } + /* Clear the cached module def. */ _extensions_cache_delete(path, name); + if (main_tstate != tstate) { + switch_back_from_main_interpreter(tstate, main_tstate, NULL); + } + return 0; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 74118030925e3e..9cc3af1f5e162c 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -121,6 +121,19 @@ sys_profile_call_or_return( Py_DECREF(meth); return res; } + else if (Py_TYPE(callable) == &PyMethod_Type) { + // CALL instruction will grab the function from the method, + // so if the function is a C function, the return event will + // be emitted. However, CALL event happens before CALL + // instruction, so we need to handle this case here. + PyObject* func = PyMethod_GET_FUNCTION(callable); + if (func == NULL) { + return NULL; + } + if (PyCFunction_Check(func)) { + return call_profile_func(self, func); + } + } Py_RETURN_NONE; } @@ -605,7 +618,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_STOP_ITERATION); PyFrameObject* frame = PyEval_GetFrame(); - if (frame->f_trace_opcodes) { + if (frame && frame->f_trace_opcodes) { int ret = _PyEval_SetOpcodeTrace(frame, true); if (ret != 0) { return ret; diff --git a/Python/lock.c b/Python/lock.c index 7c6a5175e88ff1..57675fe1873fa2 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -514,6 +514,7 @@ void _PySeqLock_LockWrite(_PySeqLock *seqlock) } else if (_Py_atomic_compare_exchange_uint32(&seqlock->sequence, &prev, prev + 1)) { // We've locked the cache + _Py_atomic_fence_release(); break; } else { @@ -547,28 +548,31 @@ uint32_t _PySeqLock_BeginRead(_PySeqLock *seqlock) return sequence; } -uint32_t _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous) +int _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous) { - // Synchronize again and validate that the entry hasn't been updated - // while we were readying the values. - if (_Py_atomic_load_uint32_acquire(&seqlock->sequence) == previous) { + // gh-121368: We need an explicit acquire fence here to ensure that + // this load of the sequence number is not reordered before any loads + // within the read lock. + _Py_atomic_fence_acquire(); + + if (_Py_atomic_load_uint32_relaxed(&seqlock->sequence) == previous) { return 1; - } + } - _Py_yield(); - return 0; + _Py_yield(); + return 0; } -uint32_t _PySeqLock_AfterFork(_PySeqLock *seqlock) +int _PySeqLock_AfterFork(_PySeqLock *seqlock) { // Synchronize again and validate that the entry hasn't been updated // while we were readying the values. - if (SEQLOCK_IS_UPDATING(seqlock->sequence)) { + if (SEQLOCK_IS_UPDATING(seqlock->sequence)) { seqlock->sequence = 0; return 1; - } + } - return 0; + return 0; } #undef PyMutex_Lock diff --git a/Python/marshal.c b/Python/marshal.c index fe97ccde2e5b13..b1708a7306f9e7 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1927,7 +1927,7 @@ machine architecture issues.\n\ Not all Python object types are supported; in general, only objects\n\ whose value is independent from a particular invocation of Python can be\n\ written and read by this module. The following types are supported:\n\ -None, integers, floating point numbers, strings, bytes, bytearrays,\n\ +None, integers, floating-point numbers, strings, bytes, bytearrays,\n\ tuples, lists, sets, dictionaries, and code objects, where it\n\ should be understood that tuples, lists and dictionaries are only\n\ supported as long as the values contained therein are themselves\n\ @@ -1938,7 +1938,7 @@ Variables:\n\ \n\ version -- indicates the format that the module uses. Version 0 is the\n\ historical format, version 1 shares interned strings and version 2\n\ - uses a binary format for floating point numbers.\n\ + uses a binary format for floating-point numbers.\n\ Version 3 shares common object references (New in version 3.4).\n\ \n\ Functions:\n\ diff --git a/Python/object_stack.c b/Python/object_stack.c index bd9696822c8a9d..bda3190f45e0ff 100644 --- a/Python/object_stack.c +++ b/Python/object_stack.c @@ -8,24 +8,11 @@ extern _PyObjectStackChunk *_PyObjectStackChunk_New(void); extern void _PyObjectStackChunk_Free(_PyObjectStackChunk *); -static struct _Py_object_stack_freelist * -get_object_stack_freelist(void) -{ - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); - return &freelists->object_stacks; -} - _PyObjectStackChunk * _PyObjectStackChunk_New(void) { - _PyObjectStackChunk *buf; - struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); - if (obj_stack_freelist->numfree > 0) { - buf = obj_stack_freelist->items; - obj_stack_freelist->items = buf->prev; - obj_stack_freelist->numfree--; - } - else { + _PyObjectStackChunk *buf = _Py_FREELIST_POP_MEM(object_stack_chunks); + if (buf == NULL) { // NOTE: we use PyMem_RawMalloc() here because this is used by the GC // during mimalloc heap traversal. In that context, it is not safe to // allocate mimalloc memory, such as via PyMem_Malloc(). @@ -43,17 +30,7 @@ void _PyObjectStackChunk_Free(_PyObjectStackChunk *buf) { assert(buf->n == 0); - struct _Py_object_stack_freelist *obj_stack_freelist = get_object_stack_freelist(); - if (obj_stack_freelist->numfree >= 0 && - obj_stack_freelist->numfree < _PyObjectStackChunk_MAXFREELIST) - { - buf->prev = obj_stack_freelist->items; - obj_stack_freelist->items = buf; - obj_stack_freelist->numfree++; - } - else { - PyMem_RawFree(buf); - } + _Py_FREELIST_FREE(object_stack_chunks, buf, PyMem_RawFree); } void @@ -87,22 +64,3 @@ _PyObjectStack_Merge(_PyObjectStack *dst, _PyObjectStack *src) dst->head = src->head; src->head = NULL; } - -void -_PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization) -{ - if (!is_finalization) { - // Ignore requests to clear the free list during GC. We use object - // stacks during GC, so emptying the free-list is counterproductive. - return; - } - - struct _Py_object_stack_freelist *freelist = &freelists->object_stacks; - while (freelist->numfree > 0) { - _PyObjectStackChunk *buf = freelist->items; - freelist->items = buf->prev; - freelist->numfree--; - PyMem_RawFree(buf); - } - freelist->numfree = -1; -} diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 6097b249c0ad0b..6b5f231e13d15a 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -42,7 +42,6 @@ static void *opcode_targets[256] = { &&TARGET_UNARY_NOT, &&TARGET_WITH_EXCEPT_START, &&TARGET_BINARY_OP, - &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_LIST, &&TARGET_BUILD_MAP, &&TARGET_BUILD_SET, @@ -148,6 +147,7 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -238,12 +238,8 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, - &&TARGET_INSTRUMENTED_RETURN_VALUE, - &&TARGET_INSTRUMENTED_RETURN_CONST, - &&TARGET_INSTRUMENTED_YIELD_VALUE, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, &&TARGET_INSTRUMENTED_FOR_ITER, - &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_CALL_KW, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_INSTRUCTION, @@ -253,6 +249,10 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + &&TARGET_INSTRUMENTED_RETURN_VALUE, + &&TARGET_INSTRUMENTED_RETURN_CONST, + &&TARGET_INSTRUMENTED_YIELD_VALUE, + &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_LINE, &&_unknown_opcode, }; diff --git a/Python/optimizer.c b/Python/optimizer.c index 561ec4efa4ee2a..7b875af2aae898 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -797,6 +797,16 @@ translate_bytecode_to_trace( if (uop == _PUSH_FRAME) { assert(i + 1 == nuops); + if (opcode == FOR_ITER_GEN || + opcode == LOAD_ATTR_PROPERTY || + opcode == SEND_GEN) + { + DPRINTF(2, "Bailing due to dynamic target\n"); + ADD_TO_TRACE(uop, oparg, 0, target); + ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); + goto done; + } + assert(_PyOpcode_Deopt[opcode] == CALL); int func_version_offset = offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) // Add one to account for the actual opcode/oparg pair: @@ -828,12 +838,6 @@ translate_bytecode_to_trace( ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } - if (opcode == FOR_ITER_GEN) { - DPRINTF(2, "Bailing due to dynamic target\n"); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0); - goto done; - } // Increment IP to the return address instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); @@ -871,6 +875,15 @@ translate_bytecode_to_trace( goto done; } + if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { + assert(i + 1 == nuops); + _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + assert(next_instr->op.code == STORE_FAST); + operand = next_instr->op.arg; + // Skip the STORE_FAST: + instr++; + } + // All other instructions ADD_TO_TRACE(uop, oparg, operand, target); } @@ -886,6 +899,11 @@ translate_bytecode_to_trace( instr++; // Add cache size for opcode instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + + if (opcode == CALL_LIST_APPEND) { + assert(instr->op.code == POP_TOP); + instr++; + } } // End for (;;) done: @@ -1135,13 +1153,15 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil *dest = buffer[i]; assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); if (opcode == _EXIT_TRACE) { - executor->exits[next_exit].target = buffer[i].target; - dest->oparg = next_exit; + _PyExitData *exit = &executor->exits[next_exit]; + exit->target = buffer[i].target; + dest->operand = (uint64_t)exit; next_exit--; } if (opcode == _DYNAMIC_EXIT) { - executor->exits[next_exit].target = 0; - dest->oparg = next_exit; + _PyExitData *exit = &executor->exits[next_exit]; + exit->target = 0; + dest->operand = (uint64_t)exit; next_exit--; } } @@ -1581,42 +1601,36 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is _Py_BloomFilter_Add(&obj_filter, obj); /* Walk the list of executors */ /* TO DO -- Use a tree to avoid traversing as many objects */ - bool no_memory = false; PyObject *invalidate = PyList_New(0); if (invalidate == NULL) { - PyErr_Clear(); - no_memory = true; + goto error; } /* Clearing an executor can deallocate others, so we need to make a list of * executors to invalidate first */ for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { assert(exec->vm_data.valid); _PyExecutorObject *next = exec->vm_data.links.next; - if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { - unlink_executor(exec); - if (no_memory) { - exec->vm_data.valid = 0; - } else { - if (PyList_Append(invalidate, (PyObject *)exec) < 0) { - PyErr_Clear(); - no_memory = true; - exec->vm_data.valid = 0; - } - } - if (is_invalidation) { - OPT_STAT_INC(executors_invalidated); - } + if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter) && + PyList_Append(invalidate, (PyObject *)exec)) + { + goto error; } exec = next; } - if (invalidate != NULL) { - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { - _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); - executor_clear(exec); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { + _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); + executor_clear(exec); + if (is_invalidation) { + OPT_STAT_INC(executors_invalidated); } - Py_DECREF(invalidate); } + Py_DECREF(invalidate); return; +error: + PyErr_Clear(); + Py_XDECREF(invalidate); + // If we're truly out of memory, wiping out everything is a fine fallback: + _Py_Executors_InvalidateAll(interp, is_invalidation); } /* Invalidate all executors */ diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 2ea839f5d6dc97..c982e37182157a 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -596,6 +596,14 @@ dummy_func(void) { } } + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { + (void)callable; + (void)self_or_null; + (void)args; + func = sym_new_not_null(ctx); + maybe_self = sym_new_not_null(ctx); + } + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ (void)callable; @@ -658,6 +666,11 @@ dummy_func(void) { ctx->done = true; } + op(_SEND_GEN_FRAME, ( -- )) { + // We are about to hit the end of the trace: + ctx->done = true; + } + op(_CHECK_STACK_SPACE, ( --)) { assert(corresponding_check_stack == NULL); corresponding_check_stack = this_instr; @@ -783,7 +796,8 @@ dummy_func(void) { ctx->done = true; } - op(_EXIT_TRACE, (--)) { + op(_EXIT_TRACE, (exit_p/4 --)) { + (void)exit_p; ctx->done = true; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 978aa911b52efc..4fa40ff861ba70 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -473,6 +473,12 @@ break; } + case _BINARY_OP_INPLACE_ADD_UNICODE: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _BINARY_SUBSCR: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); @@ -615,10 +621,6 @@ break; } - /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ - - /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ - case _GET_AITER: { _Py_UopsSymbol *iter; iter = sym_new_not_null(ctx); @@ -644,9 +646,11 @@ /* _SEND is not a viable micro-op for tier 2 */ - /* _SEND_GEN is not a viable micro-op for tier 2 */ - - /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + case _SEND_GEN_FRAME: { + // We are about to hit the end of the trace: + ctx->done = true; + break; + } case _YIELD_VALUE: { _Py_UopsSymbol *res; @@ -787,7 +791,14 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ - /* _LOAD_NAME is not a viable micro-op for tier 2 */ + case _LOAD_NAME: { + _Py_UopsSymbol *v; + v = sym_new_not_null(ctx); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _LOAD_GLOBAL: { _Py_UopsSymbol *res; @@ -910,7 +921,14 @@ break; } - /* _BUILD_SET is not a viable micro-op for tier 2 */ + case _BUILD_SET: { + _Py_UopsSymbol *set; + set = sym_new_not_null(ctx); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _BUILD_MAP: { _Py_UopsSymbol *map; @@ -925,15 +943,6 @@ break; } - case _BUILD_CONST_KEY_MAP: { - _Py_UopsSymbol *map; - map = sym_new_not_null(ctx); - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - break; - } - case _DICT_UPDATE: { stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1151,7 +1160,12 @@ break; } - /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ + case _LOAD_ATTR_PROPERTY_FRAME: { + _PyInterpreterFrame *new_frame; + new_frame = sym_new_not_null(ctx); + stack_pointer[-1] = (_Py_UopsSymbol *)new_frame; + break; + } /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ @@ -1307,6 +1321,24 @@ break; } + case _IMPORT_NAME: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IMPORT_FROM: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ @@ -1566,14 +1598,33 @@ break; } - /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + case _MAYBE_EXPAND_METHOD: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UopsSymbol *func; + _Py_UopsSymbol *maybe_self; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + (void)callable; + (void)self_or_null; + (void)args; + func = sym_new_not_null(ctx); + maybe_self = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + break; + } - /* _CALL is not a viable micro-op for tier 2 */ + /* _DO_CALL is not a viable micro-op for tier 2 */ case _CHECK_PERIODIC: { break; } + /* _MONITOR_CALL is not a viable micro-op for tier 2 */ + case _PY_FRAME_GENERAL: { _Py_UopsSymbol **args; _Py_UopsSymbol *self_or_null; @@ -1846,6 +1897,12 @@ break; } + case _CALL_LIST_APPEND: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + case _CALL_METHOD_DESCRIPTOR_O: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); @@ -2012,6 +2069,8 @@ break; } + /* _INSTRUMENTED_LINE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ @@ -2109,6 +2168,8 @@ } case _EXIT_TRACE: { + PyObject *exit_p = (PyObject *)this_instr->operand; + (void)exit_p; ctx->done = true; break; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 39eaa86098b2e5..6b641c0775f533 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -9,6 +9,7 @@ #include "pycore_dict.h" // _PyDict_Fini() #include "pycore_exceptions.h" // _PyExc_InitTypes() #include "pycore_fileutils.h" // _Py_ResetForceASCII() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_floatobject.h" // _PyFloat_InitTypes() #include "pycore_global_objects_fini_generated.h" // "_PyStaticObjects_CheckRefcnt() #include "pycore_import.h" // _PyImport_BootstrapImp() @@ -1843,7 +1844,7 @@ finalize_interp_types(PyInterpreterState *interp) #ifndef Py_GIL_DISABLED // With Py_GIL_DISABLED: // the freelists for the current thread state have already been cleared. - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + struct _Py_freelists *freelists = _Py_freelists_GET(); _PyObject_ClearFreeLists(freelists, 1); #endif diff --git a/Python/pystate.c b/Python/pystate.c index 602b13e18c71ae..6fbd17f7eaeaa9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -9,9 +9,9 @@ #include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_emscripten_trampoline.h" // _Py_EmscriptenTrampoline_Init() #include "pycore_frame.h" +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_object.h" // _PyType_InitCache() -#include "pycore_object_stack.h" // _PyObjectStackChunk_ClearFreeList() #include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_pyerrors.h" // _PyErr_Clear() #include "pycore_pylifecycle.h" // _PyAST_Fini() @@ -1499,6 +1499,8 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->previous_executor = NULL; tstate->dict_global_version = 0; + _tstate->asyncio_running_loop = NULL; + tstate->delete_later = NULL; llist_init(&_tstate->mem_free_queue); @@ -1700,6 +1702,11 @@ PyThreadState_Clear(PyThreadState *tstate) /* Don't clear tstate->pyframe: it is a borrowed reference */ + Py_CLEAR(tstate->threading_local_key); + Py_CLEAR(tstate->threading_local_sentinel); + + Py_CLEAR(((_PyThreadStateImpl *)tstate)->asyncio_running_loop); + Py_CLEAR(tstate->dict); Py_CLEAR(tstate->async_exc); @@ -1731,7 +1738,7 @@ PyThreadState_Clear(PyThreadState *tstate) #ifdef Py_GIL_DISABLED // Each thread should clear own freelists in free-threading builds. - struct _Py_object_freelists *freelists = _Py_object_freelists_GET(); + struct _Py_freelists *freelists = _Py_freelists_GET(); _PyObject_ClearFreeLists(freelists, 1); // Remove ourself from the biased reference counting table of threads. diff --git a/Python/specialize.c b/Python/specialize.c index 497feca06e4c7f..c354a9079019ac 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -29,6 +29,10 @@ GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats }; PyStats *_Py_stats = NULL; +#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID +#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" +#endif + #define ADD_STAT_TO_DICT(res, field) \ do { \ PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \ @@ -959,15 +963,10 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); goto fail; } - uint32_t version = function_get_version(fget, LOAD_ATTR); - if (version == 0) { - goto fail; - } if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); goto fail; } - write_u32(lm_cache->keys_version, version); assert(type->tp_version_tag != 0); write_u32(lm_cache->type_version, type->tp_version_tag); /* borrowed */ diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 9686d10563aa4d..4d595d98445a05 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -99,6 +99,7 @@ static const char* _Py_stdlib_module_names[] = { "_winapi", "_zoneinfo", "abc", +"annotationlib", "antigravity", "argparse", "array", diff --git a/Python/symtable.c b/Python/symtable.c index 10103dbc2582a2..88af37198bfba5 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -70,10 +70,10 @@ #define DUPLICATE_TYPE_PARAM \ "duplicate type parameter '%U'" -#define ASYNC_WITH_OUTISDE_ASYNC_FUNC \ +#define ASYNC_WITH_OUTSIDE_ASYNC_FUNC \ "'async with' outside async function" -#define ASYNC_FOR_OUTISDE_ASYNC_FUNC \ +#define ASYNC_FOR_OUTSIDE_ASYNC_FUNC \ "'async for' outside async function" #define LOCATION(x) SRC_LOCATION_FROM_AST(x) @@ -82,6 +82,8 @@ PyErr_RangedSyntaxLocationObject((FNAME), \ (L).lineno, (L).col_offset + 1, (L).end_lineno, (L).end_col_offset + 1) +#define IS_ASYNC_DEF(st) ((st)->st_cur->ste_type == FunctionBlock && (st)->st_cur->ste_coroutine) + static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, void *key, _Py_SourceLocation loc) @@ -258,6 +260,7 @@ static int symtable_visit_pattern(struct symtable *st, pattern_ty s); static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty); static int symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_SourceLocation loc); static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty); +static int symtable_add_def(struct symtable *st, PyObject *name, int flag, _Py_SourceLocation loc); /* For debugging purposes only */ #if _PY_DUMP_SYMTABLE @@ -1386,6 +1389,16 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, return 0; int result = symtable_enter_existing_block(st, ste); Py_DECREF(ste); + if (block == AnnotationBlock || block == TypeVariableBlock || block == TypeAliasBlock) { + _Py_DECLARE_STR(format, ".format"); + // We need to insert code that reads this "parameter" to the function. + if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, loc)) { + return 0; + } + if (!symtable_add_def(st, &_Py_STR(format), USE, loc)) { + return 0; + } + } return result; } @@ -1493,8 +1506,57 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s } static int -symtable_add_def(struct symtable *st, PyObject *name, int flag, _Py_SourceLocation loc) +check_name(struct symtable *st, PyObject *name, _Py_SourceLocation loc, + expr_context_ty ctx) { + if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) { + PyErr_SetString(PyExc_SyntaxError, "cannot assign to __debug__"); + SET_ERROR_LOCATION(st->st_filename, loc); + return 0; + } + if (ctx == Del && _PyUnicode_EqualToASCIIString(name, "__debug__")) { + PyErr_SetString(PyExc_SyntaxError, "cannot delete __debug__"); + SET_ERROR_LOCATION(st->st_filename, loc); + return 0; + } + return 1; +} + +static int +check_keywords(struct symtable *st, asdl_keyword_seq *keywords) +{ + for (Py_ssize_t i = 0; i < asdl_seq_LEN(keywords); i++) { + keyword_ty key = ((keyword_ty)asdl_seq_GET(keywords, i)); + if (key->arg && !check_name(st, key->arg, LOCATION(key), Store)) { + return 0; + } + } + return 1; +} + +static int +check_kwd_patterns(struct symtable *st, pattern_ty p) +{ + assert(p->kind == MatchClass_kind); + asdl_identifier_seq *kwd_attrs = p->v.MatchClass.kwd_attrs; + asdl_pattern_seq *kwd_patterns = p->v.MatchClass.kwd_patterns; + for (Py_ssize_t i = 0; i < asdl_seq_LEN(kwd_attrs); i++) { + _Py_SourceLocation loc = LOCATION(asdl_seq_GET(kwd_patterns, i)); + if (!check_name(st, asdl_seq_GET(kwd_attrs, i), loc, Store)) { + return 0; + } + } + return 1; +} + +static int +symtable_add_def_ctx(struct symtable *st, PyObject *name, int flag, + _Py_SourceLocation loc, expr_context_ty ctx) +{ + int write_mask = DEF_PARAM | DEF_LOCAL | DEF_IMPORT; + if ((flag & write_mask) && !check_name(st, name, loc, ctx)) { + return 0; + } if ((flag & DEF_TYPE_PARAM) && st->st_cur->ste_mangled_names != NULL) { if(PySet_Add(st->st_cur->ste_mangled_names, name) < 0) { return 0; @@ -1503,6 +1565,14 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag, _Py_SourceLocati return symtable_add_def_helper(st, name, flag, st->st_cur, loc); } +static int +symtable_add_def(struct symtable *st, PyObject *name, int flag, + _Py_SourceLocation loc) +{ + return symtable_add_def_ctx(st, name, flag, loc, + flag == USE ? Load : Store); +} + static int symtable_enter_type_param_block(struct symtable *st, identifier name, void *ast, int has_defaults, int has_kwdefaults, @@ -1660,12 +1730,18 @@ check_import_from(struct symtable *st, stmt_ty s) return 1; } +static bool +allows_top_level_await(struct symtable *st) +{ + return (st->st_future->ff_features & PyCF_ALLOW_TOP_LEVEL_AWAIT) && + st->st_cur->ste_type == ModuleBlock; +} + + static void maybe_set_ste_coroutine_for_module(struct symtable *st, stmt_ty s) { - if ((st->st_future->ff_features & PyCF_ALLOW_TOP_LEVEL_AWAIT) && - (st->st_cur->ste_type == ModuleBlock)) - { + if (allows_top_level_await(st)) { st->st_cur->ste_coroutine = 1; } } @@ -1749,6 +1825,9 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, type_param, s->v.ClassDef.type_params); } VISIT_SEQ(st, expr, s->v.ClassDef.bases); + if (!check_keywords(st, s->v.ClassDef.keywords)) { + VISIT_QUIT(st, 0); + } VISIT_SEQ(st, keyword, s->v.ClassDef.keywords); if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock, (void *)s, LOCATION(s))) { @@ -1863,10 +1942,11 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT(st, expr, s->v.AnnAssign.value); } break; - case AugAssign_kind: + case AugAssign_kind: { VISIT(st, expr, s->v.AugAssign.target); VISIT(st, expr, s->v.AugAssign.value); break; + } case For_kind: VISIT(st, expr, s->v.For.target); VISIT(st, expr, s->v.For.iter); @@ -2054,7 +2134,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } case AsyncWith_kind: maybe_set_ste_coroutine_for_module(st, s); - if (!symtable_raise_if_not_coroutine(st, ASYNC_WITH_OUTISDE_ASYNC_FUNC, LOCATION(s))) { + if (!symtable_raise_if_not_coroutine(st, ASYNC_WITH_OUTSIDE_ASYNC_FUNC, LOCATION(s))) { VISIT_QUIT(st, 0); } VISIT_SEQ(st, withitem, s->v.AsyncWith.items); @@ -2062,7 +2142,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case AsyncFor_kind: maybe_set_ste_coroutine_for_module(st, s); - if (!symtable_raise_if_not_coroutine(st, ASYNC_FOR_OUTISDE_ASYNC_FUNC, LOCATION(s))) { + if (!symtable_raise_if_not_coroutine(st, ASYNC_FOR_OUTSIDE_ASYNC_FUNC, LOCATION(s))) { VISIT_QUIT(st, 0); } VISIT(st, expr, s->v.AsyncFor.target); @@ -2279,6 +2359,20 @@ symtable_visit_expr(struct symtable *st, expr_ty e) if (!symtable_raise_if_annotation_block(st, "await expression", e)) { VISIT_QUIT(st, 0); } + if (!allows_top_level_await(st)) { + if (!_PyST_IsFunctionLike(st->st_cur)) { + PyErr_SetString(PyExc_SyntaxError, + "'await' outside function"); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); + VISIT_QUIT(st, 0); + } + if (!IS_ASYNC_DEF(st) && st->st_cur->ste_comprehension == NoComprehension) { + PyErr_SetString(PyExc_SyntaxError, + "'await' outside async function"); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); + VISIT_QUIT(st, 0); + } + } VISIT(st, expr, e->v.Await.value); st->st_cur->ste_coroutine = 1; break; @@ -2289,6 +2383,9 @@ symtable_visit_expr(struct symtable *st, expr_ty e) case Call_kind: VISIT(st, expr, e->v.Call.func); VISIT_SEQ(st, expr, e->v.Call.args); + if (!check_keywords(st, e->v.Call.keywords)) { + VISIT_QUIT(st, 0); + } VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords); break; case FormattedValue_kind: @@ -2304,6 +2401,9 @@ symtable_visit_expr(struct symtable *st, expr_ty e) break; /* The following exprs can be assignment targets. */ case Attribute_kind: + if (!check_name(st, e->v.Attribute.attr, LOCATION(e), e->v.Attribute.ctx)) { + VISIT_QUIT(st, 0); + } VISIT(st, expr, e->v.Attribute.value); break; case Subscript_kind: @@ -2322,9 +2422,11 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.Slice.step); break; case Name_kind: - if (!symtable_add_def(st, e->v.Name.id, - e->v.Name.ctx == Load ? USE : DEF_LOCAL, LOCATION(e))) + if (!symtable_add_def_ctx(st, e->v.Name.id, + e->v.Name.ctx == Load ? USE : DEF_LOCAL, + LOCATION(e), e->v.Name.ctx)) { VISIT_QUIT(st, 0); + } /* Special-case super: it counts as a use of __class__ */ if (e->v.Name.ctx == Load && _PyST_IsFunctionLike(st->st_cur) && @@ -2450,19 +2552,26 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) break; case MatchStar_kind: if (p->v.MatchStar.name) { - symtable_add_def(st, p->v.MatchStar.name, DEF_LOCAL, LOCATION(p)); + if (!symtable_add_def(st, p->v.MatchStar.name, DEF_LOCAL, LOCATION(p))) { + VISIT_QUIT(st, 0); + } } break; case MatchMapping_kind: VISIT_SEQ(st, expr, p->v.MatchMapping.keys); VISIT_SEQ(st, pattern, p->v.MatchMapping.patterns); if (p->v.MatchMapping.rest) { - symtable_add_def(st, p->v.MatchMapping.rest, DEF_LOCAL, LOCATION(p)); + if (!symtable_add_def(st, p->v.MatchMapping.rest, DEF_LOCAL, LOCATION(p))) { + VISIT_QUIT(st, 0); + } } break; case MatchClass_kind: VISIT(st, expr, p->v.MatchClass.cls); VISIT_SEQ(st, pattern, p->v.MatchClass.patterns); + if (!check_kwd_patterns(st, p)) { + VISIT_QUIT(st, 0); + } VISIT_SEQ(st, pattern, p->v.MatchClass.kwd_patterns); break; case MatchAs_kind: @@ -2470,7 +2579,9 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) VISIT(st, pattern, p->v.MatchAs.pattern); } if (p->v.MatchAs.name) { - symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL, LOCATION(p)); + if (!symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL, LOCATION(p))) { + VISIT_QUIT(st, 0); + } } break; case MatchOr_kind: @@ -2530,18 +2641,6 @@ symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) return 0; } } - - _Py_DECLARE_STR(format, ".format"); - // The generated __annotate__ function takes a single parameter with the - // internal name ".format". - if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, - LOCATION(annotation))) { - return 0; - } - if (!symtable_add_def(st, &_Py_STR(format), USE, - LOCATION(annotation))) { - return 0; - } } else { if (!symtable_enter_existing_block(st, parent_ste->ste_annotation_block)) { @@ -2590,14 +2689,6 @@ symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ return 0; } } - _Py_DECLARE_STR(format, ".format"); - // We need to insert code that reads this "parameter" to the function. - if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, LOCATION(o))) { - return 0; - } - if (!symtable_add_def(st, &_Py_STR(format), USE, LOCATION(o))) { - return 0; - } if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs)) return 0; if (a->args && !symtable_visit_argannotations(st, a->args)) @@ -2798,6 +2889,16 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, if (!symtable_exit_block(st)) { return 0; } + if (is_async && + !IS_ASYNC_DEF(st) && + st->st_cur->ste_comprehension == NoComprehension && + !allows_top_level_await(st)) + { + PyErr_SetString(PyExc_SyntaxError, "asynchronous comprehension outside of " + "an asynchronous function"); + SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); + return 0; + } if (is_async) { st->st_cur->ste_coroutine = 1; } diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu new file mode 100644 index 00000000000000..8242c8d17c89fb --- /dev/null +++ b/Tools/build/.warningignore_ubuntu @@ -0,0 +1,3 @@ +# Files listed will be ignored by the compiler warning checker +# for the Ubuntu/build and test job. +# Keep lines sorted lexicographically to help avoid merge conflicts. diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py new file mode 100644 index 00000000000000..f0c0067f4ab255 --- /dev/null +++ b/Tools/build/check_warnings.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +""" +Parses compiler output with -fdiagnostics-format=json and checks that warnings +exist only in files that are expected to have warnings. +""" +import argparse +import json +import re +import sys +from pathlib import Path + + +def extract_warnings_from_compiler_output(compiler_output: str) -> list[dict]: + """ + Extracts warnings from the compiler output when using + -fdiagnostics-format=json + + Compiler output as a whole is not a valid json document, but includes many + json objects and may include other output that is not json. + """ + + # Regex to find json arrays at the top level of the file + # in the compiler output + json_arrays = re.findall( + r"\[(?:[^\[\]]|\[(?:[^\[\]]|\[[^\[\]]*\])*\])*\]", compiler_output + ) + compiler_warnings = [] + for array in json_arrays: + try: + json_data = json.loads(array) + json_objects_in_array = [entry for entry in json_data] + compiler_warnings.extend( + [ + entry + for entry in json_objects_in_array + if entry.get("kind") == "warning" + ] + ) + except json.JSONDecodeError: + continue # Skip malformed JSON + + return compiler_warnings + + +def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: + """ + Returns a dictionary where the key is the file and the data is the warnings + in that file + """ + warnings_by_file = {} + for warning in warnings: + locations = warning["locations"] + for location in locations: + for key in ["caret", "start", "end"]: + if key in location: + file = location[key]["file"] + file = file.lstrip( + "./" + ) # Remove leading current directory if present + if file not in warnings_by_file: + warnings_by_file[file] = [] + warnings_by_file[file].append(warning) + + return warnings_by_file + + +def get_unexpected_warnings( + warnings: list[dict], + files_with_expected_warnings: set[str], + files_with_warnings: set[str], +) -> int: + """ + Returns failure status if warnings discovered in list of warnings + are associated with a file that is not found in the list of files + with expected warnings + """ + unexpected_warnings = [] + for file in files_with_warnings.keys(): + if file not in files_with_expected_warnings: + unexpected_warnings.extend(files_with_warnings[file]) + + if unexpected_warnings: + print("Unexpected warnings:") + for warning in unexpected_warnings: + print(warning) + return 1 + + return 0 + + +def get_unexpected_improvements( + warnings: list[dict], + files_with_expected_warnings: set[str], + files_with_warnings: set[str], +) -> int: + """ + Returns failure status if there are no warnings in the list of warnings for + a file that is in the list of files with expected warnings + """ + unexpected_improvements = [] + for file in files_with_expected_warnings: + if file not in files_with_warnings.keys(): + unexpected_improvements.append(file) + + if unexpected_improvements: + print("Unexpected improvements:") + for file in unexpected_improvements: + print(file) + return 1 + + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--compiler-output-file-path", + type=str, + required=True, + help="Path to the compiler output file", + ) + parser.add_argument( + "--warning-ignore-file-path", + type=str, + required=True, + help="Path to the warning ignore file", + ) + parser.add_argument( + "--fail-on-regression", + action="store_true", + default=False, + help="Flag to fail if new warnings are found", + ) + parser.add_argument( + "--fail-on-improvement", + action="store_true", + default=False, + help="Flag to fail if files that were expected " + "to have warnings have no warnings", + ) + + args = parser.parse_args(argv) + + exit_code = 0 + + # Check that the compiler output file is a valid path + if not Path(args.compiler_output_file_path).is_file(): + print( + "Compiler output file does not exist: " + f"{args.compiler_output_file_path}" + ) + return 1 + + # Check that the warning ignore file is a valid path + if not Path(args.warning_ignore_file_path).is_file(): + print( + "Warning ignore file does not exist: " + f"{args.warning_ignore_file_path}" + ) + return 1 + + with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: + compiler_output_file_contents = f.read() + + with Path(args.warning_ignore_file_path).open( + encoding="UTF-8" + ) as clean_files: + files_with_expected_warnings = { + file.strip() + for file in clean_files + if file.strip() and not file.startswith("#") + } + + warnings = extract_warnings_from_compiler_output( + compiler_output_file_contents + ) + files_with_warnings = get_warnings_by_file(warnings) + + status = get_unexpected_warnings( + warnings, files_with_expected_warnings, files_with_warnings + ) + if args.fail_on_regression: + exit_code |= status + + status = get_unexpected_improvements( + warnings, files_with_expected_warnings, files_with_warnings + ) + if args.fail_on_improvement: + exit_code |= status + + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index c08568f2e00326..1b000c3b16a17a 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -108,6 +108,10 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: '.gitignore:9:*.a Tools/lib.a' """ + # No paths means no filtering to be done. + if not paths: + return [] + # Filter out files in gitignore. # Non-matching files show up as '::' git_check_ignore_proc = subprocess.run( diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 8b01c91e0d6bb3..f7fccb636858f0 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -225,9 +225,9 @@ def sort_key(item): key=sort_key): write(f'EXPORT_DATA({item.name})') -REST_ROLES = { - 'function': 'function', - 'data': 'var', +ITEM_KIND_TO_DOC_ROLE = { + 'function': 'func', + 'data': 'data', 'struct': 'type', 'macro': 'macro', # 'const': 'const', # all undocumented @@ -236,22 +236,28 @@ def sort_key(item): @generator("doc_list", 'Doc/data/stable_abi.dat') def gen_doc_annotations(manifest, args, outfile): - """Generate/check the stable ABI list for documentation annotations""" + """Generate/check the stable ABI list for documentation annotations + + See ``StableABIEntry`` in ``Doc/tools/extensions/c_annotations.py`` + for a description of each field. + """ writer = csv.DictWriter( outfile, ['role', 'name', 'added', 'ifdef_note', 'struct_abi_kind'], lineterminator='\n') writer.writeheader() - for item in manifest.select(REST_ROLES.keys(), include_abi_only=False): + kinds = set(ITEM_KIND_TO_DOC_ROLE) + for item in manifest.select(kinds, include_abi_only=False): if item.ifdef: ifdef_note = manifest.contents[item.ifdef].doc else: ifdef_note = None row = { - 'role': REST_ROLES[item.kind], + 'role': ITEM_KIND_TO_DOC_ROLE[item.kind], 'name': item.name, 'added': item.added, - 'ifdef_note': ifdef_note} + 'ifdef_note': ifdef_note, + } rows = [row] if item.kind == 'struct': row['struct_abi_kind'] = item.struct_abi_kind @@ -259,7 +265,8 @@ def gen_doc_annotations(manifest, args, outfile): rows.append({ 'role': 'member', 'name': f'{item.name}.{member_name}', - 'added': item.added}) + 'added': item.added, + }) writer.writerows(rows) @generator("ctypes_test", 'Lib/test/test_stable_abi_ctypes.py') diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index f92560bd2b76b3..675dc0b9acaf45 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -27,6 +27,7 @@ class Properties: tier: int | None = None oparg_and_1: bool = False const_oparg: int = -1 + needs_prev: bool = False def dump(self, indent: str) -> None: print(indent, end="") @@ -53,6 +54,7 @@ def from_list(properties: list["Properties"]) -> "Properties": has_free=any(p.has_free for p in properties), side_exit=any(p.side_exit for p in properties), pure=all(p.pure for p in properties), + needs_prev=any(p.needs_prev for p in properties), ) @property @@ -78,7 +80,7 @@ def infallible(self) -> bool: uses_locals=False, has_free=False, side_exit=False, - pure=False, + pure=True, ) @@ -96,6 +98,20 @@ def properties(self) -> Properties: return SKIP_PROPERTIES +class Flush: + + @property + def properties(self) -> Properties: + return SKIP_PROPERTIES + + @property + def name(self) -> str: + return "flush" + + @property + def size(self) -> int: + return 0 + @dataclass class StackItem: name: str @@ -103,16 +119,19 @@ class StackItem: condition: str | None size: str peek: bool = False + used: bool = False def __str__(self) -> str: cond = f" if ({self.condition})" if self.condition else "" - size = f"[{self.size}]" if self.size != "1" else "" + size = f"[{self.size}]" if self.size else "" type = "" if self.type is None else f"{self.type} " return f"{type}{self.name}{size}{cond} {self.peek}" def is_array(self) -> bool: - return self.type == "_PyStackRef *" + return self.size != "" + def get_size(self) -> str: + return self.size if self.size else "1" @dataclass class StackEffect: @@ -131,7 +150,6 @@ class CacheEntry: def __str__(self) -> str: return f"{self.name}/{self.size}" - @dataclass class Uop: name: str @@ -193,7 +211,7 @@ def is_super(self) -> bool: return False -Part = Uop | Skip +Part = Uop | Skip | Flush @dataclass @@ -293,7 +311,7 @@ def convert_stack_item(item: parser.StackEffect, replace_op_arg_1: str | None) - if replace_op_arg_1 and OPARG_AND_1.match(item.cond): cond = replace_op_arg_1 return StackItem( - item.name, item.type, cond, (item.size or "1") + item.name, item.type, cond, item.size ) def analyze_stack(op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | None = None) -> StackEffect: @@ -301,9 +319,23 @@ def analyze_stack(op: parser.InstDef | parser.Pseudo, replace_op_arg_1: str | No convert_stack_item(i, replace_op_arg_1) for i in op.inputs if isinstance(i, parser.StackEffect) ] outputs: list[StackItem] = [convert_stack_item(i, replace_op_arg_1) for i in op.outputs] + # Mark variables with matching names at the base of the stack as "peek" + modified = False for input, output in zip(inputs, outputs): - if input.name == output.name: + if input.name == output.name and not modified: input.peek = output.peek = True + else: + modified = True + if isinstance(op, parser.InstDef): + output_names = [out.name for out in outputs] + for input in inputs: + if (variable_used(op, input.name) or + variable_used(op, "DECREF_INPUTS") or + (not input.peek and input.name in output_names)): + input.used = True + for output in outputs: + if variable_used(op, output.name): + output.used = True return StackEffect(inputs, outputs) @@ -322,7 +354,13 @@ def analyze_caches(inputs: list[parser.InputEffect]) -> list[CacheEntry]: def variable_used(node: parser.InstDef, name: str) -> bool: """Determine whether a variable with a given name is used in a node.""" return any( - token.kind == "IDENTIFIER" and token.text == name for token in node.tokens + token.kind == "IDENTIFIER" and token.text == name for token in node.block.tokens + ) + +def oparg_used(node: parser.InstDef) -> bool: + """Determine whether `oparg` is used in a node.""" + return any( + token.kind == "IDENTIFIER" and token.text == "oparg" for token in node.tokens ) def tier_variable(node: parser.InstDef) -> int | None: @@ -568,7 +606,7 @@ def compute_properties(op: parser.InstDef) -> Properties: error_without_pop=error_without_pop, deopts=deopts_if, side_exit=exits_if, - oparg=variable_used(op, "oparg"), + oparg=oparg_used(op), jumps=variable_used(op, "JUMPBY"), eval_breaker=variable_used(op, "CHECK_EVAL_BREAKER"), ends_with_eval_breaker=eval_breaker_at_end(op), @@ -582,6 +620,7 @@ def compute_properties(op: parser.InstDef) -> Properties: has_free=has_free, pure="pure" in op.annotations, tier=tier_variable(op), + needs_prev=variable_used(op, "prev_instr"), ) @@ -687,13 +726,16 @@ def desugar_inst( def add_macro( macro: parser.Macro, instructions: dict[str, Instruction], uops: dict[str, Uop] ) -> None: - parts: list[Uop | Skip] = [] + parts: list[Part] = [] for part in macro.uops: match part: case parser.OpName(): - if part.name not in uops: - analysis_error(f"No Uop named {part.name}", macro.tokens[0]) - parts.append(uops[part.name]) + if part.name == "flush": + parts.append(Flush()) + else: + if part.name not in uops: + raise analysis_error(f"No Uop named {part.name}", macro.tokens[0]) + parts.append(uops[part.name]) case parser.CacheEffect(): parts.append(Skip(part.size)) case _: @@ -758,12 +800,6 @@ def assign_opcodes( instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] - # Special case: this instruction is implemented in ceval.c - # rather than bytecodes.c, so we need to add it explicitly - # here (at least until we add something to bytecodes.c to - # declare external instructions). - instrumented.append("INSTRUMENTED_LINE") - specialized: set[str] = set() no_arg: list[str] = [] has_arg: list[str] = [] diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index e4e0c9b658c19d..587dc0d03eded5 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -5,9 +5,10 @@ Instruction, Uop, Properties, + StackItem, ) from cwriter import CWriter -from typing import Callable, Mapping, TextIO, Iterator +from typing import Callable, Mapping, TextIO, Iterator, Tuple from lexer import Token from stack import Stack @@ -24,6 +25,15 @@ def root_relative_path(filename: str) -> str: return filename +def type_and_null(var: StackItem) -> Tuple[str, str]: + if var.type: + return var.type, "NULL" + elif var.is_array(): + return "_PyStackRef *", "NULL" + else: + return "_PyStackRef", "PyStackRef_NULL" + + def write_header( generator: str, sources: list[str], outfile: TextIO, comment: str = "//" ) -> None: @@ -82,7 +92,7 @@ def replace_error( next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon out.emit(") ") - c_offset = stack.peek_offset.to_c() + c_offset = stack.peek_offset() try: offset = -int(c_offset) close = ";\n" @@ -126,7 +136,7 @@ def replace_decrefs( for var in uop.stack.inputs: if var.name == "unused" or var.name == "null" or var.peek: continue - if var.size != "1": + if var.size: out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") out.emit(f"PyStackRef_CLOSE({var.name}[_i]);\n") out.emit("}\n") diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 0f5790dc4af40f..09b9d3d211eb24 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -151,7 +151,6 @@ def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: if inst.family is not None: deopt = inst.family.name deopts.append((inst.name, deopt)) - deopts.append(("INSTRUMENTED_LINE", "INSTRUMENTED_LINE")) for name, deopt in sorted(deopts): out.emit(f"[{name}] = {deopt},\n") out.emit("};\n\n") @@ -179,7 +178,6 @@ def generate_name_table(analysis: Analysis, out: CWriter) -> None: out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit(f"const char *_PyOpcode_OpName[{table_size}] = {{\n") names = list(analysis.instructions) + list(analysis.pseudos) - names.append("INSTRUMENTED_LINE") for name in sorted(names): out.emit(f'[{name}] = "{name}",\n') out.emit("};\n") diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 277521678434c2..6a66693b93305d 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -23,7 +23,7 @@ from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token -from stack import Stack, SizeMismatch +from stack import Stack, StackError DEFAULT_OUTPUT = ROOT / "Python/optimizer_cases.c.h" DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/optimizer_bytecodes.c").absolute().as_posix() @@ -141,7 +141,7 @@ def write_uop( out.emit(stack.push(var)) out.start_line() stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True) - except SizeMismatch as ex: + except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 0bd4229e2beaf2..8957838f7a90a1 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -285,7 +285,6 @@ def stack_effect(self) -> StackEffect | None: if not (size := self.expression()): raise self.make_syntax_error("Expected expression") self.require(lx.RBRACKET) - type_text = "_PyStackRef *" size_text = size.text.strip() return StackEffect(tkn.text, type_text, cond_text, size_text) return None diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index c0e1278e519143..61dcfd3e30a510 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -28,14 +28,15 @@ def var_size(var: StackItem) -> str: if var.condition == "0": return "0" elif var.condition == "1": - return var.size - elif var.condition == "oparg & 1" and var.size == "1": + return var.get_size() + elif var.condition == "oparg & 1" and not var.size: return f"({var.condition})" else: - return f"(({var.condition}) ? {var.size} : 0)" - else: + return f"(({var.condition}) ? {var.get_size()} : 0)" + elif var.size: return var.size - + else: + return "1" @dataclass class StackOffset: @@ -48,6 +49,9 @@ class StackOffset: def empty() -> "StackOffset": return StackOffset([], []) + def copy(self) -> "StackOffset": + return StackOffset(self.popped[:], self.pushed[:]) + def pop(self, item: StackItem) -> None: self.popped.append(var_size(item)) @@ -113,7 +117,7 @@ def clear(self) -> None: self.pushed = [] -class SizeMismatch(Exception): +class StackError(Exception): pass @@ -121,39 +125,38 @@ class Stack: def __init__(self) -> None: self.top_offset = StackOffset.empty() self.base_offset = StackOffset.empty() - self.peek_offset = StackOffset.empty() self.variables: list[StackItem] = [] self.defined: set[str] = set() def pop(self, var: StackItem, extract_bits: bool = False) -> str: self.top_offset.pop(var) - if not var.peek: - self.peek_offset.pop(var) indirect = "&" if var.is_array() else "" if self.variables: popped = self.variables.pop() if popped.size != var.size: - raise SizeMismatch( + raise StackError( f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. " f"Expected {var.size} got {popped.size}" ) - if popped.name == var.name: + if var.name in UNUSED: + if popped.name not in UNUSED and popped.name in self.defined: + raise StackError(f"Value is declared unused, but is already cached by prior operation") return "" - elif popped.name in UNUSED: + if popped.name in UNUSED or popped.name not in self.defined: self.defined.add(var.name) return ( f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n" ) - elif var.name in UNUSED: - return "" else: self.defined.add(var.name) - return f"{var.name} = {popped.name};\n" + if popped.name == var.name: + return "" + else: + return f"{var.name} = {popped.name};\n" self.base_offset.pop(var) - if var.name in UNUSED: + if var.name in UNUSED or not var.used: return "" - else: - self.defined.add(var.name) + self.defined.add(var.name) cast = f"({var.type})" if (not indirect and var.type) else "" bits = ".bits" if cast and not extract_bits else "" assign = ( @@ -177,6 +180,8 @@ def push(self, var: StackItem) -> str: return f"{var.name} = &stack_pointer[{c_offset}];\n" else: self.top_offset.push(var) + if var.used: + self.defined.add(var.name) return "" def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = False) -> None: @@ -205,9 +210,16 @@ def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = self.variables = [] self.base_offset.clear() self.top_offset.clear() - self.peek_offset.clear() out.start_line() + def peek_offset(self) -> str: + peek = self.base_offset.copy() + for var in self.variables: + if not var.peek: + break + peek.push(var) + return peek.to_c() + def as_comment(self) -> str: return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */" diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index c9dce1d5f1804e..118f4b3a6eaa1c 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -12,17 +12,20 @@ Part, analyze_files, Skip, + Flush, analysis_error, + StackItem, ) from generators_common import ( DEFAULT_INPUT, ROOT, write_header, emit_tokens, + type_and_null, ) from cwriter import CWriter from typing import TextIO -from stack import Stack, SizeMismatch +from stack import Stack, StackError DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h" @@ -30,33 +33,39 @@ FOOTER = "#undef TIER_ONE\n" +def declare_variable(var: StackItem, out: CWriter) -> None: + type, null = type_and_null(var) + space = " " if type[-1].isalnum() else "" + if var.condition: + out.emit(f"{type}{space}{var.name} = {null};\n") + else: + out.emit(f"{type}{space}{var.name};\n") -def declare_variables(inst: Instruction, out: CWriter) -> None: - variables = {"unused"} - for uop in inst.parts: - if isinstance(uop, Uop): - for var in reversed(uop.stack.inputs): - if var.name not in variables: - variables.add(var.name) - type, null = (var.type, "NULL") if var.type else ("_PyStackRef", "PyStackRef_NULL") - space = " " if type[-1].isalnum() else "" - if var.condition: - out.emit(f"{type}{space}{var.name} = {null};\n") - else: - if var.is_array(): - out.emit(f"{var.type}{space}{var.name};\n") - else: - out.emit(f"{type}{space}{var.name};\n") - for var in uop.stack.outputs: - if var.name not in variables: - variables.add(var.name) - type, null = (var.type, "NULL") if var.type else ("_PyStackRef", "PyStackRef_NULL") - space = " " if type[-1].isalnum() else "" - if var.condition: - out.emit(f"{type}{space}{var.name} = {null};\n") - else: - out.emit(f"{type}{space}{var.name};\n") +def declare_variables(inst: Instruction, out: CWriter) -> None: + stack = Stack() + for part in inst.parts: + if not isinstance(part, Uop): + continue + try: + for var in reversed(part.stack.inputs): + stack.pop(var) + for var in part.stack.outputs: + stack.push(var) + except StackError as ex: + raise analysis_error(ex.args[0], part.body[0]) from None + required = set(stack.defined) + for part in inst.parts: + if not isinstance(part, Uop): + continue + for var in part.stack.inputs: + if var.name in required: + required.remove(var.name) + declare_variable(var, out) + for var in part.stack.outputs: + if var.name in required: + required.remove(var.name) + declare_variable(var, out) def write_uop( uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool @@ -66,6 +75,10 @@ def write_uop( entries = "entries" if uop.size > 1 else "entry" out.emit(f"/* Skip {uop.size} cache {entries} */\n") return offset + uop.size + if isinstance(uop, Flush): + out.emit(f"// flush\n") + stack.flush(out) + return offset try: out.start_line() if braces: @@ -100,15 +113,15 @@ def write_uop( out.emit("}\n") # out.emit(stack.as_comment() + "\n") return offset - except SizeMismatch as ex: - raise analysis_error(ex.args[0], uop.body[0]) + except StackError as ex: + raise analysis_error(ex.args[0], uop.body[0]) from None def uses_this(inst: Instruction) -> bool: if inst.properties.needs_this: return True for uop in inst.parts: - if isinstance(uop, Skip): + if not isinstance(uop, Uop): continue for cache in uop.caches: if cache.name != "unused": @@ -135,6 +148,8 @@ def generate_tier1( out.emit("\n") out.emit(f"TARGET({name}) {{\n") unused_guard = "(void)this_instr;\n" if inst.family is None else "" + if inst.properties.needs_prev: + out.emit(f"_Py_CODEUNIT *prev_instr = frame->instr_ptr;\n") if needs_this and not inst.is_target: out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;\n") out.emit(unused_guard) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index f3769bd31c295d..88ad0fd797f0cc 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -20,22 +20,23 @@ emit_tokens, emit_to, REPLACEMENT_FUNCTIONS, + type_and_null, ) from cwriter import CWriter from typing import TextIO, Iterator from lexer import Token -from stack import Stack, SizeMismatch +from stack import Stack, StackError DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h" def declare_variable( - var: StackItem, uop: Uop, variables: set[str], out: CWriter + var: StackItem, uop: Uop, required: set[str], out: CWriter ) -> None: - if var.name in variables: + if var.name not in required: return - variables.add(var.name) - type, null = (var.type, "NULL") if var.type else ("_PyStackRef", "PyStackRef_NULL") + required.remove(var.name) + type, null = type_and_null(var) space = " " if type[-1].isalnum() else "" if var.condition: out.emit(f"{type}{space}{var.name} = {null};\n") @@ -48,12 +49,16 @@ def declare_variable( def declare_variables(uop: Uop, out: CWriter) -> None: - variables = {"unused"} + stack = Stack() for var in reversed(uop.stack.inputs): - declare_variable(var, uop, variables, out) + stack.pop(var) for var in uop.stack.outputs: - declare_variable(var, uop, variables, out) - + stack.push(var) + required = set(stack.defined) + for var in reversed(uop.stack.inputs): + declare_variable(var, uop, required, out) + for var in uop.stack.outputs: + declare_variable(var, uop, required, out) def tier2_replace_error( out: CWriter, @@ -176,8 +181,8 @@ def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None: if uop.properties.stores_sp: for i, var in enumerate(uop.stack.outputs): out.emit(stack.push(var)) - except SizeMismatch as ex: - raise analysis_error(ex.args[0], uop.body[0]) + except StackError as ex: + raise analysis_error(ex.args[0], uop.body[0]) from None SKIPS = ("_EXTENDED_ARG",) diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 73d2deebbbc216..bc6f793b296f12 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -1,7 +1,7 @@ The JIT Compiler ================ -This version of CPython can be built with an experimental just-in-time compiler. While most everything you already know about building and using CPython is unchanged, you will probably need to install a compatible version of LLVM first. +This version of CPython can be built with an experimental just-in-time compiler[^pep-744]. While most everything you already know about building and using CPython is unchanged, you will probably need to install a compatible version of LLVM first. ## Installing LLVM @@ -57,6 +57,10 @@ For `PCbuild`-based builds, pass the new `--experimental-jit` option to `build.b For all other builds, pass the new `--enable-experimental-jit` option to `configure`. -Otherwise, just configure and build as you normally would. Cross-compiling "just works", since the JIT is built for the host platform. +Otherwise, just configure and build as you normally would. Cross-compiling "just works", since the JIT is built for the host platform. + +The JIT can also be enabled or disabled using the `PYTHON_JIT` environment variable, even on builds where it is enabled or disabled by default. More details about configuring CPython with the JIT and optional values for `--enable-experimental-jit` can be found [here](https://docs.python.org/dev/whatsnew/3.13.html#experimental-jit-compiler). + +[^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) [^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c index b081240ffff017..1587d53d59472e 100644 --- a/Tools/peg_generator/peg_extension/peg_extension.c +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -108,7 +108,7 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * clear_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { -#if defined(PY_DEBUG) +#if defined(Py_DEBUG) _PyPegen_clear_memo_statistics(); #endif Py_RETURN_NONE; @@ -117,7 +117,7 @@ clear_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) static PyObject * get_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { -#if defined(PY_DEBUG) +#if defined(Py_DEBUG) return _PyPegen_get_memo_statistics(); #else Py_RETURN_NONE; @@ -128,7 +128,7 @@ get_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) static PyObject * dump_memo_stats(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) { -#if defined(PY_DEBUG) +#if defined(Py_DEBUG) PyObject *list = _PyPegen_get_memo_statistics(); if (list == NULL) { return NULL; diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 534a0cedb743dd..a54e66d1212d1f 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -23,25 +23,11 @@ race:free_threadstate # These warnings trigger directly in a CPython function. -race_top:_add_to_weak_set -race_top:_in_weak_set race_top:_PyEval_EvalFrameDefault race_top:assign_version_tag -race_top:insertdict -race_top:lookup_tp_dict race_top:new_reference -# https://gist.github.com/colesbury/d13d033f413b4ad07929d044bed86c35 -race_top:set_discard_entry -race_top:_PyDict_CheckConsistency -race_top:_Py_dict_lookup_threadsafe race_top:_multiprocessing_SemLock_acquire_impl -race_top:dictiter_new -race_top:dictresize -race_top:insert_to_emptydict -race_top:insertdict race_top:list_get_item_ref -race_top:make_pending_calls -race_top:set_add_entry race_top:_Py_slot_tp_getattr_hook race_top:add_threadstate race_top:dump_traceback diff --git a/Tools/wasm/config.site-wasm32-wasi b/Tools/wasm/config.site-wasm32-wasi index 4a1a466a4ab3f1..c5d8b3e205db26 100644 --- a/Tools/wasm/config.site-wasm32-wasi +++ b/Tools/wasm/config.site-wasm32-wasi @@ -49,3 +49,11 @@ ac_cv_func_preadv=no ac_cv_func_readv=no ac_cv_func_pwritev=no ac_cv_func_writev=no + +# WASI SDK 22 added multiple stubs which we don't implement. +# https://github.com/python/cpython/issues/120371 +ac_cv_func_chmod=no +ac_cv_func_fchmod=no +ac_cv_func_fchmodat=no +ac_cv_func_statvfs=no +ac_cv_func_fstatvfs=no diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index f69299fd662806..a14f58bdac0cb2 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -26,6 +26,9 @@ LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8") +WASMTIME_VAR_NAME = "WASMTIME" +WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" + def updated_env(updates={}): """Create a new dict representing the environment to use. @@ -215,11 +218,20 @@ def configure_wasi_python(context, working_dir): # Use PYTHONPATH to include sysconfig data which must be anchored to the # WASI guest's `/` directory. - host_runner = context.host_runner.format(GUEST_DIR="/", - HOST_DIR=CHECKOUT, - ENV_VAR_NAME="PYTHONPATH", - ENV_VAR_VALUE=f"/{sysconfig_data}", - PYTHON_WASM=working_dir / "python.wasm") + args = {"GUEST_DIR": "/", + "HOST_DIR": CHECKOUT, + "ENV_VAR_NAME": "PYTHONPATH", + "ENV_VAR_VALUE": f"/{sysconfig_data}", + "PYTHON_WASM": working_dir / "python.wasm"} + # Check dynamically for wasmtime in case it was specified manually via + # `--host-runner`. + if WASMTIME_HOST_RUNNER_VAR in context.host_runner: + if wasmtime := shutil.which("wasmtime"): + args[WASMTIME_VAR_NAME] = wasmtime + else: + raise FileNotFoundError("wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime") + host_runner = context.host_runner.format_map(args) env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} build_python = os.fsdecode(build_python_path()) # The path to `configure` MUST be relative, else `python.wasm` is unable @@ -277,7 +289,7 @@ def clean_contents(context): def main(): - default_host_runner = (f"{shutil.which('wasmtime')} run " + default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " # Make sure the stack size will work for a pydebug # build. # Use 16 MiB stack. diff --git a/configure b/configure index 131ca5f7f897a7..52988f77f6d926 100755 --- a/configure +++ b/configure @@ -795,8 +795,6 @@ MODULE__POSIXSUBPROCESS_FALSE MODULE__POSIXSUBPROCESS_TRUE MODULE__PICKLE_FALSE MODULE__PICKLE_TRUE -MODULE__OPCODE_FALSE -MODULE__OPCODE_TRUE MODULE__LSPROF_FALSE MODULE__LSPROF_TRUE MODULE__JSON_FALSE @@ -930,6 +928,7 @@ DEF_MAKE_RULE DEF_MAKE_ALL_RULE JIT_STENCILS_H REGEN_JIT_COMMAND +ABI_THREAD ABIFLAGS LN MKDIR_P @@ -981,8 +980,6 @@ IPHONEOS_DEPLOYMENT_TARGET EXPORT_MACOSX_DEPLOYMENT_TARGET CONFIGURE_MACOSX_DEPLOYMENT_TARGET _PYTHON_HOST_PLATFORM -APP_STORE_COMPLIANCE_PATCH_FLAGS -APP_STORE_COMPLIANCE_PATCH_TARGET APP_STORE_COMPLIANCE_PATCH INSTALLTARGETS FRAMEWORKINSTALLAPPSPREFIX @@ -1097,6 +1094,8 @@ enable_optimizations with_lto enable_bolt with_strict_overflow +enable_safety +enable_slower_safety with_dsymutil with_address_sanitizer with_memory_sanitizer @@ -1829,6 +1828,10 @@ Optional Features: (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer (default is no) + --disable-safety disable usage of the security compiler options with + no performance overhead + --enable-slower-safety enable usage of the security compiler options with + performance overhead --enable-loadable-sqlite-extensions support loadable extensions in the sqlite3 module, see Doc/library/sqlite3.rst (default is no) @@ -4109,7 +4112,7 @@ printf "%s\n" "\"$MACHDEP\"" >&6; } # On cross-compile builds, configure will look for a host-specific compiler by # prepending the user-provided host triple to the required binary name. # -# On iOS, this results in binaries like "arm64-apple-ios12.0-simulator-gcc", +# On iOS, this results in binaries like "arm64-apple-ios13.0-simulator-gcc", # which isn't a binary that exists, and isn't very convenient, as it contains the # iOS version. As the default cross-compiler name won't exist, configure falls # back to gcc, which *definitely* won't work. We're providing wrapper scripts for @@ -4451,8 +4454,6 @@ then : Darwin|iOS) # iOS is able to share the macOS patch APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" - APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" - APP_STORE_COMPLIANCE_PATCH_FLAGS= ;; *) as_fn_error $? "no default app store compliance patch available for $ac_sys_system" "$LINENO" 5 ;; esac @@ -4461,8 +4462,6 @@ printf "%s\n" "applying default app store compliance patch" >&6; } ;; *) APP_STORE_COMPLIANCE_PATCH="${withval}" - APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" - APP_STORE_COMPLIANCE_PATCH_FLAGS= { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying custom app store compliance patch" >&5 printf "%s\n" "applying custom app store compliance patch" >&6; } ;; @@ -4474,24 +4473,12 @@ else $as_nop iOS) # Always apply the compliance patch on iOS; we can use the macOS patch APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" - APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" - APP_STORE_COMPLIANCE_PATCH_FLAGS= { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: applying default app store compliance patch" >&5 printf "%s\n" "applying default app store compliance patch" >&6; } ;; - Darwin) - # Always *check* the compliance patch on macOS - APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" - APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" - APP_STORE_COMPLIANCE_PATCH_FLAGS="--dry-run" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: checking (not applying) default app store compliance patch" >&5 -printf "%s\n" "checking (not applying) default app store compliance patch" >&6; } - ;; *) - # No app compliance patching on any other platform + # No default app compliance patching on any other platform APP_STORE_COMPLIANCE_PATCH= - APP_STORE_COMPLIANCE_PATCH_TARGET= - APP_STORE_COMPLIANCE_PATCH_FLAGS= { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: not patching for app store compliance" >&5 printf "%s\n" "not patching for app store compliance" >&6; } ;; @@ -4502,8 +4489,6 @@ fi - - if test "$cross_compiling" = yes; then case "$host" in *-*-linux*) @@ -4524,8 +4509,12 @@ if test "$cross_compiling" = yes; then _host_device=${_host_device:=os} # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking iOS deployment target" >&5 +printf %s "checking iOS deployment target... " >&6; } IPHONEOS_DEPLOYMENT_TARGET=${_host_os:3} - IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=12.0} + IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=13.0} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $IPHONEOS_DEPLOYMENT_TARGET" >&5 +printf "%s\n" "$IPHONEOS_DEPLOYMENT_TARGET" >&6; } case "$host_cpu" in aarch64) @@ -8162,7 +8151,9 @@ fi # For calculating the .so ABI tag. + ABIFLAGS="" +ABI_THREAD="" # Check for --disable-gil # --disable-gil @@ -8192,6 +8183,7 @@ printf "%s\n" "#define Py_GIL_DISABLED 1" >>confdefs.h # Add "t" for "threaded" ABIFLAGS="${ABIFLAGS}t" + ABI_THREAD="t" fi # Check for --with-pydebug @@ -9680,7 +9672,28 @@ fi # Enable flags that warn and protect for potential security vulnerabilities. # These flags should be enabled by default for all builds. -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fstack-protector-strong" >&5 + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --disable-safety" >&5 +printf %s "checking for --disable-safety... " >&6; } +# Check whether --enable-safety was given. +if test ${enable_safety+y} +then : + enableval=$enable_safety; if test "x$enable_safety" = xyes +then : + disable_safety=no +else $as_nop + disable_saftey=yes +fi +else $as_nop + disable_saftey=no +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_safety" >&5 +printf "%s\n" "$disable_safety" >&6; } + +if test "$disable_safety" = "no" +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fstack-protector-strong" >&5 printf %s "checking whether C compiler accepts -fstack-protector-strong... " >&6; } if test ${ax_cv_check_cflags__Werror__fstack_protector_strong+y} then : @@ -9719,7 +9732,7 @@ else $as_nop printf "%s\n" "$as_me: WARNING: -fstack-protector-strong not supported" >&2;} fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wtrampolines" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wtrampolines" >&5 printf %s "checking whether C compiler accepts -Wtrampolines... " >&6; } if test ${ax_cv_check_cflags__Werror__Wtrampolines+y} then : @@ -9758,6 +9771,61 @@ else $as_nop printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;} fi +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-slower-safety" >&5 +printf %s "checking for --enable-slower-safety... " >&6; } +# Check whether --enable-slower-safety was given. +if test ${enable_slower_safety+y} +then : + enableval=$enable_slower_safety; +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_slower_safety" >&5 +printf "%s\n" "$enable_slower_safety" >&6; } + +if test "$enable_slower_safety" = "yes" +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -D_FORTIFY_SOURCE=3" >&5 +printf %s "checking whether C compiler accepts -D_FORTIFY_SOURCE=3... " >&6; } +if test ${ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -D_FORTIFY_SOURCE=3" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3=yes +else $as_nop + ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" >&6; } +if test "x$ax_cv_check_cflags__Werror__D_FORTIFY_SOURCE_3" = xyes +then : + BASECFLAGS="$BASECFLAGS -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -D_FORTIFY_SOURCE=3 not supported" >&5 +printf "%s\n" "$as_me: WARNING: -D_FORTIFY_SOURCE=3 not supported" >&2;} +fi + +fi case $GCC in yes) @@ -24740,11 +24808,11 @@ fi -BINLIBDEST='$(LIBDIR)/python$(VERSION)' +BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)' # Check for --with-platlibdir -# /usr/$LIDIRNAME/python$VERSION +# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD) PLATLIBDIR="lib" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-platlibdir" >&5 @@ -24763,7 +24831,7 @@ then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } PLATLIBDIR="$withval" - BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } @@ -24777,9 +24845,9 @@ fi if test x$PLATFORM_TRIPLET = x; then - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}" else - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}" fi @@ -29231,28 +29299,6 @@ then : -fi - - - if test "$py_cv_module__opcode" != "n/a" -then : - py_cv_module__opcode=yes -fi - if test "$py_cv_module__opcode" = yes; then - MODULE__OPCODE_TRUE= - MODULE__OPCODE_FALSE='#' -else - MODULE__OPCODE_TRUE='#' - MODULE__OPCODE_FALSE= -fi - - as_fn_append MODULE_BLOCK "MODULE__OPCODE_STATE=$py_cv_module__opcode$as_nl" - if test "x$py_cv_module__opcode" = xyes -then : - - - - fi @@ -31784,10 +31830,6 @@ if test -z "${MODULE__LSPROF_TRUE}" && test -z "${MODULE__LSPROF_FALSE}"; then as_fn_error $? "conditional \"MODULE__LSPROF\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__OPCODE_TRUE}" && test -z "${MODULE__OPCODE_FALSE}"; then - as_fn_error $? "conditional \"MODULE__OPCODE\" was never defined. -Usually this means the macro was only invoked conditionally." "$LINENO" 5 -fi if test -z "${MODULE__PICKLE_TRUE}" && test -z "${MODULE__PICKLE_FALSE}"; then as_fn_error $? "conditional \"MODULE__PICKLE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 705f8752597b96..5bde6803cd5a7b 100644 --- a/configure.ac +++ b/configure.ac @@ -382,7 +382,7 @@ AC_MSG_RESULT(["$MACHDEP"]) # On cross-compile builds, configure will look for a host-specific compiler by # prepending the user-provided host triple to the required binary name. # -# On iOS, this results in binaries like "arm64-apple-ios12.0-simulator-gcc", +# On iOS, this results in binaries like "arm64-apple-ios13.0-simulator-gcc", # which isn't a binary that exists, and isn't very convenient, as it contains the # iOS version. As the default cross-compiler name won't exist, configure falls # back to gcc, which *definitely* won't work. We're providing wrapper scripts for @@ -710,8 +710,6 @@ AC_ARG_WITH( Darwin|iOS) # iOS is able to share the macOS patch APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" - APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" - APP_STORE_COMPLIANCE_PATCH_FLAGS= ;; *) AC_MSG_ERROR([no default app store compliance patch available for $ac_sys_system]) ;; esac @@ -719,8 +717,6 @@ AC_ARG_WITH( ;; *) APP_STORE_COMPLIANCE_PATCH="${withval}" - APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" - APP_STORE_COMPLIANCE_PATCH_FLAGS= AC_MSG_RESULT([applying custom app store compliance patch]) ;; esac @@ -729,29 +725,16 @@ AC_ARG_WITH( iOS) # Always apply the compliance patch on iOS; we can use the macOS patch APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" - APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" - APP_STORE_COMPLIANCE_PATCH_FLAGS= AC_MSG_RESULT([applying default app store compliance patch]) ;; - Darwin) - # Always *check* the compliance patch on macOS - APP_STORE_COMPLIANCE_PATCH="Mac/Resources/app-store-compliance.patch" - APP_STORE_COMPLIANCE_PATCH_TARGET="build/app-store-compliant" - APP_STORE_COMPLIANCE_PATCH_FLAGS="--dry-run" - AC_MSG_RESULT([checking (not applying) default app store compliance patch]) - ;; *) - # No app compliance patching on any other platform + # No default app compliance patching on any other platform APP_STORE_COMPLIANCE_PATCH= - APP_STORE_COMPLIANCE_PATCH_TARGET= - APP_STORE_COMPLIANCE_PATCH_FLAGS= AC_MSG_RESULT([not patching for app store compliance]) ;; esac ]) AC_SUBST([APP_STORE_COMPLIANCE_PATCH]) -AC_SUBST([APP_STORE_COMPLIANCE_PATCH_TARGET]) -AC_SUBST([APP_STORE_COMPLIANCE_PATCH_FLAGS]) AC_SUBST([_PYTHON_HOST_PLATFORM]) if test "$cross_compiling" = yes; then @@ -774,8 +757,10 @@ if test "$cross_compiling" = yes; then _host_device=${_host_device:=os} # IPHONEOS_DEPLOYMENT_TARGET is the minimum supported iOS version + AC_MSG_CHECKING([iOS deployment target]) IPHONEOS_DEPLOYMENT_TARGET=${_host_os:3} - IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=12.0} + IPHONEOS_DEPLOYMENT_TARGET=${IPHONEOS_DEPLOYMENT_TARGET:=13.0} + AC_MSG_RESULT([$IPHONEOS_DEPLOYMENT_TARGET]) case "$host_cpu" in aarch64) @@ -1739,7 +1724,9 @@ fi # For calculating the .so ABI tag. AC_SUBST([ABIFLAGS]) +AC_SUBST([ABI_THREAD]) ABIFLAGS="" +ABI_THREAD="" # Check for --disable-gil # --disable-gil @@ -1756,6 +1743,7 @@ then [Define if you want to disable the GIL]) # Add "t" for "threaded" ABIFLAGS="${ABIFLAGS}t" + ABI_THREAD="t" fi # Check for --with-pydebug @@ -2511,8 +2499,28 @@ AS_VAR_IF([with_strict_overflow], [yes], # Enable flags that warn and protect for potential security vulnerabilities. # These flags should be enabled by default for all builds. -AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [BASECFLAGS="$BASECFLAGS -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror]) -AX_CHECK_COMPILE_FLAG([-Wtrampolines], [BASECFLAGS="$BASECFLAGS -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror]) + +AC_MSG_CHECKING([for --disable-safety]) +AC_ARG_ENABLE([safety], + [AS_HELP_STRING([--disable-safety], [disable usage of the security compiler options with no performance overhead])], + [AS_VAR_IF([enable_safety], [yes], [disable_safety=no], [disable_saftey=yes])], [disable_saftey=no]) +AC_MSG_RESULT([$disable_safety]) + +if test "$disable_safety" = "no" +then + AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [BASECFLAGS="$BASECFLAGS -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wtrampolines], [BASECFLAGS="$BASECFLAGS -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror]) +fi + +AC_MSG_CHECKING([for --enable-slower-safety]) +AC_ARG_ENABLE([slower-safety], + [AS_HELP_STRING([--enable-slower-safety], [enable usage of the security compiler options with performance overhead])],[]) +AC_MSG_RESULT([$enable_slower_safety]) + +if test "$enable_slower_safety" = "yes" +then + AX_CHECK_COMPILE_FLAG([-D_FORTIFY_SOURCE=3], [BASECFLAGS="$BASECFLAGS -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3"], [AC_MSG_WARN([-D_FORTIFY_SOURCE=3 not supported])], [-Werror]) +fi case $GCC in yes) @@ -6234,11 +6242,11 @@ fi AC_SUBST([BINLIBDEST]) -BINLIBDEST='$(LIBDIR)/python$(VERSION)' +BINLIBDEST='$(LIBDIR)/python$(VERSION)$(ABI_THREAD)' # Check for --with-platlibdir -# /usr/$LIDIRNAME/python$VERSION +# /usr/$PLATLIBDIR/python$(VERSION)$(ABI_THREAD) AC_SUBST([PLATLIBDIR]) PLATLIBDIR="lib" AC_MSG_CHECKING([for --with-platlibdir]) @@ -6257,7 +6265,7 @@ if test -n "$withval" -a "$withval" != yes -a "$withval" != no then AC_MSG_RESULT([yes]) PLATLIBDIR="$withval" - BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' else AC_MSG_RESULT([no]) fi], @@ -6267,9 +6275,9 @@ fi], dnl define LIBPL after ABIFLAGS and LDVERSION is defined. AC_SUBST([PY_ENABLE_SHARED]) if test x$PLATFORM_TRIPLET = x; then - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}" else - LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" + LIBPL='$(prefix)'"/${PLATLIBDIR}/python${VERSION}${ABI_THREAD}/config-${LDVERSION}-${PLATFORM_TRIPLET}" fi AC_SUBST([LIBPL]) @@ -7689,7 +7697,6 @@ PY_STDLIB_MOD_SIMPLE([_csv]) PY_STDLIB_MOD_SIMPLE([_heapq]) PY_STDLIB_MOD_SIMPLE([_json]) PY_STDLIB_MOD_SIMPLE([_lsprof]) -PY_STDLIB_MOD_SIMPLE([_opcode]) PY_STDLIB_MOD_SIMPLE([_pickle]) PY_STDLIB_MOD_SIMPLE([_posixsubprocess]) PY_STDLIB_MOD_SIMPLE([_queue]) diff --git a/iOS/README.rst b/iOS/README.rst index 96cb00eb2e9bfe..4d7c344d5e9e17 100644 --- a/iOS/README.rst +++ b/iOS/README.rst @@ -188,7 +188,7 @@ especially important, as many parts of the standard library (including the ``ctypes`` module at runtime. By default, Python will be compiled with an iOS deployment target (i.e., the -minimum supported iOS version) of 12.0. To specify a different deployment +minimum supported iOS version) of 13.0. To specify a different deployment target, provide the version number as part of the ``--host`` argument - for example, ``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 simulator build with a deployment target of 15.4.