diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index e13d6701..b36a6064 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -2,9 +2,15 @@ name: MacOS Python build on: push: + branches: [main] pull_request: schedule: - cron: '13 11 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: pythonbuild: runs-on: 'macos-13' @@ -108,6 +114,19 @@ jobs: py: 'cpython-3.12' optimizations: 'pgo+lto' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.13' + optimizations: 'pgo' + - target_triple: 'aarch64-apple-darwin' + runner: macos-14 + py: 'cpython-3.13' + optimizations: 'pgo+lto' + # macOS on Intel hardware. This is pretty straightforward. We exclude # noopt because it doesn't provide any compelling advantages over PGO # or LTO builds. @@ -175,6 +194,19 @@ jobs: runner: macos-13 py: 'cpython-3.12' optimizations: 'pgo+lto' + + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.13' + optimizations: 'pgo' + - target_triple: 'x86_64-apple-darwin' + runner: macos-13 + py: 'cpython-3.13' + optimizations: 'pgo+lto' needs: - pythonbuild runs-on: ${{ matrix.build.runner }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index f0bc359b..ea30b205 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,10 +1,13 @@ - name: Check on: push: pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: check: runs-on: "ubuntu-latest" diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index d8710952..4c354cc2 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -2,9 +2,15 @@ name: Linux Python build on: push: + branches: [main] pull_request: schedule: - cron: '13 11 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: pythonbuild: runs-on: ubuntu-22.04 @@ -169,6 +175,16 @@ jobs: py: 'cpython-3.12' optimizations: 'lto' + - target_triple: 'aarch64-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 'aarch64-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'noopt' + - target_triple: 'aarch64-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'armv7-unknown-linux-gnueabi' py: 'cpython-3.9' @@ -210,6 +226,16 @@ jobs: py: 'cpython-3.12' optimizations: 'lto' + - target_triple: 'armv7-unknown-linux-gnueabi' + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 'armv7-unknown-linux-gnueabi' + py: 'cpython-3.13' + optimizations: 'noopt' + - target_triple: 'armv7-unknown-linux-gnueabi' + py: 'cpython-3.13' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'armv7-unknown-linux-gnueabihf' py: 'cpython-3.9' @@ -251,6 +277,16 @@ jobs: py: 'cpython-3.12' optimizations: 'lto' + - target_triple: 'armv7-unknown-linux-gnueabihf' + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 'armv7-unknown-linux-gnueabihf' + py: 'cpython-3.13' + optimizations: 'noopt' + - target_triple: 'armv7-unknown-linux-gnueabihf' + py: 'cpython-3.13' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'mips-unknown-linux-gnu' py: 'cpython-3.9' @@ -292,6 +328,16 @@ jobs: py: 'cpython-3.12' optimizations: 'lto' + - target_triple: 'mips-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 'mips-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'noopt' + - target_triple: 'mips-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'mipsel-unknown-linux-gnu' py: 'cpython-3.9' @@ -333,6 +379,16 @@ jobs: py: 'cpython-3.12' optimizations: 'lto' + - target_triple: 'mipsel-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 'mipsel-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'noopt' + - target_triple: 'mipsel-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 's390x-unknown-linux-gnu' py: 'cpython-3.9' @@ -374,6 +430,16 @@ jobs: py: 'cpython-3.12' optimizations: 'lto' + - target_triple: 's390x-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 's390x-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'noopt' + - target_triple: 's390x-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'lto' + # Cross-compiles can't do PGO and require Python 3.9. - target_triple: 'ppc64le-unknown-linux-gnu' py: 'cpython-3.9' @@ -415,6 +481,16 @@ jobs: py: 'cpython-3.12' optimizations: 'lto' + - target_triple: 'ppc64le-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 'ppc64le-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'noopt' + - target_triple: 'ppc64le-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'lto' + # We don't publish noopt builds when PGO is available. - target_triple: 'x86_64-unknown-linux-gnu' py: 'cpython-3.8' @@ -481,6 +557,19 @@ jobs: optimizations: 'pgo+lto' run: true + - target_triple: 'x86_64-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'debug' + run: true + - target_triple: 'x86_64-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'pgo' + run: true + - target_triple: 'x86_64-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'pgo+lto' + run: true + - target_triple: 'x86_64_v2-unknown-linux-gnu' py: 'cpython-3.9' optimizations: 'debug' @@ -533,6 +622,19 @@ jobs: optimizations: 'pgo+lto' run: true + - target_triple: 'x86_64_v2-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'debug' + run: true + - target_triple: 'x86_64_v2-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'pgo' + run: true + - target_triple: 'x86_64_v2-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'pgo+lto' + run: true + - target_triple: 'x86_64_v3-unknown-linux-gnu' py: 'cpython-3.9' optimizations: 'debug' @@ -585,6 +687,19 @@ jobs: optimizations: 'pgo+lto' run: true + - target_triple: 'x86_64_v3-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'debug' + run: true + - target_triple: 'x86_64_v3-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'pgo' + run: true + - target_triple: 'x86_64_v3-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'pgo+lto' + run: true + # GitHub Actions runners don't support x86-64-v4 so we can't PGO. - target_triple: 'x86_64_v4-unknown-linux-gnu' py: 'cpython-3.9' @@ -627,6 +742,16 @@ jobs: py: 'cpython-3.12' optimizations: 'lto' + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'noopt' + - target_triple: 'x86_64_v4-unknown-linux-gnu' + py: 'cpython-3.13' + optimizations: 'lto' + # musl doesn't support PGO. - target_triple: 'x86_64-unknown-linux-musl' py: 'cpython-3.8' @@ -693,6 +818,19 @@ jobs: optimizations: 'lto' run: true + - target_triple: 'x86_64-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'debug' + run: true + - target_triple: 'x86_64-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'noopt' + run: true + - target_triple: 'x86_64-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'lto' + run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' py: 'cpython-3.9' optimizations: 'debug' @@ -745,6 +883,19 @@ jobs: optimizations: 'lto' run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'debug' + run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'noopt' + run: true + - target_triple: 'x86_64_v2-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'lto' + run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' py: 'cpython-3.9' optimizations: 'debug' @@ -797,6 +948,19 @@ jobs: optimizations: 'lto' run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'debug' + run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'noopt' + run: true + - target_triple: 'x86_64_v3-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'lto' + run: true + - target_triple: 'x86_64_v4-unknown-linux-musl' py: 'cpython-3.9' optimizations: 'debug' @@ -837,6 +1001,16 @@ jobs: py: 'cpython-3.12' optimizations: 'lto' + - target_triple: 'x86_64_v4-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'debug' + - target_triple: 'x86_64_v4-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'noopt' + - target_triple: 'x86_64_v4-unknown-linux-musl' + py: 'cpython-3.13' + optimizations: 'lto' + needs: - pythonbuild - image diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..9d9d67e5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,74 @@ +name: Release + +on: + workflow_dispatch: + inputs: + tag: + description: "The version to release (e.g., '20240414')." + type: string + sha: + description: "The full SHA of the commit to be released (e.g., 'd09ff921d92d6da8d8a608eaa850dc8c0f638194')." + type: string + dry-run: + description: "Whether to run the release process without actually releasing." + default: false + required: false + type: boolean + +permissions: + contents: write + packages: write + +jobs: + release: + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: extractions/setup-just@v2 + + # Perform a release in dry-run mode. + - run: just release-dry-run ${{ secrets.GITHUB_TOKEN }} ${{ github.event.inputs.sha }} ${{ github.event.inputs.tag }} + if: ${{ github.event.inputs.dry-run == 'true' }} + + # Create the release itself. + - name: Configure Git identity + if: ${{ github.event.inputs.dry-run == 'false' }} + run: | + git config --global user.name "$GITHUB_ACTOR" + git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com" + + # Fetch the commit so that it exists locally. + - name: Fetch commit + if: ${{ github.event.inputs.dry-run == 'false' }} + run: git fetch origin ${{ github.event.inputs.sha }} + + # Associate the commit with the tag. + - name: Create tag + if: ${{ github.event.inputs.dry-run == 'false' }} + run: git tag ${{ github.event.inputs.tag }} ${{ github.event.inputs.sha }} + + # Push the tag to GitHub. + - name: Push tag + if: ${{ github.event.inputs.dry-run == 'false' }} + run: git push origin ${{ github.event.inputs.tag }} + + # Create a GitHub release. + - name: Create GitHub Release + if: ${{ github.event.inputs.dry-run == 'false' }} + uses: ncipollo/release-action@v1 + with: + tag: ${{ github.event.inputs.tag }} + name: ${{ github.event.inputs.tag }} + prerelease: true + body: TBD + allowUpdates: true + updateOnlyUnreleased: true + + # Uploading the relevant artifact to the GitHub release. + - run: just release-run ${{ secrets.GITHUB_TOKEN }} ${{ github.event.inputs.sha }} ${{ github.event.inputs.tag }} + if: ${{ github.event.inputs.dry-run == 'false' }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 19bdba30..413cdd3f 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -2,9 +2,15 @@ name: Windows Python build on: push: + branches: [main] pull_request: schedule: - cron: '13 11 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: pythonbuild: runs-on: 'windows-2019' @@ -48,6 +54,7 @@ jobs: - 'cpython-3.10' - 'cpython-3.11' - 'cpython-3.12' + - 'cpython-3.13' vcvars: - 'vcvars32.bat' - 'vcvars64.bat' diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 00000000..18b91b2d --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,27 @@ +============ +Contributing +============ + +Releases +======== + +To cut a release, wait for the "MacOS Python build", "Linux Python build", and +"Windows Python build" GitHub Actions to complete successfully on the target commit. + +Then, run the "Release" GitHub Action to create the release, populate the release artifacts (by +downloading the artifacts from each workflow, and uploading them to the GitHub Release), and promote +the SHA via the `latest-release` branch. + +The "Release" GitHub Action takes, as input, a tag (assumed to be a date in `YYYYMMDD` format) and +the commit SHA referenced above. + +For example, to create a release on April 19, 2024 at commit `29abc56`, run the "Release" workflow +with the tag `20240419` and the commit SHA `29abc56954fbf5ea812f7fbc3e42d87787d46825` as inputs, +once the "MacOS Python build", "Linux Python build", and "Windows Python build" workflows have +run to completion on `29abc56`. + +When the "Release" workflow is complete, populate the release notes in the GitHub UI and promote +the pre-release to a full release, again in the GitHub UI. + +At any stage, you can run the "Release" workflow in dry-run mode to avoid uploading artifacts to +GitHub. Dry-run mode can be executed before or after creating the release itself. diff --git a/Cargo.lock b/Cargo.lock index 06943f15..2b783b8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -444,6 +444,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fastrand" version = "2.0.1" @@ -628,7 +634,7 @@ checksum = "bb07a4ffed2093b118a525b1d8f5204ae274faed5604537caf7135d0f18d9887" dependencies = [ "log", "plain", - "scroll", + "scroll 0.12.0", ] [[package]] @@ -1215,6 +1221,17 @@ dependencies = [ "sha2", ] +[[package]] +name = "pdb" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82040a392923abe6279c00ab4aff62d5250d1c8555dc780e4b02783a7aa74863" +dependencies = [ + "fallible-iterator", + "scroll 0.11.0", + "uuid", +] + [[package]] name = "pem" version = "3.0.3" @@ -1325,9 +1342,10 @@ dependencies = [ "object", "octocrab", "once_cell", + "pdb", "rayon", "reqwest", - "scroll", + "scroll 0.12.0", "semver", "serde", "serde_json", @@ -1428,10 +1446,12 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "winreg", ] @@ -1587,6 +1607,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "scroll" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" + [[package]] name = "scroll" version = "0.12.0" @@ -1942,9 +1968,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", @@ -1963,9 +1989,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", @@ -2190,6 +2216,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" + [[package]] name = "vcpkg" version = "0.2.15" @@ -2289,6 +2321,19 @@ version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" +[[package]] +name = "wasm-streams" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.68" diff --git a/Cargo.toml b/Cargo.toml index d6432348..f7523b2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,12 +22,13 @@ normalize-path = "0.2.1" object = "0.32.2" octocrab = { version = "0.34.1", features = ["rustls", "stream"] } once_cell = "1.19.0" +pdb = "0.8.0" rayon = "1.8.1" -reqwest = { version = "0.11.24", features = ["rustls"] } +reqwest = { version = "0.11.24", features = ["rustls", "stream"] } scroll = "0.12.0" semver = "1.0.22" -serde_json = "1.0.114" serde = { version = "1.0.197", features = ["derive"] } +serde_json = "1.0.114" sha2 = "0.10.8" tar = "0.4.40" tempfile = "3.10.0" diff --git a/Justfile b/Justfile index a65faab3..1d6a02a3 100644 --- a/Justfile +++ b/Justfile @@ -34,11 +34,19 @@ release-download-distributions token commit: release-upload-distributions token datetime tag: cargo run --release -- upload-release-distributions --token {{token}} --datetime {{datetime}} --tag {{tag}} --dist dist +# "Upload" release artifacts to a GitHub release in dry-run mode (skip upload). +release-upload-distributions-dry-run token datetime tag: + cargo run --release -- upload-release-distributions --token {{token}} --datetime {{datetime}} --tag {{tag}} --dist dist -n + +# Promote a tag to "latest" by pushing to the `latest-release` branch. release-set-latest-release tag: #!/usr/bin/env bash set -euxo pipefail + git fetch origin git switch latest-release + git reset --hard origin/latest-release + cat << EOF > latest-release.json { "version": 1, @@ -48,24 +56,38 @@ release-set-latest-release tag: } EOF - git commit -a -m 'set latest release to {{tag}}' - git switch main + # If the branch is dirty, we add and commit. + if ! git diff --quiet; then + git add latest-release.json + git commit -m 'set latest release to {{tag}}' + git switch main - git push origin latest-release + git push origin latest-release + else + echo "No changes to commit." + fi -# Perform a release. -release token commit tag: +# Perform the release job. Assumes that the GitHub Release has been created. +release-run token commit tag: #!/bin/bash set -eo pipefail - gh release create --prerelease --notes TBD --title {{ tag }} --target {{ commit }} {{ tag }} - rm -rf dist just release-download-distributions {{token}} {{commit}} datetime=$(ls dist/cpython-3.10.*-x86_64-unknown-linux-gnu-install_only-*.tar.gz | awk -F- '{print $8}' | awk -F. '{print $1}') just release-upload-distributions {{token}} ${datetime} {{tag}} just release-set-latest-release {{tag}} +# Perform a release in dry-run mode. +release-dry-run token commit tag: + #!/bin/bash + set -eo pipefail + + rm -rf dist + just release-download-distributions {{token}} {{commit}} + datetime=$(ls dist/cpython-3.10.*-x86_64-unknown-linux-gnu-install_only-*.tar.gz | awk -F- '{print $8}' | awk -F. '{print $1}') + just release-upload-distributions-dry-run {{token}} ${datetime} {{tag}} + _download-stats mode: build/venv.*/bin/python3 -c 'import pythonbuild.utils as u; u.release_download_statistics(mode="{{mode}}")' diff --git a/LICENSE b/LICENSE index 0b8a7e86..a612ad98 100644 --- a/LICENSE +++ b/LICENSE @@ -1,27 +1,373 @@ -Copyright (c) 2018 to present, Gregory Szorc -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/check.py b/check.py index 8295590c..64ba7dcd 100755 --- a/check.py +++ b/check.py @@ -54,19 +54,9 @@ def run(): ) args = parser.parse_args() - # Lints: - # Sort imports - # Unused import - # Unused variable - check_args = ["--select", "I,F401,F841"] + check_args = [] format_args = [] - mypy_args = [ - "pythonbuild", - "check.py", - "build-linux.py", - "build-macos.py", - "build-windows.py", - ] + mypy_args = [] if args.fix: check_args.append("--fix") diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 08488956..d3225451 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -270,6 +270,8 @@ $(OUTDIR)/cpython-3.11-$(CPYTHON_3.11_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HO $(OUTDIR)/cpython-3.12-$(CPYTHON_3.12_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12-host +$(OUTDIR)/cpython-3.13-$(CPYTHON_3.13_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) + $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.13-host PYTHON_DEPENDS := \ $(PYTHON_SUPPORT_FILES) \ @@ -318,3 +320,6 @@ $(OUTDIR)/cpython-$(CPYTHON_3.11_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DE $(OUTDIR)/cpython-$(CPYTHON_3.12_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12 + +$(OUTDIR)/cpython-$(CPYTHON_3.13_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) + $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.13 diff --git a/cpython-unix/build-cpython-host.sh b/cpython-unix/build-cpython-host.sh index cfc1e8a7..338219ad 100755 --- a/cpython-unix/build-cpython-host.sh +++ b/cpython-unix/build-cpython-host.sh @@ -38,11 +38,18 @@ pushd "Python-${PYTHON_VERSION}" # configure. This is reported as https://bugs.python.org/issue45405. We nerf the # check since we know what we're doing. if [ "${CC}" = "clang" ]; then - if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_9}" ]; then + if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then + patch -p1 -i ${ROOT}/patch-disable-multiarch-13.patch + elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_9}" ]; then patch -p1 -i ${ROOT}/patch-disable-multiarch.patch else patch -p1 -i ${ROOT}/patch-disable-multiarch-legacy.patch fi +elif [ "${CC}" = "musl-clang" ]; then + # Similarly, this is a problem for musl Clang on Python 3.13+ + if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then + patch -p1 -i ${ROOT}/patch-disable-multiarch-13.patch + fi fi autoconf diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 87225429..0837843a 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -72,7 +72,9 @@ cat Makefile.extra pushd Python-${PYTHON_VERSION} # configure doesn't support cross-compiling on Apple. Teach it. -if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then +if [ "${PYTHON_MAJMIN_VERSION}" = "3.13" ]; then + patch -p1 -i ${ROOT}/patch-apple-cross-3.13.patch +elif [ "${PYTHON_MAJMIN_VERSION}" = "3.12" ]; then patch -p1 -i ${ROOT}/patch-apple-cross-3.12.patch else patch -p1 -i ${ROOT}/patch-apple-cross.patch @@ -94,7 +96,9 @@ fi # Configure nerfs RUNSHARED when cross-compiling, which prevents PGO from running when # we can in fact run the target binaries (e.g. x86_64 host and i686 target). Undo that. if [ -n "${CROSS_COMPILING}" ]; then - if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]; then + if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then + patch -p1 -i ${ROOT}/patch-dont-clear-runshared-13.patch + elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]; then patch -p1 -i ${ROOT}/patch-dont-clear-runshared.patch else patch -p1 -i ${ROOT}/patch-dont-clear-runshared-legacy.patch @@ -105,11 +109,18 @@ fi # configure. This is reported as https://bugs.python.org/issue45405. We nerf the # check since we know what we're doing. if [ "${CC}" = "clang" ]; then - if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_9}" ]; then + if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then + patch -p1 -i ${ROOT}/patch-disable-multiarch-13.patch + elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_9}" ]; then patch -p1 -i ${ROOT}/patch-disable-multiarch.patch else patch -p1 -i ${ROOT}/patch-disable-multiarch-legacy.patch fi +elif [ "${CC}" = "musl-clang" ]; then + # Similarly, this is a problem for musl Clang on Python 3.13+ + if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then + patch -p1 -i ${ROOT}/patch-disable-multiarch-13.patch + fi fi # Python 3.11 supports using a provided Python to use during bootstrapping @@ -134,9 +145,19 @@ if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_10}" ]; then patch -p1 -i ${ROOT}/patch-makesetup-deduplicate-objs.patch fi +# testembed links against Tcl/Tk and libpython which already includes Tcl/Tk leading duplicate +# symbols and warnings from objc (which then causes failures in `test_embed` during PGO). +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then + patch -p1 -i ${ROOT}/patch-make-testembed-nolink-tcltk.patch +fi + # The default build rule for the macOS dylib doesn't pick up libraries # from modules / makesetup. So patch it accordingly. -patch -p1 -i ${ROOT}/patch-macos-link-extension-modules.patch +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then + patch -p1 -i ${ROOT}/patch-macos-link-extension-modules-13.patch +else + patch -p1 -i ${ROOT}/patch-macos-link-extension-modules.patch +fi # Also on macOS, the `python` executable is linked against libraries defined by statically # linked modules. But those libraries should only get linked into libpython, not the @@ -280,7 +301,7 @@ if [ "${PYBUILD_PLATFORM}" != "macos" ]; then fi fi -# On Python 3.12 we need to link the special hacl library provided some SHA-256 +# On Python 3.12+ we need to link the special hacl library provided some SHA-256 # implementations. Since we hack up the regular extension building mechanism, we # need to reinvent this wheel. if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then @@ -336,7 +357,12 @@ fi if [ -n "${CPYTHON_OPTIMIZED}" ]; then CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-optimizations" - if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" && -n "${BOLT_CAPABLE}" ]]; then + if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && -n "${BOLT_CAPABLE}" ]]; then + # Due to a SEGFAULT when running `test_embed` with BOLT instrumented binaries, we can't use + # BOLT on Python 3.13+. + # TODO: Find a fix for this or consider skipping these tests specifically + echo "BOLT is disabled on Python 3.13+" + elif [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" && -n "${BOLT_CAPABLE}" ]]; then CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-bolt" fi fi @@ -426,6 +452,11 @@ if [ "${PYBUILD_PLATFORM}" = "macos" ]; then export MACOSX_DEPLOYMENT_TARGET="${APPLE_MIN_DEPLOYMENT_TARGET}" fi +# ptsrname_r is only available in SDK 13.4+, but we target a lower version for compatibility. +if [ "${PYBUILD_PLATFORM}" = "macos" ]; then + CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_func_ptsname_r=no" +fi + # We use ndbm on macOS and BerkeleyDB elsewhere. if [ "${PYBUILD_PLATFORM}" = "macos" ]; then CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --with-dbmliborder=ndbm" @@ -742,7 +773,13 @@ s390x-unknown-linux-gnu) PYTHON_ARCH="s390x-linux-gnu" ;; x86_64-unknown-linux-*) - PYTHON_ARCH="x86_64-linux-gnu" + # In Python 3.13+, the musl target is identified in cross compiles and the output directory + # is named accordingly. + if [ "${CC}" = "musl-clang" ] && [ "${PYTHON_MAJMIN_VERSION}" = "3.13" ]; then + PYTHON_ARCH="x86_64-linux-musl" + else + PYTHON_ARCH="x86_64-linux-gnu" + fi ;; *) echo "unhandled target triple: ${TARGET_TRIPLE}" @@ -832,7 +869,7 @@ ${BUILD_PYTHON} ${ROOT}/fix_shebangs.py ${ROOT}/out/python/install # downstream consumers. OBJECT_DIRS="Objects Parser Parser/pegen Programs Python" OBJECT_DIRS="${OBJECT_DIRS} Modules" -for ext in _blake2 cjkcodecs _ctypes _ctypes/darwin _decimal _expat _hacl _io _multiprocessing _sha3 _sqlite _sre _xxtestfuzz ; do +for ext in _blake2 cjkcodecs _ctypes _ctypes/darwin _decimal _expat _hacl _io _multiprocessing _sha3 _sqlite _sre _testinternalcapi _xxtestfuzz ; do OBJECT_DIRS="${OBJECT_DIRS} Modules/${ext}" done @@ -895,7 +932,12 @@ cp -av Python/frozen.c ${ROOT}/out/python/build/Python/ cp -av Modules/Setup* ${ROOT}/out/python/build/Modules/ # Copy the test hardness runner for convenience. -cp -av Tools/scripts/run_tests.py ${ROOT}/out/python/build/ +# As of Python 3.13, the test harness runner has been removed so we provide a compatibility script +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then + cp -av ${ROOT}/run_tests-13.py ${ROOT}/out/python/build/run_tests.py +else + cp -av Tools/scripts/run_tests.py ${ROOT}/out/python/build/ +fi mkdir ${ROOT}/out/python/licenses cp ${ROOT}/LICENSE.*.txt ${ROOT}/out/python/licenses/ diff --git a/cpython-unix/build-libffi.sh b/cpython-unix/build-libffi.sh index 33cc3d9b..7a50589e 100755 --- a/cpython-unix/build-libffi.sh +++ b/cpython-unix/build-libffi.sh @@ -13,40 +13,6 @@ tar -xf libffi-${LIBFFI_VERSION}.tar.gz pushd libffi-${LIBFFI_VERSION} -# Upstream commit ce077e5565366171aa1b4438749b0922fce887a4 to resolve a missing declaration. -patch -p1 << 'EOF' -diff --git a/include/ffi_common.h b/include/ffi_common.h -index 2bd31b0..c53a794 100644 ---- a/include/ffi_common.h -+++ b/include/ffi_common.h -@@ -128,6 +128,10 @@ void *ffi_data_to_code_pointer (void *data) FFI_HIDDEN; - static trampoline. */ - int ffi_tramp_is_present (void *closure) FFI_HIDDEN; - -+/* Return a file descriptor of a temporary zero-sized file in a -+ writable and executable filesystem. */ -+int open_temp_exec_file(void) FFI_HIDDEN; -+ - /* Extended cif, used in callback from assembly routine */ - typedef struct - { -diff --git a/src/tramp.c b/src/tramp.c -index 7e005b0..5f19b55 100644 ---- a/src/tramp.c -+++ b/src/tramp.c -@@ -39,6 +39,10 @@ - #ifdef __linux__ - #define _GNU_SOURCE 1 - #endif -+ -+#include -+#include -+ - #include - #include - #include -EOF - EXTRA_CONFIGURE= # mkostemp() was introduced in macOS 10.10 and libffi doesn't have diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index b9b3c5dc..34a90d19 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -67,6 +67,7 @@ def main(): "cpython-3.10", "cpython-3.11", "cpython-3.12", + "cpython-3.13", }, default="cpython-3.11", help="Python distribution to build", diff --git a/cpython-unix/build.py b/cpython-unix/build.py index ccea702c..5c32586b 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -435,6 +435,7 @@ def build_cpython_host( support = { "build-cpython-host.sh", "patch-disable-multiarch.patch", + "patch-disable-multiarch-13.patch", "patch-disable-multiarch-legacy.patch", } for s in sorted(support): @@ -455,7 +456,7 @@ def build_cpython_host( # Set environment variables allowing convenient testing for Python # version ranges. - for v in ("3.8", "3.9", "3.10", "3.11", "3.12"): + for v in ("3.8", "3.9", "3.10", "3.11", "3.12", "3.13"): normal_version = v.replace(".", "_") if meets_python_minimum_version(python_version, v): @@ -745,6 +746,7 @@ def build_cpython( setuptools_archive, pip_archive, SUPPORT / "build-cpython.sh", + SUPPORT / "run_tests-13.py", ): build_env.copy_file(p) @@ -782,7 +784,7 @@ def build_cpython( # Set environment variables allowing convenient testing for Python # version ranges. - for v in ("3.8", "3.9", "3.10", "3.11", "3.12"): + for v in ("3.8", "3.9", "3.10", "3.11", "3.12", "3.13"): normal_version = v.replace(".", "_") if meets_python_minimum_version(python_version, v): @@ -1195,6 +1197,7 @@ def main(): "cpython-3.10", "cpython-3.11", "cpython-3.12", + "cpython-3.13", ): build_cpython( settings, diff --git a/cpython-unix/extension-modules.yml b/cpython-unix/extension-modules.yml index 37f78101..3c580dee 100644 --- a/cpython-unix/extension-modules.yml +++ b/cpython-unix/extension-modules.yml @@ -67,6 +67,7 @@ _contextvars: - _contextvarsmodule.c _crypt: + maximum-python-version: "3.12" build-mode: shared sources: - _cryptmodule.c @@ -286,6 +287,21 @@ _heapq: _imp: config-c-only: true +_interpchannels: + minimum-python-version: "3.13" + sources: + - _interpchannelsmodule.c + +_interpqueues: + minimum-python-version: "3.13" + sources: + - _interpqueuesmodule.c + +_interpreters: + minimum-python-version: "3.13" + sources: + - _interpretersmodule.c + _io: setup-enabled: true required-targets: @@ -528,9 +544,21 @@ _struct: sources: - _struct.c +_suggestions: + setup-enabled: true + minimum-python-version: '3.13' + sources: + - _suggestions.c + _symtable: setup-enabled: true +_sysconfig: + setup-enabled: true + minimum-python-version: '3.13' + sources: + - _sysconfig.c + _testbuffer: minimum-python-version: '3.9' sources: @@ -545,6 +573,11 @@ _testcapi: sources: - _testcapimodule.c +_testexternalinspection: + minimum-python-version: '3.13' + sources: + - _testexternalinspection.c + _testimportmultiple: minimum-python-version: '3.9' sources: @@ -558,6 +591,18 @@ _testinternalcapi: - Include/internal sources: - _testinternalcapi.c + includes-conditional: + - path: _testinternalcapi/parts.h + minimum-python-version: "3.13" + sources-conditional: + - source: _testinternalcapi/pytime.c + minimum-python-version: "3.13" + - source: _testinternalcapi/set.c + minimum-python-version: "3.13" + - source: _testinternalcapi/test_critical_sections.c + minimum-python-version: "3.13" + - source: _testinternalcapi/test_lock.c + minimum-python-version: "3.13" _testmultiphase: minimum-python-version: '3.9' @@ -659,11 +704,13 @@ _weakref: _xxinterpchannels: minimum-python-version: '3.12' + maximum-python-version: '3.12' sources: - _xxinterpchannelsmodule.c _xxsubinterpreters: minimum-python-version: '3.9' + maximum-python-version: '3.12' sources: - _xxsubinterpretersmodule.c @@ -687,6 +734,7 @@ atexit: # Modules/Setup comment is ambiguous as to whether this module actually works. audioop: + maximum-python-version: '3.12' sources: - audioop.c @@ -745,6 +793,7 @@ mmap: - mmapmodule.c nis: + maximum-python-version: "3.12" disabled-targets: # NIS is not available on Apple OS. - aarch64-apple-.* @@ -767,6 +816,7 @@ nis: - nsl ossaudiodev: + maximum-python-version: "3.12" disabled-targets: # ossaudiodev not available on Apple OS. - aarch64-apple-.* @@ -806,6 +856,8 @@ readline: - readline.c defines: - USE_LIBEDIT=1 + # While some versions do not, our readline `on_startup_hook` takes arguments. + - Py_RL_STARTUP_HOOK_TAKES_ARGS includes-deps: - libedit/include - libedit/include/ncursesw @@ -828,6 +880,7 @@ select: - selectmodule.c spwd: + maximum-python-version: "3.12" sources: - spwdmodule.c diff --git a/cpython-unix/patch-apple-cross-3.13.patch b/cpython-unix/patch-apple-cross-3.13.patch new file mode 100644 index 00000000..6c4ffe44 --- /dev/null +++ b/cpython-unix/patch-apple-cross-3.13.patch @@ -0,0 +1,85 @@ +diff --git a/configure.ac b/configure.ac +index c62a565eb6..7e5d34632c 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -545,6 +545,15 @@ then + *-*-cygwin*) + ac_sys_system=Cygwin + ;; ++ *-apple-ios*) ++ ac_sys_system=iOS ++ ;; ++ *-apple-tvos*) ++ ac_sys_system=tvOS ++ ;; ++ *-apple-watchos*) ++ ac_sys_system=watchOS ++ ;; + *-*-vxworks*) + ac_sys_system=VxWorks + ;; +@@ -600,6 +609,19 @@ if test "$cross_compiling" = yes; then + *-*-cygwin*) + _host_cpu= + ;; ++ *-*-darwin*) ++ _host_cpu= ++ ;; ++ *-apple-*) ++ case "$host_cpu" in ++ arm*) ++ _host_cpu=arm ++ ;; ++ *) ++ _host_cpu=$host_cpu ++ ;; ++ esac ++ ;; + *-*-vxworks*) + _host_cpu=$host_cpu + ;; +@@ -614,6 +636,23 @@ if test "$cross_compiling" = yes; then + _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + fi + ++# The _PYTHON_HOST_PLATFORM environment variable is used to ++# override the platform name in distutils and sysconfig when ++# cross-compiling. On Apple, the platform name expansion logic ++# is non-trivial, including renaming MACHDEP=darwin to macosx ++# and including the deployment target (or current OS version if ++# not set). Here we always force an override based on the target ++# triple. We do this in all build configurations because historically ++# the automatic resolution has been brittle. ++case "$host" in ++aarch64-apple-darwin*) ++ _PYTHON_HOST_PLATFORM="macosx-${MACOSX_DEPLOYMENT_TARGET}-arm64" ++ ;; ++x86_64-apple-darwin*) ++ _PYTHON_HOST_PLATFORM="macosx-${MACOSX_DEPLOYMENT_TARGET}-x86_64" ++ ;; ++esac ++ + # Some systems cannot stand _XOPEN_SOURCE being defined at all; they + # disable features if it is defined, without any means to access these + # features as extensions. For these systems, we skip the definition of +@@ -1582,7 +1621,7 @@ if test $enable_shared = "yes"; then + BLDLIBRARY='-Wl,+b,$(LIBDIR) -L. -lpython$(LDVERSION)' + RUNSHARED=SHLIB_PATH=`pwd`${SHLIB_PATH:+:${SHLIB_PATH}} + ;; +- Darwin*) ++ Darwin*|iOS*|tvOS*|watchOS*) + LDLIBRARY='libpython$(LDVERSION).dylib' + BLDLIBRARY='-L. -lpython$(LDVERSION)' + RUNSHARED=DYLD_LIBRARY_PATH=`pwd`${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}} +@@ -3173,6 +3203,11 @@ then + Linux*|GNU*|QNX*|VxWorks*|Haiku*) + LDSHARED='$(CC) -shared' + LDCXXSHARED='$(CXX) -shared';; ++ iOS*|tvOS*|watchOS*) ++ LDSHARED='$(CC) -bundle -undefined dynamic_lookup' ++ LDCXXSHARED='$(CXX) -bundle -undefined dynamic_lookup' ++ BLDSHARED="$LDSHARED" ++ ;; + FreeBSD*) + if [[ "`$CC -dM -E - /dev/null)] +-) ++MULTIARCH= + AC_SUBST([MULTIARCH]) + + if test x$PLATFORM_TRIPLET != x && test x$MULTIARCH != x; then diff --git a/cpython-unix/patch-dont-clear-runshared-13.patch b/cpython-unix/patch-dont-clear-runshared-13.patch new file mode 100644 index 00000000..7dcba084 --- /dev/null +++ b/cpython-unix/patch-dont-clear-runshared-13.patch @@ -0,0 +1,14 @@ +diff -u 13-a/configure.ac 13-b/configure.ac +--- 13-a/configure.ac 2024-05-08 05:21:00.000000000 -0400 ++++ 13-b/configure.ac 2024-05-19 12:44:04.530770938 -0400 +@@ -1564,10 +1564,6 @@ + fi + AC_MSG_RESULT([$LDLIBRARY]) + +-if test "$cross_compiling" = yes; then +- RUNSHARED= +-fi +- + AC_MSG_CHECKING([HOSTRUNNER]) + AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) + if test -z "$HOSTRUNNER" diff --git a/cpython-unix/patch-macos-link-extension-modules-13.patch b/cpython-unix/patch-macos-link-extension-modules-13.patch new file mode 100644 index 00000000..75b0d781 --- /dev/null +++ b/cpython-unix/patch-macos-link-extension-modules-13.patch @@ -0,0 +1,12 @@ +diff -u 13-a/Makefile.pre.in 13-b/Makefile.pre.in +--- 13-a/Makefile.pre.in 2024-05-08 05:21:00.000000000 -0400 ++++ 13-b/Makefile.pre.in 2024-05-19 07:55:45.091521909 -0400 +@@ -903,7 +903,7 @@ + $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ + + libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) +- $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ ++ $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + + + libpython$(VERSION).sl: $(LIBRARY_OBJS) diff --git a/cpython-unix/patch-make-testembed-nolink-tcltk.patch b/cpython-unix/patch-make-testembed-nolink-tcltk.patch new file mode 100644 index 00000000..65c1989f --- /dev/null +++ b/cpython-unix/patch-make-testembed-nolink-tcltk.patch @@ -0,0 +1,10 @@ +diff --git a/Makefile.pre.in b/Makefile.pre.in +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -1432,6 +1432,8 @@ + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/generate_re_casefix.py $(srcdir)/Lib/re/_casefix.py + + Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) ++ $(eval MODLIBS := $(subst -Xlinker -hidden-ltcl8.6, , $(MODLIBS))) ++ $(eval MODLIBS := $(subst -Xlinker -hidden-ltk8.6, , $(MODLIBS))) + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) diff --git a/cpython-unix/run_tests-13.py b/cpython-unix/run_tests-13.py new file mode 100644 index 00000000..a78210cc --- /dev/null +++ b/cpython-unix/run_tests-13.py @@ -0,0 +1,30 @@ +""" +Run Python's test suite. + +As of Python 3.13, this script is no longer included in Python itself. +Instead, use: + + $ python -m test --slow-ci + +""" + +import os +import sys + + +def main(regrtest_args): + args = [ + sys.executable, + "-m", + "test", + "--slow-ci", + ] + + args.extend(regrtest_args) + print(" ".join(args)) + + os.execv(sys.executable, args) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index fd81e286..02e41311 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -64,6 +64,7 @@ aarch64-apple-darwin: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -153,6 +154,7 @@ aarch64-unknown-linux-gnu: - '3.10' - '3.11' - '3.12' + - '3.13' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -231,6 +233,7 @@ armv7-unknown-linux-gnueabi: - '3.10' - '3.11' - '3.12' + - '3.13' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -269,6 +272,7 @@ armv7-unknown-linux-gnueabihf: - '3.10' - '3.11' - '3.12' + - '3.13' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -308,6 +312,7 @@ i686-unknown-linux-gnu: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -351,6 +356,7 @@ mips-unknown-linux-gnu: - '3.10' - '3.11' - '3.12' + - '3.13' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -389,6 +395,7 @@ mipsel-unknown-linux-gnu: - '3.10' - '3.11' - '3.12' + - '3.13' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -427,6 +434,7 @@ ppc64le-unknown-linux-gnu: - '3.10' - '3.11' - '3.12' + - '3.13' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -465,6 +473,7 @@ s390x-unknown-linux-gnu: - '3.10' - '3.11' - '3.12' + - '3.13' docker_image_suffix: .cross host_cc: /usr/bin/x86_64-linux-gnu-gcc host_cxx: /usr/bin/x86_64-linux-gnu-g++ @@ -548,6 +557,7 @@ x86_64-apple-darwin: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true apple_sdk_platform: macosx host_cc: clang @@ -717,6 +727,7 @@ x86_64-unknown-linux-gnu: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -761,6 +772,7 @@ x86_64_v2-unknown-linux-gnu: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -806,6 +818,7 @@ x86_64_v3-unknown-linux-gnu: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -851,6 +864,7 @@ x86_64_v4-unknown-linux-gnu: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -896,6 +910,7 @@ x86_64-unknown-linux-musl: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -938,6 +953,7 @@ x86_64_v2-unknown-linux-musl: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -981,6 +997,7 @@ x86_64_v3-unknown-linux-musl: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true host_cc: clang host_cxx: clang++ @@ -1024,6 +1041,7 @@ x86_64_v4-unknown-linux-musl: - '3.10' - '3.11' - '3.12' + - '3.13' needs_toolchain: true host_cc: clang host_cxx: clang++ diff --git a/cpython-windows/build.py b/cpython-windows/build.py index bc97cd57..88bcf6e0 100644 --- a/cpython-windows/build.py +++ b/cpython-windows/build.py @@ -65,7 +65,10 @@ "_lzma": { "ignore_additional_depends": {"$(OutDir)liblzma$(PyDebugExt).lib"}, }, - "_msi": {}, + "_msi": { + # Removed in 3.13. + "ignore_missing": True, + }, "_overlapped": {}, "_multiprocessing": {}, "_socket": {}, @@ -352,6 +355,7 @@ def hack_props( xz_version = DOWNLOADS["xz"]["version"] zlib_version = DOWNLOADS["zlib"]["version"] tcltk_commit = DOWNLOADS["tk-windows-bin"]["git_commit"] + mpdecimal_version = DOWNLOADS["mpdecimal"]["version"] sqlite_path = td / ("sqlite-autoconf-%s" % sqlite_version) bzip2_path = td / ("bzip2-%s" % bzip2_version) @@ -359,6 +363,7 @@ def hack_props( tcltk_path = td / ("cpython-bin-deps-%s" % tcltk_commit) xz_path = td / ("xz-%s" % xz_version) zlib_path = td / ("zlib-%s" % zlib_version) + mpdecimal_path = td / ("mpdecimal-%s" % mpdecimal_version) openssl_root = td / "openssl" / arch openssl_libs_path = openssl_root / "lib" @@ -398,6 +403,9 @@ def hack_props( elif b"%s\\" % zlib_path + elif b"%s\\" % mpdecimal_path + lines.append(line) with python_props_path.open("wb") as fh: @@ -1155,15 +1163,17 @@ def collect_python_build_artifacts( "_ctypes_test", "_testbuffer", "_testcapi", + "_testclinic_limited", "_testclinic", "_testconsole", "_testembed", "_testimportmultiple", "_testinternalcapi", - "_testsinglephase", + "_testlimitedcapi", "_testmultiphase", - "xxlimited", + "_testsinglephase", "xxlimited_35", + "xxlimited", } other_projects = {"pythoncore"} @@ -1409,6 +1419,14 @@ def build_cpython( setuptools_wheel = download_entry("setuptools", BUILD) pip_wheel = download_entry("pip", BUILD) + # CPython 3.13+ no longer uses a bundled `mpdecimal` version so we build it + if meets_python_minimum_version(python_version, "3.13"): + mpdecimal_archive = download_entry("mpdecimal", BUILD) + else: + # TODO: Consider using the built mpdecimal for earlier versions as well, + # as we do for Unix builds. + mpdecimal_archive = None + if arch == "amd64": build_platform = "x64" build_directory = "amd64" @@ -1426,12 +1444,16 @@ def build_cpython( for a in ( python_archive, bzip2_archive, + mpdecimal_archive, openssl_archive, sqlite_archive, tk_bin_archive, xz_archive, zlib_archive, ): + if a is None: + continue + log("extracting %s to %s" % (a, td)) fs.append(e.submit(extract_tar_to_directory, a, td)) @@ -1700,10 +1722,18 @@ def build_cpython( log("copying %s to %s" % (source, dest)) shutil.copyfile(source, dest) - shutil.copyfile( - cpython_source_path / "Tools" / "scripts" / "run_tests.py", - out_dir / "python" / "build" / "run_tests.py", - ) + # CPython 3.13 removed `run_tests.py`, we provide a compatibility script + # for now. + if meets_python_minimum_version(python_version, "3.13"): + shutil.copyfile( + SUPPORT / "run_tests-13.py", + out_dir / "python" / "build" / "run_tests.py", + ) + else: + shutil.copyfile( + cpython_source_path / "Tools" / "scripts" / "run_tests.py", + out_dir / "python" / "build" / "run_tests.py", + ) licenses_dir = out_dir / "python" / "licenses" licenses_dir.mkdir() diff --git a/cpython-windows/run_tests-13.py b/cpython-windows/run_tests-13.py new file mode 100644 index 00000000..f3c165fb --- /dev/null +++ b/cpython-windows/run_tests-13.py @@ -0,0 +1,30 @@ +""" +Run Python's test suite. + +As of Python 3.13, this script is no longer included in Python itself. +Instead, use: + + $ python -m test --slow-ci + +""" + +import sys +from subprocess import call + + +def main(regrtest_args): + args = [ + sys.executable, + "-m", + "test", + "--slow-ci", + ] + + args.extend(regrtest_args) + print(" ".join(args)) + + sys.exit(call(args)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/docs/running.rst b/docs/running.rst index 69ee1932..02e4e2a4 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -151,9 +151,17 @@ Common configurations include: A debug build. No optimizations. The archive flavor denotes the content in the archive. See -:ref:`distributions` for more. Casual users will likely want to use the -``install_only`` archive, as most users do not need the build artifacts -present in the ``full`` archive. +:ref:`distributions` for more. + +Casual users will likely want to use the ``install_only`` archive, as most +users do not need the build artifacts present in the ``full`` archive. +The ``install_only`` archive doesn't include the build configuration in its +file name. It's based on the fastest available build configuration for a given +target. + +An ``install_only_stripped`` archive is also available. This archive is +equivalent to ``install_only``, but without debug symbols, which results in a +smaller download and on-disk footprint. Extracting Distributions ======================== diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..579b8167 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,9 @@ +[mypy] +warn_no_return = True +warn_redundant_casts = True +warn_return_any = True +warn_unused_configs = True +warn_unused_ignores = True +warn_unreachable = True + +files = pythonbuild,check.py,build-linux.py,build-macos.py,build-windows.py diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index 97a715d5..c499cc0f 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -41,6 +41,8 @@ "properties": { "path": {"type": "string"}, "targets": {"type": "array", "items": {"type": "string"}}, + "minimum-python-version": {"type": "string"}, + "maximum-python-version": {"type": "string"}, }, "additionalProperties": False, }, @@ -534,7 +536,19 @@ def derive_setup_local( line += f" -I{path}" for entry in info.get("includes-conditional", []): - if any(re.match(p, target_triple) for p in entry["targets"]): + if targets := entry.get("targets", []): + target_match = any(re.match(p, target_triple) for p in targets) + else: + target_match = True + + python_min_match = meets_python_minimum_version( + python_version, entry.get("minimum-python-version", "1.0") + ) + python_max_match = meets_python_maximum_version( + python_version, entry.get("maximum-python-version", "100.0") + ) + + if target_match and (python_min_match and python_max_match): line += f" -I{entry['path']}" for path in info.get("includes-deps", []): diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index ae46a85d..c526b3eb 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -71,14 +71,23 @@ "python_tag": "cp311", }, "cpython-3.12": { - "url": "https://www.python.org/ftp/python/3.12.3/Python-3.12.3.tar.xz", - "size": 20625068, - "sha256": "56bfef1fdfc1221ce6720e43a661e3eb41785dd914ce99698d8c7896af4bdaa1", - "version": "3.12.3", + "url": "https://www.python.org/ftp/python/3.12.5/Python-3.12.5.tar.xz", + "size": 20422396, + "sha256": "fa8a2e12c5e620b09f53e65bcd87550d2e5a1e2e04bf8ba991dcc55113876397", + "version": "3.12.5", "licenses": ["Python-2.0", "CNRI-Python"], "license_file": "LICENSE.cpython.txt", "python_tag": "cp312", }, + "cpython-3.13": { + "url": "https://www.python.org/ftp/python/3.13.0/Python-3.13.0rc1.tar.xz", + "size": 20881016, + "sha256": "678b884775eec0224d5159fa900879020baca2a36ce942fd95febfa1adb4a6bd", + "version": "3.13.0rc1", + "licenses": ["Python-2.0", "CNRI-Python"], + "license_file": "LICENSE.cpython.txt", + "python_tag": "cp313", + }, "expat": { "url": "https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-2.5.0.tar.xz", "size": 460560, @@ -126,10 +135,10 @@ "license_file": "LICENSE.libffi.txt", }, "libffi": { - "url": "https://github.com/libffi/libffi/releases/download/v3.4.4/libffi-3.4.4.tar.gz", - "size": 1362394, - "sha256": "d66c56ad259a82cf2a9dfc408b32bf5da52371500b84745f7fb8b645712df676", - "version": "3.4.4", + "url": "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz", + "size": 1391684, + "sha256": "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e", + "version": "3.4.6", "library_names": ["ffi"], "licenses": ["MIT"], "license_file": "LICENSE.libffi.txt", @@ -175,23 +184,26 @@ "sha256": "04cb77c660f09df017a57738ae9635ef23a506024789f2f18da1304b45af2023", "version": "14.0.3+20220508", }, - "llvm-17-x86_64-linux": { - "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240222/llvm-17.0.6+20240222-gnu_only-x86_64-unknown-linux-gnu.tar.zst", - "size": 229404408, - "sha256": "fc5e9092a8915dde438c3b491c5e6321594de541245b619f391edba719e4bd4f", - "version": "17.0.6+20240222", + # Remember to update LLVM_URL in src/release.rs whenever upgrading. + "llvm-18-x86_64-linux": { + "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-gnu_only-x86_64-unknown-linux-gnu.tar.zst", + "size": 242840506, + "sha256": "080c233fc7d75031b187bbfef62a4f9abc01188effb0c68fbc7dc4bc7370ee5b", + "version": "18.0.8+20240713", }, + # Remember to update LLVM_URL in src/release.rs whenever upgrading. "llvm-aarch64-macos": { - "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240222/llvm-17.0.6+20240222-aarch64-apple-darwin.tar.zst", - "size": 131303875, - "sha256": "e1acf616780f32787b37b5534cb342c0e1e4a56b80cb9283f537fd8c9976e0d1", - "version": "17.0.6+20240222", + "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-aarch64-apple-darwin.tar.zst", + "size": 136598617, + "sha256": "320da8d639186e020e7d54cdc35b7a5473b36cef08fdf7b22c03b59a273ba593", + "version": "18.0.8+20240713", }, + # Remember to update LLVM_URL in src/release.rs whenever upgrading. "llvm-x86_64-macos": { - "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240222/llvm-17.0.6+20240222-x86_64-apple-darwin.tar.zst", - "size": 131306317, - "sha256": "086d8c2fcb0e856b17a76f1c722a006fb62981e1c0f5f7a0ce3c912bd983c4d0", - "version": "17.0.6+20240222", + "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-x86_64-apple-darwin.tar.zst", + "size": 136599290, + "sha256": "3032161d1cadb8996b07fe5762444c956842b5a7d798b2fcfe5a04574fdf7549", + "version": "18.0.8+20240713", }, "m4": { "url": "https://ftp.gnu.org/gnu/m4/m4-1.4.19.tar.xz", @@ -215,10 +227,10 @@ "version": "1.2.5", }, "ncurses": { - "url": "https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.4.tar.gz", - "size": 3612591, - "sha256": "6931283d9ac87c5073f30b6290c4c75f21632bb4fc3603ac8100812bed248159", - "version": "6.4", + "url": "https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.5.tar.gz", + "size": 3688489, + "sha256": "136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6", + "version": "6.5", "library_names": ["ncurses", "ncursesw", "panel", "panelw"], "licenses": ["X11"], "license_file": "LICENSE.ncurses.txt", @@ -238,10 +250,10 @@ # using the latest available. # Remember to update OPENSSL_VERSION_INFO in verify_distribution.py whenever upgrading. "openssl-3.0": { - "url": "https://www.openssl.org/source/openssl-3.0.13.tar.gz", - "size": 15294843, - "sha256": "88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313", - "version": "3.0.13", + "url": "https://www.openssl.org/source/openssl-3.0.14.tar.gz", + "size": 15305497, + "sha256": "eeca035d4dd4e84fc25846d952da6297484afa0650a6f84c682e39df3a4123ca", + "version": "3.0.14", "library_names": ["crypto", "ssl"], "licenses": ["Apache-2.0"], "license_file": "LICENSE.openssl-3.txt", @@ -259,10 +271,10 @@ "version": "0.13.1", }, "pip": { - "url": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", - "size": 2110226, - "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", - "version": "24.0", + "url": "https://files.pythonhosted.org/packages/e7/54/0c1c068542cee73d8863336e974fc881e608d0170f3af15d0c0f28644531/pip-24.1.2-py3-none-any.whl", + "size": 1824406, + "sha256": "7cd207eed4c60b0f411b444cd1464198fe186671c323b6cd6d433ed80fc9d247", + "version": "24.1.2", }, "readline": { "url": "https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz", @@ -274,18 +286,18 @@ "license_file": "LICENSE.readline.txt", }, "setuptools": { - "url": "https://files.pythonhosted.org/packages/bb/0a/203797141ec9727344c7649f6d5f6cf71b89a6c28f8f55d4f18de7a1d352/setuptools-69.1.0-py3-none-any.whl", - "size": 819310, - "sha256": "c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6", - "version": "69.1.0", + "url": "https://files.pythonhosted.org/packages/ef/15/88e46eb9387e905704b69849618e699dc2f54407d8953cc4ec4b8b46528d/setuptools-70.3.0-py3-none-any.whl", + "size": 931070, + "sha256": "fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc", + "version": "70.3.0", }, # Remember to update verify_distribution.py when version changed. "sqlite": { - "url": "https://www.sqlite.org/2024/sqlite-autoconf-3450100.tar.gz", - "size": 3232682, - "sha256": "cd9c27841b7a5932c9897651e20b86c701dd740556989b01ca596fcfa3d49a0a", - "version": "3450100", - "actual_version": "3.45.1.0", + "url": "https://www.sqlite.org/2024/sqlite-autoconf-3460000.tar.gz", + "size": 3265248, + "sha256": "6f8e6a7b335273748816f9b3b62bbdc372a889de8782d7f048c653a447417a7d", + "version": "3460000", + "actual_version": "3.46.0.0", "library_names": ["sqlite3"], "licenses": [], "license_file": "LICENSE.sqlite.txt", diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index 1af9cde1..bf05d0d0 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -44,10 +44,10 @@ def supported_targets(yaml_path: pathlib.Path): targets = set() for target, settings in get_targets(yaml_path).items(): - for platform in settings["host_platforms"]: - if sys.platform == "linux" and platform == "linux64": + for host_platform in settings["host_platforms"]: + if sys.platform == "linux" and host_platform == "linux64": targets.add(target) - elif sys.platform == "darwin" and platform == "macos": + elif sys.platform == "darwin" and host_platform == "macos": targets.add(target) return targets @@ -305,9 +305,7 @@ def download_entry(key: str, dest_path: pathlib.Path, local_name=None) -> pathli assert isinstance(size, int) assert isinstance(sha256, str) - local_name = local_name or url[url.rindex("/") + 1 :] - - local_path = dest_path / local_name + local_path = dest_path / (local_name or url[url.rindex("/") + 1 :]) download_to_path(url, local_path, size, sha256) return local_path @@ -416,7 +414,7 @@ def clang_toolchain(host_platform: str, target_triple: str) -> str: if "musl" in target_triple: return "llvm-14-x86_64-linux" else: - return "llvm-17-x86_64-linux" + return "llvm-18-x86_64-linux" elif host_platform == "macos": if platform.mac_ver()[2] == "arm64": return "llvm-aarch64-macos" @@ -466,7 +464,7 @@ def add_licenses_to_extension_entry(entry): if "path_static" in link or "path_dynamic" in link: have_local_link = True - for key, value in DOWNLOADS.items(): + for value in DOWNLOADS.values(): if name not in value.get("library_names", []): continue diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..c914f91a --- /dev/null +++ b/ruff.toml @@ -0,0 +1,2 @@ +[lint] +select = ["F", "I", "B"] diff --git a/src/github.rs b/src/github.rs index f010acf2..09df552d 100644 --- a/src/github.rs +++ b/src/github.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use crate::release::{bootstrap_llvm, produce_install_only_stripped}; use { crate::release::{produce_install_only, RELEASE_TRIPLES}, anyhow::{anyhow, Result}, @@ -48,7 +49,7 @@ async fn upload_release_artifact( dry_run: bool, ) -> Result<()> { if release.assets.iter().any(|asset| asset.name == filename) { - println!("release asset {} already present; skipping", filename); + println!("release asset {filename} already present; skipping"); return Ok(()); } @@ -61,15 +62,15 @@ async fn upload_release_artifact( url.query_pairs_mut().clear().append_pair("name", &filename); - println!("uploading to {}", url); - - // Octocrab doesn't yet support release artifact upload. And the low-level HTTP API - // forces the use of strings on us. So we have to make our own HTTP client. + println!("uploading to {url}"); if dry_run { return Ok(()); } + // Octocrab doesn't yet support release artifact upload. And the low-level HTTP API + // forces the use of strings on us. So we have to make our own HTTP client. + let response = reqwest::Client::builder() .build()? .put(url) @@ -120,7 +121,7 @@ pub async fn command_fetch_release_distributions(args: &ArgMatches) -> Result<() | ".github/workflows/linux.yml" | ".github/workflows/windows.yml" ) { - workflow_names.insert(wf.id.clone(), wf.name); + workflow_names.insert(wf.id, wf.name); Some(wf.id) } else { @@ -138,26 +139,27 @@ pub async fn command_fetch_release_distributions(args: &ArgMatches) -> Result<() let mut runs: Vec = vec![]; for workflow_id in workflow_ids { + let commit = args + .get_one::("commit") + .expect("commit should be defined"); + let workflow_name = workflow_names + .get(&workflow_id) + .expect("should have workflow name"); + runs.push( workflows - .list_runs(format!("{}", workflow_id)) + .list_runs(format!("{workflow_id}")) .event("push") .status("success") .send() .await? .into_iter() .find(|run| { - run.head_sha.as_str() - == args - .get_one::("commit") - .expect("commit should be defined") + run.head_sha.as_str() == commit }) .ok_or_else(|| { anyhow!( - "could not find workflow run for commit for workflow {}", - workflow_names - .get(&workflow_id) - .expect("should have workflow name") + "could not find workflow run for commit {commit} for workflow {workflow_name}", ) })?, ); @@ -206,13 +208,15 @@ pub async fn command_fetch_release_distributions(args: &ArgMatches) -> Result<() // Iterate over `RELEASE_TRIPLES` in reverse-order to ensure that if any triple is a // substring of another, the longest match is used. - if let Some((triple, release)) = RELEASE_TRIPLES.iter().rev().find_map(|(triple, release)| { - if name.contains(triple) { - Some((triple, release)) - } else { - None - } - }) { + if let Some((triple, release)) = + RELEASE_TRIPLES.iter().rev().find_map(|(triple, release)| { + if name.contains(triple) { + Some((triple, release)) + } else { + None + } + }) + { let stripped_name = if let Some(s) = name.strip_suffix(".tar.zst") { s } else { @@ -253,9 +257,12 @@ pub async fn command_fetch_release_distributions(args: &ArgMatches) -> Result<() } } + let llvm_dir = bootstrap_llvm().await?; + install_paths .par_iter() .try_for_each(|path| -> Result<()> { + // Create the `install_only` archive. println!( "producing install_only archive from {}", path.file_name() @@ -263,7 +270,26 @@ pub async fn command_fetch_release_distributions(args: &ArgMatches) -> Result<() .to_string_lossy() ); - let dest_path = produce_install_only(&path)?; + let dest_path = produce_install_only(path)?; + + println!( + "releasing {}", + dest_path + .file_name() + .expect("should have file name") + .to_string_lossy() + ); + + // Create the `install_only_stripped` archive. + println!( + "producing install_only_stripped archive from {}", + dest_path + .file_name() + .expect("should have file name") + .to_string_lossy() + ); + + let dest_path = produce_install_only_stripped(&dest_path, &llvm_dir)?; println!( "releasing {}", @@ -302,8 +328,7 @@ pub async fn command_upload_release_distributions(args: &ArgMatches) -> Result<( .expect("repo should be specified"); let dry_run = args.get_flag("dry_run"); - let mut filenames = std::fs::read_dir(&dist_dir)? - .into_iter() + let mut filenames = std::fs::read_dir(dist_dir)? .map(|x| { let path = x?.path(); let filename = path @@ -356,6 +381,17 @@ pub async fn command_upload_release_distributions(args: &ArgMatches) -> Result<( ), format!("cpython-{}+{}-{}-install_only.tar.gz", version, tag, triple), ); + + wanted_filenames.insert( + format!( + "cpython-{}-{}-install_only_stripped-{}.tar.gz", + version, triple, datetime + ), + format!( + "cpython-{}+{}-{}-install_only_stripped.tar.gz", + version, tag, triple + ), + ); } } @@ -367,8 +403,10 @@ pub async fn command_upload_release_distributions(args: &ArgMatches) -> Result<( for f in &missing { println!("missing release artifact: {}", f); } - if !missing.is_empty() && !ignore_missing { - return Err(anyhow!("missing release artifacts")); + if missing.is_empty() { + println!("found all {} release artifacts", wanted_filenames.len()); + } else if !ignore_missing { + return Err(anyhow!("missing {} release artifacts", missing.len())); } let client = OctocrabBuilder::new() @@ -380,10 +418,14 @@ pub async fn command_upload_release_distributions(args: &ArgMatches) -> Result<( let release = if let Ok(release) = releases.get_by_tag(tag).await { release } else { - return Err(anyhow!( - "release {} does not exist; create it via GitHub web UI", - tag - )); + return if dry_run { + println!("release {tag} does not exist; exiting dry-run mode..."); + Ok(()) + } else { + Err(anyhow!( + "release {tag} does not exist; create it via GitHub web UI" + )) + }; }; let mut digests = BTreeMap::new(); @@ -445,6 +487,11 @@ pub async fn command_upload_release_distributions(args: &ArgMatches) -> Result<( // Check that content wasn't munged as part of uploading. This once happened // and created a busted release. Never again. + if dry_run { + println!("skipping SHA256SUMs check"); + return Ok(()); + } + let release = releases .get_by_tag(tag) .await diff --git a/src/json.rs b/src/json.rs index 5b656cff..7c27f46b 100644 --- a/src/json.rs +++ b/src/json.rs @@ -108,7 +108,7 @@ impl PythonJsonMain { } pub fn parse_python_json(json_data: &[u8]) -> Result { - let v: PythonJsonMain = serde_json::from_slice(&json_data)?; + let v: PythonJsonMain = serde_json::from_slice(json_data)?; Ok(v) } diff --git a/src/macho.rs b/src/macho.rs index 1f9bb067..4717fb96 100644 --- a/src/macho.rs +++ b/src/macho.rs @@ -288,11 +288,8 @@ impl TbdMetadata { let stripped = symbols .iter() .filter_map(|x| { - if let Some(stripped) = x.strip_prefix("R8289209$") { - Some(stripped.to_string()) - } else { - None - } + x.strip_prefix("R8289209$") + .map(|stripped| stripped.to_string()) }) .collect::>(); @@ -307,7 +304,7 @@ impl TbdMetadata { for (target, paths) in self.re_export_paths.iter_mut() { for path in paths.iter() { - let tbd_path = root_path.join(tbd_relative_path(&path)?); + let tbd_path = root_path.join(tbd_relative_path(path)?); let tbd_info = TbdMetadata::from_path(&tbd_path)?; if let Some(symbols) = tbd_info.symbols.get(target) { @@ -409,10 +406,7 @@ impl IndexedSdks { let empty = BTreeSet::new(); - let target_symbols = tbd_info - .symbols - .get(&symbol_target.to_string()) - .unwrap_or(&empty); + let target_symbols = tbd_info.symbols.get(symbol_target).unwrap_or(&empty); for (symbol, paths) in &symbols.symbols { if !target_symbols.contains(symbol) { diff --git a/src/main.rs b/src/main.rs index 1e217181..b12ef0fa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -85,6 +85,18 @@ fn main_impl() -> Result<()> { ), ); + let app = app.subcommand( + Command::new("convert-install-only-stripped") + .about("Convert an install_only .tar.gz archive to an install_only_stripped tar.gz archive") + .arg( + Arg::new("path") + .required(true) + .action(ArgAction::Append) + .value_parser(value_parser!(PathBuf)) + .help("Path of archive to convert"), + ), + ); + let app = app.subcommand( Command::new("upload-release-distributions") .about("Upload release distributions to a GitHub release") @@ -174,7 +186,20 @@ fn main_impl() -> Result<()> { match matches.subcommand() { Some(("convert-install-only", args)) => { for path in args.get_many::("path").unwrap() { - let dest_path = crate::release::produce_install_only(path)?; + let dest_path = release::produce_install_only(path)?; + println!("wrote {}", dest_path.display()); + } + + Ok(()) + } + Some(("convert-install-only-stripped", args)) => { + let llvm_dir = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(release::bootstrap_llvm())?; + for path in args.get_many::("path").unwrap() { + let dest_path = release::produce_install_only_stripped(path, &llvm_dir)?; println!("wrote {}", dest_path.display()); } @@ -185,18 +210,16 @@ fn main_impl() -> Result<()> { .enable_all() .build() .unwrap() - .block_on(crate::github::command_fetch_release_distributions(args)) + .block_on(github::command_fetch_release_distributions(args)) } Some(("upload-release-distributions", args)) => { tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap() - .block_on(crate::github::command_upload_release_distributions(args)) - } - Some(("validate-distribution", args)) => { - crate::validation::command_validate_distribution(args) + .block_on(github::command_upload_release_distributions(args)) } + Some(("validate-distribution", args)) => validation::command_validate_distribution(args), _ => Err(anyhow!("invalid sub-command")), } } diff --git a/src/release.rs b/src/release.rs index 2e2a6987..6c46d2e1 100644 --- a/src/release.rs +++ b/src/release.rs @@ -2,6 +2,12 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use anyhow::Context; +use futures::StreamExt; + +use object::FileKind; +use std::process::{Command, Stdio}; +use url::Url; use { crate::json::parse_python_json, anyhow::{anyhow, Result}, @@ -200,7 +206,7 @@ pub static RELEASE_TRIPLES: Lazy> = Lazy:: h }); -/// Convert a .tar.zst archive to an install only .tar.gz archive. +/// Convert a .tar.zst archive to an install-only .tar.gz archive. pub fn convert_to_install_only(reader: impl BufRead, writer: W) -> Result { let dctx = zstd::stream::Decoder::new(reader)?; @@ -241,8 +247,7 @@ pub fn convert_to_install_only(reader: impl BufRead, writer: W) -> Res // increases the size of the archive and isn't needed in most cases. if path_bytes .windows(b"/libpython".len()) - .position(|x| x == b"/libpython") - .is_some() + .any(|x| x == b"/libpython") && path_bytes.ends_with(b".a") { continue; @@ -280,6 +285,98 @@ pub fn convert_to_install_only(reader: impl BufRead, writer: W) -> Res Ok(builder.into_inner()?.finish()?) } +/// Run `llvm-strip` over the given data, returning the stripped data. +fn llvm_strip(data: &[u8], llvm_dir: &Path) -> Result> { + let mut command = Command::new(llvm_dir.join("bin/llvm-strip")) + .arg("--strip-debug") + .arg("-") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .with_context(|| "failed to spawn llvm-strip")?; + + command + .stdin + .as_mut() + .unwrap() + .write_all(data) + .with_context(|| "failed to write data to llvm-strip")?; + + let output = command + .wait_with_output() + .with_context(|| "failed to wait for llvm-strip")?; + if !output.status.success() { + return Err(anyhow!("llvm-strip failed: {}", output.status)); + } + + Ok(output.stdout) +} + +/// Given an install-only .tar.gz archive, strip the underlying build. +pub fn convert_to_stripped( + reader: impl BufRead, + writer: W, + llvm_dir: &Path, +) -> Result { + let dctx = flate2::read::GzDecoder::new(reader); + + let mut tar_in = tar::Archive::new(dctx); + + let writer = flate2::write::GzEncoder::new(writer, flate2::Compression::default()); + + let mut builder = tar::Builder::new(writer); + + for entry in tar_in.entries()? { + let mut entry = entry?; + + let mut data = vec![]; + entry.read_to_end(&mut data)?; + + let path = entry.path()?; + + // Drop PDB files. + match pdb::PDB::open(std::io::Cursor::new(&data)) { + Ok(_) => { + continue; + } + Err(err) => { + if path.extension().is_some_and(|ext| ext == "pdb") { + println!( + "file with `.pdb` extension ({}) failed to parse as PDB :{err}", + path.display() + ); + } + } + } + + // If we have an ELF, Mach-O, or PE file, strip it in-memory with `llvm-strip`, and + // return the stripped data. + if matches!( + FileKind::parse(data.as_slice()), + Ok(FileKind::Elf32 + | FileKind::Elf64 + | FileKind::MachO32 + | FileKind::MachO64 + | FileKind::MachOFat32 + | FileKind::MachOFat64 + | FileKind::Pe32 + | FileKind::Pe64) + ) { + data = llvm_strip(&data, llvm_dir) + .with_context(|| format!("failed to strip {}", path.display()))?; + } + + let mut header = entry.header().clone(); + header.set_size(data.len() as u64); + header.set_cksum(); + + builder.append(&header, std::io::Cursor::new(data))?; + } + + Ok(builder.into_inner()?.finish()?) +} + +/// Create an install-only .tar.gz archive from a .tar.zst archive. pub fn produce_install_only(tar_zst_path: &Path) -> Result { let buf = std::fs::read(tar_zst_path)?; @@ -303,7 +400,119 @@ pub fn produce_install_only(tar_zst_path: &Path) -> Result { let install_only_name = install_only_name.replace(".tar.zst", ".tar.gz"); let dest_path = tar_zst_path.with_file_name(install_only_name); - std::fs::write(&dest_path, &gz_data)?; + std::fs::write(&dest_path, gz_data)?; Ok(dest_path) } + +pub fn produce_install_only_stripped(tar_gz_path: &Path, llvm_dir: &Path) -> Result { + let buf = std::fs::read(tar_gz_path)?; + + let size_before = buf.len(); + + let gz_data = convert_to_stripped( + std::io::Cursor::new(buf), + std::io::Cursor::new(vec![]), + llvm_dir, + )? + .into_inner(); + + let size_after = gz_data.len(); + + println!( + "stripped {} from {size_before} to {size_after} bytes", + tar_gz_path.display() + ); + + // Given `cpython-3.12.4-x86_64_v3-unknown-linux-gnu-install_only-20240722T0909.tar.gz`, + // map to `cpython-3.12.4-x86_64_v3-unknown-linux-gnu-install_only_stripped-20240722T0909.tar.gz`. + let filename = tar_gz_path + .file_name() + .expect("should have filename") + .to_string_lossy(); + + let mut name_parts = filename + .split('-') + .map(|x| x.to_string()) + .collect::>(); + let parts_len = name_parts.len(); + + name_parts[parts_len - 2] = "install_only_stripped".to_string(); + + let install_only_name = name_parts.join("-"); + + let dest_path = tar_gz_path.with_file_name(install_only_name); + std::fs::write(&dest_path, gz_data)?; + + Ok(dest_path) +} + +/// URL from which to download LLVM. +/// +/// To be kept in sync with `pythonbuild/downloads.py`. +static LLVM_URL: Lazy = Lazy::new(|| { + if cfg!(target_os = "macos") { + if std::env::consts::ARCH == "aarch64" { + Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-aarch64-apple-darwin.tar.zst").unwrap() + } else if std::env::consts::ARCH == "x86_64" { + Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-x86_64-apple-darwin.tar.zst").unwrap() + } else { + panic!("unsupported macOS architecture"); + } + } else if cfg!(target_os = "linux") { + Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-gnu_only-x86_64-unknown-linux-gnu.tar.zst").unwrap() + } else { + panic!("unsupported platform"); + } +}); + +/// Bootstrap `llvm` for the current platform. +/// +/// Returns the path to the top-level `llvm` directory. +pub async fn bootstrap_llvm() -> Result { + let url = &*LLVM_URL; + let filename = url.path_segments().unwrap().last().unwrap(); + + let llvm_dir = Path::new("build").join("llvm"); + std::fs::create_dir_all(&llvm_dir)?; + + // If `llvm` is already available with the target version, return it. + if llvm_dir.join(filename).exists() { + return Ok(llvm_dir.join("llvm")); + } + + println!("Downloading LLVM tarball from: {url}"); + + // Create a temporary directory to download and extract the LLVM tarball. + let temp_dir = tempfile::TempDir::new()?; + + // Download the tarball. + let tarball_path = temp_dir + .path() + .join(url.path_segments().unwrap().last().unwrap()); + let mut tarball_file = tokio::fs::File::create(&tarball_path).await?; + let mut bytes_stream = reqwest::Client::new() + .get(url.clone()) + .send() + .await? + .bytes_stream(); + while let Some(chunk) = bytes_stream.next().await { + tokio::io::copy(&mut chunk?.as_ref(), &mut tarball_file).await?; + } + + // Decompress the tarball. + let tarball = std::fs::File::open(&tarball_path)?; + let tar = zstd::stream::Decoder::new(std::io::BufReader::new(tarball))?; + let mut archive = tar::Archive::new(tar); + archive.unpack(temp_dir.path())?; + + // Persist the directory. + match tokio::fs::remove_dir_all(&llvm_dir).await { + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err).context("failed to remove existing llvm directory"), + } + tokio::fs::rename(temp_dir.into_path(), &llvm_dir).await?; + + Ok(llvm_dir.join("llvm")) +} diff --git a/src/validation.rs b/src/validation.rs index 292a3c56..cbbf3665 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -125,6 +125,7 @@ const PE_ALLOWED_LIBRARIES: &[&str] = &[ "python310.dll", "python311.dll", "python312.dll", + "python313.dll", "sqlite3.dll", "tcl86t.dll", "tk86t.dll", @@ -287,6 +288,16 @@ static DARWIN_ALLOWED_DYLIBS: Lazy> = Lazy::new(|| { max_compatibility_version: "3.12.0".try_into().unwrap(), required: false, }, + MachOAllowedDylib { + name: "@executable_path/../lib/libpython3.13.dylib".to_string(), + max_compatibility_version: "3.13.0".try_into().unwrap(), + required: false, + }, + MachOAllowedDylib { + name: "@executable_path/../lib/libpython3.13d.dylib".to_string(), + max_compatibility_version: "3.13.0".try_into().unwrap(), + required: false, + }, MachOAllowedDylib { name: "/System/Library/Frameworks/AppKit.framework/Versions/C/AppKit".to_string(), max_compatibility_version: "45.0.0".try_into().unwrap(), @@ -638,7 +649,6 @@ const GLOBAL_EXTENSIONS: &[&str] = &[ "_weakref", "array", "atexit", - "audioop", "binascii", "builtins", "cmath", @@ -665,13 +675,15 @@ const GLOBAL_EXTENSIONS: &[&str] = &[ // _testsinglephase added in 3.12. // _sha256 and _sha512 merged into _sha2 in 3.12. // _xxinterpchannels added in 3.12. +// audioop removed in 3.13. // We didn't build ctypes_test until 3.9. // We didn't build some test extensions until 3.9. -const GLOBAL_EXTENSIONS_PYTHON_3_8: &[&str] = &["_sha256", "_sha512", "parser"]; +const GLOBAL_EXTENSIONS_PYTHON_3_8: &[&str] = &["audioop", "_sha256", "_sha512", "parser"]; const GLOBAL_EXTENSIONS_PYTHON_3_9: &[&str] = &[ + "audioop", "_peg_parser", "_sha256", "_sha512", @@ -682,6 +694,7 @@ const GLOBAL_EXTENSIONS_PYTHON_3_9: &[&str] = &[ ]; const GLOBAL_EXTENSIONS_PYTHON_3_10: &[&str] = &[ + "audioop", "_sha256", "_sha512", "_uuid", @@ -690,6 +703,7 @@ const GLOBAL_EXTENSIONS_PYTHON_3_10: &[&str] = &[ ]; const GLOBAL_EXTENSIONS_PYTHON_3_11: &[&str] = &[ + "audioop", "_sha256", "_sha512", "_tokenize", @@ -700,6 +714,7 @@ const GLOBAL_EXTENSIONS_PYTHON_3_11: &[&str] = &[ ]; const GLOBAL_EXTENSIONS_PYTHON_3_12: &[&str] = &[ + "audioop", "_sha2", "_tokenize", "_typing", @@ -708,6 +723,17 @@ const GLOBAL_EXTENSIONS_PYTHON_3_12: &[&str] = &[ "_zoneinfo", ]; +const GLOBAL_EXTENSIONS_PYTHON_3_13: &[&str] = &[ + "_interpchannels", + "_interpqueues", + "_interpreters", + "_sha2", + "_sysconfig", + "_tokenize", + "_typing", + "_zoneinfo", +]; + const GLOBAL_EXTENSIONS_MACOS: &[&str] = &["_scproxy"]; const GLOBAL_EXTENSIONS_POSIX: &[&str] = &[ @@ -729,19 +755,19 @@ const GLOBAL_EXTENSIONS_POSIX: &[&str] = &[ "termios", ]; -const GLOBAL_EXTENSIONS_LINUX: &[&str] = &["spwd"]; +const GLOBAL_EXTENSIONS_LINUX_PRE_3_13: &[&str] = &["spwd"]; const GLOBAL_EXTENSIONS_WINDOWS: &[&str] = &[ - "_msi", "_overlapped", "_winapi", - "_xxsubinterpreters", "msvcrt", "nt", "winreg", "winsound", ]; +const GLOBAL_EXTENSIONS_WINDOWS_PRE_3_13: &[&str] = &["_msi"]; + /// Extension modules not present in Windows static builds. const GLOBAL_EXTENSIONS_WINDOWS_NO_STATIC: &[&str] = &["_testinternalcapi", "_tkinter"]; @@ -792,7 +818,7 @@ impl ValidationContext { } } -fn validate_elf<'data, Elf: FileHeader>( +fn validate_elf>( context: &mut ValidationContext, json: &PythonJsonMain, target_triple: &str, @@ -985,20 +1011,18 @@ fn validate_elf<'data, Elf: FileHeader>( if let Some(version) = version_version { let parts: Vec<&str> = version.splitn(2, '_').collect(); - if parts.len() == 2 { - if parts[0] == "GLIBC" { - let v = version_compare::Version::from(parts[1]) - .expect("unable to parse version"); + if parts.len() == 2 && parts[0] == "GLIBC" { + let v = version_compare::Version::from(parts[1]) + .expect("unable to parse version"); - if &v > wanted_glibc_max_version { - context.errors.push(format!( - "{} references too new glibc symbol {:?} ({} > {})", - path.display(), - name, - v, - wanted_glibc_max_version, - )); - } + if &v > wanted_glibc_max_version { + context.errors.push(format!( + "{} references too new glibc symbol {:?} ({} > {})", + path.display(), + name, + v, + wanted_glibc_max_version, + )); } } } @@ -1026,12 +1050,12 @@ fn validate_elf<'data, Elf: FileHeader>( if let Some(filename) = path.file_name() { let filename = filename.to_string_lossy(); - if filename.starts_with("libpython") && filename.ends_with(".so.1.0") { - if matches!(symbol.st_bind(), STB_GLOBAL | STB_WEAK) - && symbol.st_visibility() == STV_DEFAULT - { - context.libpython_exported_symbols.insert(name.to_string()); - } + if filename.starts_with("libpython") + && filename.ends_with(".so.1.0") + && matches!(symbol.st_bind(), STB_GLOBAL | STB_WEAK) + && symbol.st_visibility() == STV_DEFAULT + { + context.libpython_exported_symbols.insert(name.to_string()); } } } @@ -1058,6 +1082,7 @@ fn parse_version_nibbles(v: u32) -> semver::Version { semver::Version::new(major as _, minor as _, patch as _) } +#[allow(clippy::too_many_arguments)] fn validate_macho>( context: &mut ValidationContext, target_triple: &str, @@ -1125,7 +1150,7 @@ fn validate_macho>( target_version = Some(parse_version_nibbles(v.version.get(endian))); } LoadCommandVariant::Dylib(command) => { - let raw_string = load_command.string(endian, command.dylib.name.clone())?; + let raw_string = load_command.string(endian, command.dylib.name)?; let lib = String::from_utf8(raw_string.to_vec())?; dylib_names.push(lib.clone()); @@ -1336,9 +1361,9 @@ fn validate_possible_object_file( json, triple, python_major_minor, - path.as_ref(), + path, header, - &data, + data, )?; } FileKind::Elf64 => { @@ -1349,9 +1374,9 @@ fn validate_possible_object_file( json, triple, python_major_minor, - path.as_ref(), + path, header, - &data, + data, )?; } FileKind::MachO32 => { @@ -1367,9 +1392,9 @@ fn validate_possible_object_file( json.apple_sdk_version .as_ref() .expect("apple_sdk_version should be set"), - path.as_ref(), + path, header, - &data, + data, )?; } FileKind::MachO64 => { @@ -1385,9 +1410,9 @@ fn validate_possible_object_file( json.apple_sdk_version .as_ref() .expect("apple_sdk_version should be set"), - path.as_ref(), + path, header, - &data, + data, )?; } FileKind::MachOFat32 | FileKind::MachOFat64 => { @@ -1399,11 +1424,11 @@ fn validate_possible_object_file( } FileKind::Pe32 => { let file = PeFile32::parse(data)?; - validate_pe(&mut context, path.as_ref(), &file)?; + validate_pe(&mut context, path, &file)?; } FileKind::Pe64 => { let file = PeFile64::parse(data)?; - validate_pe(&mut context, path.as_ref(), &file)?; + validate_pe(&mut context, path, &file)?; } _ => {} } @@ -1431,7 +1456,7 @@ fn validate_extension_modules( return Ok(errors); } - let mut wanted = BTreeSet::from_iter(GLOBAL_EXTENSIONS.iter().map(|x| *x)); + let mut wanted = BTreeSet::from_iter(GLOBAL_EXTENSIONS.iter().copied()); match python_major_minor { "3.8" => { @@ -1449,6 +1474,9 @@ fn validate_extension_modules( "3.12" => { wanted.extend(GLOBAL_EXTENSIONS_PYTHON_3_12); } + "3.13" => { + wanted.extend(GLOBAL_EXTENSIONS_PYTHON_3_13); + } _ => { panic!("unhandled Python version: {}", python_major_minor); } @@ -1456,12 +1484,23 @@ fn validate_extension_modules( if is_macos { wanted.extend(GLOBAL_EXTENSIONS_POSIX); + if python_major_minor == "3.13" { + wanted.remove("_crypt"); + } wanted.extend(GLOBAL_EXTENSIONS_MACOS); } if is_windows { wanted.extend(GLOBAL_EXTENSIONS_WINDOWS); + if python_major_minor == "3.8" { + wanted.insert("_xxsubinterpreters"); + } + + if matches!(python_major_minor, "3.8" | "3.9" | "3.10" | "3.11" | "3.12") { + wanted.extend(GLOBAL_EXTENSIONS_WINDOWS_PRE_3_13); + } + if static_crt { for x in GLOBAL_EXTENSIONS_WINDOWS_NO_STATIC { wanted.remove(*x); @@ -1471,14 +1510,27 @@ fn validate_extension_modules( if is_linux { wanted.extend(GLOBAL_EXTENSIONS_POSIX); - wanted.extend(GLOBAL_EXTENSIONS_LINUX); + // TODO: If there are more differences for `GLOBAL_EXTENSIONS_POSIX` in future Python + // versions, we should move the `_crypt` special-case into a constant + if python_major_minor == "3.13" { + wanted.remove("_crypt"); + } + if matches!(python_major_minor, "3.8" | "3.9" | "3.10" | "3.11" | "3.12") { + wanted.extend(GLOBAL_EXTENSIONS_LINUX_PRE_3_13); + } - if !is_linux_musl { + if !is_linux_musl && matches!(python_major_minor, "3.8" | "3.9" | "3.10" | "3.11" | "3.12") + { wanted.insert("ossaudiodev"); } } - if (is_linux || is_macos) && matches!(python_major_minor, "3.9" | "3.10" | "3.11" | "3.12") { + if (is_linux || is_macos) + && matches!( + python_major_minor, + "3.9" | "3.10" | "3.11" | "3.12" | "3.13" + ) + { wanted.extend([ "_testbuffer", "_testimportmultiple", @@ -1487,7 +1539,11 @@ fn validate_extension_modules( ]); } - if (is_linux || is_macos) && python_major_minor == "3.12" { + if (is_linux || is_macos) && python_major_minor == "3.13" { + wanted.extend(["_suggestions", "_testexternalinspection"]); + } + + if (is_linux || is_macos) && matches!(python_major_minor, "3.12" | "3.13") { wanted.insert("_testsinglephase"); } @@ -1501,7 +1557,7 @@ fn validate_extension_modules( } // _wmi is Windows only on 3.12+. - if python_major_minor == "3.12" && is_windows { + if matches!(python_major_minor, "3.12" | "3.13") && is_windows { wanted.insert("_wmi"); } @@ -1576,15 +1632,12 @@ fn validate_json(json: &PythonJsonMain, triple: &str, is_debug: bool) -> Result< .map(|x| x.as_str()) .collect::>(); - errors.extend( - validate_extension_modules( - &json.python_major_minor_version, - triple, - json.crt_features.contains(&"static".to_string()), - &have_extensions, - )? - .into_iter(), - ); + errors.extend(validate_extension_modules( + &json.python_major_minor_version, + triple, + json.crt_features.contains(&"static".to_string()), + &have_extensions, + )?); Ok(errors) } @@ -1627,6 +1680,8 @@ fn validate_distribution( "3.11" } else if dist_filename.starts_with("cpython-3.12.") { "3.12" + } else if dist_filename.starts_with("cpython-3.13.") { + "3.13" } else { return Err(anyhow!("could not parse Python version from filename")); }; @@ -1635,7 +1690,7 @@ fn validate_distribution( let is_static = triple.contains("unknown-linux-musl"); - let mut tf = crate::open_distribution_archive(&dist_path)?; + let mut tf = crate::open_distribution_archive(dist_path)?; // First entry in archive should be python/PYTHON.json. let mut entries = tf.entries()?; @@ -1701,7 +1756,7 @@ fn validate_distribution( context.merge(validate_possible_object_file( json.as_ref().unwrap(), python_major_minor, - &triple, + triple, &path, &data, )?); @@ -1726,9 +1781,9 @@ fn validate_distribution( context.merge(validate_possible_object_file( json.as_ref().unwrap(), python_major_minor, - &triple, + triple, &member_path, - &member_data, + member_data, )?); } } @@ -1841,9 +1896,7 @@ fn validate_distribution( // Ensure that some well known Python symbols are being exported from libpython. for symbol in PYTHON_EXPORTED_SYMBOLS { - let exported = context - .libpython_exported_symbols - .contains(&symbol.to_string()); + let exported = context.libpython_exported_symbols.contains(*symbol); let wanted = !is_static; if exported != wanted { @@ -1867,6 +1920,7 @@ fn validate_distribution( } } + #[allow(clippy::if_same_then_else)] // Static builds never have shared library extension modules. let want_shared = if is_static { false @@ -1904,12 +1958,18 @@ fn validate_distribution( let exported = context.libpython_exported_symbols.contains(&ext.init_fn); + #[allow(clippy::needless_bool, clippy::if_same_then_else)] // Static distributions never export symbols. let wanted = if is_static { false - // For some strange reason _PyWarnings_Init is exported as part of the ABI. + // For some strange reason _PyWarnings_Init is exported as part of the ABI } else if name == "_warnings" { - true + // But not on Python 3.13 on Windows + if triple.contains("-windows-") { + matches!(python_major_minor, "3.8" | "3.9" | "3.10" | "3.11" | "3.12") + } else { + true + } // Windows dynamic doesn't export extension module init functions. } else if triple.contains("-windows-") { false @@ -1996,7 +2056,7 @@ fn verify_distribution_behavior(dist_path: &Path) -> Result> { tf.unpack(temp_dir.path())?; let python_json_path = temp_dir.path().join("python").join("PYTHON.json"); - let python_json_data = std::fs::read(&python_json_path)?; + let python_json_data = std::fs::read(python_json_path)?; let python_json = parse_python_json(&python_json_data)?; let python_exe = temp_dir.path().join("python").join(python_json.python_exe); @@ -2005,7 +2065,7 @@ fn verify_distribution_behavior(dist_path: &Path) -> Result> { std::fs::write(&test_file, PYTHON_VERIFICATIONS.as_bytes())?; eprintln!(" running interpreter tests (output should follow)"); - let output = duct::cmd(&python_exe, &[test_file.display().to_string()]) + let output = duct::cmd(python_exe, [test_file.display().to_string()]) .stdout_to_stderr() .unchecked() .env("TARGET_TRIPLE", &python_json.target_triple) diff --git a/src/verify_distribution.py b/src/verify_distribution.py index 8b0e42d7..f21b50de 100644 --- a/src/verify_distribution.py +++ b/src/verify_distribution.py @@ -113,7 +113,7 @@ def test_hashlib(self): def test_sqlite(self): import sqlite3 - self.assertEqual(sqlite3.sqlite_version_info, (3, 45, 1)) + self.assertEqual(sqlite3.sqlite_version_info, (3, 46, 0)) # Optional SQLite3 features are enabled. conn = sqlite3.connect(":memory:") @@ -135,7 +135,7 @@ def test_ssl(self): if os.name == "nt" and sys.version_info[0:2] < (3, 11): wanted_version = (1, 1, 1, 23, 15) else: - wanted_version = (3, 0, 0, 13, 0) + wanted_version = (3, 0, 0, 14, 0) self.assertEqual(ssl.OPENSSL_VERSION_INFO, wanted_version)