global: CPython 3.12 support

I cobbled this together over several partial days of work. There's a lot of changes in there and I'm too lazy to split it back up into microcomments. I haven't performed a full audit of the produced archives and their PYTHON.json files. I suspect there's some subtle bugs lingering in here: there usually is for the initial release of a major version. But CI and our Rust validation is passing and that usually means a lot. So I'm going to commit what we have and iterate on it as needed. Closes #186.
indygreg · Oct 1, 2023 · 2a5f3a9 · 2a5f3a9
1 parent a8c0808
commit 2a5f3a9
Show file tree

Hide file tree

Showing 18 changed files with 659 additions and 28 deletions.
diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml
@@ -83,6 +83,16 @@ jobs:
             py: 'cpython-3.11'
             optimizations: 'lto'
 
+          - target_triple: 'aarch64-apple-darwin'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 'aarch64-apple-darwin'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+          - target_triple: 'aarch64-apple-darwin'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+
           # macOS on Intel hardware. This is pretty straightforward. We exclude
           # noopt because it doesn't provide any compelling advantages over PGO
           # or LTO builds.
@@ -137,6 +147,19 @@ jobs:
           - target_triple: 'x86_64-apple-darwin'
             py: 'cpython-3.11'
             optimizations: 'pgo+lto'
+
+          - target_triple: 'x86_64-apple-darwin'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 'x86_64-apple-darwin'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+          - target_triple: 'x86_64-apple-darwin'
+            py: 'cpython-3.12'
+            optimizations: 'pgo'
+          - target_triple: 'x86_64-apple-darwin'
+            py: 'cpython-3.12'
+            optimizations: 'pgo+lto'
     needs:
       - pythonbuild
     runs-on: 'macos-11'

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -157,6 +157,16 @@ jobs:
             py: 'cpython-3.11'
             optimizations: 'lto'
 
+          - target_triple: 'aarch64-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 'aarch64-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+          - target_triple: 'aarch64-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+
           # Cross-compiles can't do PGO and require Python 3.9.
           - target_triple: 'armv7-unknown-linux-gnueabi'
             py: 'cpython-3.9'
@@ -188,6 +198,16 @@ jobs:
             py: 'cpython-3.11'
             optimizations: 'lto'
 
+          - target_triple: 'armv7-unknown-linux-gnueabi'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 'armv7-unknown-linux-gnueabi'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+          - target_triple: 'armv7-unknown-linux-gnueabi'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+
           # Cross-compiles can't do PGO and require Python 3.9.
           - target_triple: 'armv7-unknown-linux-gnueabihf'
             py: 'cpython-3.9'
@@ -219,6 +239,16 @@ jobs:
             py: 'cpython-3.11'
             optimizations: 'lto'
 
+          - target_triple: 'armv7-unknown-linux-gnueabihf'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 'armv7-unknown-linux-gnueabihf'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+          - target_triple: 'armv7-unknown-linux-gnueabihf'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+
           # Cross-compiles can't do PGO and require Python 3.9.
           - target_triple: 'mips-unknown-linux-gnu'
             py: 'cpython-3.9'
@@ -250,6 +280,16 @@ jobs:
             py: 'cpython-3.11'
             optimizations: 'lto'
 
+          - target_triple: 'mips-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 'mips-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+          - target_triple: 'mips-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+
           # Cross-compiles can't do PGO and require Python 3.9.
           - target_triple: 'mipsel-unknown-linux-gnu'
             py: 'cpython-3.9'
@@ -281,6 +321,16 @@ jobs:
             py: 'cpython-3.11'
             optimizations: 'lto'
 
+          - target_triple: 'mipsel-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 'mipsel-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+          - target_triple: 'mipsel-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+
           # Cross-compiles can't do PGO and require Python 3.9.
           - target_triple: 's390x-unknown-linux-gnu'
             py: 'cpython-3.9'
@@ -312,6 +362,16 @@ jobs:
             py: 'cpython-3.11'
             optimizations: 'lto'
 
+          - target_triple: 's390x-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 's390x-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+          - target_triple: 's390x-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+
           # Cross-compiles can't do PGO and require Python 3.9.
           - target_triple: 'ppc64le-unknown-linux-gnu'
             py: 'cpython-3.9'
@@ -343,6 +403,16 @@ jobs:
             py: 'cpython-3.11'
             optimizations: 'lto'
 
+          - target_triple: 'ppc64le-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 'ppc64le-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+          - target_triple: 'ppc64le-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+
           # We don't publish noopt builds when PGO is available.
           - target_triple: 'x86_64-unknown-linux-gnu'
             py: 'cpython-3.8'
@@ -412,6 +482,23 @@ jobs:
             optimizations: 'pgo+lto'
             run: true
 
+          - target_triple: 'x86_64-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+            run: true
+          - target_triple: 'x86_64-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+            run: true
+          - target_triple: 'x86_64-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'pgo'
+            run: true
+          - target_triple: 'x86_64-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'pgo+lto'
+            run: true
+
           - target_triple: 'x86_64_v2-unknown-linux-gnu'
             py: 'cpython-3.9'
             optimizations: 'debug'
@@ -463,6 +550,23 @@ jobs:
             optimizations: 'pgo+lto'
             run: true
 
+          - target_triple: 'x86_64_v2-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+            run: true
+          - target_triple: 'x86_64_v2-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+            run: true
+          - target_triple: 'x86_64_v2-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'pgo'
+            run: true
+          - target_triple: 'x86_64_v2-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'pgo+lto'
+            run: true
+
           - target_triple: 'x86_64_v3-unknown-linux-gnu'
             py: 'cpython-3.9'
             optimizations: 'debug'
@@ -514,6 +618,23 @@ jobs:
             optimizations: 'pgo+lto'
             run: true
 
+          - target_triple: 'x86_64_v3-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+            run: true
+          - target_triple: 'x86_64_v3-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+            run: true
+          - target_triple: 'x86_64_v3-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'pgo'
+            run: true
+          - target_triple: 'x86_64_v3-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'pgo+lto'
+            run: true
+
           # GitHub Actions runners don't support x86-64-v4 so we can't PGO.
           - target_triple: 'x86_64_v4-unknown-linux-gnu'
             py: 'cpython-3.9'
@@ -546,6 +667,16 @@ jobs:
             py: 'cpython-3.11'
             optimizations: 'lto'
 
+          - target_triple: 'x86_64_v4-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 'x86_64_v4-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+          - target_triple: 'x86_64_v4-unknown-linux-gnu'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+
           # musl doesn't support PGO.
           - target_triple: 'x86_64-unknown-linux-musl'
             py: 'cpython-3.8'
@@ -599,6 +730,19 @@ jobs:
             optimizations: 'lto'
             run: true
 
+          - target_triple: 'x86_64-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+            run: true
+          - target_triple: 'x86_64-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+            run: true
+          - target_triple: 'x86_64-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+            run: true
+
           - target_triple: 'x86_64_v2-unknown-linux-musl'
             py: 'cpython-3.9'
             optimizations: 'debug'
@@ -638,6 +782,19 @@ jobs:
             optimizations: 'lto'
             run: true
 
+          - target_triple: 'x86_64_v2-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+            run: true
+          - target_triple: 'x86_64_v2-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+            run: true
+          - target_triple: 'x86_64_v2-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+            run: true
+
           - target_triple: 'x86_64_v3-unknown-linux-musl'
             py: 'cpython-3.9'
             optimizations: 'debug'
@@ -677,6 +834,19 @@ jobs:
             optimizations: 'lto'
             run: true
 
+          - target_triple: 'x86_64_v3-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+            run: true
+          - target_triple: 'x86_64_v3-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+            run: true
+          - target_triple: 'x86_64_v3-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+            run: true
+
           - target_triple: 'x86_64_v4-unknown-linux-musl'
             py: 'cpython-3.9'
             optimizations: 'debug'
@@ -707,6 +877,16 @@ jobs:
             py: 'cpython-3.11'
             optimizations: 'lto'
 
+          - target_triple: 'x86_64_v4-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'debug'
+          - target_triple: 'x86_64_v4-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'noopt'
+          - target_triple: 'x86_64_v4-unknown-linux-musl'
+            py: 'cpython-3.12'
+            optimizations: 'lto'
+
     needs:
       - pythonbuild
       - image

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -45,6 +45,7 @@ jobs:
           - 'cpython-3.9'
           - 'cpython-3.10'
           - 'cpython-3.11'
+          - 'cpython-3.12'
         vcvars:
           - 'vcvars32.bat'
           - 'vcvars64.bat'

diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile
@@ -259,6 +259,10 @@ $(OUTDIR)/cpython-3.10-$(CPYTHON_3.10_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HO
 $(OUTDIR)/cpython-3.11-$(CPYTHON_3.11_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
 	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11-host
 
+$(OUTDIR)/cpython-3.12-$(CPYTHON_3.12_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12-host
+
+
 PYTHON_DEPENDS := \
     $(PYTHON_SUPPORT_FILES) \
     $(OUTDIR)/versions/VERSION.pip \
@@ -302,3 +306,6 @@ $(OUTDIR)/cpython-$(CPYTHON_3.10_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DE
 
 $(OUTDIR)/cpython-$(CPYTHON_3.11_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS)
 	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11
+
+$(OUTDIR)/cpython-$(CPYTHON_3.12_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS)
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12
diff --git a/cpython-unix/build-cpython-host.sh b/cpython-unix/build-cpython-host.sh
@@ -69,10 +69,26 @@ case "${BUILD_TRIPLE}" in
     ;;
 esac
 
+EXTRA_CONFIGURE_FLAGS=
+
+# We may not have a usable libraries to build against. Forcefully disable extensions
+# that may not build.
+if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
+    for m in _hashlib _ssl; do
+      EXTRA_CONFIGURE_FLAGS="${EXTRA_CONFIGURE_FLAGS} py_cv_module_${m}=n/a"
+    done
+  fi
+
 CC="${HOST_CC}" CXX="${HOST_CXX}" CFLAGS="${EXTRA_HOST_CFLAGS}" CPPFLAGS="${EXTRA_HOST_CFLAGS}" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure \
   --prefix /tools/host \
-  --without-ensurepip
+  --without-ensurepip \
+  ${EXTRA_CONFIGURE_FLAGS}
 
-make -j "${NUM_CPUS}" install DESTDIR=${ROOT}/out
+# Ideally we'd do `make install` here and be done with it. But there's a race
+# condition in CPython's build system related to directory creation that gets
+# tickled when we do this. https://github.com/python/cpython/issues/109796.
+make -j "${NUM_CPUS}"
+make -j sharedinstall DESTDIR=${ROOT}/out
+make -j install DESTDIR=${ROOT}/out
 
 popd