From c3d52ba926eedd5c1c67ad14eacf68025ee12680 Mon Sep 17 00:00:00 2001 From: Jack Lovell Date: Mon, 11 Dec 2023 14:27:00 +0000 Subject: [PATCH 1/4] Pin to Cython <3.0 Cython 3.0 release introduces breaking changes. Pin to earlier Cython until the code has been updated for compatibility. Make a new post1 release of this so downstream packages will pick up the working version of 0.8.1. --- pyproject.toml | 2 +- raysect/VERSION | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e1fcde71..17d821e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools>=42.0", "wheel", "oldest-supported-numpy", "cython>=0.28"] +requires = ["setuptools>=42.0", "wheel", "oldest-supported-numpy", "cython>=0.28,<3.0"] build-backend = "setuptools.build_meta" diff --git a/raysect/VERSION b/raysect/VERSION index c18d72be..4978240f 100644 --- a/raysect/VERSION +++ b/raysect/VERSION @@ -1 +1 @@ -0.8.1 \ No newline at end of file +0.8.1.post1 \ No newline at end of file diff --git a/setup.py b/setup.py index ed029aa6..1c4f81f4 100644 --- a/setup.py +++ b/setup.py @@ -104,7 +104,7 @@ "Topic :: Multimedia :: Graphics :: 3D Rendering", "Topic :: Scientific/Engineering :: Physics" ], - install_requires=['numpy>=0.14', 'matplotlib'], + install_requires=['numpy', 'matplotlib'], packages=find_packages(), include_package_data=True, zip_safe=False, From 8afd4e5fbcba81703829f048f594158f46194c8a Mon Sep 17 00:00:00 2001 From: Jack Lovell Date: Mon, 11 Dec 2023 14:34:18 +0000 Subject: [PATCH 2/4] Switch from Travis to Github Actions --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++++ .travis.yml | 18 ------------------ 2 files changed, 28 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..4f7081a2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: CI + +on: + push: + pull_request: + +jobs: + tests: + name: Run tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + numpy-version: ["oldest-supported-numpy", "numpy"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python dependencies + run: python -m pip install --prefer-binary setuptools "cython>=0.28,<3.0" "matplotlib>=3,<4" ${{ matrix.numpy-version }} + - name: Build and install Raysect + run: dev/build.sh + - name: Run tests + run: dev/test.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5f356358..00000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -language: python -jobs: - include: - - python: 3.6 - env: NUMPY_VERSION=1.15.4 - - python: 3.7 - env: NUMPY_VERSION=1.15.4 - - python: 3.8 - env: NUMPY_VERSION=1.15.4 - - python: 3.9 - env: NUMPY_VERSION=1.19.4 - -install: - - pip install numpy==$NUMPY_VERSION scipy matplotlib>=3 cython>=0.28 - - dev/build.sh - -# command to run tests -script: dev/test.sh From d14d85c879f99ddc7b8b20f3fda7c7d3861e8be2 Mon Sep 17 00:00:00 2001 From: Jack Lovell Date: Mon, 11 Dec 2023 15:27:21 +0000 Subject: [PATCH 3/4] Build in parallel by default When installing with pip or PyPA's `build` utility, , the `-j` option for parallel builds is not passed by default. This increases the build time on systems where wheels need to be built, e.g. for releases or on systems where no pre-built wheel is available. This commit introduces a modified `build_ext` command which by default parallelises over all available CPUs, with an optional override using the `RAYSECT_BUILD_JOBS` environment variable if for any reason the number of build processes needs to be controlled manually. --- setup.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1c4f81f4..a02a8973 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ from setuptools import setup, find_packages, Extension +from setuptools.command.build_ext import build_ext as _build_ext import sys import numpy import os @@ -83,6 +84,14 @@ with open(path.join(path.dirname(__file__), 'raysect/VERSION')) as version_file: version = version_file.read().strip() +# Use multiple processes by default for building extensions +class build_ext(_build_ext): + def finalize_options(self): + super().finalize_options() + if self.parallel is None: + nproc = int(os.getenv("RAYSECT_BUILD_JOBS", str(multiprocessing.cpu_count()))) + self.parallel = nproc + setup( name="raysect", version=version, @@ -107,6 +116,7 @@ install_requires=['numpy', 'matplotlib'], packages=find_packages(), include_package_data=True, - zip_safe=False, - ext_modules=extensions + zip_safe= False, + ext_modules=extensions, + cmdclass={"build_ext": build_ext}, ) From 2b377c4ff7876549ec04d073bc852bac2b0233a7 Mon Sep 17 00:00:00 2001 From: Jack Lovell Date: Fri, 15 Dec 2023 13:27:02 +0000 Subject: [PATCH 4/4] Update notes and scripts for building wheels Wheels for Python 3.7 to 3.9 can be built using manylinux2010, as there are binary wheels of oldest-supported-numpy available for these versions. For 3.10 onwards manylinux2014 is required for numpy binary wheels. Building numpy from source in the manylinux containers is not viable. There are now two scripts, one for each manylinux version, which will build wheels for all upstream-supported Python versions. These can be run as-is: info is in the first few lines of these files. Also, the dev notes on building wheels wtih manylinux have been updated to reference these files. --- dev/build_wheels.sh | 34 --------------------- dev/build_wheels_manylinux2010.sh | 25 +++++++++++++++ dev/build_wheels_manylinux2014.sh | 26 ++++++++++++++++ dev/notes/building_bdist_with_manylinux.txt | 23 +++++++++++--- 4 files changed, 70 insertions(+), 38 deletions(-) delete mode 100755 dev/build_wheels.sh create mode 100755 dev/build_wheels_manylinux2010.sh create mode 100755 dev/build_wheels_manylinux2014.sh diff --git a/dev/build_wheels.sh b/dev/build_wheels.sh deleted file mode 100755 index 0be389ef..00000000 --- a/dev/build_wheels.sh +++ /dev/null @@ -1,34 +0,0 @@ -# to be run from within manylinux2010 docker container -# run the command below from the root of the source folder -# sudo docker run -ti -v $(pwd):/io quay.io/pypa/manylinux2010_x86_64 /bin/bash -# in the container run dev/build_wheels.sh - -VERSION=0.8.1 -PLAT=manylinux2010_x86_64 -CORES=32 - -cd /io || exit - -# python 3.7 -/opt/python/cp37-cp37m/bin/python -m pip install cython numpy==1.14.6 -/opt/python/cp37-cp37m/bin/python setup.py build_ext -j$CORES -/opt/python/cp37-cp37m/bin/python setup.py bdist_wheel -auditwheel repair dist/raysect-$VERSION-cp37-cp37m-linux_x86_64.whl --plat $PLAT - -# python 3.8 -/opt/python/cp38-cp38/bin/python -m pip install cython numpy==1.17.5 -/opt/python/cp38-cp38/bin/python setup.py build_ext -j$CORES -/opt/python/cp38-cp38/bin/python setup.py bdist_wheel -auditwheel repair dist/raysect-$VERSION-cp38-cp38-linux_x86_64.whl --plat $PLAT - -# python 3.9 -/opt/python/cp39-cp39/bin/python -m pip install cython numpy==1.19.5 -/opt/python/cp39-cp39/bin/python setup.py build_ext -j$CORES -/opt/python/cp39-cp39/bin/python setup.py bdist_wheel -auditwheel repair dist/raysect-$VERSION-cp39-cp39-linux_x86_64.whl --plat $PLAT - -# python 3.10 -/opt/python/cp310-cp310/bin/python -m pip install cython numpy==1.19.5 -/opt/python/cp310-cp310/bin/python setup.py build_ext -j$CORES -/opt/python/cp310-cp310/bin/python setup.py bdist_wheel -auditwheel repair dist/raysect-$VERSION-cp310-cp310-linux_x86_64.whl --plat $PLAT \ No newline at end of file diff --git a/dev/build_wheels_manylinux2010.sh b/dev/build_wheels_manylinux2010.sh new file mode 100755 index 00000000..6983156d --- /dev/null +++ b/dev/build_wheels_manylinux2010.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# To be run from within manylinux2010 docker container. +# Run the command below from the root of the source folder: +# sudo docker run -ti -v $(pwd):/io quay.io/pypa/manylinux2010_x86_64 ./dev/build_wheels_manylinux2010.sh +# Or, to use singularity instead of docker (e.g. on HPC, or where root not available): +# singularity run -B $(pwd):/io -W /tmp -c docker://quay.io/pypa/manylinux2010_x86_64 /io/dev/build_wheels_manylinux2010.sh + +set -e +cd /io || exit +VERSION=$(cat raysect/VERSION) +PLAT=manylinux2010_x86_64 + +# Numpy provides manylinux2010 wheels only for Python up to 3.9. + +# python 3.7 +/opt/python/cp37-cp37m/bin/python -m build . +auditwheel repair dist/raysect-$VERSION-cp37-cp37m-linux_x86_64.whl --plat $PLAT + +# python 3.8 +/opt/python/cp38-cp38/bin/python -m build . +auditwheel repair dist/raysect-$VERSION-cp38-cp38-linux_x86_64.whl --plat $PLAT + +# python 3.9 +/opt/python/cp39-cp39/bin/python -m build . +auditwheel repair dist/raysect-$VERSION-cp39-cp39-linux_x86_64.whl --plat $PLAT diff --git a/dev/build_wheels_manylinux2014.sh b/dev/build_wheels_manylinux2014.sh new file mode 100755 index 00000000..98e6e1a7 --- /dev/null +++ b/dev/build_wheels_manylinux2014.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# To be run from within manylinux2014 docker container. +# Run the command below from the root of the source folder: +# sudo docker run -ti -v $(pwd):/io quay.io/pypa/manylinux2014_x86_64 ./dev/build_wheels_manylinux2014.sh +# Or, to use singularity instead of docker (e.g. on HPC, or where root not available): +# singularity run -B $(pwd):/io -W /tmp -c docker://quay.io/pypa/manylinux2014_x86_64 /io/dev/build_wheels_manylinux2014.sh + +set -e +cd /io || exit +VERSION=$(cat raysect/VERSION) +PLAT=manylinux2014_x86_64 + +# Numpy provides manylinux2010 wheels for Python up to 3.9. So only need manylinux2014 +# wheels for 3.10 onwards + +# python 3.10 +/opt/python/cp310-cp310/bin/python -m build . +auditwheel repair dist/raysect-$VERSION-cp310-cp310-linux_x86_64.whl --plat $PLAT + +# python 3.11 +/opt/python/cp311-cp311/bin/python -m build . +auditwheel repair dist/raysect-$VERSION-cp311-cp311-linux_x86_64.whl --plat $PLAT + +# python 3.12 +/opt/python/cp312-cp312/bin/python -m build . +auditwheel repair dist/raysect-$VERSION-cp312-cp312-linux_x86_64.whl --plat $PLAT diff --git a/dev/notes/building_bdist_with_manylinux.txt b/dev/notes/building_bdist_with_manylinux.txt index 648dceb1..68b83f4c 100644 --- a/dev/notes/building_bdist_with_manylinux.txt +++ b/dev/notes/building_bdist_with_manylinux.txt @@ -13,14 +13,29 @@ Start the manylinux docker container (this will download the container, mount th This will drop you into the manylinux container terminal. The /opt/python folder in the container holds the various versions of python. This example targets python 3.7. Adjust the paths as appropriate to build bdists for different python versions. cd io/source - /opt/python/cp37-cp37m/bin/python -m pip install cython numpy==1.14.6 - /opt/python/cp37-cp37m/bin/python setup.py bdist_wheel + /opt/python/cp37-cp37m/bin/python -m pip build . auditwheel repair dist/raysect-0.8.1-cp37-cp37m-linux_x86_64.whl --plat manylinux2010_x86_64 This will compile the wheel and repair any library references to produce the manylinux wheel files in a folder ./wheelhouse. e.g. raysect-0.8.1-cp37-cp37m-manylinux1_x86_64.whl and raysect-0.8.1-cp37-cp37m-manylinux2010_x86_64.whl. -These can then be uploaded to pypi (just the manylinux2010 packages for now). +These can then be uploaded to pypi. For more info see: https://realpython.com/python-wheels/#building-a-platform-wheel-macos-and-windows - https://uwekorn.com/2019/09/15/how-we-build-apache-arrows-manylinux-wheels.html \ No newline at end of file + https://uwekorn.com/2019/09/15/how-we-build-apache-arrows-manylinux-wheels.html + +Scripts to semi-automate the wheel building process +=================================================== + +There are two scripts which automate this for different versions of Python: +- dev/build_wheels_manylinux2010.sh builds manylinux2010 wheels for Python 3.7, 3.8 and 3.9 +- dev/build_wheels_manylinux2014.sh builds manylinux2014 wheels for Python 3.10, 3.11 and 3.12 + +These can be used to produce wheels in a semi-automated fashion: +1. Install docker as above +2. Clone the raysect/source repository +3. Change into the top level of the raysect/source repository. +4. Run `sudo docker run -ti -v $(pwd):/io quay.io/pypa/manylinux2010_x86_64 /io/dev/build_wheels_manylinux2010.sh` +5. Run `sudo docker run -ti -v $(pwd):/io quay.io/pypa/manylinux2014_x86_64 /io/dev/build_wheels_manylinux2014.sh` +6. Upload the wheels in wheelhouse/, and the sdist in dist/, to PyPI using twine. +