From eb98bf311536ebb38fd8545d0992f92802d74a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Primo=C5=BE=20Godec?= Date: Sun, 21 May 2023 14:26:11 +0200 Subject: [PATCH] Build wheels with oldest supported numpy (#3467) * Use oldest-supported-numpy for build * add workflow step to test wheels against older numpy * download installwheel.py * fix installwheel command * patch installwheel.py to take processor into account * patch installwheel.py * patch installwheel.py * add step to debug test environment * work around numpy bug https://github.com/numpy/numpy/issues/23104 * add pip freeze prior to wheel build * git add .github/workflows/pipfreezedammit.py * work around cibuildwheel not outputting pip-freeze * update build-wheels.yml * Revert previous commits This reverts commit 69c33bf7c51a9c5c65942e3ff171d228c7aa2a31. This reverts commit a6cdaa0b109da1a62058ebad2e847a40665c9003. This reverts commit eef4177f753997a5b32e0134d10c01ecc9cc5d37. This reverts commit 8f63e7b57d501d68bfad209bc6b1077b38e7830c. --------- Co-authored-by: Michael Penkov --- .github/workflows/build-wheels.yml | 53 ++++++++++++++++++++++++++++-- .github/workflows/installwheel.py | 35 ++++++++++++++++++++ pyproject.toml | 11 +++++++ setup.py | 4 --- 4 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/installwheel.py create mode 100644 pyproject.toml diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index c20c01aeb8..3b0d1f603e 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -40,12 +40,12 @@ jobs: env: CIBW_ARCHS_LINUX: x86_64 aarch64 CIBW_ARCHS_MACOS: x86_64 arm64 - CIBW_ARCHS_WINDOWS: AMD64 x86 ARM64 - CIBW_BEFORE_BUILD: pip install numpy scipy + CIBW_ARCHS_WINDOWS: AMD64 x86 CIBW_SKIP: pp* cp36-* cp37-* *-win32 *_i686 *-musllinux_* CIBW_TEST_COMMAND: pytest -rfxEXs --durations=20 --disable-warnings --showlocals --pyargs gensim CIBW_TEST_REQUIRES: pytest testfixtures mock CIBW_TEST_SKIP: cp38* cp39* cp310* *_aarch64 *_arm64 *_universal2 + CIBW_BUILD_VERBOSITY: 3 - name: Upload wheels as artifacts if: always() @@ -54,6 +54,55 @@ jobs: name: wheels-${{ matrix.os }} path: wheelhouse/*.whl + test: + name: Test wheel for ${{ matrix.os }} Python ${{ matrix.python }} + needs: build_wheels + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, windows-2019, macos-11] + python: ['3.8', '3.9', '3.10', '3.11'] + + runs-on: ${{ matrix.os }} + steps: + - name: Setup up Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Downloads build artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts/ + + # + # We want to make sure our wheels run against older Numpy versions + # + - name: Install oldest-supported-numpy + run: python -m pip install oldest-supported-numpy + + # + # Avoid checking out the entire gensim repo to get just one file + # + - name: Download installwheel.py + run: curl "https://raw.githubusercontent.com/RaRe-Technologies/gensim/testwheel/.github/workflows/installwheel.py" --output installwheel.py --silent + + - name: Install wheel + run: python installwheel.py artifacts/wheels-${{ matrix.os }} + + - name: Debug test environment + run: | + pip freeze + python -c 'import numpy;print(numpy.__file__)' + python -c 'import numpy;print(numpy.__version__)' + + # + # If the wheel was incorrectly built, then this will fail. + # https://github.com/RaRe-Technologies/gensim/issues/3097 + # + - name: Test wheel + run: python -c 'import gensim' + upload: name: Upload to S3 if: always() diff --git a/.github/workflows/installwheel.py b/.github/workflows/installwheel.py new file mode 100644 index 0000000000..43a6b7c833 --- /dev/null +++ b/.github/workflows/installwheel.py @@ -0,0 +1,35 @@ +"""Install a wheel for the current platform.""" +import os +import platform +import subprocess +import sys + + +def main(): + subdir = sys.argv[1] + vi = sys.version_info + + if platform.system() in ('Linux', 'Darwin'): + arch = 'x86_64' + else: + arch = 'amd64' + + want = f'-cp{vi.major}{vi.minor}-' + suffix = f'_{arch}.whl' + + files = sorted(os.listdir(subdir)) + for f in files: + if want in f and f.endswith(suffix): + command = [sys.executable, '-m', 'pip', 'install', os.path.join(subdir, f)] + subprocess.check_call(command) + return 0 + + print(f'no matches for {want} / {suffix} in {subdir}:') + print('\n'.join(files)) + + return 1 + + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..91ae97c0c7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[build-system] +requires = [ + "Cython>=0.29.32", + # oldest supported Numpy for this platform is 1.17 but the oldest supported by Gensim + # is 1.18.5, remove the line when they increase oldest supported Numpy for this platform + "numpy==1.18.5; python_version=='3.8' and platform_machine not in 'arm64|aarch64'", + "oldest-supported-numpy; python_version>'3.8' or platform_machine in 'arm64|aarch64'", + "scipy", + "setuptools", + "wheel", +] \ No newline at end of file diff --git a/setup.py b/setup.py index 314ee87711..d47dc7e508 100644 --- a/setup.py +++ b/setup.py @@ -346,11 +346,8 @@ def run(self): 'smart_open >= 1.8.1', ] -setup_requires = [NUMPY_STR] - if need_cython(): install_requires.append(CYTHON_STR) - setup_requires.append(CYTHON_STR) setup( name='gensim', @@ -399,7 +396,6 @@ def run(self): test_suite="gensim.test", python_requires='>=3.8', - setup_requires=setup_requires, install_requires=install_requires, tests_require=linux_testenv, extras_require={