Build wheels with oldest supported numpy (#3467)

* Use oldest-supported-numpy for build

* add workflow step to test wheels against older numpy

* download installwheel.py

* fix installwheel command

* patch installwheel.py to take processor into account

* patch installwheel.py

* patch installwheel.py

* add step to debug test environment

* work around numpy bug

numpy/numpy#23104

* add pip freeze prior to wheel build

* git add .github/workflows/pipfreezedammit.py

* work around cibuildwheel not outputting pip-freeze

* update build-wheels.yml

* Revert previous commits

This reverts commit 69c33bf.
This reverts commit a6cdaa0.
This reverts commit eef4177.
This reverts commit 8f63e7b.

---------

Co-authored-by: Michael Penkov <m@penkov.dev>

.github/workflows/build-wheels.yml

@@ -40,12 +40,12 @@ jobs:
         env:
           CIBW_ARCHS_LINUX: x86_64 aarch64
           CIBW_ARCHS_MACOS: x86_64 arm64
-          CIBW_ARCHS_WINDOWS: AMD64 x86 ARM64
-          CIBW_BEFORE_BUILD: pip install numpy scipy
+          CIBW_ARCHS_WINDOWS: AMD64 x86
           CIBW_SKIP: pp* cp36-* cp37-* *-win32 *_i686 *-musllinux_*
           CIBW_TEST_COMMAND: pytest -rfxEXs --durations=20 --disable-warnings --showlocals --pyargs gensim
           CIBW_TEST_REQUIRES: pytest testfixtures mock
           CIBW_TEST_SKIP: cp38* cp39* cp310* *_aarch64 *_arm64 *_universal2
+          CIBW_BUILD_VERBOSITY: 3
 
       - name: Upload wheels as artifacts
         if: always()
@@ -54,6 +54,55 @@ jobs:
           name: wheels-${{ matrix.os }}
           path: wheelhouse/*.whl
 
+  test:
+    name: Test wheel for ${{ matrix.os }} Python ${{ matrix.python }}
+    needs: build_wheels
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, windows-2019, macos-11]
+        python: ['3.8', '3.9', '3.10', '3.11']
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Setup up Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Downloads build artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts/
+
+      #
+      # We want to make sure our wheels run against older Numpy versions
+      #
+      - name: Install oldest-supported-numpy
+        run: python -m pip install oldest-supported-numpy
+
+      #
+      # Avoid checking out the entire gensim repo to get just one file
+      #
+      - name: Download installwheel.py
+        run: curl "https://raw.githubusercontent.com/RaRe-Technologies/gensim/testwheel/.github/workflows/installwheel.py" --output installwheel.py --silent
+
+      - name: Install wheel
+        run: python installwheel.py artifacts/wheels-${{ matrix.os }}
+
+      - name: Debug test environment
+        run: |
+          pip freeze
+          python -c 'import numpy;print(numpy.__file__)'
+          python -c 'import numpy;print(numpy.__version__)'
+
+      #
+      # If the wheel was incorrectly built, then this will fail.
+      # https://github.com/RaRe-Technologies/gensim/issues/3097
+      #
+      - name: Test wheel
+        run: python -c 'import gensim'
+
   upload:
     name: Upload to S3
     if: always()

.github/workflows/installwheel.py

@@ -0,0 +1,35 @@
+"""Install a wheel for the current platform."""
+import os
+import platform
+import subprocess
+import sys
+
+
+def main():
+    subdir = sys.argv[1]
+    vi = sys.version_info
+
+    if platform.system() in ('Linux', 'Darwin'):
+        arch = 'x86_64'
+    else:
+        arch = 'amd64'
+
+    want = f'-cp{vi.major}{vi.minor}-'
+    suffix = f'_{arch}.whl'
+
+    files = sorted(os.listdir(subdir))
+    for f in files:
+        if want in f and f.endswith(suffix):
+            command = [sys.executable, '-m', 'pip', 'install', os.path.join(subdir, f)]
+            subprocess.check_call(command)
+            return 0
+
+    print(f'no matches for {want} / {suffix} in {subdir}:')
+    print('\n'.join(files))
+
+    return 1
+
+
+
+if __name__ == '__main__':
+    sys.exit(main())

pyproject.toml

@@ -0,0 +1,11 @@
+[build-system]
+requires = [
+	"Cython>=0.29.32",
+	# oldest supported Numpy for this platform is 1.17 but the oldest supported by Gensim
+	# is 1.18.5, remove the line when they increase oldest supported Numpy for this platform
+	"numpy==1.18.5; python_version=='3.8' and platform_machine not in 'arm64|aarch64'",
+	"oldest-supported-numpy; python_version>'3.8' or platform_machine in 'arm64|aarch64'", 
+	"scipy",
+	"setuptools", 
+	"wheel", 
+]

setup.py

@@ -346,11 +346,8 @@ def run(self):
     'smart_open >= 1.8.1',
 ]
 
-setup_requires = [NUMPY_STR]
-
 if need_cython():
     install_requires.append(CYTHON_STR)
-    setup_requires.append(CYTHON_STR)
 
 setup(
     name='gensim',
@@ -399,7 +396,6 @@ def run(self):
 
     test_suite="gensim.test",
     python_requires='>=3.8',
-    setup_requires=setup_requires,
     install_requires=install_requires,
     tests_require=linux_testenv,
     extras_require={