From 0f63f353d1dc663516b2c46a8ec9756bbc741d65 Mon Sep 17 00:00:00 2001 From: Nicolas Karolak Date: Mon, 6 Mar 2023 08:19:23 +0100 Subject: [PATCH] ci: enable arm64/aarch64 wheel builds (#3448) * ci: move linters into reusable workflow Move lint steps into its own workflow that can be called from other workflows (`on.workflow_dispatch`). * ci: enable builds for macos-arm64 and linux-aarch64 * ci: replace multibuild by qemu+cibuildwheel * ci: configure dependabot for github-actions This allow dependabot to create PR to update thrid-party GitHub Actions. * ci: run wheel test only on cp311 and native archs --- .github/dependabot.yml | 6 + .github/workflows/build-wheels.yml | 351 ++++------------------------- .github/workflows/linters.yml | 32 +++ .github/workflows/tests.yml | 25 +- .gitmodules | 3 - multibuild | 1 - 6 files changed, 81 insertions(+), 337 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/linters.yml delete mode 100644 .gitmodules delete mode 160000 multibuild diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..8ac6b8c498 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index dab4957b2f..31544d6b26 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -9,328 +9,59 @@ on: - cron: '0 0 * * sun,wed' jobs: - # - # The linters job duplicates tests.yml, can't think of a way to avoid this right now. - # - linters: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Setup up Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Update pip - run: python -m pip install -U pip - - - name: Install dependencies - run: python -m pip install flake8 flake8-rst - - - name: Run flake8 linter (source) - run: flake8 --ignore E12,W503 --max-line-length 120 --show-source gensim - # - name: Run flake8 linter (documentation) - # run: flake8 --ignore E202,E402,E302,E305,F821 --max-line-length 120 --filename '*.py,*.rst' docs - - - name: Check Sphinx Gallery cache - run: python docs/src/check_gallery.py + linters: + uses: ./.github/workflows/linters.yml - multibuild: - timeout-minutes: 35 + build_wheels: + needs: linters + name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} defaults: run: shell: bash - - needs: [linters] - - strategy: - fail-fast: false - matrix: - include: - # - # We want the _oldest_ possible manylinux version to ensure our - # wheels work on the widest possible range of distros. Version 1 - # seems to break for certain Python versions under Linux and Windows, - # so we use 2010, which is the next oldest. - # - # When selecting the numpy version to build against, we need to satisfy - # two conditions. First, we want the wheel to be available for the - # version of Python we're building against, because building numpy - # wheels on our own is too much work. - # - # Second, in order to guarantee compatibility with the greatest range - # of numpy versions, we want to build against the oldest possible numpy - # version, as long as it's 1.17.0 or newer. Building versions earlier - # than 1.17.0 yields wheels that are incompatible with some newer - # versions of numpy. See https://github.com/RaRe-Technologies/gensim/issues/3226 - # for details. - # - # The logic for numpy version selection is based on - # https://github.com/scipy/oldest-supported-numpy/blob/master/setup.cfg - # with the exception that we enforce the minimum version to be 1.17.0. - # - - os: ubuntu-latest - manylinux-version: 2010 - python-version: "3.8" - build-depends: numpy==1.17.3 - - - os: ubuntu-latest - manylinux-version: 2010 - python-version: "3.9" - build-depends: numpy==1.19.3 - - - os: ubuntu-latest - manylinux-version: 2014 - python-version: "3.10" - build-depends: numpy==1.22.2 scipy==1.8.0 - - - os: ubuntu-latest - manylinux-version: 2014 - python-version: "3.11" - build-depends: numpy==1.23.2 scipy==1.9.2 - - - os: macos-latest - travis-os-name: osx - manylinux-version: 1 - python-version: "3.8" - build-depends: numpy==1.17.3 - - - os: macos-latest - travis-os-name: osx - manylinux-version: 1 - python-version: "3.9" - build-depends: numpy==1.19.3 - - - os: macos-latest - travis-os-name: osx - manylinux-version: 1 - python-version: "3.10" - build-depends: numpy==1.22.2 scipy==1.8.0 - - - os: macos-latest - travis-os-name: osx - manylinux-version: 1 - python-version: "3.11" - build-depends: numpy==1.23.2 scipy==1.9.2 - - env: - SKIP_NETWORK_TESTS: 1 - TEST_DEPENDS: pytest mock testfixtures - BUILD_DEPENDS: ${{ matrix.build-depends }} - - # - # For multibuild - # - BUILD_COMMIT: HEAD - DOCKER_TEST_IMAGE: multibuild/xenial_x86_64 - MB_ML_VER: ${{ matrix.manylinux-version }} - MB_PYTHON_VERSION: ${{ matrix.python-version }} # MB_PYTHON_VERSION is needed by Multibuild - PKG_NAME: gensim - PLAT: x86_64 - REPO_DIR: gensim - TRAVIS_OS_NAME: ${{ matrix.travis-os-name }} - UNICODE_WIDTH: 32 - - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - fetch-depth: 0 - - name: Print environment variables - run: | - echo "PLAT: ${PLAT}" - echo "MB_ML_VER: ${MB_ML_VER}" - echo "DOCKER_TEST_IMAGE: ${DOCKER_TEST_IMAGE}" - echo "TEST_DEPENDS: ${TEST_DEPENDS}" - echo "TRAVIS_OS_NAME: ${TRAVIS_OS_NAME}" - echo "SKIP_NETWORK_TESTS: ${SKIP_NETWORK_TESTS}" - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install virtualenv - - name: Build Wheel - run: | - echo ::group::Set up Multibuild - source multibuild/common_utils.sh - source multibuild/travis_steps.sh - source config.sh - echo ::endgroup:: - echo ::group::Before install - before_install - echo ::endgroup:: - echo ::group::Build wheel - find . -type f -name "*.egg" -exec rm -v {} \; - build_wheel $REPO_DIR ${{ matrix.PLAT }} - echo ::endgroup:: - - - name: Prepare for testing - run: | - # - # FIXME: Why are these eggs here? - # - # These eggs prevent the wheel from building and running on Py3.10 - # - find . -type f -name "*.egg" -exec rm -v {} \; - python -m venv test_environment - - # - # Multibuild has a test step but it essentially just installs the wheel - # and runs the test, and requires a lot of magic to get it working. - # It also does not work under Windows. - # So, we create our own simple test step here. - # - - name: Install and Test Wheel - run: | - . test_environment/bin/activate - python -m pip install --upgrade pip - pip install pytest testfixtures mock - pip install wheelhouse/*.whl - cd test_environment - python -c 'import gensim;print(gensim.__version__)' - # - # This part relies on the wheel containing tests and required data. - # If we remove that from the wheel, we'll need to rewrite this step. - # - pytest -rfxEXs --durations=20 --disable-warnings --showlocals --pyargs gensim - - - name: Upload wheels to s3://gensim-wheels - # - # Only do this if the credentials are set. - # This means that PRs will still build wheels, but not upload them. - # (PRs do not have access to secrets). - # - # The always() ensures this step runs even if a previous step fails. - # We want to upload wheels whenever possible (even if e.g. tests failed) - # because we don't want an innocuous test failure from blocking a release. - # - if: ${{ always() && env.WHEELHOUSE_UPLOADER_USERNAME && env.WHEELHOUSE_UPLOADER_SECRET }} - run: | - python -m pip install wheelhouse-uploader - ls wheelhouse/*.whl - python -m wheelhouse_uploader upload --local-folder wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn - env: - WHEELHOUSE_UPLOADER_USERNAME: ${{ secrets.AWS_ACCESS_KEY_ID }} - WHEELHOUSE_UPLOADER_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - # - # The build process for windows is different to that of Linux and MacOS. - # First, we cannot use multibuild (it does not support Windows). - # This means we have to write our own building and testing steps, but in a - # way it's simpler, because we don't need to care about configuring - # multibuild ourselves. - # Second, the syntax to enable virtual environments, etc. is different. - # - build_windows: - timeout-minutes: 35 - runs-on: windows-latest - defaults: - run: - shell: bash - - needs: [linters] - strategy: fail-fast: false matrix: - include: - - python-version: "3.8" - build-depends: numpy==1.17.3 - - - python-version: "3.9" - build-depends: numpy==1.19.3 - - - python-version: "3.10" - build-depends: numpy==1.22.2 scipy==1.8.0 - - - python-version: "3.11" - build-depends: numpy==1.23.2 scipy==1.9.2 - - env: - SKIP_NETWORK_TESTS: 1 - TEST_DEPENDS: pytest mock testfixtures - BUILD_DEPENDS: ${{ matrix.build-depends }} - + os: [ubuntu-20.04, windows-2019, macos-11] steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install virtualenv - - name: Build Wheel - run: | - echo ::group::Set up dependencies - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" - python -m pip install -U pip setuptools wheel wheelhouse_uploader ${{ env.BUILD_DEPENDS }} - echo ::endgroup:: - echo ::group::Build wheel - python setup.py bdist_wheel - echo ::endgroup - echo ::group::Install run - ls dist - python continuous_integration/install_wheel.py - echo ::endgroup:: - # - # For consistency with the multibuild step. The wheel uploader expects - # the wheels to be under wheelhouse. - # - mv dist wheelhouse + - name: Checkout + uses: actions/checkout@v3 - - name: Prepare for testing - run: | + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 + with: + platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.12.0 + env: + CIBW_ARCHS_LINUX: x86_64 aarch64 + CIBW_ARCHS_MACOS: x86_64 arm64 + CIBW_ARCHS_WINDOWS: AMD64 x86 ARM64 + CIBW_BEFORE_BUILD: pip install numpy scipy + CIBW_SKIP: pp* cp36-* cp37-* *-win32 *_i686 *-musllinux_* + CIBW_TEST_COMMAND: pytest -rfxEXs --durations=20 --disable-warnings --showlocals --pyargs gensim + CIBW_TEST_REQUIRES: pytest testfixtures mock + CIBW_TEST_SKIP: cp38* cp39* cp310* *_aarch64 *_arm64 *_universal2 + + - name: Upload wheels to s3://gensim-wheels # - # FIXME: Why are these eggs here? + # Only do this if the credentials are set. + # This means that PRs will still build wheels, but not upload them. + # (PRs do not have access to secrets). # - # These eggs prevent the wheel from building and running on Py3.10 + # The always() ensures this step runs even if a previous step fails. + # We want to upload wheels whenever possible (even if e.g. tests failed) + # because we don't want an innocuous test failure from blocking a release. # - find . -type f -name "*.egg" -exec rm -v {} \; - python -m venv test_environment - - # - # We need a separate testing step for windows because the command for - # activating the virtual environment is slightly different - # - - name: Install and Test Wheel (Windows) - run: | - test_environment/Scripts/activate.bat - python -m pip install --upgrade pip - pip install pytest testfixtures mock - pip install wheelhouse/*.whl - cd test_environment - python -c 'import gensim;print(gensim.__version__)' - pytest -rfxEXs --durations=20 --disable-warnings --showlocals --pyargs gensim - - - name: Upload wheels to s3://gensim-wheels - # - # Only do this if the credentials are set. - # This means that PRs will still build wheels, but not upload them. - # (PRs do not have access to secrets). - # - # The always() ensures this step runs even if a previous step fails. - # We want to upload wheels whenever possible (even if e.g. tests failed) - # because we don't want an innocuous test failure from blocking a release. - # - if: ${{ always() && env.WHEELHOUSE_UPLOADER_USERNAME && env.WHEELHOUSE_UPLOADER_SECRET }} - run: | - python -m pip install wheelhouse-uploader - ls wheelhouse/*.whl - python -m wheelhouse_uploader upload --local-folder wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn - env: - WHEELHOUSE_UPLOADER_USERNAME: ${{ secrets.AWS_ACCESS_KEY_ID }} - WHEELHOUSE_UPLOADER_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + if: ${{ always() && env.WHEELHOUSE_UPLOADER_USERNAME && env.WHEELHOUSE_UPLOADER_SECRET }} + run: | + python -m pip install wheelhouse-uploader + ls wheelhouse/*.whl + python -m wheelhouse_uploader upload --local-folder wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn + env: + WHEELHOUSE_UPLOADER_USERNAME: ${{ secrets.AWS_ACCESS_KEY_ID }} + WHEELHOUSE_UPLOADER_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml new file mode 100644 index 0000000000..05693fd252 --- /dev/null +++ b/.github/workflows/linters.yml @@ -0,0 +1,32 @@ +name: Linters + +on: + workflow_call: + +jobs: + + linters: + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v3 + + - name: Setup up Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Update pip + run: python -m pip install -U pip + + - name: Install dependencies + run: python -m pip install flake8 flake8-rst + + - name: Run flake8 linter (source) + run: flake8 --ignore E12,W503 --max-line-length 120 --show-source gensim + + # - name: Run flake8 linter (documentation) + # run: flake8 --ignore E202,E402,E302,E305,F821 --max-line-length 120 --filename '*.py,*.rst' docs + + - name: Check Sphinx Gallery cache + run: python docs/src/check_gallery.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9d52759538..4a1d90f808 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,30 +6,9 @@ on: branches: [ develop ] jobs: - linters: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Setup up Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - name: Update pip - run: python -m pip install -U pip - - - name: Install dependencies - run: python -m pip install flake8 flake8-rst - - - name: Run flake8 linter (source) - run: flake8 --ignore E12,W503 --max-line-length 120 --show-source gensim - - # - name: Run flake8 linter (documentation) - # run: flake8 --ignore E202,E402,E302,E305,F821 --max-line-length 120 --filename '*.py,*.rst' docs - - - name: Check Sphinx Gallery cache - run: python docs/src/check_gallery.py + linters: + uses: ./.github/workflows/linters.yml docs: name: build documentation diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 52a1b1716c..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "multibuild"] - path = multibuild - url = https://github.com/multi-build/multibuild diff --git a/multibuild b/multibuild deleted file mode 160000 index 8f9229a442..0000000000 --- a/multibuild +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8f9229a442dcaad809ecde96e23f872e56d6ad60