From 83ac6c3cf8a2d9408b70f21f071d242d4c0951a6 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 20 Jan 2023 07:26:36 +0100 Subject: [PATCH] Maximize wheel build parallellization --- .github/workflows/wheel.yml | 104 ++++++++++++++++++++---------------- packages/vaex-core/setup.py | 2 + 2 files changed, 59 insertions(+), 47 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 20436927eb..8fcf06793f 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -7,84 +7,94 @@ on: pull_request: jobs: + build_wheels_matrix: + runs-on: ubuntu-18.04 + outputs: + include: ${{ steps.set-matrix.outputs.include }} + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - run: pip install cibuildwheel==2.12.0 + + - id: set-matrix + run: | + MATRIX_INCLUDE=$( + { + cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64 | grep cp | jq -Rc '{"only": inputs, "os": "ubuntu-latest"}' \ + && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,arm64 | grep cp | jq -Rc '{"only": inputs, "os": "macos-latest"}' \ + && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | grep cp | jq -Rc '{"only": inputs, "os": "windows-latest"}' + } | jq -sc + ) + echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT + working-directory: packages/vaex-core/ + build_wheels: - name: Build wheel on ${{ matrix.os }} + needs: build_wheels_matrix runs-on: ${{ matrix.os }} + name: Build ${{ matrix.only }} + strategy: - fail-fast: false matrix: - os: [ubuntu-18.04, macOS-latest, windows-latest] - python-version: [36, 37, 38, 39, 310] + include: ${{ fromJson(needs.build_wheels_matrix.outputs.include) }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: submodules: true - name: Copy dll - if: (matrix.os == 'windows-latest') + if: matrix.os == 'windows-latest' uses: ./ci/actions/windll - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.7' - - - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel==2.8.1 - - - name: Install twine/wheel - run: | - python -m pip install twine wheel - - name: chores if: matrix.os != 'windows-latest' run: | - mkdir dist mkdir packages/vaex-core/bin cp bin/install_pcre.sh packages/vaex-core/bin/ - - name: Build vaex-core (windows) - if: (matrix.os == 'windows-latest') - env: - CIBW_BUILD: cp${{ matrix.python-version }}*64 - CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'" - run: | - python -m cibuildwheel --output-dir dist packages/vaex-core + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 - - name: Build vaex-core (linux, osx) - if: (matrix.os != 'windows-latest') + - uses: pypa/cibuildwheel@v2.12.0 + with: + only: ${{ matrix.only }} + package-dir: packages/vaex-core/ + output-dir: packages/vaex-core/dist/ env: - CIBW_BEFORE_BUILD: bin/install_pcre.sh - CIBW_BUILD: "cp${{ matrix.python-version }}*64 cp${{ matrix.python-version }}-macosx-*" - CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'" + CIBW_BEFORE_BUILD: ${{ matrix.os == 'windows-latest' && '' || 'bin/install_pcre.sh' }} CIBW_BUILD_VERBOSITY: 2 - # needed for m1 builds + CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'" CIBW_ENVIRONMENT_MACOS: "CFLAGS='-I/usr/local/include -L/usr/local/lib' CXXFLAGS='-I/usr/local/include -L/usr/local/lib' LDFLAGS='-L/usr/local/lib'" - CIBW_ARCHS_MACOS: "x86_64 arm64" - run: | - python -m cibuildwheel --output-dir dist packages/vaex-core - - name: Make source distribution - shell: bash - run: | - cd packages/vaex-core && python setup.py sdist + - uses: actions/setup-python@v4 + with: + python-version: "3.x" - - name: Copy build artifacts - shell: bash - run: | - cp packages/vaex-core/dist/* dist/ + - name: Make source distribution + run: python setup.py sdist + working-directory: packages/vaex-core/ - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 with: name: distributions - path: ./dist + path: packages/vaex-core/dist - name: Publish a Python distribution to PyPI + if: github.event_name == 'push' env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.pypi_password_vaex_core }} run: | + pip install twine wheel openssl sha256 dist/* - twine upload --skip-existing dist/vaex?core* + twine upload --skip-existing + working-directory: packages/vaex-core/ diff --git a/packages/vaex-core/setup.py b/packages/vaex-core/setup.py index b5c98c18ca..f590ee90b9 100644 --- a/packages/vaex-core/setup.py +++ b/packages/vaex-core/setup.py @@ -183,12 +183,14 @@ def __str__(self): include_package_data=True, ext_modules=([extension_vaexfast] if on_rtd else [extension_vaexfast, extension_strings, extension_superutils, extension_superagg]) if not use_skbuild else [], zip_safe=False, + python_requires=">=3.6", classifiers=[ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], extras_require={ 'all': ["gcsfs>=0.6.2", "s3fs"]