From bdb8fe573e42950eaea18405950a66ce7b23d7a4 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 20 Jan 2023 07:26:36 +0100 Subject: [PATCH 1/7] Maximize wheel build parallellization --- .github/workflows/wheel.yml | 106 ++++++++++++++++++++---------------- packages/vaex-core/setup.py | 2 + 2 files changed, 61 insertions(+), 47 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 20436927eb..fee851602f 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -7,84 +7,96 @@ on: pull_request: jobs: + build_wheels_matrix: + runs-on: ubuntu-latest + outputs: + include: ${{ steps.set-matrix.outputs.include }} + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - run: pip install cibuildwheel==2.12.0 + + - id: set-matrix + run: | + MATRIX_INCLUDE=$( + { + cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64 | grep cp | jq -Rc '{"only": ., "os": "ubuntu-latest"}' \ + && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,arm64 | grep cp | jq -Rc '{"only": ., "os": "macos-latest"}' \ + && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | grep cp | jq -Rc '{"only": ., "os": "windows-latest"}' + } | jq -sc + ) + echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT + working-directory: packages/vaex-core/ + build_wheels: - name: Build wheel on ${{ matrix.os }} + needs: build_wheels_matrix runs-on: ${{ matrix.os }} + name: Build ${{ matrix.only }} + strategy: - fail-fast: false matrix: - os: [ubuntu-18.04, macOS-latest, windows-latest] - python-version: [36, 37, 38, 39, 310] + include: ${{ fromJson(needs.build_wheels_matrix.outputs.include) }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: submodules: true - name: Copy dll - if: (matrix.os == 'windows-latest') + if: matrix.os == 'windows-latest' uses: ./ci/actions/windll - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.7' - - - name: Install cibuildwheel - run: | - python -m pip install cibuildwheel==2.8.1 - - - name: Install twine/wheel - run: | - python -m pip install twine wheel - - name: chores if: matrix.os != 'windows-latest' run: | - mkdir dist mkdir packages/vaex-core/bin cp bin/install_pcre.sh packages/vaex-core/bin/ - - name: Build vaex-core (windows) - if: (matrix.os == 'windows-latest') - env: - CIBW_BUILD: cp${{ matrix.python-version }}*64 - CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'" - run: | - python -m cibuildwheel --output-dir dist packages/vaex-core + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 - - name: Build vaex-core (linux, osx) - if: (matrix.os != 'windows-latest') + - uses: pypa/cibuildwheel@v2.12.0 + with: + only: ${{ matrix.only }} + package-dir: packages/vaex-core/ + output-dir: packages/vaex-core/dist/ env: - CIBW_BEFORE_BUILD: bin/install_pcre.sh - CIBW_BUILD: "cp${{ matrix.python-version }}*64 cp${{ matrix.python-version }}-macosx-*" - CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'" + CIBW_BEFORE_BUILD: ${{ matrix.os == 'windows-latest' && '' || 'bin/install_pcre.sh' }} CIBW_BUILD_VERBOSITY: 2 - # needed for m1 builds + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'" CIBW_ENVIRONMENT_MACOS: "CFLAGS='-I/usr/local/include -L/usr/local/lib' CXXFLAGS='-I/usr/local/include -L/usr/local/lib' LDFLAGS='-L/usr/local/lib'" - CIBW_ARCHS_MACOS: "x86_64 arm64" - run: | - python -m cibuildwheel --output-dir dist packages/vaex-core - - name: Make source distribution - shell: bash - run: | - cd packages/vaex-core && python setup.py sdist + - uses: actions/setup-python@v4 + with: + python-version: "3.x" - - name: Copy build artifacts - shell: bash - run: | - cp packages/vaex-core/dist/* dist/ + - name: Make source distribution + run: python setup.py sdist + working-directory: packages/vaex-core/ - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v3 with: name: distributions - path: ./dist + path: packages/vaex-core/dist - name: Publish a Python distribution to PyPI + if: github.event_name == 'push' env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.pypi_password_vaex_core }} run: | + pip install twine wheel openssl sha256 dist/* - twine upload --skip-existing dist/vaex?core* + twine upload --skip-existing + working-directory: packages/vaex-core/ diff --git a/packages/vaex-core/setup.py b/packages/vaex-core/setup.py index b5c98c18ca..f590ee90b9 100644 --- a/packages/vaex-core/setup.py +++ b/packages/vaex-core/setup.py @@ -183,12 +183,14 @@ def __str__(self): include_package_data=True, ext_modules=([extension_vaexfast] if on_rtd else [extension_vaexfast, extension_strings, extension_superutils, extension_superagg]) if not use_skbuild else [], zip_safe=False, + python_requires=">=3.6", classifiers=[ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], extras_require={ 'all': ["gcsfs>=0.6.2", "s3fs"] From 682fbaf7b83292c5b63be4e15742545f9eb31ec2 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 20 Jan 2023 08:11:47 +0100 Subject: [PATCH 2/7] git submodule update --remote --merge packages/vaex-core/vendor/pybind11 --- packages/vaex-core/vendor/pybind11 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vaex-core/vendor/pybind11 b/packages/vaex-core/vendor/pybind11 index 25abf7efba..a500f439d0 160000 --- a/packages/vaex-core/vendor/pybind11 +++ b/packages/vaex-core/vendor/pybind11 @@ -1 +1 @@ -Subproject commit 25abf7efba0b2990f5a6dfb0a31bc65c0f2f4d17 +Subproject commit a500f439d06d220ee2c680cdd2c8828eac8e7dfc From 7e8f5ab5a23461268746855fba64d0cf8afe12c4 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 20 Jan 2023 08:17:15 +0100 Subject: [PATCH 3/7] Fix gcc error --- packages/vaex-core/src/superstring.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/vaex-core/src/superstring.hpp b/packages/vaex-core/src/superstring.hpp index b62d492bc2..924be203bc 100644 --- a/packages/vaex-core/src/superstring.hpp +++ b/packages/vaex-core/src/superstring.hpp @@ -3,6 +3,7 @@ #include #include +#include typedef nonstd::string_view string_view; typedef std::string string; #include From 3a7dd5e44004aa338602d111b27a756e14309197 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 20 Jan 2023 08:26:10 +0100 Subject: [PATCH 4/7] Fix workflow syntax --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index fee851602f..3d6ab3c5a5 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -70,7 +70,7 @@ jobs: package-dir: packages/vaex-core/ output-dir: packages/vaex-core/dist/ env: - CIBW_BEFORE_BUILD: ${{ matrix.os == 'windows-latest' && '' || 'bin/install_pcre.sh' }} + CIBW_BEFORE_BUILD: ${{ ( matrix.os != 'windows-latest' && 'bin/install_pcre.sh' ) || '' }} CIBW_BUILD_VERBOSITY: 2 CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 From 37ece87c778ef248183f1d4e19fa5d80726262e8 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 20 Jan 2023 08:44:18 +0100 Subject: [PATCH 5/7] Remove redefinition https://github.com/pybind/pybind11/blob/769fd3b889fef6cddb060f2a0be26aee62b4da05/include/pybind11/pytypes.h#L859 https://github.com/ddelange/vaex/actions/runs/3965609112/jobs/6795506653#step:6:2110 --- packages/vaex-core/src/hash_object.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/vaex-core/src/hash_object.cpp b/packages/vaex-core/src/hash_object.cpp index dfd99c8db4..a86287cc3a 100644 --- a/packages/vaex-core/src/hash_object.cpp +++ b/packages/vaex-core/src/hash_object.cpp @@ -13,15 +13,6 @@ typedef SSIZE_T ssize_t; namespace py = pybind11; -namespace std { - template<> - struct hash { - size_t operator()(const PyObject *const &o) const { - return PyObject_Hash((PyObject*)o); - } - }; -} - namespace vaex { struct CompareObjects From 5438dc1594885e6c11553e1592d62e59ea38410a Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 20 Jan 2023 08:50:35 +0100 Subject: [PATCH 6/7] Disable win32 https://github.com/ddelange/vaex/actions/runs/3965689146/jobs/6795667118#step:6:538 --- .github/workflows/wheel.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 3d6ab3c5a5..17e48dc090 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -27,9 +27,9 @@ jobs: run: | MATRIX_INCLUDE=$( { - cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64 | grep cp | jq -Rc '{"only": ., "os": "ubuntu-latest"}' \ - && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,arm64 | grep cp | jq -Rc '{"only": ., "os": "macos-latest"}' \ - && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | grep cp | jq -Rc '{"only": ., "os": "windows-latest"}' + cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64 | grep cp | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ + && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,arm64 | grep cp | jq -nRc '{"only": inputs, "os": "macos-latest"}' \ + && cibuildwheel --print-build-identifiers --platform windows --arch AMD64 | grep cp | jq -nRc '{"only": inputs, "os": "windows-latest"}' } | jq -sc ) echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT From beff33a426a9b853347b07bb4924d77523b6cc9e Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Fri, 20 Jan 2023 09:15:43 +0100 Subject: [PATCH 7/7] Remove testing leftovers --- .github/workflows/wheel.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 17e48dc090..6e6ef8c063 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -4,7 +4,6 @@ on: push: tags: - core-* - pull_request: jobs: build_wheels_matrix: @@ -91,7 +90,6 @@ jobs: path: packages/vaex-core/dist - name: Publish a Python distribution to PyPI - if: github.event_name == 'push' env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.pypi_password_vaex_core }}