Skip to content

Fix workflow runners on macos #241

Fix workflow runners on macos

Fix workflow runners on macos #241

name: Python cibuildwheel
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
workflow_dispatch:
jobs:
build_wheels:
name: cibuildwheel ${{ matrix.os }}/${{ matrix.arch }}/${{ matrix.flavor }}/${{ matrix.target }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-22.04, macos-12]
# separate archs, so they use individual caches
arch: [ 'x86_64', 'arm64' ]
flavor: ['cpython', 'pypy']
# separate musl and many on linux, for mac we just skip one of those
target: [ 'many', 'musl' ]
exclude:
- os: macos-12
target: musl
- os: ubuntu-22.04
target: musl
flavor: pypy
- os: macos-12
arch: arm64
flavor: pypy
steps:
- uses: actions/checkout@v4
- name: Set up QEMU
if: ${{ (runner.os == 'Linux') && (matrix.arch == 'arm64') }}
uses: docker/setup-qemu-action@v3
with:
platforms: all
- name: ccache
uses: hendrikmuhs/ccache-action@v1.2.11
with:
key: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.target }}-${{ matrix.flavor }}-python
- name: Sets env for x86_64
run: |
echo "CIBW_ARCHS_LINUX=auto64" >> $GITHUB_ENV
echo "CIBW_ARCHS_MACOS=x86_64" >> $GITHUB_ENV
if: matrix.arch == 'x86_64'
- name: Sets env for arm64
run: |
echo "CIBW_ARCHS_LINUX=aarch64" >> $GITHUB_ENV
echo "CIBW_ARCHS_MACOS=arm64" >> $GITHUB_ENV
if: matrix.arch == 'arm64'
- name: Skip manylinux for musllinux target
if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }}
run: |
echo "CIBW_SKIP=*manylinux*" >> $GITHUB_ENV
- name: Skip musllinux for manylinux target
if: ${{ (runner.os == 'Linux') && (matrix.target == 'many') }}
run: |
echo "CIBW_SKIP=*musllinux*" >> $GITHUB_ENV
- name: Skip pypy for cpython
if: ${{ matrix.flavor == 'cpython' }}
run: |
echo "CIBW_SKIP=${{ env.CIBW_SKIP }} pp*" >> $GITHUB_ENV
- name: Skip cpython for pypy
if: ${{ matrix.flavor == 'pypy' }}
run: |
echo "CIBW_SKIP=${{ env.CIBW_SKIP }} cp*" >> $GITHUB_ENV
- name: install mac dependencies
if: ${{ runner.os == 'macOS' }}
# 2nd command: workaround https://github.com/actions/setup-python/issues/577
run: |
brew update && \
brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \
brew install ccache
- name: install mac dependencies X86_64
if: ${{ (runner.os == 'macOS') && (matrix.arch == 'x86_64') }}
run: |
brew update && \
brew install zlib snappy boost
- name: install mac dependencies arm64
if: ${{ (runner.os == 'macOS') && (matrix.arch == 'arm64') }}
run: |
set -e
echo "MACOSX_DEPLOYMENT_TARGET=12.3.0" >> $GITHUB_ENV
echo "_CMAKE_PREFIX_PATH=${{ github.workspace }}/arm64-homebrew" >> $GITHUB_ENV
echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=DYLD_LIBRARY_PATH=${{ github.workspace }}/arm64-homebrew delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" >> $GITHUB_ENV
mkdir arm64-homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm64-homebrew
PACKAGES=( icu4c xz lz4 zstd zlib snappy boost )
for PACKAGE in "${PACKAGES[@]}"
do
response=$(arm64-homebrew/bin/brew fetch --force --bottle-tag=arm64_monterey $PACKAGE | grep Downloaded )
download_path=$(echo $response | xargs -n 1 | tail -1)
arm64-homebrew/bin/brew reinstall -vd $download_path
done
arm64-homebrew/bin/brew config
ls /Users/runner/work/keyvi/keyvi/arm64-homebrew
- name: Build python wheels for ${{ matrix.os }} on ${{ matrix.arch }}
uses: pypa/cibuildwheel@v2.16.5
env:
# Skip CPython 3.6 and CPython 3.7
CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-*
# skip testing all python versions on linux arm, only test 3.12
# skip tests on pypy, currently fails for indexer tests
CIBW_TEST_SKIP: "*p{38,39,310,311}-m*linux_aarch64 pp*"
# (many)linux custom docker images
CIBW_MANYLINUX_X86_64_IMAGE: 'keyvidev/manylinux-builder-x86_64'
CIBW_MANYLINUX_AARCH64_IMAGE: 'keyvidev/manylinux-builder-aarch64'
CIBW_MUSLLINUX_X86_64_IMAGE: 'keyvidev/musllinux-builder-x86_64'
CIBW_MUSLLINUX_AARCH64_IMAGE: 'keyvidev/musllinux-builder-aarch64'
# ccache using path
CIBW_ENVIRONMENT_MACOS: PATH=/usr/local/opt/ccache/libexec:$PATH
CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf
# python dependencies
CIBW_BEFORE_BUILD: pip install -r python/requirements.txt
# testing
CIBW_TEST_REQUIRES: pytest
CIBW_TEST_COMMAND: >
python -m pytest {package}/tests &&
python -m pytest {package}/integration-tests
# for debugging set this to 1,2 or 3
# CIBW_BUILD_VERBOSITY: 2
with:
package-dir: python
- uses: actions/upload-artifact@v4
with:
name: artifact-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.flavor }}-${{ matrix.target }}
path: ./wheelhouse/*.whl
build_sdist:
name: sdist
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: install Linux deps
run: |
sudo apt-get update && \
sudo apt-get install -y libsnappy-dev libzzip-dev zlib1g-dev libboost-all-dev ccache
- name: ccache
uses: hendrikmuhs/ccache-action@v1.2.11
with:
key: ${{ matrix.os }}-sdist-python
- name: Build SDist
run: |
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
cd python && \
python -m pip install -r requirements.txt && \
python setup.py sdist -d wheelhouse && \
python -m pip uninstall -y autowrap && \
python -m pip install wheelhouse/*.tar.gz -v && \
python -m pytest tests && \
python -m pip uninstall -y keyvi
- uses: actions/upload-artifact@v4
with:
name: artifact-sdist
path: python/wheelhouse/*.tar.gz
upload_all:
needs: [build_wheels, build_sdist]
runs-on: ubuntu-latest
if: github.event_name == 'release' && github.event.action == 'published'
steps:
- uses: actions/download-artifact@v4
with:
pattern: artifact-*
merge-multiple: true
path: dist
- uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.pypi_password }}