Python Build Release Candidate #55
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| name: "Python Build Release Candidate" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: 'Version (e.g, 0.8.0)' | |
| type: string | |
| required: true | |
| rc: | |
| description: 'Release Candidate (RC) (e.g, rc1)' | |
| type: string | |
| required: true | |
| env: | |
| RELEASE_CANDIDATE: "${{ github.event.inputs.version }}${{ github.event.inputs.rc }}" | |
| jobs: | |
| validate-inputs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Validate Input Version and RC | |
| id: validate | |
| run: | | |
| # Validate version (e.g., 1.0.0) | |
| if [[ ! "${{ github.event.inputs.version }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| echo "Error: version must be in the format number.number.number" | |
| exit 1 | |
| fi | |
| # Validate rc (e.g., rc1) | |
| if [[ ! "${{ github.event.inputs.rc }}" =~ ^rc[0-9]+$ ]]; then | |
| echo "Error: rc must be in the format rc<number>" | |
| exit 1 | |
| fi | |
| - name: Echo Release Candidate Version | |
| run: | | |
| echo "Running Release Candidate Version: $RELEASE_CANDIDATE" | |
| validate-library-version: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.12 | |
| - name: Install Poetry | |
| run: | | |
| pip install poetry | |
| - name: Validate current pyiceberg version | |
| run: | | |
| # Extract the current version from Poetry | |
| current_pyiceberg_version=$(poetry version --short) | |
| echo "Detected Poetry version: $current_pyiceberg_version" | |
| # Compare the input version with the Poetry version | |
| input_version="${{ github.event.inputs.version }}" | |
| # Check if the input version matches the Poetry version | |
| if [[ "$input_version" != "$current_pyiceberg_version" ]]; then | |
| echo "Error: Input version ($input_version) does not match the Poetry version ($current_pyiceberg_version)" | |
| exit 1 | |
| fi | |
| # SVN | |
| svn_build_artifacts: | |
| name: Build artifacts for SVN on ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| needs: | |
| - validate-inputs | |
| - validate-library-version | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: | | |
| 3.9 | |
| 3.10 | |
| 3.11 | |
| 3.12 | |
| - name: Install poetry | |
| run: pip install poetry | |
| - name: Cache Poetry | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pypoetry | |
| key: ${{ runner.os }}-poetry-${{ hashFiles('pyproject.toml', 'poetry.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-poetry- | |
| # Publish the source distribution with the version that's in | |
| # the repository, otherwise the tests will fail | |
| - name: Compile source distribution | |
| run: python3 -m poetry build --format=sdist | |
| if: startsWith(matrix.os, 'ubuntu') | |
| - name: Set up cache for cibuildwheel output | |
| uses: actions/cache@v3 | |
| with: | |
| path: wheelhouse | |
| key: ${{ runner.os }}-cibuildwheel-${{ hashFiles('pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cibuildwheel- | |
| - name: Build wheels | |
| uses: pypa/cibuildwheel@v2.22.0 | |
| with: | |
| output-dir: wheelhouse | |
| config-file: "pyproject.toml" | |
| env: | |
| # Ignore 32 bit architectures | |
| CIBW_ARCHS: "auto64" | |
| CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13" | |
| CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1" | |
| CIBW_TEST_EXTRAS: "s3fs,glue" | |
| CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py" | |
| # There is an upstream issue with installing on MacOSX | |
| # https://github.com/pypa/cibuildwheel/issues/1603 | |
| # Ignore tests for pypy since not all dependencies are compiled for it | |
| # and would require a local rust build chain | |
| CIBW_TEST_SKIP: "pp* *macosx*" | |
| - name: Add source distribution | |
| if: startsWith(matrix.os, 'ubuntu') | |
| run: ls -lah dist/* && cp dist/* wheelhouse/ | |
| # TODO: upload to SVN | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: "svn-release-candidate-${{ matrix.os }}" | |
| path: ./wheelhouse/* | |
| svn_generate_checksums: | |
| name: Generate SHA512 checksums | |
| runs-on: ubuntu-latest | |
| needs: svn_build_artifacts | |
| steps: | |
| - name: Download All SVN Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| pattern: svn-release-candidate-* | |
| merge-multiple: true | |
| - name: Generate SHA512 checksums | |
| run: | | |
| cd artifacts | |
| for name in $(ls pyiceberg-*.whl pyiceberg-*.tar.gz) | |
| do | |
| shasum -a 512 "${name}" > "${name}.sha512" | |
| done | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: svn-release-candidate-sha-checksums | |
| path: ./artifacts/*.sha512 | |
| svn_generate_signatures: | |
| name: Generate GPG Signatures | |
| runs-on: ubuntu-latest | |
| needs: svn_build_artifacts | |
| steps: | |
| - name: Download All SVN Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| pattern: svn-release-candidate-* | |
| merge-multiple: true | |
| - name: Import GPG Key | |
| id: import_gpg | |
| uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 | |
| with: | |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | |
| passphrase: ${{ secrets.GPG_PASSPHRASE }} | |
| - name: Generate GPG signatures | |
| run: | | |
| cd artifacts | |
| for name in $(ls pyiceberg-*.whl pyiceberg-*.tar.gz) | |
| do | |
| gpg --yes --armor --output "${name}.asc" --detach-sig "${name}" | |
| done | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: svn-release-candidate-gpg-signatures | |
| path: ./artifacts/*.asc | |
| svn_merge_artifacts: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - svn_build_artifacts | |
| - svn_generate_checksums | |
| - svn_generate_signatures | |
| steps: | |
| - name: Merge Artifacts | |
| uses: actions/upload-artifact/merge@v4 | |
| with: | |
| name: "svn-release-candidate-${{ env.RELEASE_CANDIDATE }}" | |
| pattern: svn-release-candidate* | |
| delete-merged: true | |
| # PyPi | |
| pypi_build_artifacts: | |
| name: Build artifacts for PyPi on ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| needs: | |
| - validate-inputs | |
| - validate-library-version | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: | | |
| 3.9 | |
| 3.10 | |
| 3.11 | |
| 3.12 | |
| - name: Install poetry | |
| run: pip install poetry | |
| - name: Set version with RC | |
| run: python -m poetry version "${{ env.RELEASE_CANDIDATE }}" | |
| - name: Cache Poetry | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pypoetry | |
| key: ${{ runner.os }}-poetry-${{ hashFiles('pyproject.toml', 'poetry.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-poetry- | |
| # Publish the source distribution with the version that's in | |
| # the repository, otherwise the tests will fail | |
| - name: Compile source distribution | |
| run: python3 -m poetry build --format=sdist | |
| if: startsWith(matrix.os, 'ubuntu') | |
| - name: Set up cache for cibuildwheel output | |
| uses: actions/cache@v3 | |
| with: | |
| path: wheelhouse | |
| key: ${{ runner.os }}-cibuildwheel-${{ hashFiles('pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cibuildwheel- | |
| - name: Build wheels | |
| uses: pypa/cibuildwheel@v2.22.0 | |
| with: | |
| output-dir: wheelhouse | |
| config-file: "pyproject.toml" | |
| env: | |
| CIBW_BUILD_FRONTEND: "build" | |
| # Ignore 32 bit architectures | |
| CIBW_ARCHS: "auto64" | |
| CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13" | |
| CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1" | |
| CIBW_TEST_EXTRAS: "s3fs,glue" | |
| CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py" | |
| # There is an upstream issue with installing on MacOSX | |
| # https://github.com/pypa/cibuildwheel/issues/1603 | |
| # Ignore tests for pypy since not all dependencies are compiled for it | |
| # and would require a local rust build chain | |
| CIBW_TEST_SKIP: "pp* *macosx*" | |
| - name: Add source distribution | |
| if: startsWith(matrix.os, 'ubuntu') | |
| run: ls -lah dist/* && cp dist/* wheelhouse/ | |
| # TODO: upload to PyPi | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: "pypi-release-candidate-${{ matrix.os }}" | |
| path: ./wheelhouse/* | |
| pypi_merge_artifacts: | |
| runs-on: ubuntu-latest | |
| needs: pypi_build_artifacts | |
| steps: | |
| - name: Merge Artifacts | |
| uses: actions/upload-artifact/merge@v4 | |
| with: | |
| name: "pypi-release-candidate-${{ env.RELEASE_CANDIDATE }}" | |
| pattern: pypi-release-candidate* | |
| delete-merged: true |