Publishing Release #424
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publishing Release | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| # Manually trigger the release workflow, a version must be provided | |
| inputs: | |
| version: | |
| description: 'The version to release (e.g. v0.8.0), or leave it empty for nightly build' | |
| type: string | |
| required: false | |
| env: | |
| PROD_PWD: ${{ secrets.PYPI_PWD_PROD }} | |
| NIGHT_PWD: ${{ secrets.TAICHI_PYPI_PWD }} | |
| NIGHT_USERNAME: ${{ secrets.TAICHI_PYPI_USERNAME }} | |
| METADATA_USERNAME: ${{ secrets.METADATA_USERNAME }} | |
| METADATA_PASSWORD: ${{ secrets.METADATA_PASSWORD }} | |
| METADATA_URL: ${{ secrets.METADATA_URL }} | |
| RELEASE_VERSION: ${{ github.event.inputs.version }} | |
| TI_CI: 1 | |
| jobs: | |
| add_version_to_database: | |
| name: Add version to database | |
| # Skip running release workflow on forks | |
| if: github.repository_owner == 'taichi-dev' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Save new version | |
| run: | | |
| if [ -z "$RELEASE_VERSION" ]; then | |
| echo "Not a production release" | |
| exit 0 | |
| fi | |
| python3 -m pip install requests==2.26 | |
| python3 misc/save_new_version.py | |
| # This job set environment matrix with respect to production release and nightly release. | |
| matrix_prep: | |
| runs-on: ubuntu-latest | |
| needs: add_version_to_database | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| matrix_osx: ${{ steps.set-matrix.outputs.matrix_osx }} | |
| steps: | |
| - id: set-matrix | |
| run: | | |
| if [ -n "$RELEASE_VERSION" ]; then | |
| # For production release, we run on five python versions. | |
| echo '::set-output name=matrix::{"include":[{"name":"taichi","python":"3.6","conda_python":"py36"},{"name":"taichi","python":"3.7","conda_python":"py37"},{"name":"taichi","python":"3.8","conda_python":"py38"},{"name":"taichi","python":"3.9","conda_python":"py39"},{"name":"taichi","python":"3.10","conda_python":"py310"}]}"' | |
| # M1 only supports py38, py39, and py310(conda), so change matrix. | |
| echo '::set-output name=matrix_osx::{"include":[{"name":"taichi","python":"3.8"},{"name":"taichi","python":"3.9"},{"name":"taichi","python":"3.10"}]}"' | |
| else | |
| # For nightly release, we run on three python versions. | |
| echo '::set-output name=matrix::{"include":[{"name":"taichi-nightly","python":"3.6","conda_python":"py36"},{"name":"taichi-nightly","python":"3.7","conda_python":"py37"},{"name":"taichi-nightly","python":"3.8","conda_python":"py38"},{"name":"taichi-nightly","python":"3.9","conda_python":"py39"},{"name":"taichi-nightly","python":"3.10","conda_python":"py310"}]}"' | |
| # M1 only supports py38 and py310(conda), so change matrix. | |
| echo '::set-output name=matrix_osx::{"include":[{"name":"taichi-nightly","python":"3.8"},{"name":"taichi-nightly","python":"3.9"},{"name":"taichi-nightly","python":"3.10"}]}"' | |
| fi | |
| build_and_test_linux: | |
| name: Build and Upload (linux only) | |
| needs: matrix_prep | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }} | |
| runs-on: [self-hosted, cuda, vulkan, cn, release] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: 'recursive' | |
| - name: Get sccache cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: sccache_cache | |
| key: sccache-linux-gpu-${{ github.sha }} | |
| restore-keys: | | |
| sccache-linux-gpu- | |
| - name: Build | |
| run: | | |
| mkdir -m777 shared | |
| docker create --user dev --name taichi_build --gpus all -v /tmp/.X11-unix:/tmp/.X11-unix \ | |
| -e DISPLAY -e PY -e GPU_BUILD -e TAICHI_CMAKE_ARGS -e PROJECT_NAME \ | |
| registry.taichigraphics.com/taichidev-ubuntu18.04:v0.3.0 \ | |
| /home/dev/taichi/.github/workflows/scripts/unix_build.sh | |
| tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - taichi_build:/home/dev/ | |
| docker start -a taichi_build | |
| docker cp taichi_build:/home/dev/taichi/dist shared/dist | |
| docker cp taichi_build:/home/dev/taichi/build shared/build | |
| env: | |
| PY: ${{ matrix.conda_python }} | |
| GPU_BUILD: ON | |
| TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache | |
| PROJECT_NAME: ${{ matrix.name }} | |
| DISPLAY: ':1' | |
| - name: Archive Wheel Artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist | |
| path: shared/dist/*.whl | |
| retention-days: 20 | |
| - name: Test | |
| run: | | |
| docker create --user dev --name taichi_test --gpus all -v /tmp/.X11-unix:/tmp/.X11-unix \ | |
| -e DISPLAY -e PY -e GPU_TEST registry.taichigraphics.com/taichidev-ubuntu18.04:v0.3.0 \ | |
| /home/dev/unix_test.sh | |
| docker cp .github/workflows/scripts/unix_test.sh taichi_test:/home/dev/unix_test.sh | |
| docker cp ./requirements_test.txt taichi_test:/home/dev/requirements_test.txt | |
| docker cp shared/dist/ taichi_test:/home/dev/ | |
| docker cp shared/build/ taichi_test:/home/dev/ | |
| docker cp pyproject.toml taichi_test:/home/dev/ | |
| docker cp tests/ taichi_test:/home/dev/ | |
| docker start -a taichi_test | |
| env: | |
| PY: ${{ matrix.conda_python }} | |
| GPU_TEST: ON | |
| DISPLAY: ':1' | |
| - name: clean docker container | |
| if: always() | |
| run: | | |
| docker rm taichi_build taichi_test -f | |
| build_and_test_manylinux2014: | |
| name: Build and Upload (manylinux2014) | |
| needs: matrix_prep | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }} | |
| runs-on: [self-hosted, cuda, cn, release] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: "recursive" | |
| - name: Get sccache cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: sccache_cache | |
| key: sccache-manylinux-${{ github.sha }} | |
| restore-keys: | | |
| sccache-manylinux- | |
| - name: Build | |
| run: | | |
| mkdir -m777 shared | |
| docker create --user dev --name taichi_build --gpus all \ | |
| -e PY -e GPU_BUILD -e TAICHI_CMAKE_ARGS -e PROJECT_NAME \ | |
| registry.taichigraphics.com/taichidev-manylinux2014-cuda:v0.0.0 \ | |
| /home/dev/taichi/.github/workflows/scripts/unix_build.sh | |
| tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - taichi_build:/home/dev/ | |
| docker start -a taichi_build | |
| docker cp taichi_build:/home/dev/taichi/dist shared/dist | |
| docker cp taichi_build:/home/dev/taichi/build shared/build | |
| env: | |
| PY: ${{ matrix.conda_python }} | |
| GPU_BUILD: ON | |
| TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache | |
| PROJECT_NAME: ${{ matrix.name }} | |
| - name: Archive Wheel Artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist | |
| path: shared/dist/*.whl | |
| retention-days: 20 | |
| - name: Test | |
| run: | | |
| docker create --user dev --name taichi_test --gpus all -e TI_WANTED_ARCHS \ | |
| -e PY -e GPU_TEST registry.taichigraphics.com/taichidev-manylinux2014-cuda:v0.0.0 \ | |
| /home/dev/unix_test.sh | |
| docker cp .github/workflows/scripts/unix_test.sh taichi_test:/home/dev/unix_test.sh | |
| docker cp ./requirements_test.txt taichi_test:/home/dev/requirements_test.txt | |
| docker cp shared/dist/ taichi_test:/home/dev/ | |
| docker cp shared/build/ taichi_test:/home/dev/ | |
| docker cp pyproject.toml taichi_test:/home/dev/ | |
| docker cp tests/ taichi_test:/home/dev/ | |
| docker start -a taichi_test | |
| env: | |
| PY: ${{ matrix.conda_python }} | |
| GPU_TEST: ON | |
| TI_WANTED_ARCHS: "cuda,cpu" | |
| - name: clean docker container | |
| if: always() | |
| run: | | |
| docker rm taichi_build taichi_test -f | |
| build_and_test_mac: | |
| name: Build and Upload (macOS only) | |
| needs: matrix_prep | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }} | |
| runs-on: macos-10.15 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: 'recursive' | |
| - name: Get sccache cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: sccache_cache | |
| key: sccache-mac-${{ github.sha }} | |
| restore-keys: | | |
| sccache-mac- | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Download Pre-Built LLVM 10.0.0 | |
| run: python misc/ci_download.py | |
| env: | |
| CI_PLATFORM: macos-10.15 | |
| - name: Create Python Wheel | |
| run: | | |
| brew install molten-vk | |
| export PATH=$(pwd)/taichi-llvm/bin/:$PATH | |
| bash .github/workflows/scripts/unix_build.sh | |
| brew uninstall molten-vk | |
| env: | |
| TAICHI_CMAKE_ARGS: -DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache | |
| PROJECT_NAME: ${{ matrix.name }} | |
| CXX: clang++ | |
| - name: Archive Wheel Artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist | |
| path: dist/*.whl | |
| retention-days: 20 | |
| - name: Test | |
| run: .github/workflows/scripts/unix_test.sh | |
| env: | |
| TI_WANTED_ARCHS: 'cpu' | |
| build_and_test_m1: | |
| name: Build and Upload (Apple M1) | |
| needs: matrix_prep | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix_osx) }} | |
| runs-on: [self-hosted, m1] | |
| defaults: | |
| run: | |
| shell: '/usr/bin/arch -arch arm64e /bin/bash --noprofile --norc -eo pipefail {0}' | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: 'recursive' | |
| - name: Get sccache cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: sccache_cache | |
| key: sccache-m1-${{ github.sha }} | |
| restore-keys: | | |
| sccache-m1- | |
| - name: Build | |
| run: | | |
| brew install molten-vk | |
| # We hacked here because conda activate in CI won't update python PATH | |
| # automatically. So we don't activate and use desired python version | |
| # directly. | |
| export PATH=/Users/github/miniforge3/envs/$PYTHON/bin:$PATH | |
| bash .github/workflows/scripts/unix_build.sh | |
| brew uninstall molten-vk | |
| env: | |
| TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache | |
| PROJECT_NAME: ${{ matrix.name }} | |
| PYTHON: ${{ matrix.python }} | |
| CXX: clang++ | |
| - name: Archive Wheel Artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist | |
| path: dist/*.whl | |
| retention-days: 20 | |
| - name: Test | |
| run: | | |
| export PATH=/Users/github/miniforge3/envs/$PYTHON/bin:$PATH | |
| .github/workflows/scripts/unix_test.sh | |
| env: | |
| TI_WANTED_ARCHS: 'metal,vulkan,cpu' | |
| PYTHON: ${{ matrix.python }} | |
| GPU_TEST: ON | |
| build_and_test_macos_1014: | |
| name: Build and Upload (macos 1014) | |
| needs: matrix_prep | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }} | |
| runs-on: [self-hosted, macos10.14] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: 'recursive' | |
| - name: Get sccache cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: sccache_cache | |
| key: sccache-1014-${{ github.sha }} | |
| restore-keys: | | |
| sccache-1014- | |
| - name: Build | |
| run: | | |
| # We hacked here because conda activate in CI won't update python PATH | |
| # automatically. So we don't activate and use desired python version | |
| # directly. | |
| export PATH=/Users/buildbot6/miniconda3/envs/$PYTHON/bin:$PATH | |
| export LLVM_DIR=/Users/buildbot6/taichi-llvm-10.0.0-macos | |
| export PATH=$LLVM_DIR/bin:$PATH | |
| bash .github/workflows/scripts/unix_build.sh | |
| env: | |
| TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache | |
| PROJECT_NAME: ${{ matrix.name }} | |
| PYTHON: ${{ matrix.python }} | |
| CXX: clang++ | |
| - name: Archive Wheel Artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist | |
| path: dist/*.whl | |
| retention-days: 20 | |
| - name: Test | |
| run: | | |
| export PATH=/Users/buildbot6/miniconda3/envs/$PYTHON/bin:$PATH | |
| .github/workflows/scripts/unix_test.sh | |
| env: | |
| TI_WANTED_ARCHS: 'cpu' | |
| PYTHON: ${{ matrix.python }} | |
| build_and_test_windows: | |
| name: Build and Upload (Windows only) | |
| needs: matrix_prep | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }} | |
| runs-on: windows-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: 'recursive' | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Add Visual Studio Shell to ENV | |
| uses: egor-tensin/vs-shell@v2 | |
| with: | |
| arch: x64 | |
| - name: Get sccache cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: ccache_cache | |
| key: ccache-win64-clang-${{ github.sha }} | |
| restore-keys: | | |
| ccache-win64-clang- | |
| - name: Build Python Wheel | |
| shell: powershell | |
| run: | | |
| .\.github\workflows\scripts\win_build.ps1 -installVulkan -libsDir C:\ | |
| venv\Scripts\python -m pip install $(dir dist\*.whl) | |
| env: | |
| PROJECT_NAME: ${{ matrix.name }} | |
| - name: Archive Wheel Artifacts | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: dist | |
| path: dist/*.whl | |
| retention-days: 20 | |
| - name: Test | |
| shell: powershell | |
| run: | | |
| $env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH | |
| . venv\Scripts\activate.ps1 | |
| python -c "import taichi" | |
| pip install torch | |
| pip install -r requirements_test.txt | |
| ti diagnose | |
| python tests/run_tests.py -vr2 -t2 | |
| env: | |
| TI_SKIP_VERSION_CHECK: ON | |
| upload_to_pypi: | |
| name: Upload release to PyPI | |
| needs: [build_and_test_linux, build_and_test_manylinux2014, build_and_test_mac, build_and_test_m1, build_and_test_macos_1014, build_and_test_windows] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - name: Get dist files | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: dist | |
| path: dist | |
| - name: Upload to PyPI | |
| run: | | |
| ls -l dist/ | |
| if [ -z "$RELEASE_VERSION" ]; then | |
| export PROJECT_NAME="taichi-nightly" | |
| else | |
| export PROJECT_NAME="taichi" | |
| fi | |
| python -m pip install requests twine | |
| python misc/upload_release.py | |
| create_release: | |
| name: Create tag and publish release | |
| needs: upload_to_pypi | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event.inputs.version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - name: Generate Changelog | |
| id: changelog | |
| run: | | |
| pip3 install gitpython | |
| content=$(python3 misc/make_changelog.py) | |
| echo $content | |
| # Escape multiline strings: | |
| # https://renehernandez.io/snippets/multiline-strings-as-a-job-output-in-github-actions/ | |
| content="${content//'%'/'%25'}" | |
| content="${content//$'\n'/'%0A'}" | |
| content="${content//$'\r'/'%0D'}" | |
| echo "::set-output name=content::$content" | |
| - name: Create tag | |
| run: | | |
| git config user.email "taichigardener@gmail.com" | |
| git config user.name "Taichi Gardener" | |
| git tag -a ${RELEASE_VERSION} -m "Release ${RELEASE_VERSION}" | |
| git push origin --tags | |
| - name: Publish release | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| body: ${{ steps.changelog.outputs.content }} | |
| tag_name: ${{ github.event.inputs.version }} | |
| - name: Bump version | |
| run: | | |
| version_parts=(${RELEASE_VERSION//./ }) | |
| version_parts[2]=$(expr ${version_parts[2]} + 1) | |
| next_version=$(IFS=.; echo "${version_parts[*]}") | |
| # Update version.txt | |
| git checkout -b "bump/$next_version" | |
| echo "$next_version" > version.txt | |
| git add version.txt | |
| # Commit and push changes | |
| git commit -m "Bump version to $next_version" | |
| git push origin "bump/$next_version" | |
| # Create pull request | |
| gh pr create -B master -t "[misc] Bump version to $next_version" -b "Bump version to $next_version" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GARDENER_PAT }} |