From c2d9e77ce15619e55f0cf1a283f8fab20cf239d0 Mon Sep 17 00:00:00 2001 From: lin-hitonami <90667349+lin-hitonami@users.noreply.github.com> Date: Wed, 1 Dec 2021 19:44:21 +0800 Subject: [PATCH] [ci] Add sccache to ci building process on Linux and Mac jobs (#3559) * add sccache to cpu * add curl * add curl * change to wget * change to wget * fix * fix * stat * add cache * fix * fix * fix * fix * fix * fix * fix * fix * fix * test * test * add postsubmit * add mac and m1 * debug * debug * debug * debug * debug * debug * fix * fix * test postsubmit * test * fix * test * test * test * fix * Update unix_docker_build.sh * Update presubmit.yml * Update unix_docker_build.sh * Update presubmit.yml * Update unix_docker_build.sh * Update presubmit.yml * Update presubmit.yml * Update presubmit.yml * Update presubmit.yml * Update unix_docker_build.sh * Update unix_docker_build.sh * Update presubmit.yml * Update unix_docker_build.sh * Update unix_docker_build.sh * Update presubmit.yml * Update unix_docker_build.sh * Update presubmit.yml * Update unix_docker_build.sh * Update presubmit.yml * Update postsubmit.yml * Update postsubmit.yml * Update postsubmit.yml * Update postsubmit.yml * Update presubmit.yml * fix typo * test done * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * finish * fix --- .github/workflows/scripts/unix_build.sh | 23 ++++++- .../workflows/scripts/unix_docker_build.sh | 15 +++- .github/workflows/testing.yml | 68 ++++++++++++++++--- 3 files changed, 94 insertions(+), 12 deletions(-) diff --git a/.github/workflows/scripts/unix_build.sh b/.github/workflows/scripts/unix_build.sh index 2affd03a891f9..cb340ca899f07 100755 --- a/.github/workflows/scripts/unix_build.sh +++ b/.github/workflows/scripts/unix_build.sh @@ -4,7 +4,26 @@ python3 -m pip uninstall taichi taichi-nightly -y python3 -m pip install -r requirements_dev.txt python3 -m pip install -r requirements_test.txt git fetch origin master - +export SCCACHE_DIR=`pwd`/sccache_cache +export SCCACHE_CACHE_SIZE="128M" +echo sccache dir: $SCCACHE_DIR +if [[ $OSTYPE == "linux-"* ]]; then + wget https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz + tar -xzf sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz + chmod +x sccache-v0.2.15-x86_64-unknown-linux-musl/sccache + export PATH=$(pwd)/sccache-v0.2.15-x86_64-unknown-linux-musl:$PATH +elif [[ `uname -m` == "arm64" ]]; then + wget https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-aarch64-apple-darwin.tar.gz + tar -xzf sccache-v0.2.15-aarch64-apple-darwin.tar.gz + chmod +x sccache-v0.2.15-aarch64-apple-darwin/sccache + export PATH=$(pwd)/sccache-v0.2.15-aarch64-apple-darwin:$PATH +else + wget https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-apple-darwin.tar.gz + tar -xzf sccache-v0.2.15-x86_64-apple-darwin.tar.gz + chmod +x sccache-v0.2.15-x86_64-apple-darwin/sccache + export PATH=$(pwd)/sccache-v0.2.15-x86_64-apple-darwin:$PATH +fi +sccache -s PROJECT_TAGS="" EXTRA_ARGS="" if [ $PROJECT_NAME = "taichi-nightly" ]; then @@ -17,8 +36,10 @@ fi python3 misc/make_changelog.py origin/master ./ True python3 setup.py $PROJECT_TAGS bdist_wheel $EXTRA_ARGS +sccache -s export NUM_WHL=`ls dist/*.whl | wc -l` if [ $NUM_WHL -ne 1 ]; then echo `ERROR: created more than 1 whl.` && exit 1; fi python3 -m pip install dist/*.whl +chmod -R 777 $SCCACHE_DIR rm -f python/CHANGELOG.md diff --git a/.github/workflows/scripts/unix_docker_build.sh b/.github/workflows/scripts/unix_docker_build.sh index 1e3cd9bb6c228..cb28a5221a58b 100755 --- a/.github/workflows/scripts/unix_docker_build.sh +++ b/.github/workflows/scripts/unix_docker_build.sh @@ -7,10 +7,19 @@ PY=$1 GPU_BUILD=$2 PROJECT_NAME=$3 CI_SETUP_CMAKE_ARGS=$4 +export SCCACHE_DIR=/home/dev/taichi/sccache_cache +export SCCACHE_CACHE_SIZE="128M" +export SCCACHE_LOG=error +export SCCACHE_ERROR_LOG=/home/dev/sccache_error +wget https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz +tar -xzf sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz +chmod +x sccache-v0.2.15-x86_64-unknown-linux-musl/sccache +export PATH=$(pwd)/sccache-v0.2.15-x86_64-unknown-linux-musl:$PATH +sccache -s +ls -la /home/dev/taichi/sccache_cache source /home/dev/miniconda/etc/profile.d/conda.sh conda activate $PY - python3 -m pip uninstall taichi taichi-nightly -y cd taichi @@ -37,6 +46,8 @@ TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS PROJECT_NAME=$PROJECT_NAME python3 setup. CUR_DIR=`pwd` TI_LIB_DIR=$CUR_DIR/python/taichi/lib ./build/taichi_cpp_tests - +cat /home/dev/sccache_error +sccache -s +chmod -R 777 $SCCACHE_DIR cp dist/*.whl /wheel/ rm -f python/CHANGELOG.md diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 12e08dfb0baad..2817ab5e79da9 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -160,6 +160,14 @@ jobs: with: submodules: "recursive" + - name: Get sccache cache + uses: actions/cache@v2 + with: + path: sccache_cache + key: sccache-linux-${{matrix.with_cc}}-${{ github.sha }} + restore-keys: | + sccache-linux-${{matrix.with_cc}}- + - name: Get docker images run: | if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then @@ -177,14 +185,19 @@ jobs: exit 0 fi mkdir -m777 wheel + mkdir -p sccache_cache docker create -v `pwd`/wheel:/wheel --user dev --name taichi_build ghcr.io/taichi-dev/taichidev-cpu-ubuntu18.04:v0.1.0 /home/dev/taichi/.github/workflows/scripts/unix_docker_build.sh $PY $GPU_BUILD $PROJECT_NAME "$CI_SETUP_CMAKE_ARGS" - tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner dev --group dev | docker cp - taichi_build:/home/dev/ + # A tarball is needed because sccache needs some permissions that only the file owner has. 1000 is the uid and gid of user "dev" in the container. + # If the uid or gid of the user inside the docker changes, please change the uid and gid in the following line. + tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - taichi_build:/home/dev/ docker start -a taichi_build + rm -rf sccache_cache + docker cp taichi_build:/home/dev/taichi/sccache_cache sccache_cache env: PY: ${{ matrix.python }} GPU_BUILD: OFF PROJECT_NAME: taichi - CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON + CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - name: Test run: | @@ -202,6 +215,11 @@ jobs: GPU_TEST: OFF TI_WANTED_ARCHS: ${{ matrix.wanted_archs }} + - name: clean docker container + if: always() + run: | + docker rm taichi_build taichi_test -f + build_and_test_cpu_mac: name: Build and Test macos (CPU) needs: [check_code_format, check_files] @@ -224,6 +242,14 @@ jobs: with: python-version: ${{ matrix.python }} + - name: Get sccache cache + uses: actions/cache@v2 + with: + path: sccache_cache + key: sccache-mac-${{ github.sha }} + restore-keys: | + sccache-mac- + - name: Download Pre-Built LLVM 10.0.0 run: | if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then @@ -238,10 +264,11 @@ jobs: if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then exit 0 fi + mkdir -p sccache_cache export PATH=`pwd`/taichi-llvm/bin/:$PATH .github/workflows/scripts/unix_build.sh env: - TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }} + TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache CXX: clang++ # [DEBUG] Copy this step around to enable debugging inside Github Action instances. #- name: Setup tmate session @@ -269,20 +296,34 @@ jobs: with: submodules: "recursive" - - name: Build & Install + - name: Get sccache cache + uses: actions/cache@v2 + with: + path: sccache_cache + key: sccache-linux-gpu-${{ github.sha }} + restore-keys: | + sccache-linux-gpu- + + - name: Build & Instal run: | if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then exit 0 fi mkdir -m777 wheel + mkdir -p sccache_cache + chmod -R 777 sccache_cache docker create -v `pwd`/wheel:/wheel --user dev --name taichi_build --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix registry.taichigraphics.com/taichidev-ubuntu18.04:v0.1.1 /home/dev/taichi/.github/workflows/scripts/unix_docker_build.sh $PY $GPU_BUILD $PROJECT_NAME "$CI_SETUP_CMAKE_ARGS" - tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner dev --group dev | docker cp - taichi_build:/home/dev/ + # A tarball is needed because sccache needs some permissions that only the file owner has. 1000 is the uid and gid of user "dev" in the container. + # If the uid or gid of the user inside the docker changes, please change the uid and gid in the following line. + tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - taichi_build:/home/dev/ docker start -a taichi_build + rm -rf sccache_cache + docker cp taichi_build:/home/dev/taichi/sccache_cache sccache_cache env: PY: py38 GPU_BUILD: ON PROJECT_NAME: taichi - CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=ON -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON + CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache DISPLAY: :1 - name: Test @@ -299,7 +340,7 @@ jobs: PY: py38 GPU_TEST: ON DISPLAY: :1 - TI_WANTED_ARCHS: "cpu,cc,cuda,vulkan,opengl" + TI_WANTED_ARCHS: "cpu,cuda,vulkan,opengl" - name: clean docker container if: always() @@ -368,15 +409,24 @@ jobs: with: submodules: "recursive" + - name: Get sccache cache + uses: actions/cache@v2 + with: + path: sccache_cache + key: sccache-m1-${{ github.sha }} + restore-keys: | + sccache-m1- + - name: Build run: | if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then exit 0 fi rm -rf $HOME/Library/Python/3.8/lib/python/site-packages/taichi + mkdir -p sccache_cache .github/workflows/scripts/unix_build.sh env: - TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON + TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache CXX: clang++ - name: Test @@ -390,6 +440,6 @@ jobs: TI_LIB_DIR=`python3 -c "import taichi;print(taichi.__path__[0])" | tail -1` TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests echo wanted archs: $TI_WANTED_ARCHS - python3 tests/run_tests.py -vr2 -t4 -x + python3 tests/run_tests.py -vr2 -t4 -x -a "$TI_WANTED_ARCHS" env: TI_WANTED_ARCHS: "metal,cpu"