[ci] Add sccache to ci building process on Linux and Mac jobs (#3559)

* add sccache to cpu * add curl * add curl * change to wget * change to wget * fix * fix * stat * add cache * fix * fix * fix * fix * fix * fix * fix * fix * fix * test * test * add postsubmit * add mac and m1 * debug * debug * debug * debug * debug * debug * fix * fix * test postsubmit * test * fix * test * test * test * fix * Update unix_docker_build.sh * Update presubmit.yml * Update unix_docker_build.sh * Update presubmit.yml * Update unix_docker_build.sh * Update presubmit.yml * Update presubmit.yml * Update presubmit.yml * Update presubmit.yml * Update unix_docker_build.sh * Update unix_docker_build.sh * Update presubmit.yml * Update unix_docker_build.sh * Update unix_docker_build.sh * Update presubmit.yml * Update unix_docker_build.sh * Update presubmit.yml * Update unix_docker_build.sh * Update presubmit.yml * Update postsubmit.yml * Update postsubmit.yml * Update postsubmit.yml * Update postsubmit.yml * Update presubmit.yml * fix typo * test done * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * finish * fix
taichi-dev · Dec 1, 2021 · c2d9e77 · c2d9e77
1 parent 4fe5481
commit c2d9e77
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 12 deletions.
diff --git a/.github/workflows/scripts/unix_build.sh b/.github/workflows/scripts/unix_build.sh
@@ -4,7 +4,26 @@ python3 -m pip uninstall taichi taichi-nightly -y
 python3 -m pip install -r requirements_dev.txt
 python3 -m pip install -r requirements_test.txt
 git fetch origin master
-
+export SCCACHE_DIR=`pwd`/sccache_cache
+export SCCACHE_CACHE_SIZE="128M"
+echo sccache dir: $SCCACHE_DIR
+if [[ $OSTYPE == "linux-"* ]]; then
+    wget https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz
+    tar -xzf sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz
+    chmod +x sccache-v0.2.15-x86_64-unknown-linux-musl/sccache
+    export PATH=$(pwd)/sccache-v0.2.15-x86_64-unknown-linux-musl:$PATH
+elif [[ `uname -m` == "arm64" ]]; then
+    wget https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-aarch64-apple-darwin.tar.gz
+    tar -xzf sccache-v0.2.15-aarch64-apple-darwin.tar.gz
+    chmod +x sccache-v0.2.15-aarch64-apple-darwin/sccache
+    export PATH=$(pwd)/sccache-v0.2.15-aarch64-apple-darwin:$PATH
+else
+    wget https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-apple-darwin.tar.gz
+    tar -xzf sccache-v0.2.15-x86_64-apple-darwin.tar.gz
+    chmod +x sccache-v0.2.15-x86_64-apple-darwin/sccache
+    export PATH=$(pwd)/sccache-v0.2.15-x86_64-apple-darwin:$PATH
+fi
+sccache -s
 PROJECT_TAGS=""
 EXTRA_ARGS=""
 if [ $PROJECT_NAME = "taichi-nightly" ]; then
@@ -17,8 +36,10 @@ fi
 python3 misc/make_changelog.py origin/master ./ True
 python3 setup.py $PROJECT_TAGS bdist_wheel $EXTRA_ARGS
 
+sccache -s
 export NUM_WHL=`ls dist/*.whl | wc -l`
 if [ $NUM_WHL -ne 1 ]; then echo `ERROR: created more than 1 whl.` && exit 1; fi
 python3 -m pip install dist/*.whl
 
+chmod -R 777 $SCCACHE_DIR
 rm -f python/CHANGELOG.md
diff --git a/.github/workflows/scripts/unix_docker_build.sh b/.github/workflows/scripts/unix_docker_build.sh
@@ -7,10 +7,19 @@ PY=$1
 GPU_BUILD=$2
 PROJECT_NAME=$3
 CI_SETUP_CMAKE_ARGS=$4
+export SCCACHE_DIR=/home/dev/taichi/sccache_cache
+export SCCACHE_CACHE_SIZE="128M"
+export SCCACHE_LOG=error
+export SCCACHE_ERROR_LOG=/home/dev/sccache_error
+wget https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz
+tar -xzf sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz
+chmod +x sccache-v0.2.15-x86_64-unknown-linux-musl/sccache
+export PATH=$(pwd)/sccache-v0.2.15-x86_64-unknown-linux-musl:$PATH
+sccache -s
+ls -la /home/dev/taichi/sccache_cache
 
 source /home/dev/miniconda/etc/profile.d/conda.sh
 conda activate $PY
-
 python3 -m pip uninstall taichi taichi-nightly -y
 
 cd taichi
@@ -37,6 +46,8 @@ TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS PROJECT_NAME=$PROJECT_NAME python3 setup.
 
 CUR_DIR=`pwd`
 TI_LIB_DIR=$CUR_DIR/python/taichi/lib ./build/taichi_cpp_tests
-
+cat /home/dev/sccache_error
+sccache -s
+chmod -R 777 $SCCACHE_DIR
 cp dist/*.whl /wheel/
 rm -f python/CHANGELOG.md
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -160,6 +160,14 @@ jobs:
         with:
           submodules: "recursive"
 
+      - name: Get sccache cache
+        uses: actions/cache@v2
+        with:
+          path: sccache_cache
+          key: sccache-linux-${{matrix.with_cc}}-${{ github.sha }}
+          restore-keys: |
+            sccache-linux-${{matrix.with_cc}}-
+
       - name: Get docker images
         run: |
           if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then
@@ -177,14 +185,19 @@ jobs:
             exit 0
           fi
           mkdir -m777 wheel
+          mkdir -p sccache_cache
           docker create -v `pwd`/wheel:/wheel --user dev --name taichi_build ghcr.io/taichi-dev/taichidev-cpu-ubuntu18.04:v0.1.0 /home/dev/taichi/.github/workflows/scripts/unix_docker_build.sh $PY $GPU_BUILD $PROJECT_NAME "$CI_SETUP_CMAKE_ARGS"
-          tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner dev --group dev | docker cp - taichi_build:/home/dev/
+          # A tarball is needed because sccache needs some permissions that only the file owner has. 1000 is the uid and gid of user "dev" in the container.
+          # If the uid or gid of the user inside the docker changes, please change the uid and gid in the following line.
+          tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - taichi_build:/home/dev/
           docker start -a taichi_build
+          rm -rf sccache_cache
+          docker cp taichi_build:/home/dev/taichi/sccache_cache sccache_cache
         env:
           PY: ${{ matrix.python }}
           GPU_BUILD: OFF
           PROJECT_NAME: taichi
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
 
       - name: Test
         run: |
@@ -202,6 +215,11 @@ jobs:
           GPU_TEST: OFF
           TI_WANTED_ARCHS: ${{ matrix.wanted_archs }}
 
+      - name: clean docker container
+        if: always()
+        run: |
+          docker rm taichi_build taichi_test -f
+
   build_and_test_cpu_mac:
     name: Build and Test macos (CPU)
     needs: [check_code_format, check_files]
@@ -224,6 +242,14 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
+      - name: Get sccache cache
+        uses: actions/cache@v2
+        with:
+          path: sccache_cache
+          key: sccache-mac-${{ github.sha }}
+          restore-keys: |
+            sccache-mac-
+
       - name: Download Pre-Built LLVM 10.0.0
         run: |
           if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then
@@ -238,10 +264,11 @@ jobs:
           if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then
             exit 0
           fi
+          mkdir -p sccache_cache
           export PATH=`pwd`/taichi-llvm/bin/:$PATH
           .github/workflows/scripts/unix_build.sh
         env:
-          TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
+          TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
           CXX: clang++
       # [DEBUG] Copy this step around to enable debugging inside Github Action instances.
       #- name: Setup tmate session
@@ -269,20 +296,34 @@ jobs:
         with:
           submodules: "recursive"
 
-      - name: Build & Install
+      - name: Get sccache cache
+        uses: actions/cache@v2
+        with:
+          path: sccache_cache
+          key: sccache-linux-gpu-${{ github.sha }}
+          restore-keys: |
+            sccache-linux-gpu-
+
+      - name: Build & Instal
         run: |
           if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then
             exit 0
           fi
           mkdir -m777 wheel
+          mkdir -p sccache_cache
+          chmod -R 777 sccache_cache
           docker create -v `pwd`/wheel:/wheel --user dev --name taichi_build --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix registry.taichigraphics.com/taichidev-ubuntu18.04:v0.1.1 /home/dev/taichi/.github/workflows/scripts/unix_docker_build.sh $PY $GPU_BUILD $PROJECT_NAME "$CI_SETUP_CMAKE_ARGS"
-          tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner dev --group dev | docker cp - taichi_build:/home/dev/
+          # A tarball is needed because sccache needs some permissions that only the file owner has. 1000 is the uid and gid of user "dev" in the container.
+          # If the uid or gid of the user inside the docker changes, please change the uid and gid in the following line.
+          tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - taichi_build:/home/dev/
           docker start -a taichi_build
+          rm -rf sccache_cache
+          docker cp taichi_build:/home/dev/taichi/sccache_cache sccache_cache
         env:
           PY: py38
           GPU_BUILD: ON
           PROJECT_NAME: taichi
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=ON -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
           DISPLAY: :1
 
       - name: Test
@@ -299,7 +340,7 @@ jobs:
           PY: py38
           GPU_TEST: ON
           DISPLAY: :1
-          TI_WANTED_ARCHS: "cpu,cc,cuda,vulkan,opengl"
+          TI_WANTED_ARCHS: "cpu,cuda,vulkan,opengl"
 
       - name: clean docker container
         if: always()
@@ -368,15 +409,24 @@ jobs:
         with:
           submodules: "recursive"
 
+      - name: Get sccache cache
+        uses: actions/cache@v2
+        with:
+          path: sccache_cache
+          key: sccache-m1-${{ github.sha }}
+          restore-keys: |
+            sccache-m1-
+
       - name: Build
         run: |
           if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then
             exit 0
           fi
           rm -rf $HOME/Library/Python/3.8/lib/python/site-packages/taichi
+          mkdir -p sccache_cache
           .github/workflows/scripts/unix_build.sh
         env:
-          TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
+          TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
           CXX: clang++
 
       - name: Test
@@ -390,6 +440,6 @@ jobs:
           TI_LIB_DIR=`python3 -c "import taichi;print(taichi.__path__[0])" | tail -1`
           TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests
           echo wanted archs: $TI_WANTED_ARCHS
-          python3 tests/run_tests.py -vr2 -t4 -x
+          python3 tests/run_tests.py -vr2 -t4 -x -a "$TI_WANTED_ARCHS"
         env:
           TI_WANTED_ARCHS: "metal,cpu"