From 34994a9ebc1d34be79031c2b5e857f9ad24f268e Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 15 Jun 2020 15:12:43 +0200 Subject: [PATCH] ci: allow gating gha on everything but macOS In our GitHub Actions setup macOS is too unreliable to gate on it, but the other builders work fine. This commit splits the macOS builders into a separate job (called auto-fallible), allowing us to gate on the auto job without failing due to macOS spurious failures. --- .github/workflows/ci.yml | 155 ++++++++++++++++++++++++++++------- src/ci/github-actions/ci.yml | 81 ++++++++++-------- 2 files changed, 171 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ffc8a7ad5241..a38761d166567 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -363,35 +363,6 @@ jobs: env: DEPLOY_TOOLSTATES_JSON: toolstates-linux.json os: ubuntu-latest-xl - - name: dist-x86_64-apple - env: - SCRIPT: "./x.py dist" - RUST_CONFIGURE_ARGS: "--target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - os: macos-latest - - name: dist-x86_64-apple-alt - env: - SCRIPT: "./x.py dist" - RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - os: macos-latest - - name: x86_64-apple - env: - SCRIPT: "./x.py test" - RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - os: macos-latest - name: x86_64-msvc-1 env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler" @@ -571,6 +542,132 @@ jobs: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" + auto-fallible: + name: auto-fallible + env: + CI_JOB_NAME: "${{ matrix.name }}" + SCCACHE_BUCKET: rust-lang-gha-caches + DEPLOY_BUCKET: rust-lang-gha + TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate" + TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/pietroalbini/rust-toolstate/issues" + TOOLSTATE_PUBLISH: 1 + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF + CACHE_DOMAIN: ci-caches-gha.rust-lang.org + if: "github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" + strategy: + matrix: + include: + - name: dist-x86_64-apple + env: + SCRIPT: "./x.py dist" + RUST_CONFIGURE_ARGS: "--target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc" + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + os: macos-latest + - name: dist-x86_64-apple-alt + env: + SCRIPT: "./x.py dist" + RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc" + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + os: macos-latest + - name: x86_64-apple + env: + SCRIPT: "./x.py test" + RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc" + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + os: macos-latest + timeout-minutes: 600 + runs-on: "${{ matrix.os }}" + steps: + - name: disable git crlf conversion + run: git config --global core.autocrlf false + shell: bash + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: configure GitHub Actions to kill the build when outdated + uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master + with: + github_token: "${{ secrets.github_token }}" + if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" + - name: add extra environment variables + run: src/ci/scripts/setup-environment.sh + env: + EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" + if: success() && !env.SKIP_JOB + - name: decide whether to skip this job + run: src/ci/scripts/should-skip-this.sh + if: success() && !env.SKIP_JOB + - name: collect CPU statistics + run: src/ci/scripts/collect-cpu-stats.sh + if: success() && !env.SKIP_JOB + - name: show the current environment + run: src/ci/scripts/dump-environment.sh + if: success() && !env.SKIP_JOB + - name: install awscli + run: src/ci/scripts/install-awscli.sh + if: success() && !env.SKIP_JOB + - name: install sccache + run: src/ci/scripts/install-sccache.sh + if: success() && !env.SKIP_JOB + - name: install clang + run: src/ci/scripts/install-clang.sh + if: success() && !env.SKIP_JOB + - name: install WIX + run: src/ci/scripts/install-wix.sh + if: success() && !env.SKIP_JOB + - name: ensure the build happens on a partition with enough space + run: src/ci/scripts/symlink-build-dir.sh + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + if: success() && !env.SKIP_JOB + - name: install MSYS2 + run: src/ci/scripts/install-msys2.sh + if: success() && !env.SKIP_JOB + - name: install MinGW + run: src/ci/scripts/install-mingw.sh + if: success() && !env.SKIP_JOB + - name: install ninja + run: src/ci/scripts/install-ninja.sh + if: success() && !env.SKIP_JOB + - name: enable ipv6 on Docker + run: src/ci/scripts/enable-docker-ipv6.sh + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + if: success() && !env.SKIP_JOB + - name: checkout submodules + run: src/ci/scripts/checkout-submodules.sh + if: success() && !env.SKIP_JOB + - name: ensure line endings are correct + run: src/ci/scripts/verify-line-endings.sh + if: success() && !env.SKIP_JOB + - name: run the build + run: src/ci/scripts/run-build-from-ci.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" + TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" + if: success() && !env.SKIP_JOB + - name: upload artifacts to S3 + run: src/ci/scripts/upload-artifacts.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" + if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" master: name: master runs-on: ubuntu-latest diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 0614d5e31d1ee..502949d5e4c13 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -410,42 +410,6 @@ jobs: DEPLOY_TOOLSTATES_JSON: toolstates-linux.json <<: *job-linux-xl - #################### - # macOS Builders # - #################### - - - name: dist-x86_64-apple - env: - SCRIPT: ./x.py dist - RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - <<: *job-macos-xl - - - name: dist-x86_64-apple-alt - env: - SCRIPT: ./x.py dist - RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - <<: *job-macos-xl - - - name: x86_64-apple - env: - SCRIPT: ./x.py test - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - <<: *job-macos-xl - ###################### # Windows Builders # ###################### @@ -591,6 +555,51 @@ jobs: SCRIPT: python x.py dist <<: *job-windows-xl + auto-fallible: + <<: *base-ci-job + name: auto-fallible + env: + <<: [*shared-ci-variables, *prod-variables] + if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust' + strategy: + matrix: + include: + #################### + # macOS Builders # + #################### + + - name: dist-x86_64-apple + env: + SCRIPT: ./x.py dist + RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-macos-xl + + - name: dist-x86_64-apple-alt + env: + SCRIPT: ./x.py dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + <<: *job-macos-xl + + - name: x86_64-apple + env: + SCRIPT: ./x.py test + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + <<: *job-macos-xl + master: name: master runs-on: ubuntu-latest