Skip to content

Commit

Permalink
Use a dynamic environment for PR jobs
Browse files Browse the repository at this point in the history
  • Loading branch information
Kobzol committed Apr 19, 2024
1 parent 05f3c0a commit 65141ab
Show file tree
Hide file tree
Showing 15 changed files with 104 additions and 189 deletions.
153 changes: 7 additions & 146 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,23 +45,23 @@ jobs:
- name: Checkout the source code
uses: actions/checkout@v4
- name: Calculate the CI job matrix
run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
env:
GITHUB_CTX: "${{ toJSON(github) }}"
run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
id: jobs
pr:
name: "PR - ${{ matrix.name }}"
job:
name: "${{ matrix.name }}"
needs:
- calculate_matrix
env:
PR_CI_JOB: 1
CI_JOB_NAME: "${{ matrix.name }}"
CI_JOB_NAME: "${{ matrix.image }}"
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
SCCACHE_BUCKET: rust-lang-ci-sccache2
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
CACHE_DOMAIN: ci-caches.rust-lang.org
if: "github.event_name == 'pull_request'"
continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
continue-on-error: "${{ matrix.continue_on_error || false }}"
strategy:
matrix:
include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
Expand Down Expand Up @@ -589,145 +589,6 @@ jobs:
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
try:
name: "try - ${{ matrix.name }}"
env:
DIST_TRY_BUILD: 1
CI_JOB_NAME: "${{ matrix.name }}"
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
SCCACHE_BUCKET: rust-lang-ci-sccache2
DEPLOY_BUCKET: rust-lang-ci2
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues"
TOOLSTATE_PUBLISH: 1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHE_DOMAIN: ci-caches.rust-lang.org
if: "github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))"
strategy:
matrix:
include:
- name: dist-x86_64-linux
env:
CODEGEN_BACKENDS: "llvm,cranelift"
os: ubuntu-20.04-16core-64gb
defaults:
run:
shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
timeout-minutes: 600
runs-on: "${{ matrix.os }}"
steps:
- if: "contains(matrix.os, 'windows')"
uses: msys2/setup-msys2@v2.22.0
with:
msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
update: false
release: true
path-type: inherit
install: "make dos2unix diffutils\n"
- name: disable git crlf conversion
run: git config --global core.autocrlf false
- name: checkout the source code
uses: actions/checkout@v4
with:
fetch-depth: 2
- name: configure the PR in which the error message will be posted
run: "echo \"[CI_PR_NUMBER=$num]\""
env:
num: "${{ github.event.number }}"
if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'"
- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
env:
EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
if: success() && !env.SKIP_JOB
- name: decide whether to skip this job
run: src/ci/scripts/should-skip-this.sh
if: success() && !env.SKIP_JOB
- name: ensure the channel matches the target branch
run: src/ci/scripts/verify-channel.sh
if: success() && !env.SKIP_JOB
- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
if: success() && !env.SKIP_JOB
- name: show the current environment
run: src/ci/scripts/dump-environment.sh
if: success() && !env.SKIP_JOB
- name: install awscli
run: src/ci/scripts/install-awscli.sh
if: success() && !env.SKIP_JOB
- name: install sccache
run: src/ci/scripts/install-sccache.sh
if: success() && !env.SKIP_JOB
- name: select Xcode
run: src/ci/scripts/select-xcode.sh
if: success() && !env.SKIP_JOB
- name: install clang
run: src/ci/scripts/install-clang.sh
if: success() && !env.SKIP_JOB
- name: install tidy
run: src/ci/scripts/install-tidy.sh
if: success() && !env.SKIP_JOB
- name: install WIX
run: src/ci/scripts/install-wix.sh
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
if: success() && !env.SKIP_JOB
- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh
if: success() && !env.SKIP_JOB
- name: install MSYS2
run: src/ci/scripts/install-msys2.sh
if: success() && !env.SKIP_JOB
- name: install MinGW
run: src/ci/scripts/install-mingw.sh
if: success() && !env.SKIP_JOB
- name: install ninja
run: src/ci/scripts/install-ninja.sh
if: success() && !env.SKIP_JOB
- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
if: success() && !env.SKIP_JOB
- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
if: success() && !env.SKIP_JOB
- name: ensure backported commits are in upstream branches
run: src/ci/scripts/verify-backported-commits.sh
if: success() && !env.SKIP_JOB
- name: ensure the stable version number is correct
run: src/ci/scripts/verify-stable-version-number.sh
if: success() && !env.SKIP_JOB
- name: run the build
run: src/ci/scripts/run-build-from-ci.sh 2>&1
env:
AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
if: success() && !env.SKIP_JOB
- name: create github artifacts
run: src/ci/scripts/create-doc-artifacts.sh
if: success() && !env.SKIP_JOB
- name: upload artifacts to github
uses: actions/upload-artifact@v4
with:
name: "${{ env.DOC_ARTIFACT_NAME }}"
path: obj/artifacts/doc
if-no-files-found: ignore
retention-days: 5
if: success() && !env.SKIP_JOB
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
master:
name: master
runs-on: ubuntu-latest
Expand Down
32 changes: 29 additions & 3 deletions src/ci/github-actions/calculate-job-matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,41 @@
"""

import json
import os
from pathlib import Path
from typing import List, Dict

import yaml

JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"


def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
for job in jobs:
job["name"] = f"{prefix} - {job['image']}"
return jobs


if __name__ == "__main__":
github_ctx = json.loads(os.environ["GITHUB_CTX"])

with open(JOBS_YAML_PATH) as f:
jobs = yaml.safe_load(f)
job_output = jobs["pr"]
print(f"jobs={json.dumps(job_output)}")
data = yaml.safe_load(f)

event_name = github_ctx["event_name"]
ref = github_ctx["ref"]
repository = github_ctx["repository"]

old_bors_try_build = ref in ("refs/heads/try", "refs/heads/try-perf") and repository == "rust-lang-ci/rust"
new_bors_try_build = ref == "refs/heads/automation/bors/try" and repository == "rust-lang/rust"
try_build = old_bors_try_build or new_bors_try_build

jobs = []
# Pull request CI jobs. Their name is 'PR - <image>'
if event_name == "pull_request":
jobs = name_jobs(data["pr"], "PR")
# Try builds
elif event_name == "push" and try_build:
jobs = name_jobs(data["try"], "try")

print(f"jobs={json.dumps(jobs)}")
38 changes: 14 additions & 24 deletions src/ci/github-actions/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -351,17 +351,24 @@ jobs:
- name: Checkout the source code
uses: actions/checkout@v4
- name: Calculate the CI job matrix
run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
env:
GITHUB_CTX: ${{ toJSON(github) }}
run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
id: jobs
pr:
job:
<<: *base-ci-job
name: PR - ${{ matrix.name }}
name: ${{ matrix.name }}
needs: [ calculate_matrix ]
env:
<<: [*shared-ci-variables, *public-variables]
PR_CI_JOB: 1
if: github.event_name == 'pull_request'
continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
CI_JOB_NAME: ${{ matrix.image }}
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
# commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SCCACHE_BUCKET: rust-lang-ci-sccache2
TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
CACHE_DOMAIN: ci-caches.rust-lang.org
continue-on-error: ${{ matrix.continue_on_error || false }}
strategy:
matrix:
# Check the `calculate_matrix` job to see how is the matrix defined.
Expand Down Expand Up @@ -751,23 +758,6 @@ jobs:
SCRIPT: python x.py dist bootstrap --include-default-paths
<<: *job-windows-8c

try:
<<: *base-ci-job
name: try - ${{ matrix.name }}
env:
DIST_TRY_BUILD: 1
<<: [*shared-ci-variables, *prod-variables]
if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))
strategy:
matrix:
include:
- &dist-x86_64-linux
name: dist-x86_64-linux
env:
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-linux-16c


master:
name: master
runs-on: ubuntu-latest
Expand Down
48 changes: 43 additions & 5 deletions src/ci/github-actions/jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# dynamically in CI from ci.yml.
# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
# modify this file.
shared_defs:
runners:
- &base-job
env: { }

Expand Down Expand Up @@ -37,14 +37,52 @@ shared_defs:
- &job-aarch64-linux
os: [ self-hosted, ARM64, linux ]

envs:
- &pr-env
PR_CI_JOB: 1

- &prod-variables
SCCACHE_BUCKET: rust-lang-ci-sccache2
DEPLOY_BUCKET: rust-lang-ci2
TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
TOOLSTATE_PUBLISH: 1
# AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
# AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
# rotate them in a single branch while keeping the old key in another
# branch, which wouldn't be possible if the key was named with the kind
# (caches, artifacts...).
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
CACHE_DOMAIN: ci-caches.rust-lang.org

# Pull request jobs
pr:
- name: mingw-check
- image: mingw-check
env:
<<: *pr-env
<<: *job-linux-4c
- name: mingw-check-tidy
- image: mingw-check-tidy
continue_on_error: true
env:
<<: *pr-env
<<: *job-linux-4c
- name: x86_64-gnu-llvm-17
- image: x86_64-gnu-llvm-17
env:
ENABLE_GCC_CODEGEN: "1"
<<: *pr-env
<<: *job-linux-16c
- name: x86_64-gnu-tools
- image: x86_64-gnu-tools
env:
<<: *pr-env
<<: *job-linux-16c

# Try build jobs
try:
- image: dist-x86_64-linux
env:
DIST_TRY_BUILD: 1
CODEGEN_BACKENDS: llvm,cranelift
<<: *prod-variables
<<: *job-linux-16c
2 changes: 1 addition & 1 deletion src/doc/book
2 changes: 1 addition & 1 deletion src/doc/edition-guide
2 changes: 1 addition & 1 deletion src/doc/embedded-book
2 changes: 1 addition & 1 deletion src/doc/nomicon
2 changes: 1 addition & 1 deletion src/doc/reference
2 changes: 1 addition & 1 deletion src/doc/rust-by-example
2 changes: 1 addition & 1 deletion src/llvm-project
Submodule llvm-project updated 82 files
+1 −1 .github/workflows/release-lit.yml
+0 −7 clang/docs/ReleaseNotes.rst
+2 −2 clang/include/clang/Sema/ScopeInfo.h
+16 −27 clang/lib/CodeGen/CGDecl.cpp
+10 −10 clang/lib/Format/TokenAnnotator.cpp
+0 −4 clang/lib/Headers/__stddef_unreachable.h
+26 −47 clang/lib/Sema/SemaDecl.cpp
+1 −21 clang/lib/Sema/SemaTemplate.cpp
+2 −2 clang/test/CodeGen/aapcs-align.cpp
+4 −4 clang/test/CodeGen/aapcs64-align.cpp
+18 −6 clang/test/CodeGen/attr-counted-by.c
+12 −15 clang/test/CodeGenCXX/auto-var-init.cpp
+8 −1 clang/test/CodeGenOpenCL/amdgpu-printf.cl
+3 −1 clang/test/OpenMP/bug54082.c
+3 −89 clang/test/SemaCXX/warn-shadow-in-lambdas.cpp
+0 −26 clang/test/SemaTemplate/concepts-friends.cpp
+1 −1 clang/test/SemaTemplate/ctad.cpp
+0 −14 clang/unittests/Format/TokenAnnotatorTest.cpp
+0 −42 compiler-rt/lib/builtins/riscv/restore.S
+0 −42 compiler-rt/lib/builtins/riscv/save.S
+1 −1 compiler-rt/lib/msan/msan.cpp
+11 −24 compiler-rt/lib/msan/msan.h
+0 −3 compiler-rt/lib/msan/msan_allocator.cpp
+13 −54 compiler-rt/lib/msan/msan_linux.cpp
+25 −18 compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+1 −1 libcxx/include/__format/formatter_floating_point.h
+19 −10 libcxx/include/stddef.h
+2 −13 libcxx/modules/CMakeLists.txt
+0 −7 lld/COFF/Chunks.cpp
+3 −4 lld/ELF/Arch/X86_64.cpp
+0 −41 lld/test/COFF/autoimport-gc.s
+1 −11 lld/test/ELF/x86-64-gotpc-relax-too-far.s
+1 −1 llvm/CMakeLists.txt
+0 −2 llvm/docs/LangRef.rst
+7 −44 llvm/include/llvm/Support/FormattedStream.h
+1 −2 llvm/lib/Analysis/InstructionSimplify.cpp
+1 −6 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+2 −2 llvm/lib/IR/ConstantRange.cpp
+0 −3 llvm/lib/Support/FormattedStream.cpp
+34 −8 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+0 −3 llvm/lib/Target/AArch64/AArch64ISelLowering.h
+4 −7 llvm/lib/Target/ARM/ARMFrameLowering.cpp
+0 −4 llvm/lib/Target/ARM/ARMFrameLowering.h
+13 −10 llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+0 −5 llvm/lib/Target/LoongArch/LoongArchTargetMachine.h
+6 −0 llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+0 −44 llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
+0 −34 llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+1 −3 llvm/lib/Target/X86/X86MCInstLower.cpp
+27 −53 llvm/lib/Transforms/Scalar/SROA.cpp
+1 −20 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+0 −56 llvm/test/CodeGen/ARM/ldst-opt-lr-restored.ll
+0 −47 llvm/test/CodeGen/LoongArch/addrspacecast.ll
+45 −284 llvm/test/CodeGen/PowerPC/crsave.ll
+0 −9 llvm/test/CodeGen/SPARC/inlineasm-bad.ll
+0 −9 llvm/test/CodeGen/SPARC/inlineasm.ll
+0 −22 llvm/test/CodeGen/SystemZ/int-usub-12.ll
+0 −2 llvm/test/CodeGen/SystemZ/int-usub-13.ll
+0 −23 llvm/test/CodeGen/X86/addcarry.ll
+0 −17 llvm/test/CodeGen/X86/patchable-prologue.ll
+0 −20 llvm/test/Transforms/Float2Int/pr79158.ll
+0 −42 llvm/test/Transforms/InstSimplify/pr87042.ll
+0 −82 llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll
+0 −62 llvm/test/Transforms/SROA/vector-promotion.ll
+5 −0 llvm/tools/llvm-mc/llvm-mc.cpp
+7 −0 llvm/tools/llvm-objdump/llvm-objdump.cpp
+0 −18 llvm/unittests/IR/ConstantRangeTest.cpp
+1 −1 llvm/utils/lit/lit/__init__.py
+1 −1 llvm/utils/release/github-upload-release.py
+0 −1 mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+0 −1 mlir/test/Interfaces/TilingInterface/lower-to-loops-using-interface.mlir
+1 −1 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+1 −3 openmp/runtime/src/kmp.h
+8 −121 openmp/runtime/src/kmp_affinity.cpp
+2 −71 openmp/runtime/src/kmp_affinity.h
+1 −2 openmp/runtime/src/kmp_os.h
+6 −31 openmp/runtime/src/z_Linux_util.cpp
+1 −1 openmp/runtime/test/lit.cfg
+1 −1 polly/lib/Exchange/JSONExporter.cpp
+1 −1 polly/lib/Transform/DeLICM.cpp
+1 −1 polly/lib/Transform/FlattenSchedule.cpp
+1 −1 polly/lib/Transform/ForwardOpTree.cpp
2 changes: 1 addition & 1 deletion src/tools/cargo
Submodule cargo updated 307 files

0 comments on commit 65141ab

Please sign in to comment.