Skip to content

Commit

Permalink
Merge branch 'main' into mg/musl-werror
Browse files Browse the repository at this point in the history
  • Loading branch information
DilumAluthge authored Apr 8, 2022
2 parents 28ad08f + f5f7815 commit 2870ee5
Show file tree
Hide file tree
Showing 21 changed files with 176 additions and 152 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,6 @@ repo.key
# Ignore any agent keys (public or private) we have stored
agent_key*
agent.key*

# Ignore a local clone of the `cryptic-buildkite-plugin` repository
cryptic-buildkite-plugin/
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
export PATH := $(shell pwd)/cryptic-buildkite-plugin/bin:$(PATH)

decrypt:
cd .buildkite/cryptic_repo_root && decrypt --repo-root=$$(pwd) --verbose

Expand Down
4 changes: 2 additions & 2 deletions pipelines/main/0_webui.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect.
# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps.
steps:
- group: ":buildkite: Infrastructure"
- group: "Web UI"
steps:
- label: ":unlock: Unlock secrets, launch pipelines"
- label: "Unlock secrets, launch pipelines"
agents:
queue: "julia"
cryptic_capable: "true"
Expand Down
54 changes: 30 additions & 24 deletions pipelines/main/launch_signed_jobs.yml
Original file line number Diff line number Diff line change
@@ -1,29 +1,35 @@
# This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3

steps:
- label: ":buildkite: :unlock: Launch signed pipelines"
plugins:
- JuliaCI/external-buildkite#v1:
version: "./.buildkite-external-version"
repo_url: "https://github.com/JuliaCI/julia-buildkite"
commands: |
# Explicitly pass along the cryptic token to child pipelines
export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
- group: "Upload"
notify:
- github_commit_status:
context: "Upload"
steps:
- label: "Launch signed pipelines"
plugins:
- JuliaCI/external-buildkite#v1:
version: "./.buildkite-external-version"
repo_url: "https://github.com/JuliaCI/julia-buildkite"
commands: |
# Explicitly pass along the cryptic token to child pipelines
export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
# Launch `upload_linux` jobs to store tarballs into S3 once tests are done
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/upload_linux.arches \
.buildkite/pipelines/main/platforms/upload_linux.yml
# Launch `upload_linux` jobs to store tarballs into S3 once tests are done
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/upload_linux.arches \
.buildkite/pipelines/main/platforms/upload_linux.yml
# Launch `upload_macos` jobs to store tarballs into S3 once tests are done
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/upload_macos.arches \
.buildkite/pipelines/main/platforms/upload_macos.yml
# Launch `upload_macos` jobs to store tarballs into S3 once tests are done
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/upload_macos.arches \
.buildkite/pipelines/main/platforms/upload_macos.yml
# Don't share this with buildkite's env display
unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
agents:
queue: "julia"
cryptic_capable: "true"
env:
# Receive cryptic token from parent job
BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}
# Don't share this with buildkite's env display
unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
agents:
queue: "julia"
cryptic_capable: "true"
env:
# Receive cryptic token from parent job
BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}
Binary file modified pipelines/main/launch_signed_jobs.yml.signature
Binary file not shown.
55 changes: 34 additions & 21 deletions pipelines/main/launch_unsigned_builders.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,48 +13,61 @@
# something about the privileged steps.

steps:
- group: ":buildkite: Infrastructure"
- group: "Build"
notify:
- github_commit_status:
context: "Build"
steps:
- label: ":buildkite: Launch misc. jobs"
- label: "Launch build jobs"
plugins:
- JuliaCI/external-buildkite#v1:
version: "./.buildkite-external-version"
repo_url: "https://github.com/JuliaCI/julia-buildkite"
commands: |
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
# Launch Linux build jobs
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/build_linux.arches \
.buildkite/pipelines/main/platforms/build_linux.yml
# Launch macOS packaging jobs
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/build_macos.arches \
.buildkite/pipelines/main/platforms/build_macos.yml
agents:
queue: "julia"

- group: ":linux: Linux"
- group: "Check"
notify:
- github_commit_status:
context: "Check"
steps:
- label: ":buildkite: Launch jobs"
- label: "Launch check jobs"
plugins:
- JuliaCI/external-buildkite#v1:
version: "./.buildkite-external-version"
repo_url: "https://github.com/JuliaCI/julia-buildkite"
commands: |
# Launch the miscellaneous jobs in alphabetical order.
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml
# Launch Linux packaging jobs
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/build_linux.arches \
.buildkite/pipelines/main/platforms/build_linux.yml
# Launch macos packaging jobs
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/build_macos.arches \
.buildkite/pipelines/main/platforms/build_macos.yml
# Launch Linux testing jobs
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
agents:
queue: "julia"
- group: "Test"
notify:
- github_commit_status:
context: "Test"
steps:
- label: "Launch test jobs"
plugins:
- JuliaCI/external-buildkite#v1:
version: "./.buildkite-external-version"
repo_url: "https://github.com/JuliaCI/julia-buildkite"
commands: |
# Launch Linux test jobs
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/test_linux.arches \
.buildkite/pipelines/main/platforms/test_linux.yml
# Launch macos testing jobs
# Launch macOS test jobs
bash .buildkite/utilities/arches_pipeline_upload.sh \
.buildkite/pipelines/main/platforms/test_macos.arches \
.buildkite/pipelines/main/platforms/test_macos.yml
Expand Down
2 changes: 1 addition & 1 deletion pipelines/main/misc/doctest.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
steps:
- group: ":linux: Linux"
- group: "Check"
steps:
- label: "doctest"
key: doctest
Expand Down
2 changes: 1 addition & 1 deletion pipelines/main/misc/embedding.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
steps:
- group: ":linux: Linux"
- group: "Check"
steps:
- label: "embedding"
key: "embedding"
Expand Down
3 changes: 1 addition & 2 deletions pipelines/main/misc/llvmpasses.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
steps:
- group: ":linux: Linux"
- group: "Check"
steps:
- label: "analyzegc"
key: "analyzegc"
Expand Down Expand Up @@ -28,7 +28,6 @@ steps:
queue: "julia"
sandbox_capable: "true"
os: "linux"

- label: "llvmpasses"
key: "llvmpasses"
plugins:
Expand Down
3 changes: 1 addition & 2 deletions pipelines/main/misc/sanitizers.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
steps:
- group: ":linux: Linux"
- group: "Check"
steps:
- label: "asan"
key: "asan"
Expand Down Expand Up @@ -36,7 +36,6 @@ steps:
queue: "julia"
sandbox_capable: "true"
os: "linux"

- label: "tsan"
key: "tsan"
plugins:
Expand Down
4 changes: 2 additions & 2 deletions pipelines/main/misc/whitespace.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
steps:
- group: ":buildkite: Infrastructure"
- group: "Check"
steps:
- label: ":buildkite: Whitespace check"
- label: "whitespace"
key: "whitespace"
plugins:
- JuliaCI/external-buildkite#v1:
Expand Down
2 changes: 1 addition & 1 deletion pipelines/main/platforms/build_linux.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
steps:
- group: ":linux: Linux"
- group: "Build"
steps:
- label: ":linux: build ${TRIPLET?}"
key: "build_${TRIPLET?}"
Expand Down
3 changes: 1 addition & 2 deletions pipelines/main/platforms/build_macos.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
steps:
- group: ":macos: Macos"
- group: "Build"
steps:
- label: ":macos: build ${TRIPLET?}"
key: "build_${TRIPLET?}"
Expand All @@ -17,4 +17,3 @@ steps:
# Receive some environment mappings from our templating engine
TRIPLET: "${TRIPLET?}"
MAKE_FLAGS: "${MAKE_FLAGS?}"

5 changes: 2 additions & 3 deletions pipelines/main/platforms/test_linux.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
steps:
- group: ":linux: Linux"
- group: "Test"
steps:
- label: ":linux: :test_tube: test ${TRIPLET?}${USE_RR-}"
- label: ":linux: test ${TRIPLET?}${USE_RR-}"
key: "test_${TRIPLET?}${USE_RR-}"
depends_on:
- "build_${TRIPLET?}"
Expand Down Expand Up @@ -33,4 +33,3 @@ steps:
JULIA_SHELL: "/bin/bash"
TRIPLET: "${TRIPLET?}"
USE_RR: "${USE_RR?}"

4 changes: 2 additions & 2 deletions pipelines/main/platforms/test_macos.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
steps:
- group: ":macos: Macos"
- group: "Test"
steps:
- label: ":macos: :test_tube: test ${TRIPLET?}"
- label: ":macos: test ${TRIPLET?}"
key: "test_${TRIPLET?}"
depends_on:
- "build_${TRIPLET?}"
Expand Down
96 changes: 49 additions & 47 deletions pipelines/main/platforms/upload_linux.yml
Original file line number Diff line number Diff line change
@@ -1,48 +1,50 @@
steps:
- label: ":linux: upload ${TRIPLET?}"
key: "upload_${TRIPLET?}"
# We only upload to S3 if one of the following criteria are true:
# 1. The branch is `master`.
# 2. The branch is `release-*`.
# 3. The build is a tag build AND the tag is `v*`.
# 4. The pipeline is `julia-buildkite`.
if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite")
depends_on:
# Wait for the builder to finish
- "build_${TRIPLET?}"
# Wait for the tester to finish
- "test_${TRIPLET?}"
# Prevent multiple pipelines from uploading to S3 simultaneously
# It is okay for two different triplets to upload simultaneously
concurrency: 1
concurrency_group: 'upload/julialangnightlies/${TRIPLET?}'
plugins:
- JuliaCI/external-buildkite#v1:
version: "./.buildkite-external-version"
repo_url: "https://github.com/JuliaCI/julia-buildkite"
- JuliaCI/julia#v1:
# Drop default "registries" directory, so it is not persisted from execution to execution
persist_depot_dirs: packages,artifacts,compiled
version: '1'
- staticfloat/sandbox#v1:
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.7/aws_uploader.x86_64.tar.gz
rootfs_treehash: "0d5ad94ba902eea4d9a7d220b49c144626a102ea"
uid: 1000
gid: 1000
- staticfloat/cryptic#v2:
variables:
- AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1"
- AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg=="
files:
- .buildkite/secrets/tarball_signing.gpg
timeout_in_minutes: ${TIMEOUT?}
commands: "bash .buildkite/utilities/upload_julia.sh"
agents:
queue: "julia"
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
sandbox_capable: "true"
os: "linux"
env:
# Receive cryptic token from parent job
BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}
TRIPLET: "${TRIPLET?}"
- group: "Upload"
steps:
- label: ":linux: upload ${TRIPLET?}"
key: "upload_${TRIPLET?}"
# We only upload to S3 if one of the following criteria are true:
# 1. The branch is `master`.
# 2. The branch is `release-*`.
# 3. The build is a tag build AND the tag is `v*`.
# 4. The pipeline is `julia-buildkite`.
if: (build.branch == "master") || (build.branch =~ /^release-/) || (build.tag =~ /^v/) || (pipeline.slug == "julia-buildkite")
depends_on:
# Wait for the builder to finish
- "build_${TRIPLET?}"
# Wait for the tester to finish
- "test_${TRIPLET?}"
# Prevent multiple pipelines from uploading to S3 simultaneously
# It is okay for two different triplets to upload simultaneously
concurrency: 1
concurrency_group: 'upload/julialangnightlies/${TRIPLET?}'
plugins:
- JuliaCI/external-buildkite#v1:
version: "./.buildkite-external-version"
repo_url: "https://github.com/JuliaCI/julia-buildkite"
- JuliaCI/julia#v1:
# Drop default "registries" directory, so it is not persisted from execution to execution
persist_depot_dirs: packages,artifacts,compiled
version: '1'
- staticfloat/sandbox#v1:
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v5.7/aws_uploader.x86_64.tar.gz
rootfs_treehash: "0d5ad94ba902eea4d9a7d220b49c144626a102ea"
uid: 1000
gid: 1000
- staticfloat/cryptic#v2:
variables:
- AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1"
- AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg=="
files:
- .buildkite/secrets/tarball_signing.gpg
timeout_in_minutes: ${TIMEOUT?}
commands: "bash .buildkite/utilities/upload_julia.sh"
agents:
queue: "julia"
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
sandbox_capable: "true"
os: "linux"
env:
# Receive cryptic token from parent job
BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}
TRIPLET: "${TRIPLET?}"
Loading

0 comments on commit 2870ee5

Please sign in to comment.