From f35e9187851d9d2225a6e78fccecab490bd3e35c Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 16 Apr 2022 17:04:41 -0400 Subject: [PATCH] Setup buildkite as an alternative to AZP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mosè Giordano --- .buildkite/0_webui.yml | 23 ++ .buildkite/Manifest.toml | 162 ++++++++ .buildkite/Project.toml | 6 + .buildkite/build.sh | 17 + .buildkite/cryptic_repo_keys/.gitignore | 7 + .../cryptic_repo_keys/repo_key.2297e5e7 | Bin 0 -> 256 bytes .buildkite/generator.jl | 133 +++++++ .buildkite/init.sh | 14 + .buildkite/path_processors/per-project | 9 + .buildkite/pipeline.yml | 28 ++ .buildkite/pipeline.yml.signature | 1 + .buildkite/register.sh | 20 + .buildkite/utils.jl | 159 ++++++++ .ci/azp_agent/.env | Bin 488 -> 0 bytes .ci/azp_agent/.gitattributes | 2 - .ci/azp_agent/agent_startup.conf | 33 -- .ci/azp_agent/debug_agent.sh | 17 - .ci/azp_agent/install_agents.sh | 102 ----- .ci/azp_agent/loopback_startup.conf | 8 - .ci/azp_agent/restart_agents.sh | 6 - .ci/azp_agent/run_agent.sh | 30 -- .ci/azp_agent/setup_loopback.sh | 15 - .ci/azp_agent/stop_agents.sh | 6 - .ci/azp_agent/yggdrasil_rsa | Bin 3269 -> 0 bytes .ci/register_package.jl | 30 +- azure-pipelines.yml | 366 ------------------ 26 files changed, 595 insertions(+), 599 deletions(-) create mode 100644 .buildkite/0_webui.yml create mode 100644 .buildkite/Manifest.toml create mode 100644 .buildkite/Project.toml create mode 100755 .buildkite/build.sh create mode 100644 .buildkite/cryptic_repo_keys/.gitignore create mode 100644 .buildkite/cryptic_repo_keys/repo_key.2297e5e7 create mode 100755 .buildkite/generator.jl create mode 100755 .buildkite/init.sh create mode 100755 .buildkite/path_processors/per-project create mode 100644 .buildkite/pipeline.yml create mode 100644 .buildkite/pipeline.yml.signature create mode 100755 .buildkite/register.sh create mode 100644 .buildkite/utils.jl delete mode 100644 .ci/azp_agent/.env delete mode 100644 .ci/azp_agent/.gitattributes delete mode 100644 .ci/azp_agent/agent_startup.conf delete mode 100755 .ci/azp_agent/debug_agent.sh delete mode 100755 .ci/azp_agent/install_agents.sh delete mode 100644 .ci/azp_agent/loopback_startup.conf delete mode 100755 .ci/azp_agent/restart_agents.sh delete mode 100755 .ci/azp_agent/run_agent.sh delete mode 100755 .ci/azp_agent/setup_loopback.sh delete mode 100755 .ci/azp_agent/stop_agents.sh delete mode 100644 .ci/azp_agent/yggdrasil_rsa delete mode 100644 azure-pipelines.yml diff --git a/.buildkite/0_webui.yml b/.buildkite/0_webui.yml new file mode 100644 index 00000000000..f86454f9b1a --- /dev/null +++ b/.buildkite/0_webui.yml @@ -0,0 +1,23 @@ +# This file represents what is put into the webUI. +# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. +steps: + - label: ":buildkite: Pipeline upload" + command: buildkite-agent pipeline upload + branches: "!gh-pages" + agents: + queue: "yggdrasil" + plugins: + - JuliaCI/merge-commit: ~ + - staticfloat/cryptic#v2: + signed_pipelines: + - pipeline: .buildkite/pipeline.yml + signature_file: .buildkite/pipeline.yml.signature + inputs: + - .buildkite/generator.jl + - .buildkite/utils.jl + - .buildkite/Project.toml + - .buildkite/Manifest.toml + - .buildkite/path_processors/per-project + - .buildkite/init.sh + - .buildkite/register.sh + command: "true" diff --git a/.buildkite/Manifest.toml b/.buildkite/Manifest.toml new file mode 100644 index 00000000000..116eb377d96 --- /dev/null +++ b/.buildkite/Manifest.toml @@ -0,0 +1,162 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.8.0-beta3" +manifest_format = "2.0" +project_hash = "85881da52ac27331aa61f08c30c93ab8b41b5a1c" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.JLLWrappers]] +deps = ["Preferences"] +git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.4.1" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.81.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.16.1+1" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.0+0" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.2.1" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.8.0" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "d3538e7f8a790dc8903519090857ef8e1283eecd" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.2.5" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA", "Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.StringEncodings]] +deps = ["Libiconv_jll"] +git-tree-sha1 = "50ccd5ddb00d19392577902f0079267a72c5ab04" +uuid = "69024149-9ee7-55f6-a4c4-859efe599b68" +version = "0.3.5" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.0" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.YAML]] +deps = ["Base64", "Dates", "Printf", "StringEncodings"] +git-tree-sha1 = "3c6e8b9f5cdaaa21340f841653942e1a6b6561e5" +uuid = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" +version = "0.4.7" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.12+1" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.41.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "16.2.1+1" diff --git a/.buildkite/Project.toml b/.buildkite/Project.toml new file mode 100644 index 00000000000..ab10ed56c7c --- /dev/null +++ b/.buildkite/Project.toml @@ -0,0 +1,6 @@ +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" diff --git a/.buildkite/build.sh b/.buildkite/build.sh new file mode 100755 index 00000000000..1e7c97a81ab --- /dev/null +++ b/.buildkite/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Fail on error +set -e + +export JULIA_PROJECT="${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci" + +echo "--- Setup Julia packages" +julia --color=yes -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()' + +# Cleanup temporary things that might have been left-over +echo "--- Cleanup" +./clean_builds.sh +./clean_products.sh + +echo "+++ Build" +cd "${PROJECT}" +julia ./build_tarballs.jl --verbose "${PLATFORM}" diff --git a/.buildkite/cryptic_repo_keys/.gitignore b/.buildkite/cryptic_repo_keys/.gitignore new file mode 100644 index 00000000000..f84d08964da --- /dev/null +++ b/.buildkite/cryptic_repo_keys/.gitignore @@ -0,0 +1,7 @@ + +# Ignore the unencrypted repo_key +repo_key + +# Ignore any agent keys (public or private) we have stored +agent_key* + diff --git a/.buildkite/cryptic_repo_keys/repo_key.2297e5e7 b/.buildkite/cryptic_repo_keys/repo_key.2297e5e7 new file mode 100644 index 0000000000000000000000000000000000000000..fbc212d5fa84765816a6435851a521f0dacb4cb9 GIT binary patch literal 256 zcmV+b0ssD#z`1y77x`$k*g}-yIqkGJM0iaT;eKTrHFyGm%BUoNF9=_w#wi0Z;>O@r zNbPU0OIwizMaLTKb&6+1em*_^U+cs5ZKYC{FcS*un@YLDU*a~aY4>N5RqHB~J57}E z&A>v_uw?JtFOdOa3qy>fTbiwUKixy>#CtGdql+^5u2G{7#kECdWS6Z`3If~;q30st zPVUSe99cABV+|@NKFK~7cgorJE$J=MvjvOEb*!#k#&_FnLAs~~KY}%a=Sn+7KeN$C-gy8^{@hL#%IpDgdg2hG GR9sQsQ+&Vx literal 0 HcmV?d00001 diff --git a/.buildkite/generator.jl b/.buildkite/generator.jl new file mode 100755 index 00000000000..1bf523b1a9e --- /dev/null +++ b/.buildkite/generator.jl @@ -0,0 +1,133 @@ +#!/bin/env julia +if VERSION < v"1.8.0" + Base.ACTIVE_PROJECT[] = @__DIR__ +else + Base.set_active_project(@__DIR__) +end +import Pkg +Pkg.instantiate() + +using Downloads, Dates, SHA + +const PROJECT = only(ARGS) +const DEBUG = !haskey(ENV, "BUILDKITE") + +include("utils.jl") + +if !isnothing(Pkg.pkg_server()) + resp = try + headers = Pkg.PlatformEngines.get_metadata_headers(Pkg.pkg_server()) + Downloads.request("$(Pkg.pkg_server())/registries"; headers) + catch e + # Let us know the download of the registry went wrong, but do not hard fail + @error "Could not download the registry" exception=(e, catch_backtrace()) + end + last_mod_idx = findfirst(h -> first(h) == "last-modified", resp.headers) + msg = "PkgServer: " * resp.url + delay = if !isnothing(last_mod_idx) + last_mod = last(resp.headers[last_mod_idx]) + msg *= " -- last updated: " * last_mod + # Manually strip the "GMT" timezone and hope it never changes. + # Do not error out if parsing fails. + dt = tryparse(DateTime, replace(last_mod, " GMT"=>""), dateformat"e, d u y H:M:S") + # If parsing did fail, set the delay to 0. + isnothing(dt) ? Second(0) : now(UTC) - dt + else + Second(0) + end + delay > Second(0) && (msg *= " (" * string(Dates.canonicalize(round(delay, Second))) * " ago)") + @info(msg) + annotate(msg, style="info", context="pkg") + tolerance = Hour(1) + if delay > tolerance + @warn "The PkgServer registry is older than $(tolerance)" + annotate("The PkgServer registry is older than $(tolerance)", style = "warning", context="pkg") + end +end + +# If there are scary projects we need to exclude, we list them here. (Used to contain `LLVM`) +EXCLUDED_NAMES = Set{String}([]) + +if PROJECT ∈ EXCLUDED_NAMES + @info "Skipping project since it is excluded." PROJECT + exit() +end + +# Remove secret from environment +sanitize(cmd) = addenv(cmd, Dict("BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET" => nothing)) +exec(cmd) = @assert success(pipeline(sanitize(cmd), stderr=stderr, stdout=stdout)) + +YGGDRASIL_BASE = dirname(@__DIR__) +julia(args) = `$(Base.julia_cmd()) --project=$(YGGDRASIL_BASE)/.ci $args` + +# Next, we're going to ensure that our BB is up to date and precompiled +julia(`-e "import Pkg; Pkg.instantiate(); Pkg.precompile()"`) |> exec + +# Next, for each project, download its sources. We do this by generating meta.json +# files, then parsing them with `download_sources.jl` + +TEMP = mktempdir() + +const NAME = basename(PROJECT) + +# We always invoke a `build_tarballs.jl` file from its own directory +cd(PROJECT) do + println("Generating meta.json...") + JSON_PATH = "$(TEMP)/$(NAME).meta.json" + julia(`--compile=min ./build_tarballs.jl --meta-json="$(JSON_PATH)"`) |> exec + + println("Downloading sources...") + julia(`$(YGGDRASIL_BASE)/.ci/download_sources.jl "$(JSON_PATH)" $(TEMP)/$(NAME).platforms.list`) |> exec +end + +println("Determining builds to queue...") + +# Load in the platforms +PLATFORMS = split(readchomp(joinpath(TEMP, "$(NAME).platforms.list"))) +if isempty(PLATFORMS) + @error "Unable to determine the proper platforms" NAME +end + +# Create the BUILD_STEPS +BUILD_STEPS = Any[] +for PLATFORM in PLATFORMS + println(" $(PLATFORM): building") + + push!(BUILD_STEPS, build_step(NAME, PLATFORM, PROJECT)) +end + +const IS_PR = get(ENV, "BUILDKITE_PULL_REQUEST", "false") != "false" +const SKIP_BUILD_COOKIE="[skip build]" + +if IS_PR + # If we're on a PR though, we look at the entire branch at once + BASE_BRANCH = get(ENV, "BUILDKITE_PULL_REQUEST_BASE_BRANCH", "master") + @assert !isempty(BASE_BRANCH) + + PR_NUMBER = ENV["BUILDKITE_PULL_REQUEST"] + exec(`git fetch origin "refs/pull/$(PR_NUMBER)/head:refs/remotes/origin/pr/$(PR_NUMBER)"`) + + COMMIT_MSG = readchomp(`git show -s --format=%B origin/pr/$(PR_NUMBER)`) +else + COMMIT_MSG = readchomp(`git show -s --format=%B`) +end +# This variable will tell us whether we want to skip the build +const SKIP_BUILD = contains(COMMIT_MSG, SKIP_BUILD_COOKIE) + +STEPS = Any[] +if !IS_PR + push!(STEPS, jll_init_step(NAME, PROJECT)) + push!(STEPS, wait_step()) +end +if !SKIP_BUILD + push!(STEPS, group_step(NAME, BUILD_STEPS)) +end +if !IS_PR + push!(STEPS, wait_step()) + push!(STEPS, register_step(NAME, PROJECT, SKIP_BUILD)) +end + +definition = Dict( + :steps => STEPS +) +upload_pipeline(definition) diff --git a/.buildkite/init.sh b/.buildkite/init.sh new file mode 100755 index 00000000000..b9562dd9dad --- /dev/null +++ b/.buildkite/init.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Fail on error +set -e + +export JULIA_PROJECT="${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci" + +echo "--- Setup Julia packages" +GITHUB_TOKEN="" julia --color=yes -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()' + +cd ${PROJECT} +echo "--- Generating meta.json..." +GITHUB_TOKEN="" julia --compile=min ./build_tarballs.jl --meta-json=${NAME}.meta.json +echo "--- Initializing JLL package..." +julia ${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci/jll_init.jl ${NAME}.meta.json diff --git a/.buildkite/path_processors/per-project b/.buildkite/path_processors/per-project new file mode 100755 index 00000000000..905c1b75e6e --- /dev/null +++ b/.buildkite/path_processors/per-project @@ -0,0 +1,9 @@ +#!/bin/bash + +# De-duplicate directories +declare -A DIRS +for f in "$@"; do + DIR="$(dirname ${f})" + DIRS["${DIR}"]=1 +done +echo "${!DIRS[@]}" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 00000000000..4c7632384f1 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,28 @@ +steps: + - label: ":runner: Dynamically launch Pipelines" + plugins: + - JuliaCI/julia#v1: + persist_depot_dirs: packages,artifacts,compiled + version: "1.7" + depot_hard_size_limit: "68719476736" # 64GB + - JuliaCI/merge-commit: ~ + - staticfloat/forerunner#v1: + watch: + - "**/*/build_tarballs.jl" + path_processor: .buildkite/path_processors/per-project + target: .buildkite/generator.jl + target_type: command + agents: + queue: "yggdrasil" + arch: "x86_64" + os: "linux" + # Only run on "sandbox.jl" machines + sandbox_capable: "true" + if: build.message !~ /\[skip tests\]/ + timeout_in_minutes: 10 + env: + JULIA_PKG_SERVER: us-east.pkg.julialang.org + # Use eager registry to not have to wait for updates of the conservative registry + JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager + # Forward JOB_ID_SECRET + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/.buildkite/pipeline.yml.signature b/.buildkite/pipeline.yml.signature new file mode 100644 index 00000000000..7eb6e7a8d75 --- /dev/null +++ b/.buildkite/pipeline.yml.signature @@ -0,0 +1 @@ +Salted__ä³älTÜ=} ª/ô ¶¿ãTÌÎRŒ>bTN]u„¹î9]ÑÌ©­Üí-ÄË@¡´šÙˆ§‚}aL¨-{׌¼õÊ…ý¶Oý™Í9ñÇ ûÖ±º“+½ \ No newline at end of file diff --git a/.buildkite/register.sh b/.buildkite/register.sh new file mode 100755 index 00000000000..c65f4192f7e --- /dev/null +++ b/.buildkite/register.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Fail on error +set -e + +export JULIA_PROJECT="${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci" + +echo "--- Setup Julia packages" +GITHUB_TOKEN="" julia --color=yes -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()' + +cd "${PROJECT}" +echo "--- Generating meta.json..." +GITHUB_TOKEN="" julia --compile=min ./build_tarballs.jl --meta-json=${NAME}.meta.json + +echo "--- Setting up git" +git config --global user.name "jlbuild" +git config --global user.email "juliabuildbot@gmail.com" + +echo "--- Registering ${NAME}..." +julia ${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci/register_package.jl "${NAME}.meta.json" --verbose + diff --git a/.buildkite/utils.jl b/.buildkite/utils.jl new file mode 100644 index 00000000000..110c2b2ec25 --- /dev/null +++ b/.buildkite/utils.jl @@ -0,0 +1,159 @@ +import YAML + +function upload_pipeline(definition) + try + open(`buildkite-agent pipeline upload --no-interpolation`, stdout, write=true) do io + YAML.write(io, definition) + end + catch + println(stderr, "pipeline upload failed:") + YAML.write(stderr, definition) + println(stderr) + rethrow() + end +end + +function annotate(annotation; context="default", style="info", append=true) + @assert style in ("success", "info", "warning", "error") + append = append ? `--append` : `` + cmd = `buildkite-agent annotate --style $(style) --context $(context) $(append)` + open(cmd, stdout, write=true) do io + write(io, annotation) + end +end + +agent() = Dict( + :queue => "yggdrasil", + :arch => "x86_64", + :os => "linux", + :sandbox_capable => "true" +) + +plugins() = Pair{String, Union{Nothing, Dict}}[ + "JuliaCI/julia#v1" => Dict( + "persist_depot_dirs" => "packages,artifacts,compiled", + "version" => "1.7", + "depot_hard_size_limit" => "68719476736", #64GB + ), + "JuliaCI/merge-commit" => nothing +] + +env(NAME, PROJECT) = Dict( + "JULIA_PKG_SERVER" => "us-east.pkg.julialang.org", + "JULIA_PKG_SERVER_REGISTRY_PREFERENCE" => "eager", + "NAME" => NAME, + "PROJECT" => PROJECT, + "YGGDRASIL" => "true", + # Inherit the secret so that we can decrypt cryptic secrets + "BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET" => get(ENV, "BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET", ""), +) + +wait_step() = Dict(:wait => nothing) +group_step(name, steps) = Dict(:group => name, :steps => steps) + +function jll_init_step(NAME, PROJECT) + script = raw""" + # Don't share secrets with build_tarballs.jl + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET="" .buildkite/init.sh + """ + + init_plugins = plugins() + push!(init_plugins, + "staticfloat/cryptic#v2" => Dict( + "variables" => [ + "GITHUB_TOKEN=\"U2FsdGVkX19pZyo9s0+7a8o2ShJ7rk9iDq/27GGmg+tg692sK0ezyqzVDmVfjtUd+NGfVbh+z+Bk3UWf8xwM8Q==\"", + ] + )) + + Dict( + :label => "jll_init -- $NAME", + :agents => agent(), + :plugins => init_plugins, + :timeout_in_minutes => 10, + :concurrency => 1, + :concurrency_group => "yggdrasil/jll_init", + :commands => [script], + :env => env(NAME, PROJECT) + ) +end + +function build_step(NAME, PLATFORM, PROJECT) + script = raw""" + # Don't share secrets with build_tarballs.jl + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET="" AWS_SECRET_ACCESS_KEY="" .buildkite/build.sh + """ + + build_plugins = plugins() + push!(build_plugins, + "staticfloat/cryptic#v2" => Dict( + "variables" => [ + "AWS_SECRET_ACCESS_KEY=\"U2FsdGVkX1846b0BRbZjwIWSFV+Fiv1C/Hds/vB3aTkxubHPnRP6lVxGkAkOcFuvAntkoLF6J64QrOHWvjz8xg==\"", + ] + ), + "staticfloat/coppermind#v2" => Dict( + "inputs" => [ + PROJECT, + ".ci/", + # ?meta.json + ], + "s3_prefix" => "s3://julia-bb-buildcache/" + ), + ) + build_env = env(NAME, PROJECT) + merge!(build_env, Dict( + "PLATFORM" => PLATFORM, + "BINARYBUILDER_AUTOMATIC_APPLE" => "true", + "BINARYBUILDER_USE_CCACHE" => "true", + "BINARYBUILDER_STORAGE_DIR" => "/cache/yggdrasil", + "BINARYBUILDER_NPROC" => "16", # Limit parallelism somewhat to avoid OOM for LLVM + "AWS_ACCESS_KEY_ID" => "AKIA4WZGSTHCB2YWWN46", + "AWS_DEFAULT_REGION" => "us-east-1", + )) + + + Dict( + :key => "$NAME--$PLATFORM", + :label => "build -- $NAME -- $PLATFORM", + :agents => agent(), + :plugins => build_plugins, + :timeout_in_minutes => 180, + :priority => -1, + :concurrency => 16, + :concurrency_group => "yggdrasil/build/$NAME", # Could use ENV["BUILDKITE_JOB_ID"] + :commands => [script], + :env => build_env, + :artifacts => [ + "**/products/$NAME*.tar.gz" + ] + ) +end + +function register_step(NAME, PROJECT, SKIP_BUILD) + script = raw""" + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET="" .buildkite/register.sh + """ + + register_plugins = plugins() + push!(register_plugins, + "staticfloat/cryptic#v2" => Dict( + "variables" => [ + "GITHUB_TOKEN=\"U2FsdGVkX19pZyo9s0+7a8o2ShJ7rk9iDq/27GGmg+tg692sK0ezyqzVDmVfjtUd+NGfVbh+z+Bk3UWf8xwM8Q==\"", + ] + )) + + register_env = env(NAME, PROJECT) + if SKIP_BUILD + register_env["SKIP_BUILD"] = "true" + end + + Dict( + :label => "register -- $NAME", + :agents => agent(), + :plugins => register_plugins, + :timeout_in_minutes => 30, + :concurrency => 1, + :concurrency_group => "yggdrasil/register", + :commands => [script], + :env => register_env + ) +end diff --git a/.ci/azp_agent/.env b/.ci/azp_agent/.env deleted file mode 100644 index 9871c6dcb661de3e1c9e512ab0ee5a6e7cd9492c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 488 zcmV#C6FnELDM0y`0wJFAr1-5Nj6^w$=<#RKk;T9i|Y zrNmdFI@ij;6Xlu`iq7%O_)8jaI8X(mPS$K8{li8}d+UyRaIBxU-zQGmh0p;C9!&{6 zt05ETV$li&VNR}Itz+DydYZ^(of%G;*FRmb<<_MJ~Q$-3)Rh%_#U0ox-r#g zDQjZC`mc9MM!Ux1L}m|1Gw}uf@v2U@PVIZT9X(noh%Hy5#?RB*AG<;jaWP%6wUBOX zDWSShexA(q<7#tf2F(x+Ap6A(AWR~DaI3d!g+ReggH{lE3G`B94x4e%O{W-4ldmZ3 z;2C~=S*phUO8y$2h;jAi8u>c~B6p_^bvW_y-}mku_<~E!eW@uHu^Ic(&(?JUs1Ax1 zI7Y9Un^$E__uw@=DZ>Ga5|4@#0bZL7)*RjKKIgf%tmF{Y415%$`NohD47 e(U8;*&%^InnpepLLttA81*6OLVpaB~WzFZ3Nbd6h diff --git a/.ci/azp_agent/.gitattributes b/.ci/azp_agent/.gitattributes deleted file mode 100644 index de63977fba3..00000000000 --- a/.ci/azp_agent/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -.env filter=git-crypt diff=git-crypt -*_rsa filter=git-crypt diff=git-crypt diff --git a/.ci/azp_agent/agent_startup.conf b/.ci/azp_agent/agent_startup.conf deleted file mode 100644 index 6e672791bce..00000000000 --- a/.ci/azp_agent/agent_startup.conf +++ /dev/null @@ -1,33 +0,0 @@ -[Unit] -Description = Yggdrasil AZP Agent ${AGENT_IDX} -StartLimitIntervalSec = 0 -StartLimitBurst = 0 -Wants = loopback_startup.service - -[Service] -PermissionsStartOnly = true -Environment = AGENT_IDX=${AGENT_IDX} -EnvironmentFile = ${SRC_DIR}/.env -ExecStartPre = /bin/bash -c "mkdir -p ${STORAGE_DIR}/depot" -ExecStartPre = /bin/bash -c "mkdir -p ${STORAGE_DIR}/logs/agent_${AGENT_IDX}" -ExecStartPre = /bin/bash -c "mkdir -p ${STORAGE_DIR}/work/agent_${AGENT_IDX}" -ExecStartPre = /bin/bash -c "mkdir -p ${STORAGE_DIR}/tmp/agent_${AGENT_IDX}" -ExecStartPre = /bin/bash -c "mkdir -p ${STORAGE_DIR}/tmp/agent_${AGENT_IDX}_registries" -ExecStartPre = /bin/bash -c "mkdir -p ${STORAGE_DIR}/tmp/agent_${AGENT_IDX}_registries_binarybuilder" -ExecStart = ${STORAGE_DIR}/rootfs/sandbox --verbose --rootfs=${STORAGE_DIR}/rootfs \ - --map=${STORAGE_DIR}/rootfs:/rootfs \ - --workspace=${STORAGE_DIR}/logs/agent_${AGENT_IDX}:/agent/_diag \ - --workspace=${STORAGE_DIR}/work/agent_${AGENT_IDX}:/agent/_work \ - --workspace=${STORAGE_DIR}/tmp/agent_${AGENT_IDX}:/tmp \ - --workspace=${STORAGE_DIR}/tmp/agent_${AGENT_IDX}_registries:/depot/registries \ - --workspace=${STORAGE_DIR}/tmp/agent_${AGENT_IDX}_registries_binarybuilder:/depot/registries_binarybuilder \ - --workspace=${STORAGE_DIR}/depot:/depot \ - --workspace=${STORAGE_DIR}/tmp/agent_${AGENT_IDX}:/tmp \ - -- /bin/bash -c "/run_agent.sh" - -# Always restart, don't limit restarts at all, restart once per second -Restart = always -RestartSec = 1 - -[Install] -WantedBy = default.target diff --git a/.ci/azp_agent/debug_agent.sh b/.ci/azp_agent/debug_agent.sh deleted file mode 100755 index 7ef894826a7..00000000000 --- a/.ci/azp_agent/debug_agent.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# First, source .env to get the environment -source .env - -# Next, extract the heinous `sandbox` invocation, with some even more heinous `sed` commands: -SANDBOX_CMD=$(sed -n -E '/ExecStart\s+=/,/bin\/bash/p' agent_startup.conf | sed -E 's/ExecStart\s+=//' | sed -E 's_/bin/bash -c .*_/bin/bash_') - -# If an agent index has been given, use it, otherwise default to agent 1 -if [[ ! -z "$1" ]]; then - export AGENT_IDX=$(echo "$1" | sed -e 's/agent_//') -else - export AGENT_IDX=1 -fi - -# Run the `sandbox` command: -eval "${SANDBOX_CMD}" diff --git a/.ci/azp_agent/install_agents.sh b/.ci/azp_agent/install_agents.sh deleted file mode 100755 index 56df372e7a2..00000000000 --- a/.ci/azp_agent/install_agents.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -set -e - -REQUIRED_TOOLS="debootstrap jq" -for TOOL in ${REQUIRED_TOOLS}; do - if [[ -z $(PATH=/usr/sbin:$PATH which "${TOOL}") ]]; then - echo "Must install '${TOOL}'" - fi -done - -NUM_AGENTS=16 -SRC_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -mkdir -p ${HOME}/.config/systemd/user -source .env - -# Create a nice little rootfs for our agents -if [[ ! -d "${STORAGE_DIR}/rootfs" ]] || [[ ! -d "${STORAGE_DIR}/rootfs/etc" ]]; then - echo "Setting up rootfs..." - mkdir -p "${STORAGE_DIR}/rootfs" - sudo debootstrap --variant=minbase --include=ssh,curl,libicu63,git,xz-utils,bzip2,unzip,p7zip,zstd,expect,locales,libgomp1 buster "${STORAGE_DIR}/rootfs" - - # Remove special `dev` files - sudo rm -rf "${STORAGE_DIR}/rootfs/dev/*" - # take ownership - sudo chown $(id -u):$(id -g) -R "${STORAGE_DIR}/rootfs" - # Remove `_apt` user so that `apt` doesn't try to `setgroups()` - sed '/_apt:/d' -i "${STORAGE_DIR}/rootfs/etc/passwd" - - # Set up the one true locale - echo "en_US.UTF-8 UTF-8" >> ${STORAGE_DIR}/rootfs/etc/locale.gen - sudo chroot ${STORAGE_DIR}/rootfs locale-gen -fi - -if [[ ! -f "${STORAGE_DIR}/rootfs/etc/gitconfig" ]]; then - # Add `git` username - echo "[user]" > ${STORAGE_DIR}/rootfs/etc/gitconfig - echo " email = juliabuildbot@gmail.com" >> ${STORAGE_DIR}/rootfs/etc/gitconfig - echo " name = jlbuild" >> ${STORAGE_DIR}/rootfs/etc/gitconfig -fi - -# Add SSH keys -SSH_DIR="${STORAGE_DIR}/rootfs/root/.ssh" -if [[ ! -d "${SSH_DIR}" ]]; then - mkdir -p "${SSH_DIR}" - cp -a ./yggdrasil_rsa "${SSH_DIR}/id_rsa" - chmod 0600 "${SSH_DIR}/id_rsa" - chmod 0700 "${SSH_DIR}" -fi - -if [[ ! -f "${STORAGE_DIR}/rootfs/usr/local/bin/julia" ]]; then - # Install Julia into the rootfs - echo "Installing Julia..." - JULIA_URL="https://julialang-s3.julialang.org/bin/linux/x64/1.7/julia-1.7.1-linux-x86_64.tar.gz" - curl -# -L "$JULIA_URL" | tar --strip-components=1 -zx -C "${STORAGE_DIR}/rootfs/usr/local" -fi - -if [[ ! -f "${STORAGE_DIR}/rootfs/sandbox" ]]; then - # Install `sandbox` and `run_agent.sh` into the rootfs - echo "Installing sandbox..." - SANDBOX_URL="https://github.com/JuliaPackaging/Yggdrasil/raw/master/0_RootFS/Rootfs/bundled/utils/sandbox" - curl -# -L "${SANDBOX_URL}" -o "${STORAGE_DIR}/rootfs/sandbox" - chmod +x "${STORAGE_DIR}/rootfs/sandbox" - cp -a "${SRC_DIR}/run_agent.sh" "${STORAGE_DIR}/rootfs/run_agent.sh" -fi - -if [[ ! -d "${STORAGE_DIR}/rootfs/agent" ]]; then - # Install agent executable - AZP_AGENTPACKAGE_URL=$( - curl -LsS -u "user:${AZP_TOKEN}" \ - -H 'Accept:application/json;api-version=3.0-preview' \ - "${AZP_URL}/_apis/distributedtask/packages/agent?platform=linux-x64" | - jq -r '.value | map([.version.major,.version.minor,.version.patch,.downloadUrl]) | sort | .[length-1] | .[3]' - ) - - if [ -z "$AZP_AGENTPACKAGE_URL" -o "$AZP_AGENTPACKAGE_URL" == "null" ]; then - echo 1>&2 "error: could not determine a matching Azure Pipelines agent - check that account '$AZP_URL' is correct and the token is valid for that account" - exit 1 - fi - - echo "Installing AZP agent..." - mkdir -p "${STORAGE_DIR}/rootfs/agent" - curl -LsS "$AZP_AGENTPACKAGE_URL" | tar -xz -C "${STORAGE_DIR}/rootfs/agent" -fi - -# Install mknod startup unit -envsubst "\$SRC_DIR" <"loopback_startup.conf" >"${HOME}/.config/systemd/user/loopback_startup.unit" - -for AGENT_IDX in $(seq 1 $NUM_AGENTS); do - export SRC_DIR STORAGE_DIR AGENT_IDX - envsubst "\$SRC_DIR \$STORAGE_DIR \$AGENT_IDX" <"agent_startup.conf" >"${HOME}/.config/systemd/user/azp_agent_${AGENT_IDX}.service" -done - -# Reload systemd user daemon -systemctl --user daemon-reload - -for AGENT_IDX in $(seq 1 ${NUM_AGENTS}); do - systemctl --user stop azp_agent_${AGENT_IDX} || true - # Enable and start AZP agents - systemctl --user enable azp_agent_${AGENT_IDX} - systemctl --user start azp_agent_${AGENT_IDX} -done diff --git a/.ci/azp_agent/loopback_startup.conf b/.ci/azp_agent/loopback_startup.conf deleted file mode 100644 index 2692722ed5d..00000000000 --- a/.ci/azp_agent/loopback_startup.conf +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description = Loopback setup - -[Service] -ExecStart = ${SRC_DIR}/setup_loopback.sh - -[Install] -WantedBy = default.target diff --git a/.ci/azp_agent/restart_agents.sh b/.ci/azp_agent/restart_agents.sh deleted file mode 100755 index 4c01ef03990..00000000000 --- a/.ci/azp_agent/restart_agents.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -NUM_AGENTS=8 -for AGENT_IDX in $(seq 1 ${NUM_AGENTS}); do - systemctl --user restart azp_agent_${AGENT_IDX} -done diff --git a/.ci/azp_agent/run_agent.sh b/.ci/azp_agent/run_agent.sh deleted file mode 100755 index 21d741521f9..00000000000 --- a/.ci/azp_agent/run_agent.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -set -e - -if [ -z "$AZP_URL" ]; then - echo 1>&2 "error: missing AZP_URL environment variable" - exit 1 -fi - -if [ -z "$AZP_TOKEN" ]; then - echo 1>&2 "error: missing AZP_TOKEN environment variable" - exit 1 -fi - -# Tell the agent to ignore the token env variables -export VSO_AGENT_IGNORE=AZP_TOKEN -export AGENT_ALLOW_RUNASROOT=1 - -cd /agent -./config.sh --unattended \ - --agent "${HOSTNAME}.${AGENT_IDX}" \ - --url "${AZP_URL}" \ - --auth PAT \ - --token "${AZP_TOKEN}" \ - --pool "${AZP_POOL:-Default}" \ - --work _work \ - --replace \ - --acceptTeeEula & wait $! - -exec ./externals/node/bin/node ./bin/AgentService.js interactive diff --git a/.ci/azp_agent/setup_loopback.sh b/.ci/azp_agent/setup_loopback.sh deleted file mode 100755 index 90173e56527..00000000000 --- a/.ci/azp_agent/setup_loopback.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -NUM_LOOPBACKS=64 -echo -n "Creating $NUM_LOOPBACKS loopback devices..." -for idx in $(seq 1 $((NUM_LOOPBACKS - 1))); do - if [ -e /dev/loop${idx} ]; then - continue - fi - - sudo mknod /dev/loop${idx} b 7 ${idx} - sudo chown --reference=/dev/loop0 /dev/loop${idx} - sudo chmod --reference=/dev/loop0 /dev/loop${idx} - echo -n "." -done -echo \ No newline at end of file diff --git a/.ci/azp_agent/stop_agents.sh b/.ci/azp_agent/stop_agents.sh deleted file mode 100755 index 2fb6c75aa66..00000000000 --- a/.ci/azp_agent/stop_agents.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -NUM_AGENTS=8 -for AGENT_IDX in $(seq 1 ${NUM_AGENTS}); do - systemctl --user stop azp_agent_${AGENT_IDX} -done diff --git a/.ci/azp_agent/yggdrasil_rsa b/.ci/azp_agent/yggdrasil_rsa deleted file mode 100644 index 60de46f771ed58bfe05f0dee50a87e9e5e32139b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3269 zcmV;$3_9}wM@dveQdv+`0I51kD8&gy^IrT3H9DZTEf@MOP4=n;lXHD26dMs(OphO?)XQd-lDd;opu(Lz6?0 z^3T8E4}O~AK$XGV_Za`Y@Q$1s%5RRaU%43x)Q3mcUHE|R4)$!KSJ2K6%V^F8) zEtvvea?h}H{L}{3=d!L;5yWH);#m8QJ8n%Gg$7S`^+*nzFWZ&Ol%W^o{C}?B<1b=t zGB3r*l=Do6g#_H3Kv>*uU)DCq0S)IS#WRNGt&s07a?MUz)pig-#2x0Xfb9SS>b)@{HyP0c)ZefdbO1*y9poEA0sh4(sve zV3kg_t>P8}^qaVe0R|fhf?X4|zFo5`+$*FF|o@dqXQ5`K~*{1hAxj0GddL8k_q zy!7O;+hNm3XJkP!cRMy)Ay71!`!aeFwLlyj{TJ8i|akO zsmU_HPz7vBL!ANEpzPISb_$r(Qt~9`Hu;)zJ|OI1nEyc;>R1e=2*oq_HLjX)>W4+c z+KCTVu58VFWo%ypeK)YCeZATZl~Oa@0WO z9~JSh89Vc2B82g&3wZEOGy(<1zl2z<$qnotQWB>wZr|#}Bn%QyM!Dj;9)M7rm-mE zO)U`BrL3g7#O)^;2p7&^M*j4oQa1*2j_7%FCGf`I09_w-qytvp!l)<~XmxGOKwVIa zmq`gip6|mZ`yt}>ax+Oe^PhTHXT)|fiF~f2+U1-E)nuT>)VJU&-+iKP=u%rqVq{84 za=NMETw&l9sWUIB?d+`@LF>HvUxqYG9YBn^z9QT-*iK^RF60iTp`Ar!fa9(wsy6G{o zn9Fi&U1Gpjwx@_eeb36`70T|kh4y!=MA|Hv#0rPX{tcT0)gvUr_9g~1Uc?5(^W8BT zcQJF0E9fh0X}akylNqV*BMoGDiNFK_qC8nI>Xtfa)BQ&Ebs3Qoio-R~p9T-BxxCLBY-=aC985COG}?294JCzyZd`MOI$5%y$a9V;6?>qheB2 zOYA?u?9NQTw_+!3Vh8|in+hrFhaJ*)W%%rt-*9dieL(3?{^o2P&+A-SB9oOgwC{&9 zQv?huO2K#bpJ>jI!}IgJr_hbBEu3zD8@V8@Xvi<&sRc2h(OPg;4KD;;*T8GGQ0sTT z&_nB?7J{~OP%E{&*2W4a13Tz1@-*=%1+!gJeJX(4_8WM_y3e(p<$-c|{JO)&vbpvw zC@a6N>F&#wxIng|cP9{~h7^@VYDNDD3!72or`G>=XBSW3p|Hf~J1J6h!l z^9zZ~OyF}*>B3m(mfcu_U=jXzSpP)5uAKN!pHZTwWu3f+ZiUh{oKP=7 zAbxz(Rt-(+pgC0n946oI#rX{W9R(-J+bUeNwrOZ@7c!CNe2e1EGQ8Zs3Cb6<{8Y*b z@J~t~)5EPgGiKW({|tnW&{1@x1JDH+KZwUGKIgpYyesc>X>ZdfdUd31#+2)qBFvr$ z1J+DUOf$!r^o$jRsB2DyqK*~yu2r_Y8g^AOkZ{6-_PjJ>S9M2wRpCq~69-oFf}^8! zyuNDfuDh!6^$)-Kf`Fxg*a1}@kx@|g&TmsX{Dd?~Wx<~NB#74_mJ2U)7U4C03R^4+ z#4WZd+Xi?LsUH+h<NEyvX$e)QGNZZ+$_6Kpc zi=~{EzI_Kz2B=dAB+wKlPJt%tk0~}S;}JXY0bY>SHM1kNPHj?ldIL5Cn{SJH!4P zcVJ)KW0-(A>dYhMU{YlQOKrUiaad0Ps6sn)K001t-Gkf1 zPY3qb5)cIC)uW8H022NSCyUcBl^_kZnCfRfT9MOPowrk^1Jp>?GNYYE-@+a>rxs8@r6$xT>dacJW zGh+&X8ML7w-6jU(3sHF4Ay8>)s*-ox@r8hrK)y|I?m99da-wI1-**U#JaQzL`=(A!MmuDJ2%a{F`YfYQnx_Z|)W{QX|HJW+_!kz7AtMWkVkl z>TlbG7N>2L8e&k}8d`bo5GelAj;OM2Y-q;0_U*W!JRp}!C}=*tN0#m$dz|394TMab zIFfS@QI~IDl6f`+I6>7~qj3vw==aj`l%VLdJP|#r>FXit*HHAJa0>&6?9_2<%2<+I zqKgHheF5O57S60<;E2HKSFp?#%@%B%T+-RXVt~Jks7eLb-w{>91da@h(3)pHp|wZ5 zd-WZ0tZLK`+EH}`G*@~5~?DI{@YK7@C-Mgdg%6IHm(=I z|A+=^tYGeZi&lNJLO7+WsQlk(bsLj9gWaKhyK34luMKvX)p@I#l)sveOqYYMHQnv_ zI)z8C8)<5^Rw!3rVcxxWe(c|2JDpYKzzu#PN|o1n+s5R-O@{;8#TTx9eI@5Cuiyj^ zuI#<3<2Tz>obwQ14(%GvjPJC~9iEqiTs?dg=iiT8lC>hu7E{I9YEN)hA!LKf%X=CbZxRu1e@5k|%IsBcf(TkUf(sI_78ueYc;OpPmWkix3G?=a0gr6_ z2}tl{aY~cipViT0vKmSms3(0c5<1UuMXw`>5^-9UZve~33j&C>fPlJj``ohV?NFdC z6-tU|8q|p0ap)Y?SKQp1MC4)K03PbT$Vi6sRY>?PBt(H~fhxWIzZbur_|#9W>T;I7 z3zhiVu)`@z52%8I_qG!MlPOPd%J~w2b_o~hAGnDcL8_>y#OA_HF<1~5R=fOev_ZO3 z(W=szQ};KC*`eHz=MA)OezV;qqF9sK3*9o}nG%dRgyXoUf1a83^fX_`wO@z~-?6C- znnwILn-V}vScM`!J^Ny0?oM6Fmm}SO6gVRukHu_H%0z_rDE)X|eeBb70<2#ppFY|2 z7T|QLjf_sjpN`0%lu$n>uBxXdQuWHT*p~Jzam2eU3vRzp4z3P9tUdS3VVvoox8L|U z`aeNvUR`Rh3-IccRuK_WiO5(&CZ#Bm(+De=^RX){rWDAvo{RhBOt-2ka>#P3mxb2- zUlli1HFGFc;&l-HttULKi6$*|j6A0X#w?-1$IFbR-q@Q1&9aNuv{i!P$XhN~Ge7!1 zKxk%}t}UDMr|U8b{sru^3o-%j(ox0HZfHbjE?BC<2Po?facNqn*L_6eU6}veQ6A#c z3}M^hNMZCPOOH=ymU<%L3`z(0?pNDaraAf|&Rghyp$oq8_w(;P%SMV`LyqM9Fr_g7 Da=lK$ diff --git a/.ci/register_package.jl b/.ci/register_package.jl index 94c178ba810..1221af5a5ac 100644 --- a/.ci/register_package.jl +++ b/.ci/register_package.jl @@ -48,19 +48,21 @@ function reset_downloader() end end -function download_cached_binaries(download_dir, platforms) - # Grab things out of the aether for maximum consistency - bb_hash = ENV["BB_HASH"] - proj_hash = ENV["PROJ_HASH"] - - for platform in platforms, suffix in ("", "-logs") - url = "https://julia-bb-buildcache.s3.amazonaws.com/$(bb_hash)/$(proj_hash)/$(triplet(platform))$(suffix).tar.gz" - filename = "$(name)$(suffix).v$(version).$(triplet(platform)).tar.gz" - reset_downloader() - println("Downloading $url...") - Downloads.download(url, joinpath(download_dir, filename)) - println() +function mvdir(src, dest) + for file in readdir(src) + mv(joinpath(src, file), joinpath(dest, file)) + end +end + +function download_cached_binaries(download_dir) + NAME = ENV["NAME"] + PROJECT = ENV["PROJECT"] + artifacts = "$(PROJECT)/products/$(NAME)*.tar.gz" + cmd = `buildkite-agent artifact download $artifacts $download_dir` + if !success(pipeline(cmd; stderr)) + error("Download failed") end + mvdir(joinpath(download_dir, PROJECT, "products"), download_dir) end function download_binaries_from_release(download_dir) @@ -74,7 +76,7 @@ function download_binaries_from_release(download_dir) println("done") end - # Doownload the tarballs reading the information in the current `Artifacts.toml`. + # Download the tarballs reading the information in the current `Artifacts.toml`. artifacts = Pkg.Artifacts.load_artifacts_toml(joinpath(code_dir, "Artifacts.toml"))[name] if artifacts isa Dict # If it's a Dict, that means this is an AnyPlatform artifact, act accordingly. @@ -103,7 +105,7 @@ mktempdir() do download_dir else # We are going to publish the new binaries we've just baked, take them # out of the cache while they're hot. - download_cached_binaries(download_dir, merged["platforms"]) + download_cached_binaries(download_dir) end # Push up the JLL package (pointing to as-of-yet missing tarballs) diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index c5327a323ea..00000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,366 +0,0 @@ -# Trigger on pushes to `master` -trigger: -- master - -# Trigger on PRs against `master` -pr: -- master - -# By default, use the `Native` pool of agents -pool: Native - -variables: - JULIA: unbuffer julia --project=$(Build.SourcesDirectory)/.ci --color=yes - # We limit our parallelism somewhat in order to avoid strange OOM errors while building LLVM - BINARYBUILDER_NPROC: 16 - JULIA_PKG_SERVER: us-east.pkg.julialang.org - # Use eager registry to not have to wait for updates of the conservative registry - JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager - -jobs: -- job: generator - steps: - - checkout: self - fetchDepth: 99999 - clean: true - - bash: | - # Be fragile, like a beautiful porcelain doll - set -e - - # Normally we look at the last pushed commit - COMPARE_AGAINST="HEAD~1" - # Keyword to be used in the commit message to skip a rebuild - SKIP_BUILD_COOKIE="[skip build]" - # This variable will tell us whether we want to skip the build - export SKIP_BUILD="false" - - if [[ $(Build.Reason) == "PullRequest" ]]; then - # If we're on a PR though, we look at the entire branch at once - TARGET_BRANCH="remotes/origin/$(System.PullRequest.TargetBranch)" - COMPARE_AGAINST=$(git merge-base --fork-point ${TARGET_BRANCH} HEAD) - git fetch origin "refs/pull/$(System.PullRequest.PullRequestNumber)/head:refs/remotes/origin/pr/$(System.PullRequest.PullRequestNumber)" - if [[ "$(git show -s --format=%B origin/pr/$(System.PullRequest.PullRequestNumber))" == *"${SKIP_BUILD_COOKIE}"* ]]; then - SKIP_BUILD="true" - fi - else - if [[ "$(git show -s --format=%B)" == *"${SKIP_BUILD_COOKIE}"* ]]; then - SKIP_BUILD="true" - fi - fi - - $(JULIA) --compile=min -O0 -e 'using InteractiveUtils, Pkg, Downloads, Dates - versioninfo() - if !isnothing(Pkg.pkg_server()) - resp = try - headers = Pkg.PlatformEngines.get_metadata_headers(Pkg.pkg_server()) - Downloads.request("$(Pkg.pkg_server())/registries"; headers) - catch e - # Let us know the download of the registry went wrong, but do not hard fail - @error "Could not download the registry" exception=(e, catch_backtrace()) - exit(0) - end - last_mod_idx = findfirst(h -> first(h) == "last-modified", resp.headers) - msg = "PkgServer: " * resp.url - delay = if !isnothing(last_mod_idx) - last_mod = last(resp.headers[last_mod_idx]) - msg *= " -- last updated: " * last_mod - # Manually strip the "GMT" timezone and hope it never changes. - # Do not error out if parsing fails. - dt = tryparse(DateTime, replace(last_mod, " GMT"=>""), dateformat"e, d u y H:M:S") - # If parsing did fail, set the delay to 0. - isnothing(dt) ? Second(0) : now(UTC) - dt - else - Second(0) - end - delay > Second(0) && (msg *= " (" * string(Dates.canonicalize(round(delay, Second))) * " ago)") - @info(msg) - tolerance = Hour(1) - delay > tolerance && @warn "The PkgServer registry is older than $(tolerance)" - end' - - # Get the directories holding changed files - # 1. All changed files - # 2. Only files in directories - # 3. dirname - # 4. Unique the directories - PROJECTS=$(git diff-tree --no-commit-id --name-only -r HEAD "${COMPARE_AGAINST}" | grep -E ".+/.+"| sed 's#/[^/]*$##' | sort -u) - - # If there are scary projects we need to exclude, we list them here. (Used to contain `LLVM`) - EXCLUDED_NAMES=" " - - # This is the dynamic mapping we're going to build up, if it's empty we don't do anything - PROJECTS_ACCEPTED=() - for PROJECT in ${PROJECTS}; do - NAME=$(basename "${PROJECT}") - echo "Considering ${PROJECT}" - # Only accept things that contain a `build_tarballs.jl` - while [[ ! -f "${PROJECT}/build_tarballs.jl" ]] && [[ "${PROJECT}" == */* ]]; do - echo " --> ${PROJECT} does not contain a build_tarballs.jl, moving up a directory" - PROJECT="$(dirname "${PROJECT}")" - done - if [[ "${PROJECT}" != */* ]]; then - echo " --> Skipping as we could not find a build_tarballs.jl" - continue - fi - - # Ignore RootFS stuff, we'll do that manually - if [[ "${PROJECT}" == "0_RootFS/"* ]]; then - echo " --> Skipping as it's within 0_RootFS/" - continue - fi - - # Ignore stuff in our excluded projects - if [[ "${EXCLUDED_NAMES}" == *" ${NAME} "* ]]; then - echo " --> Skipping as it's excluded" - continue - fi - - # Otherwise, emit a build with `PROJECT` set to `${PROJECT}` - if [[ " ${PROJECTS_ACCEPTED[@]} " =~ " ${PROJECT} " ]]; then - echo " --> Already in accepted projects, skipping" - else - echo " --> Accepted!" - PROJECTS_ACCEPTED+=("${PROJECT}") - fi - done - if [[ -n "${PROJECTS_ACCEPTED[@]}" ]]; then - if [[ ${#PROJECTS_ACCEPTED[@]} -gt 20 ]]; then - echo "Too many projects requested" - exit 1 - fi - - # Next, we're going to ensure that our BB is up to date and precompiled - $(JULIA) -e "import Pkg; Pkg.instantiate(); Pkg.precompile()" - - # We're going to snarf out the BB and BBB tree hashes and combine them to be used later in our build cache - BB_HASH=$($(JULIA) -e "using Pkg, SHA; \ - gethash(uuid) = collect(Pkg.Types.Context().env.manifest[Pkg.Types.UUID(uuid)].tree_hash.bytes); \ - print(bytes2hex(sha256(vcat( \ - gethash(\"7f725544-6523-48cd-82d1-3fa08ff4056e\"), \ - gethash(\"12aac903-9f7c-5d81-afc2-d9565ea332ae\"), \ - ))));") - - # Next, for each project, download its sources. We do this by generating meta.json - # files, then parsing them with `download_sources.jl` - for PROJECT in "${PROJECTS_ACCEPTED[@]}"; do - NAME=$(basename ${PROJECT}) - - # We always invoke a `build_tarballs.jl` file from its own directory - pushd ${PROJECT} >/dev/null - echo "Generating meta.json..." - JSON_PATH="$(Agent.TempDirectory)/${NAME}.meta.json" - $(JULIA) --compile=min ./build_tarballs.jl --meta-json="${JSON_PATH}" - - echo "Downloading sources..." - $(JULIA) $(Build.SourcesDirectory)/.ci/download_sources.jl "${JSON_PATH}" $(Agent.TempDirectory)/${NAME}.platforms.list - - # Pop back up to the overworld - popd >/dev/null - done - - # Emit project variable declarations - for PROJECT in "${PROJECTS_ACCEPTED[@]}"; do - NAME=$(basename ${PROJECT}) - done - - # Emit project/platform joint variable declarations - VAR_PROJECTS="##vso[task.setVariable variable=projects;isOutput=true]{" - VAR_PROJPLATFORMS="##vso[task.setVariable variable=projplatforms;isOutput=true]{" - echo "Determining builds to queue..." - for PROJECT in "${PROJECTS_ACCEPTED[@]}"; do - NAME=$(basename ${PROJECT}) - - # "project source hash" is a combination of meta.json (to absorb - # changes from include()'ing a `common.jl`) as well as the entire - # tree the project lives in (to absorb changes from patches) - # In order to support symlinked directories as dependencies we calculate - # the tree hash on a TMP_PROJECT using `cp -RL` to resolve the symlink and - # hash the actual content. - TMP_PROJECT=$(mktemp -d) - cp -RL ${PROJECT} ${TMP_PROJECT} - TREE_HASH=$($(JULIA) -e "using Pkg; print(bytes2hex(Pkg.GitTools.tree_hash(\"${TMP_PROJECT}\")))") - META_HASH=$(shasum -a 256 "$(Agent.TempDirectory)/${NAME}.meta.json" | cut -d' ' -f1) - PROJ_HASH=$(echo -n ${TREE_HASH}${META_HASH} | shasum -a 256 | cut -d' ' -f1) - - # Load in the platforms - PLATFORMS=$(cat $(Agent.TempDirectory)/${NAME}.platforms.list) - if [[ -z "${PLATFORMS}" ]]; then - echo "##vso[task.logissue type=error]Unable to determine the proper platforms for ${NAME}" - continue - fi - - # That's everything we need to know for `$(PROJECTS)` later on down - VAR_PROJECTS="${VAR_PROJECTS} '${NAME}':{ \ - 'NAME': '${NAME}', \ - 'PROJECT':'${PROJECT}', \ - 'PLATFORMS':'${PLATFORMS}', \ - 'BB_HASH':'${BB_HASH}', \ - 'PROJ_HASH':'${PROJ_HASH}', \ - 'SKIP_BUILD':'${SKIP_BUILD}' \ - }, " - - # Some debugging info - echo " ---> ${NAME}: ${BB_HASH}/${PROJ_HASH} (${TREE_HASH} + ${META_HASH})" - - # For $(PROJPLATFORMS)`, we need to know more... - for PLATFORM in ${PLATFORMS}; do - if [[ "${SKIP_BUILD}" == "true" ]]; then - echo "The commit messages contains ${SKIP_BUILD_COOKIE}, skipping build" - break - fi - - check_exists() { - CACHE_URL="https://julia-bb-buildcache.s3.amazonaws.com/${BB_HASH}/${PROJ_HASH}/${1}.tar.gz" - curl --output /tmp/curl_${PROJ_HASH}_${1}.log --silent --include --HEAD "${CACHE_URL}" --write-out '%{http_code}' - } - - # Here, we hit the build cache to see if we can skip this particular combo - if [[ "$(check_exists ${PLATFORM})" == "200" && "$(check_exists ${PLATFORM}-logs)" == "200" ]]; then - echo " ${PLATFORM}: skipping, existant" - continue; - fi - echo " ${PLATFORM}: building" - - # # Debugging: let's see why `curl` failed: - # echo "CACHE_URL: ${CACHE_URL}" - # cat /tmp/curl_${PROJ_HASH}_${PLATFORM}.log || true - - # Otherwise, emit the build - VAR_PROJPLATFORMS="${VAR_PROJPLATFORMS} '${NAME}-${PLATFORM}':{ \ - 'NAME': '${NAME}', \ - 'PROJECT':'${PROJECT}', \ - 'PLATFORM':'${PLATFORM}', \ - 'PROJ_HASH':'${PROJ_HASH}', \ - 'BB_HASH':'${BB_HASH}' \ - }, " - done - rm -f /tmp/curl_${PROJ_HASH}*.log - done - # Add closing parens - VAR_PROJECTS="${VAR_PROJECTS} }" - VAR_PROJPLATFORMS="${VAR_PROJPLATFORMS} }" - - # Actually output the variables - echo "${VAR_PROJECTS}" - echo "${VAR_PROJPLATFORMS}" - fi - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) - name: mtrx - -- job: jll_init - dependsOn: generator - timeoutInMinutes: 10 - cancelTimeoutInMinutes: 2 - strategy: - matrix: $[ dependencies.generator.outputs['mtrx.projects'] ] - variables: - projects: $[ dependencies.generator.outputs['mtrx.projects'] ] - steps: - - script: | - # Fail on error - set -e - - cd $(PROJECT) - echo "Generating meta.json..." - $(JULIA) --compile=min ./build_tarballs.jl --meta-json=$(Agent.TempDirectory)/$(NAME).meta.json - echo "Initializing JLL package..." - $(JULIA) $(Build.SourcesDirectory)/.ci/jll_init.jl "$(Agent.TempDirectory)/${NAME}.meta.json" - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) - displayName: "initialize JLL package" - condition: and(and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master')), ne(variables['projects'], '')) - -- job: build - dependsOn: - - generator - - jll_init - timeoutInMinutes: 240 - cancelTimeoutInMinutes: 2 - strategy: - matrix: $[ dependencies.generator.outputs['mtrx.projplatforms'] ] - variables: - projplatforms: $[ dependencies.generator.outputs['mtrx.projplatforms'] ] - steps: - - script: | - # Fail on error - set -e - - # Cleanup temporary things that might have been left-over - ./clean_builds.sh - ./clean_products.sh - - cd $(PROJECT) - $(JULIA) ./build_tarballs.jl --verbose $(PLATFORM) - - # After building, we take the single tarball produced with the proper NAME, and upload it: - TARBALLS=( ./products/${NAME%@*}*${PLATFORM}*.tar.gz ) - if [[ "${#TARBALLS[@]}" != 2 ]]; then - echo "Not exactly 2 tarballs? This isn't right!" >&2 - exit 1 - fi - - upload() { - for file in "${@}"; do - # Upload with curl - ACL="x-amz-acl:public-read" - CONTENT_TYPE="application/x-gtar" - BUCKET="julia-bb-buildcache" - if [[ "${file}" == *-logs.* ]]; then - BUCKET_PATH="${BB_HASH}/${PROJ_HASH}/${PLATFORM}-logs.tar.gz" - else - BUCKET_PATH="${BB_HASH}/${PROJ_HASH}/${PLATFORM}.tar.gz" - fi - DATE="$(date -R)" - S3SIGNATURE=$(echo -en "PUT\n\n${CONTENT_TYPE}\n${DATE}\n${ACL}\n/${BUCKET}/${BUCKET_PATH}" | openssl sha1 -hmac "${S3SECRET}" -binary | base64) - HOST="${BUCKET}.s3.amazonaws.com" - echo "Uploading artifact to https://${HOST}/${BUCKET_PATH}" - curl -X PUT -T "${file}" \ - -H "Host: ${HOST}" \ - -H "Date: ${DATE}" \ - -H "Content-Type: ${CONTENT_TYPE}" \ - -H "${ACL}" \ - -H "Authorization: AWS ${S3KEY}:${S3SIGNATURE}" \ - "https://${HOST}/${BUCKET_PATH}" - done - } - - upload "${TARBALLS[@]}" - - if [[ "$?" != 0 ]]; then - echo "Failed to upload artifact!" >&2 - exit 1 - fi - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) - S3KEY: $(S3KEY) - S3SECRET: $(S3SECRET) - BINARYBUILDER_NPROC: $(BINARYBUILDER_NPROC) - displayName: "run build_tarballs.jl" - condition: and(ne(variables['projplatforms'], ''), ne(variables['projplatforms'], '{ }')) - -- job: register - dependsOn: - - generator - - build - strategy: - matrix: $[ dependencies.generator.outputs['mtrx.projects'] ] - maxParallel: 1 - variables: - projects: $[ dependencies.generator.outputs['mtrx.projects'] ] - steps: - - script: | - # Fail on error - set -e - - cd $(PROJECT) - echo "Generating meta.json..." - $(JULIA) --compile=min ./build_tarballs.jl --meta-json=$(Agent.TempDirectory)/$(NAME).meta.json - echo "Registering $(NAME)..." - export BB_HASH PROJ_HASH - $(JULIA) $(Build.SourcesDirectory)/.ci/register_package.jl $(Agent.TempDirectory)/$(NAME).meta.json --verbose - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) - displayName: "register JLL package" - # We only register if this is on `master`; same as setting `${DEPLOY}` above. - condition: and(and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master')), ne(variables['projects'], ''))