From 61b6994884642f8fc7e18844d37dc70b9dfdb9ab Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 21 Jul 2021 13:10:03 -0400 Subject: [PATCH] CI: delete the `.buildkite/rootfs_images` directory (#41648) --- .buildkite/rootfs_images/Manifest.toml | 134 ----------------------- .buildkite/rootfs_images/Project.toml | 5 - .buildkite/rootfs_images/README.md | 5 - .buildkite/rootfs_images/llvm-passes.jl | 31 ------ .buildkite/rootfs_images/rootfs_utils.jl | 123 --------------------- 5 files changed, 298 deletions(-) delete mode 100644 .buildkite/rootfs_images/Manifest.toml delete mode 100644 .buildkite/rootfs_images/Project.toml delete mode 100644 .buildkite/rootfs_images/README.md delete mode 100755 .buildkite/rootfs_images/llvm-passes.jl delete mode 100644 .buildkite/rootfs_images/rootfs_utils.jl diff --git a/.buildkite/rootfs_images/Manifest.toml b/.buildkite/rootfs_images/Manifest.toml deleted file mode 100644 index d24e9a4ee166e..0000000000000 --- a/.buildkite/rootfs_images/Manifest.toml +++ /dev/null @@ -1,134 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -[[ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" - -[[Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" - -[[Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[Downloads]] -deps = ["ArgTools", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" - -[[InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[JLLWrappers]] -deps = ["Preferences"] -git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.3.0" - -[[LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" - -[[LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" - -[[LibGit2]] -deps = ["Base64", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" - -[[Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" - -[[MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" - -[[NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" - -[[Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" - -[[Preferences]] -deps = ["TOML"] -git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.2.2" - -[[Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[Random]] -deps = ["Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" - -[[Scratch]] -deps = ["Dates"] -git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda" -uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.1.0" - -[[Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" - -[[Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" - -[[UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" - -[[Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" - -[[ghr_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "f5c8cb306d4fe2d1fff90443a088fc5ba536c134" -uuid = "07c12ed4-43bc-5495-8a2a-d5838ef8d533" -version = "0.13.0+1" - -[[nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" - -[[p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/.buildkite/rootfs_images/Project.toml b/.buildkite/rootfs_images/Project.toml deleted file mode 100644 index 1dbde5ed9df66..0000000000000 --- a/.buildkite/rootfs_images/Project.toml +++ /dev/null @@ -1,5 +0,0 @@ -[deps] -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" -Scratch = "6c6a2e73-6563-6170-7368-637461726353" -ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533" diff --git a/.buildkite/rootfs_images/README.md b/.buildkite/rootfs_images/README.md deleted file mode 100644 index 1d3962c2bee3e..0000000000000 --- a/.buildkite/rootfs_images/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## Rootfs images - -Our CI setup makes use of rootfs images that contain our build tools. -These rootfs images are built using the fairly simple scripts held within this directory. -Most images are based on Debian, making use of `debootstrap` to provide a quick and easy rootfs with packages installed through an initial `apt` invocation. diff --git a/.buildkite/rootfs_images/llvm-passes.jl b/.buildkite/rootfs_images/llvm-passes.jl deleted file mode 100755 index bc6d57eb2f87e..0000000000000 --- a/.buildkite/rootfs_images/llvm-passes.jl +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env julia - -## This rootfs includes enough of a host toolchain to build the LLVM passes. -## Eventually, this image will probably be replaced with the actual builder image, -## as that will have the necessary toolchains as well, but that image is not built yet. - -include("rootfs_utils.jl") - -const tag_name, force_overwrite = get_arguments(ARGS, @__FILE__) - -# Build debian-based image with the following extra packages: -packages = [ - "bash", - "build-essential", - "cmake", - "curl", - "gfortran", - "git", - "less", - "libatomic1", - "m4", - "perl", - "pkg-config", - "python", - "python3", - "wget", -] -tarball_path = debootstrap("llvm-passes"; packages) - -# Upload it -upload_rootfs_image(tarball_path; tag_name, force_overwrite) diff --git a/.buildkite/rootfs_images/rootfs_utils.jl b/.buildkite/rootfs_images/rootfs_utils.jl deleted file mode 100644 index 82baeec7a0933..0000000000000 --- a/.buildkite/rootfs_images/rootfs_utils.jl +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env julia - -# This is an example invocation of `debootstrap` to generate a Debian/Ubuntu-based rootfs -using Scratch, Pkg, Pkg.Artifacts, ghr_jll, SHA, Dates - -# Utility functions -getuid() = ccall(:getuid, Cint, ()) -getgid() = ccall(:getgid, Cint, ()) - -function debootstrap(name::String; release::String="buster", variant::String="minbase", - packages::Vector{String}=String[], force::Bool=false) - if Sys.which("debootstrap") === nothing - error("Must install `debootstrap`!") - end - - tarball_path = joinpath(@get_scratch!("rootfs-images"), "$(name).tar.gz") - if !force && isfile(tarball_path) - @error("Refusing to overwrite tarball without `force` set", tarball_path) - error() - end - - artifact_hash = create_artifact() do rootfs - packages_string = join(push!(packages, "locales"), ",") - @info("Running debootstrap", release, variant, packages) - run(`sudo debootstrap --variant=$(variant) --include=$(packages_string) $(release) "$(rootfs)"`) - - # Remove special `dev` files - @info("Cleaning up `/dev`") - for f in readdir(joinpath(rootfs, "dev"); join=true) - # Keep the symlinks around (such as `/dev/fd`), as they're useful - if !islink(f) - run(`sudo rm -rf "$(f)"`) - end - end - - # take ownership of the entire rootfs - @info("Chown'ing rootfs") - run(`sudo chown $(getuid()):$(getgid()) -R "$(rootfs)"`) - - # Write out rootfs-info to contain a minimally-identifying string - open(joinpath(rootfs, "etc", "rootfs-info"), write=true) do io - write(io, """ - rootfs_type=debootstrap - release=$(release) - variant=$(variant) - packages=$(packages_string) - build_date=$(Dates.now()) - """) - end - - # Write out a reasonable default resolv.conf - open(joinpath(rootfs, "etc", "resolv.conf"), write=true) do io - write(io, """ - nameserver 1.1.1.1 - nameserver 8.8.8.8 - """) - end - - # Remove `_apt` user so that `apt` doesn't try to `setgroups()` - @info("Removing `_apt` user") - open(joinpath(rootfs, "etc", "passwd"), write=true, read=true) do io - filtered_lines = filter(l -> !startswith(l, "_apt:"), readlines(io)) - truncate(io, 0) - seek(io, 0) - for l in filtered_lines - println(io, l) - end - end - - # Set up the one true locale - @info("Setting up UTF-8 locale") - open(joinpath(rootfs, "etc", "locale.gen"), "a") do io - println(io, "en_US.UTF-8 UTF-8") - end - run(`sudo chroot --userspec=$(getuid()):$(getgid()) $(rootfs) locale-gen`) - end - - # Archive it into a `.tar.gz` file - @info("Archiving", tarball_path, artifact_hash) - archive_artifact(artifact_hash, tarball_path) - - return tarball_path -end - -function upload_rootfs_image(tarball_path::String; - github_repo::String="JuliaCI/rootfs-images", - tag_name::String, - force_overwrite::Bool) - # Upload it to `github_repo` - tarball_url = "https://github.com/$(github_repo)/releases/download/$(tag_name)/$(basename(tarball_path))" - @info("Uploading to $(github_repo)@$(tag_name)", tarball_url) - cmd = ghr_jll.ghr() - append!(cmd.exec, ["-u", dirname(github_repo), "-r", basename(github_repo)]) - force_overwrite && push!(cmd.exec, "-replace") - append!(cmd.exec, [tag_name, tarball_path]) - run(cmd) - return tarball_url -end - -# process command-line arguments - -function get_arguments(args::AbstractVector, script_file::AbstractString) - usage = "Usage: $(basename(script_file)) [--force-overwrite]" - length(args) < 1 && throw(ArgumentError(usage)) - length(args) > 2 && throw(ArgumentError(usage)) - tag_name = get_tag_name(args; usage) - force_overwrite = get_force_overwrite(args; usage) - return (; tag_name, force_overwrite) -end - -function get_tag_name(args::AbstractVector; usage::AbstractString) - tag_name = convert(String, strip(args[1]))::String - isempty(tag_name) && throw(ArgumentError(usage)) - startswith(tag_name, "--") && throw(ArgumentError(usage)) - return tag_name -end - -function get_force_overwrite(args::AbstractVector; usage::AbstractString) - force_overwrite_string = strip(get(args, 2, "")) - force_overwrite_string == "" && return false - force_overwrite_string == "--force-overwrite" && return true - throw(ArgumentError(usage)) -end