Add the ability to skip tarball creation when developing locally

linux/agent_linux.jl

@@ -1,31 +1,25 @@
-## This rootfs includes just enough of the tools for our buildkite agent
-## to run inside of.  Most CI steps will be run within a different image
-## nested inside of this one.
-
 using RootfsUtils: parse_build_args, debootstrap, chroot, upload_gha, test_sandbox
 
-arch, image, = parse_build_args(ARGS, @__FILE__)
+args         = parse_build_args(ARGS, @__FILE__)
+arch         = args.arch
+archive      = args.archive
+image        = args.image
 
-# Build debian-based image with the following extra packages:
 packages = [
-    # General package getting/installing packages
     "apt-transport-https",
     "curl",
-    "gnupg2",
-    "openssh-client",
-    "wget",
-    # We use these in our buildkite plugins a lot
     "git",
+    "gnupg2",
     "jq",
     "locales",
+    "openssh-client",
     "openssl",
     "python3",
-    # Debugging
     "vim",
+    "wget",
 ]
 
-artifact_hash, tarball_path, = debootstrap(arch, image; packages) do rootfs
-    # Also download buildkite-agent
+artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs
     @info("Installing buildkite-agent...")
     buildkite_install_cmd = """
     echo 'deb https://apt.buildkite.com/buildkite-agent stable main' >> /etc/apt/sources.list && \\
@@ -36,8 +30,5 @@ artifact_hash, tarball_path, = debootstrap(arch, image; packages) do rootfs
     chroot(rootfs, "bash", "-c", buildkite_install_cmd; uid=0, gid=0)
 end
 
-# Upload it
 upload_gha(tarball_path)
-
-# Test that we can use our new rootfs image with Sandbox.jl
 test_sandbox(artifact_hash)

linux/debian_minimal.jl

@@ -1,13 +1,13 @@
 using RootfsUtils: parse_build_args, debootstrap, chroot, upload_gha, test_sandbox
 
-arch, image, = parse_build_args(ARGS, @__FILE__)
+args         = parse_build_args(ARGS, @__FILE__)
+arch         = args.arch
+archive      = args.archive
+image        = args.image
 
-# Build debian-based image with the following extra packages:
 packages = String[]
-artifact_hash, tarball_path, = debootstrap(arch, image; packages, locale = false)
+locale = false
 
-# Upload it
+artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages, locale)
 upload_gha(tarball_path)
-
-# Test that we can use our new rootfs image with Sandbox.jl
 test_sandbox(artifact_hash)

linux/llvm_passes.jl

@@ -1,10 +1,10 @@
-## This rootfs includes enough of a host toolchain to build the LLVM passes (such as `analyzegc`).
-
 using RootfsUtils: parse_build_args, debootstrap, chroot, upload_gha, test_sandbox
 
-arch, image, = parse_build_args(ARGS, @__FILE__)
+args         = parse_build_args(ARGS, @__FILE__)
+arch         = args.arch
+archive      = args.archive
+image        = args.image
 
-# Build debian-based image with the following extra packages:
 packages = [
     "build-essential",
     "cmake",
@@ -21,10 +21,7 @@ packages = [
     "python3",
     "wget",
 ]
-artifact_hash, tarball_path, = debootstrap(arch, image; packages)
 
-# Upload it
+artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages)
 upload_gha(tarball_path)
-
-# Test that we can use our new rootfs image with Sandbox.jl
 test_sandbox(artifact_hash)

linux/package_linux.jl

@@ -1,11 +1,10 @@
-## This rootfs includes everything that must be installed to build Julia
-## within a debian-based environment with GCC 9.
-
 using RootfsUtils: parse_build_args, debootstrap, chroot, upload_gha, test_sandbox
 
-arch, image, = parse_build_args(ARGS, @__FILE__)
+args         = parse_build_args(ARGS, @__FILE__)
+arch         = args.arch
+archive      = args.archive
+image        = args.image
 
-# Build debian-based image with the following extra packages:
 packages = [
     "automake",
     "bash",
@@ -26,10 +25,11 @@ packages = [
     "pkg-config",
     "python",
     "python3",
-    "wget",
     "vim",
+    "wget",
 ]
-artifact_hash, tarball_path, = debootstrap(arch, image; packages) do rootfs
+
+artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs
     # Install GCC 9, specifically
     @info("Installing gcc-9")
     gcc_install_cmd = """
@@ -47,8 +47,5 @@ artifact_hash, tarball_path, = debootstrap(arch, image; packages) do rootfs
     chroot(rootfs, "bash", "-c", gcc_install_cmd; uid=0, gid=0)
 end
 
-# Upload it
 upload_gha(tarball_path)
-
-# Test that we can use our new rootfs image with Sandbox.jl
 test_sandbox(artifact_hash)

linux/package_musl.jl

@@ -1,16 +1,13 @@
-## This rootfs includes everything that must be installed to build Julia
-## within an alpine-based environment with GCC 9.
-
 using RootfsUtils: AlpinePackage, parse_build_args, alpine_bootstrap, chroot, upload_gha, test_sandbox
 
-arch, image, = parse_build_args(ARGS, @__FILE__)
+args         = parse_build_args(ARGS, @__FILE__)
+archive      = args.archive
+image        = args.image
 
-# Build alpine-based image with the following extra packages:
 packages = [
     AlpinePackage("bash"),
     AlpinePackage("cmake"),
     AlpinePackage("curl"),
-    AlpinePackage("gdb"),
     AlpinePackage("git"),
     AlpinePackage("less"),
     AlpinePackage("lldb"),
@@ -19,15 +16,12 @@ packages = [
     AlpinePackage("python3"),
     AlpinePackage("wget"),
 
-    # Install gcc/g++/gfortran v9, which comes from the Alpine v3.11 line
     AlpinePackage("g++~9", "v3.11"),
     AlpinePackage("gcc~9", "v3.11"),
+    AlpinePackage("gdb"),
     AlpinePackage("gfortran~9", "v3.11"),
 ]
-artifact_hash, tarball_path, = alpine_bootstrap(image; packages)
 
-# Upload it
+artifact_hash, tarball_path, = alpine_bootstrap(image; archive, packages)
 upload_gha(tarball_path)
-
-# Test that we can use our new rootfs image with Sandbox.jl
 test_sandbox(artifact_hash)

linux/pkgserver_logsync.jl

@@ -1,21 +1,20 @@
 using RootfsUtils: parse_build_args, debootstrap, chroot, upload_gha, test_sandbox
 
-arch, image, = parse_build_args(ARGS, @__FILE__)
+args         = parse_build_args(ARGS, @__FILE__)
+arch         = args.arch
+archive      = args.archive
+image        = args.image
 
-# Build debian-based image with the following extra packages:
 packages = String[
-    "openssh-client",
     "awscli",
-    "rsync",
     "curl",
     "jq",
-    "zstd",
     "locales",
+    "openssh-client",
+    "rsync",
+    "zstd",
 ]
-artifact_hash, tarball_path, = debootstrap(arch, image; packages)
 
-# Upload it
+artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages)
 upload_gha(tarball_path)
-
-# Test that we can use our new rootfs image with Sandbox.jl
 test_sandbox(artifact_hash)

linux/tester_linux.jl

@@ -1,21 +1,18 @@
-## This rootfs does not include the compiler toolchain.
-
 using RootfsUtils: parse_build_args, debootstrap, chroot, upload_gha, test_sandbox
 
-arch, image, = parse_build_args(ARGS, @__FILE__)
+args         = parse_build_args(ARGS, @__FILE__)
+arch         = args.arch
+archive      = args.archive
+image        = args.image
 
-# Build debian-based image with the following extra packages:
 packages = [
     "bash",
     "curl",
     "gdb",
     "locales",
     "vim",
 ]
-artifact_hash, tarball_path, = debootstrap(arch, image; packages)
 
-# Upload it
+artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages)
 upload_gha(tarball_path)
-
-# Test that we can use our new rootfs image with Sandbox.jl
 test_sandbox(artifact_hash)

linux/tester_musl.jl

@@ -1,20 +1,18 @@
-## This rootfs does not include the compiler toolchain.
-
 using RootfsUtils: AlpinePackage, parse_build_args, alpine_bootstrap, chroot, upload_gha, test_sandbox
 
-arch, image, = parse_build_args(ARGS, @__FILE__)
+args         = parse_build_args(ARGS, @__FILE__)
+arch         = args.arch
+archive      = args.archive
+image        = args.image
 
-# Build alpine-based image with the following extra packages:
+# Build alpine-based args.image with the following extra packages:
 packages = [
     AlpinePackage("bash"),
     AlpinePackage("curl"),
     AlpinePackage("gdb"),
     AlpinePackage("vim"),
 ]
-artifact_hash, tarball_path, = alpine_bootstrap(image; packages)
 
-# Upload it
+artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages)
 upload_gha(tarball_path)
-
-# Test that we can use our new rootfs image with Sandbox.jl
 test_sandbox(artifact_hash)

src/build_img/alpine.jl

@@ -5,14 +5,14 @@ function repository_arg(repo)
     return "--repository=http://dl-cdn.alpinelinux.org/alpine/$(repo)/main"
 end
 
-function alpine_bootstrap(f::Function,
-                          name::String;
+function alpine_bootstrap(f::Function, name::String;
+                          archive::Bool = true,
                           force::Bool = false,
                           git_rev = "97bdddbacbe7f7fa6165ed2bdfa86d7d0ab43420", # v3.13
                           packages::Vector{AlpinePackage} = AlpinePackage[],
                           release::VersionNumber = v"3.13.6",
                           variant::String = "minirootfs")
-    return create_rootfs(name; force) do rootfs
+    return create_rootfs(name; archive, force) do rootfs
         rootfs_url = "https://github.com/alpinelinux/docker-alpine/raw/$(git_rev)/x86_64/alpine-$(variant)-$(release)-x86_64.tar.gz"
         @info("Downloading Alpine rootfs", url=rootfs_url)
         rm(rootfs)

src/build_img/args.jl

@@ -11,10 +11,14 @@ function parse_build_args(args::AbstractVector, file::AbstractString)
             arg_type = String
             required = true
             help = "The architecture for which you would like to build"
+        "--no-archive"
+            action = :store_true
+            help = "When this flag is provided, the .tar.gz archive will not be created"
     end
     parsed_args = ArgParse.parse_args(args, settings)
-    # arch = parsed_args["arch"]
-    arch = _process_required_string_arg(parsed_args, "arch")
-    image = generate_image_name(arch, file)
-    return (; arch, image)
+    arch = _process_required_string_arg(parsed_args, "arch")::String
+    image = generate_image_name(arch, file)::String
+    no_archive = parsed_args["no-archive"]::Bool
+    archive = !no_archive
+    return (; arch, archive, image)
 end

src/build_img/common.jl

@@ -81,7 +81,9 @@ function cleanup_rootfs(rootfs; rootfs_info=nothing)
     force_relative(rootfs)
 end
 
-function create_rootfs(f::Function, name::String; force::Bool=false)
+function create_rootfs(f::Function, name::String;
+                       archive::Bool=true,
+                       force::Bool=false)
     tarball_path = joinpath(Scratch.@get_scratch!("rootfs-images"), "$(name).tar.gz")
     if !force && isfile(tarball_path)
         @error("Refusing to overwrite tarball without `force` set", tarball_path)
@@ -90,12 +92,21 @@ function create_rootfs(f::Function, name::String; force::Bool=false)
 
     artifact_hash = Pkg.Artifacts.create_artifact(f)
 
-    # Archive it into a `.tar.gz` file (but only if this is not a pull request build).
+    # If the `--no-archive` command-line flag was passed, we will skip the tarball creation.
+    if !archive
+        info_msg = "Skipping tarball creation because the `--no-archive` command-line flag was passed"
+        @info info_msg artifact_hash basename(tarball_path)
+        return (; artifact_hash, tarball_path = nothing)
+    end
+
+    # If this is a pull request build, we will skip the tarball creation.
     if is_github_actions_pr()
         info_msg = "Skipping tarball creation because the build is a `pull_request` build"
         @info info_msg artifact_hash basename(tarball_path)
         return (; artifact_hash, tarball_path = nothing)
     end
+
+    # Archive the artifact into a `.tar.gz` file.
     @info "Archiving" artifact_hash tarball_path
     Pkg.Artifacts.archive_artifact(artifact_hash, tarball_path)
     if is_github_actions()
@@ -201,6 +212,11 @@ end
 github_actions_set_output(p::Pair) = github_actions_set_output(stdout, p)
 
 function upload_gha(tarball_path::Nothing)
+    if !is_github_actions()
+        @info "Skipping release artifact upload because this is not a GitHub Actions build"
+        return nothing
+    end
+
     if !is_github_actions_pr()
         error_msg = "You are only allowed to skip tarball creation if the build is a `pull_request` build"
         throw(ErrorException(error_msg))

src/build_img/debian.jl

@@ -10,6 +10,7 @@ function debian_arch(image_arch::String)
 end
 
 function debootstrap(f::Function, arch::String, name::String;
+                     archive::Bool = true,
                      force::Bool = false,
                      locale::Bool = true,
                      packages::Vector{String} = String[],
@@ -37,7 +38,7 @@ function debootstrap(f::Function, arch::String, name::String;
         error("Must install qemu-user-static and binfmt_misc!")
     end
 
-    return create_rootfs(name; force) do rootfs
+    return create_rootfs(name; archive, force) do rootfs
         @info("Running debootstrap", release, variant, packages)
         debootstrap_cmd = `sudo debootstrap`
         push!(debootstrap_cmd.exec, "--arch=$(debian_arch(arch))")

test_rootfs.jl

@@ -3,14 +3,13 @@ using RootfsUtils: parse_test_args, ensure_artifact_exists_locally
 using Sandbox: Sandbox, SandboxConfig, with_executor
 
 args            = parse_test_args(ARGS, @__FILE__)
-
 command         = args.command
 multiarch       = args.multiarch
 read_write_maps = args.read_write_maps
 tmpfs_size      = args.tmpfs_size
 treehash        = args.treehash
-url             = args.url
 working_dir     = args.working_dir
+url             = args.url
 
 # If the artifact is not locally existent, download it
 ensure_artifact_exists_locally(; treehash, url)