ci/repo: Add deb fetcher

.bazelrc

@@ -4,3 +4,7 @@ common:ci --noshow_progress
 common:ci --noshow_loading_progress
 common:ci --test_output=errors
 common:ci --//:aptly-custom=//:.aptly-ci-override
+
+common:debs-ci --config=ci
+common:debs-ci --//debs:excludes=//debs:custom-excludes.txt
+# common:debs-ci --//debs:token=//debs:token.txt

build-repository.sh

@@ -6,6 +6,8 @@ BOLD="\e[1m"
 UNDERLINE="\e[4m"
 NORMAL="\e[0m"
 
+EXCLUDE_FILE=debs/custom-excludes.txt
+DEBS_ROOT=/opt/build/cache/repository
 
 bold () {
     echo -n "${BOLD}${*}${NORMAL}"
@@ -23,5 +25,15 @@ import_public_key () {
     gpg --no-default-keyring --keyring trustedkeys.gpg --import envoy-maintainers-public.key
 }
 
+create_excludes () {
+    # Prevent re-downloading cached files
+    if [[ -e "${DEBS_ROOT}" ]]; then
+        ls "${DEBS_ROOT}" | (grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' || echo '') | sort -u > "$EXCLUDE_FILE"
+    else
+        touch "$EXCLUDE_FILE"
+    fi
+}
+
 import_public_key
-bazel run --config=ci //debs:publish
+create_excludes
+bazel run --config=debs-ci //debs:publish

debs/BUILD

@@ -1,16 +1,162 @@
 load("@aspect_bazel_lib//lib:jq.bzl", "jq")
 
+MAINTAINER = "Envoy maintainers <envoy-maintainers@googlegroups.com>"
+
+exports_files([
+    "custom-excludes.txt",
+    "token.txt",
+])
+
+jq(
+    name = "envoy_versions",
+    srcs = ["@envoy_repo//:project"],
+    out = "envoy_versions.json",
+    filter = """
+    .releases as $releases
+    | {
+      latest_releases: [
+        .stable_versions[]
+        | . as $minor
+        | {
+            version: $minor,
+            releases: (
+              $releases
+              | map(select(startswith("v" + $minor + ".")))
+            )
+          }
+      ]
+    }
+    """,
+    visibility = ["//visibility:public"],
+)
+
+jq(
+    name = "minor_versions",
+    srcs = [":envoy_versions"],
+    out = "minor_versions.txt",
+    filter = """
+    .latest_releases[] | .version
+    """,
+    args = ["-r"],
+    visibility = ["//visibility:public"],
+)
+
+jq(
+    name = "patch_versions",
+    srcs = [":envoy_versions"],
+    out = "patch_versions.txt",
+    filter = """
+    .latest_releases[] | .releases[]
+    """,
+    args = ["-r"],
+    visibility = ["//visibility:public"],
+)
+
+jq(
+    name = "deb_checksum_downloads",
+    srcs = [":envoy_versions"],
+    out = "deb_checksum_downloads.txt",
+    filter = """
+    reduce .latest_releases[].releases[] as $item ({};
+      .[$item] = {"signature": "%s"})
+    | with_entries(
+        {"key": "https://github.com/envoyproxy/envoy/releases/download/\\(.key)/checksums.txt.asc",
+         "value": .value})
+    """ % MAINTAINER,
+    args = ["-r"],
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "published_checksums",
+    outs = ["published_checksums.txt"],
+    cmd = """
+    $(location //tools/fetch) $(location :deb_checksum_downloads) --output=json > $@
+    """,
+    tools = [
+        "//tools/fetch",
+        ":deb_checksum_downloads",
+    ],
+)
+
+jq(
+    name = "debs_downloads",
+    srcs = [":published_checksums"],
+    out = "debs_downloads.json",
+    filter = """
+    with_entries(
+      .key as $key
+        | .value as $value
+        | ($key | capture("v(?<version>[0-9.]+)") | .version) as $version
+        | {key: ("https://github.com/envoyproxy/envoy/releases/download/v\\($version)/debs.tar.gz"),
+           value: {
+             "path": "v\\($version)",
+             "checksum": (
+               $value
+               | split("\n")
+               | map(select(endswith("debs.tar.gz")))
+               | first
+               | split(" ")
+               | .[0]
+             )}})
+    """,
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "empty",
+    outs = ["empty.txt"],
+    cmd = """
+    touch $@
+    """,
+)
+
+label_flag(
+    name = "excludes",
+    build_setting_default = ":empty",
+)
+
+label_flag(
+    name = "token",
+    build_setting_default = ":empty",
+)
+
+genrule(
+    name = "debs",
+    outs = ["debs.tar.gz"],
+    cmd = """
+    $(location //tools/fetch) $(location :debs_downloads) \
+        --concurrency 4 \
+        --excludes=$(location :excludes) \
+        --token-path=$(location :token) \
+        --extract-downloads \
+        --output-path=$@
+    if [[ ! -e $@ ]]; then
+        touch $@
+    fi
+    """,
+    tools = [
+        "//tools/fetch",
+        ":debs_downloads",
+        ":excludes",
+        ":token",
+    ],
+)
+
 sh_binary(
     name = "publish",
     srcs = ["publish.sh"],
     env = {
         "APTLY_BIN": "$(location @aptly)",
         "MAINTAINER_KEY": "$(location //:envoy-maintainers-public.key)",
         "APTLY_CONF": "$(location //:aptly-config)",
+        "DEBS": "$(location :debs)",
+        "DEBS_ROOT_DEFAULT": "/opt/build/cache/repository",
     },
     data = [
         "@aptly",
         "//:aptly-config",
         "//:envoy-maintainers-public.key",
+        ":debs"
     ],
 )

debs/publish.sh

@@ -6,18 +6,37 @@ APTLY_BIN="$APTLY_BIN"
 APTLY_CONF="${APTLY_CONF:-${APTLY_CONF}}"
 APTLY=("$APTLY_BIN" -config="${APTLY_CONF}")
 
-publish_root () {
+DEBS_ROOT="${DEBS_ROOT:-${DEBS_ROOT_DEFAULT}}"
+
+publish_dir () {
     "${APTLY[@]}" config show \
         | jq -r '.FileSystemPublishEndpoints.public.rootDir'
 }
 
-publish_repository () {
-    PUBLIC_DIR="$(publish_root)"
+create_dirs () {
+    PUBLIC_DIR="$(publish_dir)"
     mkdir -p "${PUBLIC_DIR}"
+    mkdir -p "${DEBS_ROOT}"
+}
+
+unpack_debs () {
+    if [[ -s "$DEBS" ]]; then
+        tar xf "$DEBS" -C "$DEBS_ROOT"
+    fi
+}
+
+publish_repository () {
+    PUBLIC_DIR="$(publish_dir)"
     KEY_URL="${DEPLOY_PRIME_URL}/envoy-maintainer-public.key"
     cat "$MAINTAINER_KEY" > "${PUBLIC_DIR}/envoy-maintainer-public.key"
     echo "<h1>COMING SOON: ${DEPLOY_PRIME_URL}</h1>" > "${PUBLIC_DIR}/index.html"
     echo "<div>Signing key: <a href=\"${KEY_URL}\">${KEY_URL}</div>" >> "${PUBLIC_DIR}/index.html"
 }
 
-publish_repository
+publish () {
+    create_dirs
+    unpack_debs
+    publish_repository
+}
+
+publish

tools/requirements.in

@@ -1 +1 @@
-envoy.base.utils>=0.5.4
+envoy.base.utils>=0.5.5

tools/requirements.txt

@@ -220,9 +220,9 @@ cryptography==43.0.0 \
     --hash=sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5 \
     --hash=sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0
     # via pyjwt
-envoy-base-utils==0.5.4 \
-    --hash=sha256:90882337c2ce509b19fc21f5d66395dad44a706e4d616baabd1ffaea256ee358 \
-    --hash=sha256:c1167f567994596a3071027d57ca895ac4a8bd8969d4de846b0f2cda499fa5ad
+envoy-base-utils==0.5.5 \
+    --hash=sha256:2341b7618c92969e3ac5fc43dd16fcc39d132981ffff3f0ddbabf08a336a6fa1 \
+    --hash=sha256:a72af1df5680d4fcdeb91f06d19b7495d740f8b9a8a1549c012272e10fdd59ea
     # via -r requirements.in
 frozendict==2.4.4 \
     --hash=sha256:07c3a5dee8bbb84cba770e273cdbf2c87c8e035903af8f781292d72583416801 \