feat: support oci_image_index in oci_tarball (#325)

examples/multi_arch_go/BUILD

@@ -0,0 +1,114 @@
+load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_binary")
+load("@aspect_bazel_lib//lib:testing.bzl", "assert_json_matches")
+load("@bazel_skylib//rules:write_file.bzl", "write_file")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
+load("@rules_oci//oci:defs.bzl", "oci_image", "oci_image_index", "oci_push", "oci_tarball")
+
+go_library(
+    name = "lib",
+    srcs = ["main.go"],
+    importpath = "main",
+)
+
+go_binary(
+    name = "bin-x86_64",
+    embed = [":lib"],
+    goarch = "amd64",
+    goos = "linux",
+)
+
+pkg_tar(
+    name = "bin-x86_64_tar",
+    srcs = [":bin-x86_64"],
+    package_dir = "usr/local/bin",
+)
+
+go_binary(
+    name = "bin-arm64",
+    embed = [":lib"],
+    goarch = "arm64",
+    goos = "linux",
+)
+
+pkg_tar(
+    name = "bin-arm64_tar",
+    srcs = [":bin-arm64"],
+    package_dir = "usr/local/bin",
+)
+
+oci_image(
+    name = "image-x86_64",
+    base = "@ubuntu_linux_amd64",
+    entrypoint = ["/usr/local/bin/bin-x86_64"],
+    tars = [":bin-x86_64_tar"],
+)
+
+repo_tags = [
+    "gcr.io/empty_base:latest",
+    "two:is_a_company",
+    "three:is_a_crowd",
+]
+
+oci_tarball(
+    name = "image-x86_64-tar",
+    image = ":image-x86_64",
+    repo_tags = repo_tags,
+)
+
+oci_image(
+    name = "image-arm64",
+    base = "@ubuntu_linux_arm64_v8",
+    entrypoint = ["/usr/local/bin/bin-arm64"],
+    tars = [":bin-arm64_tar"],
+)
+
+oci_image_index(
+    name = "image-multiarch",
+    images = [
+        ":image-arm64",
+        ":image-x86_64",
+    ],
+)
+
+oci_tarball(
+    name = "image-multiarch-tar",
+    format = "oci",
+    image = ":image-multiarch",
+    repo_tags = repo_tags,
+)
+
+write_file(
+    name = "expected_RepoTags",
+    out = "expected_RepoTags.json",
+    content = [str(repo_tags)],
+)
+
+genrule(
+    name = "tar_multiarch_index",
+    srcs = [":image-multiarch-tar"],
+    outs = ["multiarch_index.json"],
+    cmd = "tar -xOf ./$(location :image-multiarch-tar) index.json > $@",
+)
+
+assert_json_matches(
+    name = "check_multiarch_tags",
+    file1 = ":tar_multiarch_index",
+    file2 = ":expected_RepoTags",
+    filter1 = ".manifests[].annotations[\"org.opencontainers.image.ref.name\"]",
+    filter2 = ".[]",
+)
+
+genrule(
+    name = "tar_x86_64_index",
+    srcs = [":image-x86_64-tar"],
+    outs = ["x86_64_index.json"],
+    cmd = "tar -xOf ./$(location :image-x86_64-tar) manifest.json > $@",
+)
+
+assert_json_matches(
+    name = "check_x86_64_tags",
+    file1 = ":tar_x86_64_index",
+    file2 = ":expected_RepoTags",
+    filter1 = ".[0].RepoTags",
+)

examples/multi_arch_go/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "fmt"
+
+func main() {
+    fmt.Println("yo")
+}

oci/private/tarball.bzl

@@ -26,6 +26,11 @@ Passing anything other than oci_image to the image attribute will lead to build
 """
 
 attrs = {
+    "format": attr.string(
+        default = "docker",
+        doc = "Format of image to generate. Options are: docker, oci. Currently, when the input image is an image_index, only oci is supported, and when the input image is an image, only docker is supported. Conversions between formats may be supported in the future.",
+        values = ["docker", "oci"],
+    ),
     "image": attr.label(mandatory = True, allow_single_file = True, doc = "Label of a directory containing an OCI layout, typically `oci_image`"),
     "repo_tags": attr.label(
         doc = """\
@@ -53,6 +58,7 @@ def _tarball_impl(ctx):
     repo_tags = ctx.file.repo_tags
 
     substitutions = {
+        "{{format}}": ctx.attr.format,
         "{{yq}}": yq_bin.path,
         "{{image_dir}}": image.path,
         "{{tarball_path}}": tarball.path,

oci/private/tarball.sh.tpl

@@ -1,26 +1,132 @@
 #!/usr/bin/env bash
 set -o pipefail -o errexit -o nounset
 
+readonly FORMAT="{{format}}"
 readonly STAGING_DIR=$(mktemp -d)
 readonly YQ="{{yq}}"
 readonly IMAGE_DIR="{{image_dir}}"
 readonly BLOBS_DIR="${STAGING_DIR}/blobs"
 readonly TARBALL_PATH="{{tarball_path}}"
 readonly REPOTAGS=($(cat "{{tags}}"))
+readonly INDEX_FILE="${IMAGE_DIR}/index.json"
+
+cp_f_with_mkdir() {
+  SRC="$1"
+  DST="$2"
+  mkdir -p "$(dirname "${DST}")"
+  cp -f "${SRC}" "${DST}"
+}
+
+MANIFEST_DIGEST=$(${YQ} eval '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr  -d '"')
+
+MANIFESTS_LENGTH=$("${YQ}" eval '.manifests | length' "${INDEX_FILE}")
+if [[ "${MANIFESTS_LENGTH}" != 1 ]]; then
+  echo >&2 "Expected exactly one manifest in ${INDEX_FILE}"
+  exit 1
+fi
+
+MEDIA_TYPE=$("${YQ}" eval ".manifests[0].mediaType" "${INDEX_FILE}")
+
+# Check that we know how to generate the output format given the input format.
+# We may expand the supported options here in the future, but for now,
+if [[ "${FORMAT}" != "docker" && "${FORMAT}" != "oci" ]]; then
+  echo >&2 "Unknown format: ${FORMAT}. Only support docker|oci"
+  exit 1
+fi
+if [[ "${FORMAT}" == "oci" && "${MEDIA_TYPE}" != "application/vnd.oci.image.index.v1+json" && "${MEDIA_TYPE}" != "application/vnd.docker.distribution.manifest.v2+json" ]]; then
+  echo >&2 "Format oci is only supported for oci_image_index targets but saw ${MEDIA_TYPE}"
+  exit 1
+fi
+if [[ "${FORMAT}" == "docker" && "${MEDIA_TYPE}" != "application/vnd.oci.image.manifest.v1+json" && "${MEDIA_TYPE}" != "application/vnd.docker.distribution.manifest.v2+json" ]]; then
+  echo >&2 "Format docker is only supported for oci_image targets but saw ${MEDIA_TYPE}"
+  exit 1
+fi
+
+if [[ "${FORMAT}" == "oci" ]]; then
+  # Handle multi-architecture image indexes.
+  # Ideally the toolchains we rely on would output these for us, but they don't seem to.
+
+  echo -n '{"imageLayoutVersion": "1.0.0"}' > "${STAGING_DIR}/oci-layout"
+
+  INDEX_FILE_MANIFEST_DIGEST=$("${YQ}" eval '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr  -d '"')
+  INDEX_FILE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${INDEX_FILE_MANIFEST_DIGEST}"
+
+  cp_f_with_mkdir "${INDEX_FILE_MANIFEST_BLOB_PATH}" "${BLOBS_DIR}/${INDEX_FILE_MANIFEST_DIGEST}"
+
+  IMAGE_MANIFESTS_DIGESTS=($("${YQ}" '.manifests[] | .digest | sub(":"; "/")' "${INDEX_FILE_MANIFEST_BLOB_PATH}"))
+
+  for IMAGE_MANIFEST_DIGEST in "${IMAGE_MANIFESTS_DIGESTS[@]}"; do
+    IMAGE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${IMAGE_MANIFEST_DIGEST}"
+    cp_f_with_mkdir "${IMAGE_MANIFEST_BLOB_PATH}" "${BLOBS_DIR}/${IMAGE_MANIFEST_DIGEST}"
+
+    CONFIG_DIGEST=$("${YQ}" eval '.config.digest  | sub(":"; "/")' ${IMAGE_MANIFEST_BLOB_PATH})
+    CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}"
+    cp_f_with_mkdir "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}"
+
+    LAYER_DIGESTS=$("${YQ}" eval '.layers | map(.digest | sub(":"; "/"))' "${IMAGE_MANIFEST_BLOB_PATH}")
+    for LAYER_DIGEST in $("${YQ}" ".[]" <<< $LAYER_DIGESTS); do
+      cp_f_with_mkdir "${IMAGE_DIR}/blobs/${LAYER_DIGEST}" ${BLOBS_DIR}/${LAYER_DIGEST}
+    done
+  done
+
+  # Fill in repo tags as per https://github.com/opencontainers/image-spec/issues/796
+  # If there's more than one repo tag, we need to duplicate the manifest entry, so we have one copy per repo tag.
+  MANIFEST_COPIES=".manifests"
+  if [[ "${#REPOTAGS[@]}" -gt 1 ]]; then
+    for i in $(seq 2 "${#REPOTAGS[@]}"); do
+      MANIFEST_COPIES="${MANIFEST_COPIES} + .manifests"
+    done
+  fi
+  # Convert:
+  # {
+  #   "schemaVersion": 2,
+  #   "manifests": [
+  #     {
+  #       "mediaType": "application/vnd.oci.image.index.v1+json",
+  #       "size": 668,
+  #       "digest": "sha256:41981de3b7207f5260fd94fac77272218518d58a6335d843136d88d91341e3d9"
+  #     }
+  #   ]
+  # }
+  # Into:
+  # {
+  #   "schemaVersion": 2,
+  #   "manifests": [
+  #     {
+  #       "mediaType": "application/vnd.oci.image.index.v1+json",
+  #       "size": 668,
+  #       "digest": "sha256:41981de3b7207f5260fd94fac77272218518d58a6335d843136d88d91341e3d9",
+  #       "annotations": {
+  #         "org.opencontainers.image.ref.name": "repo-tag:1"
+  #       }
+  #     },
+  #     {
+  #       "mediaType": "application/vnd.oci.image.index.v1+json",
+  #       "size": 668,
+  #       "digest": "sha256:41981de3b7207f5260fd94fac77272218518d58a6335d843136d88d91341e3d9",
+  #       "annotations": {
+  #         "org.opencontainers.image.ref.name": "repo-tag:2"
+  #       }
+  #     }
+  #   ]
+  # }
+  repo_tags="${REPOTAGS[@]}" "${YQ}" -o json eval "(.manifests = ${MANIFEST_COPIES}) *d {\"manifests\": (env(repo_tags) | split \" \" | map {\"annotations\": {\"org.opencontainers.image.ref.name\": .}})}" "${INDEX_FILE}" > "${STAGING_DIR}/index.json"
+
+  tar -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" index.json blobs oci-layout
+  exit 0
+fi
 
-MANIFEST_DIGEST=$(${YQ} eval '.manifests[0].digest | sub(":"; "/")' "${IMAGE_DIR}/index.json" | tr  -d '"')
 MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${MANIFEST_DIGEST}"
 
 CONFIG_DIGEST=$(${YQ} eval '.config.digest  | sub(":"; "/")' ${MANIFEST_BLOB_PATH})
 CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}"
 
 LAYERS=$(${YQ} eval '.layers | map(.digest | sub(":"; "/"))' ${MANIFEST_BLOB_PATH})
 
-mkdir -p $(dirname "${BLOBS_DIR}/${CONFIG_DIGEST}")
-cp "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}"
+cp_f_with_mkdir "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}"
 
-for LAYER in $(${YQ} ".[]" <<< $LAYERS); do 
-    cp -f "${IMAGE_DIR}/blobs/${LAYER}" "${BLOBS_DIR}/${LAYER}.tar.gz"
+for LAYER in $(${YQ} ".[]" <<< $LAYERS); do
+  cp_f_with_mkdir "${IMAGE_DIR}/blobs/${LAYER}" "${BLOBS_DIR}/${LAYER}.tar.gz"
 done
 
 repo_tags="${REPOTAGS[@]}" \