diff --git a/docs/tarball.md b/docs/tarball.md index c4dbbcde..54609312 100644 --- a/docs/tarball.md +++ b/docs/tarball.md @@ -25,7 +25,7 @@ docker run --rm my-repository:latest ## oci_tarball
-oci_tarball(name, image, repo_tags)
+oci_tarball(name, format, image, repo_tags)
 
Creates tarball from OCI layouts that can be loaded into docker daemon without needing to publish the image first. @@ -39,6 +39,7 @@ Passing anything other than oci_image to the image attribute will lead to build | Name | Description | Type | Mandatory | Default | | :------------- | :------------- | :------------- | :------------- | :------------- | | name | A unique name for this target. | Name | required | | +| format | Format of image to generate. Options are: docker, oci. Currently, when the input image is an image_index, only oci is supported, and when the input image is an image, only docker is supported. Conversions between formats may be supported in the future. | String | optional | "docker" | | image | Label of a directory containing an OCI layout, typically oci_image | Label | required | | | repo_tags | a file containing repo_tags, one per line. | Label | required | | diff --git a/examples/multi_arch_go/BUILD b/examples/multi_arch_go/BUILD new file mode 100644 index 00000000..11e40f15 --- /dev/null +++ b/examples/multi_arch_go/BUILD @@ -0,0 +1,114 @@ +load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_binary") +load("@aspect_bazel_lib//lib:testing.bzl", "assert_json_matches") +load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_image_index", "oci_push", "oci_tarball") + +go_library( + name = "lib", + srcs = ["main.go"], + importpath = "main", +) + +go_binary( + name = "bin-x86_64", + embed = [":lib"], + goarch = "amd64", + goos = "linux", +) + +pkg_tar( + name = "bin-x86_64_tar", + srcs = [":bin-x86_64"], + package_dir = "usr/local/bin", +) + +go_binary( + name = "bin-arm64", + embed = [":lib"], + goarch = "arm64", + goos = "linux", +) + +pkg_tar( + name = "bin-arm64_tar", + srcs = [":bin-arm64"], + package_dir = "usr/local/bin", +) + +oci_image( + name = "image-x86_64", + base = "@ubuntu_linux_amd64", + entrypoint = ["/usr/local/bin/bin-x86_64"], + tars = [":bin-x86_64_tar"], +) + +repo_tags = [ + "gcr.io/empty_base:latest", + "two:is_a_company", + "three:is_a_crowd", +] + +oci_tarball( + name = "image-x86_64-tar", + image = ":image-x86_64", + repo_tags = repo_tags, +) + +oci_image( + name = "image-arm64", + base = "@ubuntu_linux_arm64_v8", + entrypoint = ["/usr/local/bin/bin-arm64"], + tars = [":bin-arm64_tar"], +) + +oci_image_index( + name = "image-multiarch", + images = [ + ":image-arm64", + ":image-x86_64", + ], +) + +oci_tarball( + name = "image-multiarch-tar", + format = "oci", + image = ":image-multiarch", + repo_tags = repo_tags, +) + +write_file( + name = "expected_RepoTags", + out = "expected_RepoTags.json", + content = [str(repo_tags)], +) + +genrule( + name = "tar_multiarch_index", + srcs = [":image-multiarch-tar"], + outs = ["multiarch_index.json"], + cmd = "tar -xOf ./$(location :image-multiarch-tar) index.json > $@", +) + +assert_json_matches( + name = "check_multiarch_tags", + file1 = ":tar_multiarch_index", + file2 = ":expected_RepoTags", + filter1 = ".manifests[].annotations[\"org.opencontainers.image.ref.name\"]", + filter2 = ".[]", +) + +genrule( + name = "tar_x86_64_index", + srcs = [":image-x86_64-tar"], + outs = ["x86_64_index.json"], + cmd = "tar -xOf ./$(location :image-x86_64-tar) manifest.json > $@", +) + +assert_json_matches( + name = "check_x86_64_tags", + file1 = ":tar_x86_64_index", + file2 = ":expected_RepoTags", + filter1 = ".[0].RepoTags", +) diff --git a/examples/multi_arch_go/main.go b/examples/multi_arch_go/main.go new file mode 100644 index 00000000..84dcf5cb --- /dev/null +++ b/examples/multi_arch_go/main.go @@ -0,0 +1,7 @@ +package main + +import "fmt" + +func main() { + fmt.Println("yo") +} diff --git a/oci/private/tarball.bzl b/oci/private/tarball.bzl index dc9d89c9..d2b0118b 100644 --- a/oci/private/tarball.bzl +++ b/oci/private/tarball.bzl @@ -26,6 +26,11 @@ Passing anything other than oci_image to the image attribute will lead to build """ attrs = { + "format": attr.string( + default = "docker", + doc = "Format of image to generate. Options are: docker, oci. Currently, when the input image is an image_index, only oci is supported, and when the input image is an image, only docker is supported. Conversions between formats may be supported in the future.", + values = ["docker", "oci"], + ), "image": attr.label(mandatory = True, allow_single_file = True, doc = "Label of a directory containing an OCI layout, typically `oci_image`"), "repo_tags": attr.label( doc = """\ @@ -53,6 +58,7 @@ def _tarball_impl(ctx): repo_tags = ctx.file.repo_tags substitutions = { + "{{format}}": ctx.attr.format, "{{yq}}": yq_bin.path, "{{image_dir}}": image.path, "{{tarball_path}}": tarball.path, diff --git a/oci/private/tarball.sh.tpl b/oci/private/tarball.sh.tpl index 9bbc3788..deb3ef47 100644 --- a/oci/private/tarball.sh.tpl +++ b/oci/private/tarball.sh.tpl @@ -1,14 +1,121 @@ #!/usr/bin/env bash set -o pipefail -o errexit -o nounset +readonly FORMAT="{{format}}" readonly STAGING_DIR=$(mktemp -d) readonly YQ="{{yq}}" readonly IMAGE_DIR="{{image_dir}}" readonly BLOBS_DIR="${STAGING_DIR}/blobs" readonly TARBALL_PATH="{{tarball_path}}" readonly REPOTAGS=($(cat "{{tags}}")) +readonly INDEX_FILE="${IMAGE_DIR}/index.json" + +cp_f_with_mkdir() { + SRC="$1" + DST="$2" + mkdir -p "$(dirname "${DST}")" + cp -f "${SRC}" "${DST}" +} + +MANIFEST_DIGEST=$(${YQ} eval '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"') + +MANIFESTS_LENGTH=$("${YQ}" eval '.manifests | length' "${INDEX_FILE}") +if [[ "${MANIFESTS_LENGTH}" != 1 ]]; then + echo >&2 "Expected exactly one manifest in ${INDEX_FILE}" + exit 1 +fi + +MEDIA_TYPE=$("${YQ}" eval ".manifests[0].mediaType" "${INDEX_FILE}") + +# Check that we know how to generate the output format given the input format. +# We may expand the supported options here in the future, but for now, +if [[ "${FORMAT}" != "docker" && "${FORMAT}" != "oci" ]]; then + echo >&2 "Unknown format: ${FORMAT}. Only support docker|oci" + exit 1 +fi +if [[ "${FORMAT}" == "oci" && "${MEDIA_TYPE}" != "application/vnd.oci.image.index.v1+json" && "${MEDIA_TYPE}" != "application/vnd.docker.distribution.manifest.v2+json" ]]; then + echo >&2 "Format oci is only supported for oci_image_index targets but saw ${MEDIA_TYPE}" + exit 1 +fi +if [[ "${FORMAT}" == "docker" && "${MEDIA_TYPE}" != "application/vnd.oci.image.manifest.v1+json" && "${MEDIA_TYPE}" != "application/vnd.docker.distribution.manifest.v2+json" ]]; then + echo >&2 "Format docker is only supported for oci_image targets but saw ${MEDIA_TYPE}" + exit 1 +fi + +if [[ "${FORMAT}" == "oci" ]]; then + # Handle multi-architecture image indexes. + # Ideally the toolchains we rely on would output these for us, but they don't seem to. + + echo -n '{"imageLayoutVersion": "1.0.0"}' > "${STAGING_DIR}/oci-layout" + + INDEX_FILE_MANIFEST_DIGEST=$("${YQ}" eval '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"') + INDEX_FILE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${INDEX_FILE_MANIFEST_DIGEST}" + + cp_f_with_mkdir "${INDEX_FILE_MANIFEST_BLOB_PATH}" "${BLOBS_DIR}/${INDEX_FILE_MANIFEST_DIGEST}" + + IMAGE_MANIFESTS_DIGESTS=($("${YQ}" '.manifests[] | .digest | sub(":"; "/")' "${INDEX_FILE_MANIFEST_BLOB_PATH}")) + + for IMAGE_MANIFEST_DIGEST in "${IMAGE_MANIFESTS_DIGESTS[@]}"; do + IMAGE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${IMAGE_MANIFEST_DIGEST}" + cp_f_with_mkdir "${IMAGE_MANIFEST_BLOB_PATH}" "${BLOBS_DIR}/${IMAGE_MANIFEST_DIGEST}" + + CONFIG_DIGEST=$("${YQ}" eval '.config.digest | sub(":"; "/")' ${IMAGE_MANIFEST_BLOB_PATH}) + CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}" + cp_f_with_mkdir "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}" + + LAYER_DIGESTS=$("${YQ}" eval '.layers | map(.digest | sub(":"; "/"))' "${IMAGE_MANIFEST_BLOB_PATH}") + for LAYER_DIGEST in $("${YQ}" ".[]" <<< $LAYER_DIGESTS); do + cp_f_with_mkdir "${IMAGE_DIR}/blobs/${LAYER_DIGEST}" ${BLOBS_DIR}/${LAYER_DIGEST} + done + done + + # Fill in repo tags as per https://github.com/opencontainers/image-spec/issues/796 + # If there's more than one repo tag, we need to duplicate the manifest entry, so we have one copy per repo tag. + MANIFEST_COPIES=".manifests" + if [[ "${#REPOTAGS[@]}" -gt 1 ]]; then + for i in $(seq 2 "${#REPOTAGS[@]}"); do + MANIFEST_COPIES="${MANIFEST_COPIES} + .manifests" + done + fi + # Convert: + # { + # "schemaVersion": 2, + # "manifests": [ + # { + # "mediaType": "application/vnd.oci.image.index.v1+json", + # "size": 668, + # "digest": "sha256:41981de3b7207f5260fd94fac77272218518d58a6335d843136d88d91341e3d9" + # } + # ] + # } + # Into: + # { + # "schemaVersion": 2, + # "manifests": [ + # { + # "mediaType": "application/vnd.oci.image.index.v1+json", + # "size": 668, + # "digest": "sha256:41981de3b7207f5260fd94fac77272218518d58a6335d843136d88d91341e3d9", + # "annotations": { + # "org.opencontainers.image.ref.name": "repo-tag:1" + # } + # }, + # { + # "mediaType": "application/vnd.oci.image.index.v1+json", + # "size": 668, + # "digest": "sha256:41981de3b7207f5260fd94fac77272218518d58a6335d843136d88d91341e3d9", + # "annotations": { + # "org.opencontainers.image.ref.name": "repo-tag:2" + # } + # } + # ] + # } + repo_tags="${REPOTAGS[@]}" "${YQ}" -o json eval "(.manifests = ${MANIFEST_COPIES}) *d {\"manifests\": (env(repo_tags) | split \" \" | map {\"annotations\": {\"org.opencontainers.image.ref.name\": .}})}" "${INDEX_FILE}" > "${STAGING_DIR}/index.json" + + tar -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" index.json blobs oci-layout + exit 0 +fi -MANIFEST_DIGEST=$(${YQ} eval '.manifests[0].digest | sub(":"; "/")' "${IMAGE_DIR}/index.json" | tr -d '"') MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${MANIFEST_DIGEST}" CONFIG_DIGEST=$(${YQ} eval '.config.digest | sub(":"; "/")' ${MANIFEST_BLOB_PATH}) @@ -16,11 +123,10 @@ CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}" LAYERS=$(${YQ} eval '.layers | map(.digest | sub(":"; "/"))' ${MANIFEST_BLOB_PATH}) -mkdir -p $(dirname "${BLOBS_DIR}/${CONFIG_DIGEST}") -cp "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}" +cp_f_with_mkdir "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}" -for LAYER in $(${YQ} ".[]" <<< $LAYERS); do - cp -f "${IMAGE_DIR}/blobs/${LAYER}" "${BLOBS_DIR}/${LAYER}.tar.gz" +for LAYER in $(${YQ} ".[]" <<< $LAYERS); do + cp_f_with_mkdir "${IMAGE_DIR}/blobs/${LAYER}" "${BLOBS_DIR}/${LAYER}.tar.gz" done repo_tags="${REPOTAGS[@]}" \