diff --git a/anago b/anago index d911b35939f..79f8889de34 100755 --- a/anago +++ b/anago @@ -259,12 +259,11 @@ copy_logs_to_workdir () { } ############################################################################### -# Ensures all registries that will be used during both mock and --nomock -# runs allow write access so we don't fall over later -# @param registries - A space separated list of registries +# Ensures we have write access to a specified registry +# @param registry - A registry to check the ACLs for # ensure_registry_acls () { - local registries=($1) + local registry="$1" local emptyfile="$TMPDIR/empty-file.$$" local gs_path local r @@ -276,27 +275,30 @@ ensure_registry_acls () { # Short of creating a hardcoded map of project-id to registry, translating # _ to - seems to be a simple rule to keep this, well, simple. - for r in ${registries[*]//_/-}; do - # In this context, "google-containers" is still used - if [[ "$r" == "$GCRIO_PATH_PROD" ]]; then - artifact_namespace="google-containers" - else - artifact_namespace="${r/gcr.io\//}" - fi + r=${registry//_/-} - gs_path="gs://artifacts.$artifact_namespace.appspot.com/containers" - logecho -n "Checking write access to registry $r: " - if logrun $GSUTIL -q cp $emptyfile $gs_path && \ - logrun $GSUTIL -q rm $gs_path/${emptyfile##*/}; then - logecho $OK - else - logecho $FAILED - ((retcode++)) - fi + # When we are no-mock mode we need to perform an image promotion, so it's + # unnecessary to check for write access to the production container registry. + if ((FLAGS_nomock)); then + logecho -n "Skipping container registry ACL check on $GCRIO_PATH_PROD in no-mock mode: " + logecho $OK + return 0 + else + artifact_namespace="${r/gcr.io\//}" + fi - # Always reset back to $USER - ((FLAGS_gcb)) || logrun $GCLOUD config set account $GCP_USER - done + gs_path="gs://artifacts.$artifact_namespace.appspot.com/containers" + logecho -n "Checking write access to registry $r: " + if logrun $GSUTIL -q cp $emptyfile $gs_path && \ + logrun $GSUTIL -q rm $gs_path/${emptyfile##*/}; then + logecho $OK + else + logecho $FAILED + ((retcode++)) + fi + + # Always reset back to $USER + ((FLAGS_gcb)) || logrun $GCLOUD config set account $GCP_USER logrun rm -f $emptyfile @@ -378,7 +380,7 @@ check_prerequisites () { # Verify write access to all container registries that might be used # to ensure both mock and --nomock runs will work. - ensure_registry_acls "${ALL_CONTAINER_REGISTRIES[*]}" || return 1 + ensure_registry_acls "$GCRIO_PATH" || return 1 logecho -n "Checking cloud project state: " GCLOUD_PROJECT=$($GCLOUD config get-value project 2>/dev/null) @@ -1444,8 +1446,16 @@ push_all_artifacts () { gs://$RELEASE_BUCKET/$BUCKET_TYPE/$version || return 1 fi - common::runstep release::docker::release \ - $KUBE_DOCKER_REGISTRY $version $BUILD_OUTPUT-$version || return 1 + # When we are no-mock mode we need to perform an image promotion, so + # instead of pushing to the production container registry, we validate + # that the manifest is populated on the remote registry. + if ! ((FLAGS_nomock)); then + common::runstep release::docker::release \ + $KUBE_DOCKER_REGISTRY $version $BUILD_OUTPUT-$version || return 1 + fi + + common::runstep release::docker::validate_remote_manifests \ + $KUBE_DOCKER_REGISTRY $version $BUILD_OUTPUT-$version || return 1 common::runstep release::gcs::publish_version \ $BUCKET_TYPE $version $BUILD_OUTPUT-$version $RELEASE_BUCKET || return 1 diff --git a/lib/releaselib.sh b/lib/releaselib.sh index a87610f65fa..5ec347d9e04 100644 --- a/lib/releaselib.sh +++ b/lib/releaselib.sh @@ -31,8 +31,10 @@ readonly CI_BUCKET="kubernetes-release-dev" # TODO(vdf): Need to reference K8s Infra registries here readonly GCRIO_PATH_PROD="k8s.gcr.io" +# TODO(vdf): Remove all GCRIO_PATH_PROD_PUSH logic once the k8s.gcr.io vanity +# domain flip (VDF) is successful readonly GCRIO_PATH_PROD_PUSH="gcr.io/google-containers" -readonly GCRIO_PATH_TEST="gcr.io/$TEST_PROJECT" +readonly GCRIO_PATH_TEST="gcr.io/k8s-staging-kubernetes" readonly KUBE_CROSS_REGISTRY="us.gcr.io/k8s-artifacts-prod/build-image" readonly KUBE_CROSS_IMAGE="${KUBE_CROSS_REGISTRY}/kube-cross" @@ -650,7 +652,6 @@ release::gcs::locally_stage_release_artifacts() { # --release-kind used by push-build.sh local release_kind=${4:-"kubernetes"} local platform - local platforms local release_stage=$build_output/release-stage local release_tars=$build_output/release-tars local gcs_stage=$build_output/gcs-stage/$version @@ -745,7 +746,7 @@ release::gcs::locally_stage_release_artifacts() { # Upload the "naked" binaries to GCS. This is useful for install scripts that # download the binaries directly and don't need tars. - platforms=($(cd "$release_stage/client"; echo *)) + mapfile -t platforms < <(find "${release_stage}/client" -maxdepth 1 -mindepth 1 -type f -exec basename {} \;) for platform in "${platforms[@]}"; do src="$release_stage/client/$platform/$release_kind/client/bin/*" dst="bin/${platform/-//}/" @@ -1020,21 +1021,22 @@ release::gcs::publish () { # @param build_output - build output directory # @return 1 on failure release::docker::release () { - local registry=$1 - local push_registry=$registry - local version=$2 - local build_output=$3 - local release_images=$build_output/release-images - local docker_target + local registry="$1" + local push_registry="$registry" + local version="$2" + local build_output="$3" + local release_images="$build_output/release-images" local arch - local -a arches local tarfile local orig_tag - local -a new_tags local new_tag local binary local -A manifest_images + common::argc_validate 3 + + # TODO(vdf): Remove all GCRIO_PATH_PROD_PUSH logic once the k8s.gcr.io vanity + # domain flip (VDF) is successful if [[ "$registry" == "$GCRIO_PATH_PROD" ]]; then # Switch to the push alias if using the $GCRIO_PATH_PROD alias push_registry="$GCRIO_PATH_PROD_PUSH" @@ -1042,8 +1044,8 @@ release::docker::release () { logecho "Send docker containers from release-images to $push_registry..." - arches=($(cd "$release_images"; echo *)) - for arch in ${arches[@]}; do + mapfile -t arches < <(find "${release_images}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) + for arch in "${arches[@]}"; do for tarfile in $release_images/$arch/*.tar; do # There may be multiple tags; just get the first orig_tag=$(tar xf $tarfile manifest.json -O | jq -r '.[0].RepoTags[0]') @@ -1099,6 +1101,89 @@ release::docker::release () { return 0 } +# TODO(vdf): Consider collapsing this into release::docker::release and renaming +# that function AFTER the k8s.gcr.io Vanity Domain Flip (VDF). +############################################################################### +# Validates that image manifests have been pushed to a specified remote registry. +# Uses 'skopeo inspect'. +# +# @param registry - docker registry +# @param version - version tag +# @param build_output - build output directory +# @return 1 on failure +release::docker::validate_remote_manifests () { + local registry="$1" + local push_registry="$registry" + local version="$2" + local build_output="$3" + local release_images="$build_output/release-images" + local arch + local tarfile + local orig_tag + local new_tag + local binary + local -A manifest_images + + common::argc_validate 3 + + # TODO(vdf): Remove all GCRIO_PATH_PROD_PUSH logic once the k8s.gcr.io vanity + # domain flip (VDF) is successful + if [[ "$registry" == "$GCRIO_PATH_PROD" ]]; then + # Switch to the push alias if using the $GCRIO_PATH_PROD alias + push_registry="$GCRIO_PATH_PROD_PUSH" + fi + + logecho "Validating image manifests in $push_registry..." + + mapfile -t arches < <(find "${release_images}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) + for arch in "${arches[@]}"; do + for tarfile in $release_images/$arch/*.tar; do + # There may be multiple tags; just get the first + orig_tag=$(tar xf $tarfile manifest.json -O | jq -r '.[0].RepoTags[0]') + if [[ ! "$orig_tag" =~ ^.+/(.+):.+$ ]]; then + logecho "$FAILED: malformed tag in $tarfile:" + logecho $orig_tag + return 1 + fi + binary=${BASH_REMATCH[1]} + + new_tag="$push_registry/${binary/-$arch/}" + manifest_images["${new_tag}"]+=" $arch" + done + done + + for image in "${!manifest_images[@]}"; do + local archs + local manifest + local digest + + logecho "Validating manifest list exists for ${image}:${version}..." + + archs=$(echo "${manifest_images[$image]}" | sed -e 's/^[[:space:]]*//') + + if ! manifest=$(skopeo inspect "docker://${image}:${version}" --raw); then + logecho "Could not find manifest list for ${image}:${version}" + return 1 + fi + + for arch in ${archs}; do + logecho "Checking image digest for ${image} on ${arch} architecture..." + + digest=$(echo -n "${manifest}" \ + | jq --arg a "${arch}" -r '.manifests[] | select(.platform.architecture==$a)' \ + | jq -r '.digest') + + if [[ -n "$digest" ]]; then + logecho "Digest for ${image} on ${arch}: ${digest}" + else + logecho "Could not find the image digest for ${image} on ${arch}. Exiting..." + return 1 + fi + done + done + + return 0 +} ############################################################################### # Get the kubecross image version for a given release branch. @@ -1277,9 +1362,6 @@ release::send_announcement () { # READ_RELEASE_BUCKETS - array of readable buckets for multiple sourcing of # mock staged builds # GCRIO_PATH - GCR path based on mock or --nomock -# ALL_CONTAINER_REGISTRIES - when running mock (via GCB) this array also -# contains k8s.gcr.io so we can check access in mock -# mode before an actual release occurs release::set_globals () { logecho -n "Setting global variables: " @@ -1306,7 +1388,6 @@ release::set_globals () { fi GCRIO_PATH="${FLAGS_gcrio_path:-$GCRIO_PATH_TEST}" - ALL_CONTAINER_REGISTRIES=("$GCRIO_PATH") if ((FLAGS_nomock)); then RELEASE_BUCKET="$PROD_BUCKET" @@ -1339,8 +1420,6 @@ release::set_globals () { WRITE_RELEASE_BUCKETS=("$RELEASE_BUCKET") READ_RELEASE_BUCKETS+=("$RELEASE_BUCKET") - ALL_CONTAINER_REGISTRIES=("$GCRIO_PATH") - # TODO: # These KUBE_ globals extend beyond the scope of the new release refactored # tooling so to pass these through as flags will require fixes across