Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[VDF] anago: Publish container images on K8s Infra (k8s-staging-kubernetes) #1199

Merged
merged 4 commits into from
Apr 1, 2020
Merged

[VDF] anago: Publish container images on K8s Infra (k8s-staging-kubernetes) #1199

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
dcaf5c0
anago: Skip ACL check on production container registry
justaugustus Mar 3, 2020
4343e86
anago: Skip image pushes in --nomock mode
justaugustus Mar 4, 2020
29b8e1d
lib/release: Set GCRIO_PATH_TEST to 'gcr.io/k8s-staging-kubernetes'
justaugustus Mar 19, 2020
9df2381
lib/release: Add image validation
justaugustus Mar 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 36 additions & 26 deletions anago
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,12 +259,11 @@ copy_logs_to_workdir () {
}

###############################################################################
# Ensures all registries that will be used during both mock and --nomock
# runs allow write access so we don't fall over later
# @param registries - A space separated list of registries
# Ensures we have write access to a specified registry
# @param registry - A registry to check the ACLs for
#
ensure_registry_acls () {
local registries=($1)
local registry="$1"
local emptyfile="$TMPDIR/empty-file.$$"
local gs_path
local r
Expand All @@ -276,27 +275,30 @@ ensure_registry_acls () {

# Short of creating a hardcoded map of project-id to registry, translating
# _ to - seems to be a simple rule to keep this, well, simple.
for r in ${registries[*]//_/-}; do
# In this context, "google-containers" is still used
if [[ "$r" == "$GCRIO_PATH_PROD" ]]; then
artifact_namespace="google-containers"
else
artifact_namespace="${r/gcr.io\//}"
fi
r=${registry//_/-}

gs_path="gs://artifacts.$artifact_namespace.appspot.com/containers"
logecho -n "Checking write access to registry $r: "
if logrun $GSUTIL -q cp $emptyfile $gs_path && \
logrun $GSUTIL -q rm $gs_path/${emptyfile##*/}; then
logecho $OK
else
logecho $FAILED
((retcode++))
fi
# When we are no-mock mode we need to perform an image promotion, so it's
# unnecessary to check for write access to the production container registry.
if ((FLAGS_nomock)); then
logecho -n "Skipping container registry ACL check on $GCRIO_PATH_PROD in no-mock mode: "
logecho $OK
return 0
else
artifact_namespace="${r/gcr.io\//}"
fi

# Always reset back to $USER
((FLAGS_gcb)) || logrun $GCLOUD config set account $GCP_USER
done
gs_path="gs://artifacts.$artifact_namespace.appspot.com/containers"
logecho -n "Checking write access to registry $r: "
if logrun $GSUTIL -q cp $emptyfile $gs_path && \
logrun $GSUTIL -q rm $gs_path/${emptyfile##*/}; then
logecho $OK
else
logecho $FAILED
((retcode++))
fi

# Always reset back to $USER
((FLAGS_gcb)) || logrun $GCLOUD config set account $GCP_USER

logrun rm -f $emptyfile

Expand Down Expand Up @@ -378,7 +380,7 @@ check_prerequisites () {

# Verify write access to all container registries that might be used
# to ensure both mock and --nomock runs will work.
ensure_registry_acls "${ALL_CONTAINER_REGISTRIES[*]}" || return 1
ensure_registry_acls "$GCRIO_PATH" || return 1

logecho -n "Checking cloud project state: "
GCLOUD_PROJECT=$($GCLOUD config get-value project 2>/dev/null)
Expand Down Expand Up @@ -1444,8 +1446,16 @@ push_all_artifacts () {
gs://$RELEASE_BUCKET/$BUCKET_TYPE/$version || return 1
fi

common::runstep release::docker::release \
$KUBE_DOCKER_REGISTRY $version $BUILD_OUTPUT-$version || return 1
# When we are no-mock mode we need to perform an image promotion, so
# instead of pushing to the production container registry, we validate
# that the manifest is populated on the remote registry.
if ! ((FLAGS_nomock)); then
common::runstep release::docker::release \
$KUBE_DOCKER_REGISTRY $version $BUILD_OUTPUT-$version || return 1
fi

common::runstep release::docker::validate_remote_manifests \
$KUBE_DOCKER_REGISTRY $version $BUILD_OUTPUT-$version || return 1

common::runstep release::gcs::publish_version \
$BUCKET_TYPE $version $BUILD_OUTPUT-$version $RELEASE_BUCKET || return 1
Expand Down
117 changes: 98 additions & 19 deletions lib/releaselib.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ readonly CI_BUCKET="kubernetes-release-dev"

# TODO(vdf): Need to reference K8s Infra registries here
readonly GCRIO_PATH_PROD="k8s.gcr.io"
# TODO(vdf): Remove all GCRIO_PATH_PROD_PUSH logic once the k8s.gcr.io vanity
# domain flip (VDF) is successful
readonly GCRIO_PATH_PROD_PUSH="gcr.io/google-containers"
readonly GCRIO_PATH_TEST="gcr.io/$TEST_PROJECT"
readonly GCRIO_PATH_TEST="gcr.io/k8s-staging-kubernetes"

readonly KUBE_CROSS_REGISTRY="us.gcr.io/k8s-artifacts-prod/build-image"
readonly KUBE_CROSS_IMAGE="${KUBE_CROSS_REGISTRY}/kube-cross"
Expand Down Expand Up @@ -650,7 +652,6 @@ release::gcs::locally_stage_release_artifacts() {
# --release-kind used by push-build.sh
local release_kind=${4:-"kubernetes"}
local platform
local platforms
local release_stage=$build_output/release-stage
local release_tars=$build_output/release-tars
local gcs_stage=$build_output/gcs-stage/$version
Expand Down Expand Up @@ -745,7 +746,7 @@ release::gcs::locally_stage_release_artifacts() {

# Upload the "naked" binaries to GCS. This is useful for install scripts that
# download the binaries directly and don't need tars.
platforms=($(cd "$release_stage/client"; echo *))
mapfile -t platforms < <(find "${release_stage}/client" -maxdepth 1 -mindepth 1 -type f -exec basename {} \;)
for platform in "${platforms[@]}"; do
src="$release_stage/client/$platform/$release_kind/client/bin/*"
dst="bin/${platform/-//}/"
Expand Down Expand Up @@ -1020,30 +1021,31 @@ release::gcs::publish () {
# @param build_output - build output directory
# @return 1 on failure
release::docker::release () {
local registry=$1
local push_registry=$registry
local version=$2
local build_output=$3
local release_images=$build_output/release-images
local docker_target
local registry="$1"
local push_registry="$registry"
local version="$2"
local build_output="$3"
local release_images="$build_output/release-images"
local arch
local -a arches
local tarfile
local orig_tag
local -a new_tags
local new_tag
local binary
local -A manifest_images

common::argc_validate 3

# TODO(vdf): Remove all GCRIO_PATH_PROD_PUSH logic once the k8s.gcr.io vanity
# domain flip (VDF) is successful
if [[ "$registry" == "$GCRIO_PATH_PROD" ]]; then
# Switch to the push alias if using the $GCRIO_PATH_PROD alias
push_registry="$GCRIO_PATH_PROD_PUSH"
fi

logecho "Send docker containers from release-images to $push_registry..."

arches=($(cd "$release_images"; echo *))
for arch in ${arches[@]}; do
mapfile -t arches < <(find "${release_images}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;)
for arch in "${arches[@]}"; do
for tarfile in $release_images/$arch/*.tar; do
# There may be multiple tags; just get the first
orig_tag=$(tar xf $tarfile manifest.json -O | jq -r '.[0].RepoTags[0]')
Expand Down Expand Up @@ -1099,6 +1101,89 @@ release::docker::release () {
return 0
}

# TODO(vdf): Consider collapsing this into release::docker::release and renaming
# that function AFTER the k8s.gcr.io Vanity Domain Flip (VDF).
###############################################################################
# Validates that image manifests have been pushed to a specified remote registry.
# Uses 'skopeo inspect'.
#
# @param registry - docker registry
# @param version - version tag
# @param build_output - build output directory
# @return 1 on failure
release::docker::validate_remote_manifests () {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know in other places in this file it's not being checked but when you are writing new one you should check if the function receives enough arguments it expects and if not fail.

Copy link
Member Author

@justaugustus justaugustus Mar 30, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks! I've added common::argc_validate to both functions. :)

local registry="$1"
local push_registry="$registry"
local version="$2"
local build_output="$3"
local release_images="$build_output/release-images"
local arch
local tarfile
local orig_tag
local new_tag
local binary
local -A manifest_images

common::argc_validate 3

# TODO(vdf): Remove all GCRIO_PATH_PROD_PUSH logic once the k8s.gcr.io vanity
# domain flip (VDF) is successful
if [[ "$registry" == "$GCRIO_PATH_PROD" ]]; then
# Switch to the push alias if using the $GCRIO_PATH_PROD alias
push_registry="$GCRIO_PATH_PROD_PUSH"
fi

logecho "Validating image manifests in $push_registry..."

mapfile -t arches < <(find "${release_images}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;)
for arch in "${arches[@]}"; do
for tarfile in $release_images/$arch/*.tar; do
# There may be multiple tags; just get the first
orig_tag=$(tar xf $tarfile manifest.json -O | jq -r '.[0].RepoTags[0]')
if [[ ! "$orig_tag" =~ ^.+/(.+):.+$ ]]; then
logecho "$FAILED: malformed tag in $tarfile:"
logecho $orig_tag
return 1
fi
binary=${BASH_REMATCH[1]}

new_tag="$push_registry/${binary/-$arch/}"
manifest_images["${new_tag}"]+=" $arch"
done
done

for image in "${!manifest_images[@]}"; do
local archs
local manifest
local digest

logecho "Validating manifest list exists for ${image}:${version}..."

archs=$(echo "${manifest_images[$image]}" | sed -e 's/^[[:space:]]*//')

if ! manifest=$(skopeo inspect "docker://${image}:${version}" --raw); then
logecho "Could not find manifest list for ${image}:${version}"
return 1
fi

for arch in ${archs}; do
logecho "Checking image digest for ${image} on ${arch} architecture..."

digest=$(echo -n "${manifest}" \
| jq --arg a "${arch}" -r '.manifests[] | select(.platform.architecture==$a)' \
| jq -r '.digest')

if [[ -n "$digest" ]]; then
logecho "Digest for ${image} on ${arch}: ${digest}"
else
logecho "Could not find the image digest for ${image} on ${arch}. Exiting..."
return 1
fi
done
done

return 0
}

###############################################################################
# Get the kubecross image version for a given release branch.
Expand Down Expand Up @@ -1277,9 +1362,6 @@ release::send_announcement () {
# READ_RELEASE_BUCKETS - array of readable buckets for multiple sourcing of
# mock staged builds
# GCRIO_PATH - GCR path based on mock or --nomock
# ALL_CONTAINER_REGISTRIES - when running mock (via GCB) this array also
# contains k8s.gcr.io so we can check access in mock
# mode before an actual release occurs
release::set_globals () {
logecho -n "Setting global variables: "

Expand All @@ -1306,7 +1388,6 @@ release::set_globals () {
fi

GCRIO_PATH="${FLAGS_gcrio_path:-$GCRIO_PATH_TEST}"
ALL_CONTAINER_REGISTRIES=("$GCRIO_PATH")

if ((FLAGS_nomock)); then
RELEASE_BUCKET="$PROD_BUCKET"
Expand Down Expand Up @@ -1339,8 +1420,6 @@ release::set_globals () {
WRITE_RELEASE_BUCKETS=("$RELEASE_BUCKET")
READ_RELEASE_BUCKETS+=("$RELEASE_BUCKET")

ALL_CONTAINER_REGISTRIES=("$GCRIO_PATH")

# TODO:
# These KUBE_ globals extend beyond the scope of the new release refactored
# tooling so to pass these through as flags will require fixes across
Expand Down