From 505e3151356d5ab6bc10d76b428e5597b2ec8f33 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 23 Feb 2021 09:51:20 +0000 Subject: [PATCH] [PACKAGING] Push docker images with the architecture in the version (#24121) (#24159) --- .ci/packaging.groovy | 69 ++++++++++++++++++++++------------ .ci/scripts/docker-tag-push.sh | 12 ++++++ Jenkinsfile | 67 ++++++++++++++++++++------------- 3 files changed, 98 insertions(+), 50 deletions(-) create mode 100755 .ci/scripts/docker-tag-push.sh diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 6d33b5b880f..5acbbc7bcf6 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -152,7 +152,9 @@ pipeline { withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") { deleteDir() release() - pushCIDockerImages() + dir("${BASE_DIR}"){ + pushCIDockerImages(arch: 'amd64') + } } prepareE2ETestForPackage("${BEATS_FOLDER}") } @@ -234,7 +236,9 @@ pipeline { withGithubNotify(context: "Packaging linux/arm64 ${BEATS_FOLDER}") { deleteWorkspace() release() - pushCIDockerImages() + dir("${BASE_DIR}"){ + pushCIDockerImages(arch: 'arm64') + } } } post { @@ -270,27 +274,37 @@ pipeline { } } -def pushCIDockerImages(){ +/** +* @param arch what architecture +*/ +def pushCIDockerImages(Map args = [:]) { + def arch = args.get('arch', 'amd64') catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') { if (env?.BEATS_FOLDER?.endsWith('auditbeat')) { - tagAndPush('auditbeat') + tagAndPush(beatName: 'auditbeat', arch: arch) } else if (env?.BEATS_FOLDER?.endsWith('filebeat')) { - tagAndPush('filebeat') + tagAndPush(beatName: 'filebeat', arch: arch) } else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) { - tagAndPush('heartbeat') + tagAndPush(beatName: 'heartbeat', arch: arch) } else if ("${env.BEATS_FOLDER}" == "journalbeat"){ - tagAndPush('journalbeat') + tagAndPush(beatName: 'journalbeat', arch: arch) } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) { - tagAndPush('metricbeat') + tagAndPush(beatName: 'metricbeat', arch: arch) } else if ("${env.BEATS_FOLDER}" == "packetbeat"){ - tagAndPush('packetbeat') + tagAndPush(beatName: 'packetbeat', arch: arch) } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") { - tagAndPush('elastic-agent') + tagAndPush(beatName: 'elastic-agent', arch: arch) } } } -def tagAndPush(beatName){ +/** +* @param beatName name of the Beat +* @param arch what architecture +*/ +def tagAndPush(Map args = [:]) { + def beatName = args.beatName + def arch = args.get('arch', 'amd64') def libbetaVer = env.BEAT_VERSION def aliasVersion = "" if("${env.SNAPSHOT}" == "true"){ @@ -307,14 +321,22 @@ def tagAndPush(beatName){ dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + // supported tags + def tags = [tagName, "${env.GIT_BASE_COMMIT}"] + if (!isPR() && aliasVersion != "") { + tags << aliasVersion + } // supported image flavours def variants = ["", "-oss", "-ubi8"] variants.each { variant -> - doTagAndPush(beatName, variant, libbetaVer, tagName) - doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}") - - if (!isPR() && aliasVersion != "") { - doTagAndPush(beatName, variant, libbetaVer, aliasVersion) + tags.each { tag -> + // TODO: + // For backward compatibility let's ensure we tag only for amd64, then E2E can benefit from until + // they support the versioning with the architecture + if ("${arch}" == "amd64") { + doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}") + } + doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}") } } } @@ -325,18 +347,19 @@ def tagAndPush(beatName){ * @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace * @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace */ -def doTagAndPush(beatName, variant, sourceTag, targetTag) { +def doTagAndPush(Map args = [:]) { + def beatName = args.beatName + def variant = args.variant + def sourceTag = args.sourceTag + def targetTag = args.targetTag def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}" def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}" - def iterations = 0 retryWithSleep(retries: 3, seconds: 5, backoff: true) { iterations++ - def status = sh(label: "Change tag and push ${targetName}", script: """ - docker tag ${sourceName} ${targetName} - docker push ${targetName} - """, returnStatus: true) - + def status = sh(label: "Change tag and push ${targetName}", + script: ".ci/scripts/docker-tag-push.sh ${sourceName} ${targetName}", + returnStatus: true) if ( status > 0 && iterations < 3) { error("tag and push failed for ${beatName}, retry") } else if ( status > 0 ) { diff --git a/.ci/scripts/docker-tag-push.sh b/.ci/scripts/docker-tag-push.sh new file mode 100755 index 00000000000..49886422c34 --- /dev/null +++ b/.ci/scripts/docker-tag-push.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -exuo pipefail +MSG="parameter missing." +SOURCE_IMAGE=${1:?$MSG} +TARGET_IMAGE=${2:?$MSG} + +if docker image inspect "${SOURCE_IMAGE}" &> /dev/null ; then + docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}" + docker push "${TARGET_IMAGE}" +else + echo "docker image ${SOURCE_IMAGE} does not exist" +fi diff --git a/Jenkinsfile b/Jenkinsfile index 8e8d67c782b..b317251ce53 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -321,23 +321,26 @@ def uploadPackages(bucketUri, beatsFolder){ /** * Push the docker images for the given beat. * @param beatsFolder beats folder +* @param arch what architecture */ -def pushCIDockerImages(beatsFolder){ +def pushCIDockerImages(Map args = [:]) { + def arch = args.get('arch', 'amd64') + def beatsFolder = args.beatsFolder catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') { if (beatsFolder.endsWith('auditbeat')) { - tagAndPush('auditbeat') + tagAndPush(beatName: 'auditbeat', arch: arch) } else if (beatsFolder.endsWith('filebeat')) { - tagAndPush('filebeat') + tagAndPush(beatName: 'filebeat', arch: arch) } else if (beatsFolder.endsWith('heartbeat')) { - tagAndPush('heartbeat') + tagAndPush(beatName: 'heartbeat', arch: arch) } else if ("${beatsFolder}" == "journalbeat"){ - tagAndPush('journalbeat') + tagAndPush(beatName: 'journalbeat', arch: arch) } else if (beatsFolder.endsWith('metricbeat')) { - tagAndPush('metricbeat') + tagAndPush(beatName: 'metricbeat', arch: arch) } else if ("${beatsFolder}" == "packetbeat"){ - tagAndPush('packetbeat') + tagAndPush(beatName: 'packetbeat', arch: arch) } else if ("${beatsFolder}" == "x-pack/elastic-agent") { - tagAndPush('elastic-agent') + tagAndPush(beatName: 'elastic-agent', arch: arch) } } } @@ -346,7 +349,9 @@ def pushCIDockerImages(beatsFolder){ * Tag and push all the docker images for the given beat. * @param beatName name of the Beat */ -def tagAndPush(beatName){ +def tagAndPush(Map args = [:]) { + def beatName = args.beatName + def arch = args.get('arch', 'amd64') def libbetaVer = env.VERSION if("${env?.SNAPSHOT.trim()}" == "true"){ aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version @@ -360,41 +365,40 @@ def tagAndPush(beatName){ tagName = "pr-${env.CHANGE_ID}" } + // supported tags + def tags = [tagName, "${env.GIT_BASE_COMMIT}"] + if (!isPR() && aliasVersion != "") { + tags << aliasVersion + } // supported image flavours def variants = ["", "-oss", "-ubi8"] variants.each { variant -> - doTagAndPush(beatName, variant, libbetaVer, tagName) - doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}") - - if (!isPR() && aliasVersion != "") { - doTagAndPush(beatName, variant, libbetaVer, aliasVersion) + tags.each { tag -> + doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}") } } } /** -* Tag and push the given sourceTag docker image with the tag name targetTag. * @param beatName name of the Beat * @param variant name of the variant used to build the docker image name * @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace * @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace */ -def doTagAndPush(beatName, variant, sourceTag, targetTag) { +def doTagAndPush(Map args = [:]) { + def beatName = args.beatName + def variant = args.variant + def sourceTag = args.sourceTag + def targetTag = args.targetTag def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}" def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}" def iterations = 0 retryWithSleep(retries: 3, seconds: 5, backoff: true) { iterations++ - def status = sh(label: "Change tag and push ${targetName}", script: """#!/usr/bin/env bash - docker images - if docker image inspect "${sourceName}" &> /dev/null ; then - docker tag ${sourceName} ${targetName} - docker push ${targetName} - else - echo 'docker image ${sourceName} does not exist' - fi - """, returnStatus: true) + def status = sh(label: "Change tag and push ${targetName}", + script: ".ci/scripts/docker-tag-push.sh ${sourceName} ${targetName}", + returnStatus: true) if ( status > 0 && iterations < 3) { error("tag and push failed for ${beatName}, retry") } else if ( status > 0 ) { @@ -462,6 +466,7 @@ def target(Map args = [:]) { def isMage = args.get('isMage', false) def isE2E = args.e2e?.get('enabled', false) def isPackaging = args.get('package', false) + def dockerArch = args.get('dockerArch', 'amd64') withNode(args.label) { withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { @@ -482,7 +487,7 @@ def target(Map args = [:]) { // TODO: // push docker images should happen only after the e2e? if (isPackaging) { - pushCIDockerImages("${directory}") + pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch) } } } @@ -918,7 +923,15 @@ class RunCommand extends co.elastic.beats.BeatsFunction { steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) } if(args?.content?.containsKey('packaging-linux')) { - steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'), package: true) + steps.packagingLinux(context: args.context, + command: args.content.get('packaging-linux'), + directory: args.project, + label: args.label, + isMage: true, + id: args.id, + e2e: args.content.get('e2e'), + package: true, + dockerArch: 'amd64') } if(args?.content?.containsKey('k8sTest')) { steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)