From e1a9e2c9a55578cef1030b184a4334486aedc292 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 9 Nov 2020 09:19:51 +0000 Subject: [PATCH] [packaging][beats-tester] use commit id binaries (#22027) (#22472) --- .ci/beats-tester.groovy | 30 +++++++++++++++++++++--------- .ci/jobs/beats-tester.yml | 4 ++-- .ci/packaging.groovy | 38 +++++++++++++++++++++++++++++++++++--- Jenkinsfile | 7 +++++++ Makefile | 5 +++++ 5 files changed, 70 insertions(+), 14 deletions(-) diff --git a/.ci/beats-tester.groovy b/.ci/beats-tester.groovy index 91781a98d31..3650246e966 100644 --- a/.ci/beats-tester.groovy +++ b/.ci/beats-tester.groovy @@ -54,7 +54,6 @@ pipeline { options { skipDefaultCheckout() } when { branch 'master' } steps { - // TODO: to use the git commit that triggered the upstream build runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") } } @@ -62,7 +61,6 @@ pipeline { options { skipDefaultCheckout() } when { branch '*.x' } steps { - // TODO: to use the git commit that triggered the upstream build runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") } } @@ -79,7 +77,7 @@ pipeline { options { skipDefaultCheckout() } when { not { - allOf { + anyOf { branch comparator: 'REGEXP', pattern: '(master|.*x)' changeRequest() } @@ -96,14 +94,28 @@ pipeline { } def runBeatsTesterJob(Map args = [:]) { - if (args.apm && args.beats) { + def apm = args.get('apm', '') + def beats = args.get('beats', '') + def version = args.version + + if (isUpstreamTrigger()) { + copyArtifacts(filter: 'beats-tester.properties', + flatten: true, + projectName: "Beats/packaging/${env.JOB_BASE_NAME}", + selector: upstream(fallbackToLastSuccessful: true)) + def props = readProperties(file: 'beats-tester.properties') + apm = props.get('APM_URL_BASE', '') + beats = props.get('BEATS_URL_BASE', '') + version = props.get('VERSION', '8.0.0-SNAPSHOT') + } + if (apm?.trim() || beats?.trim()) { build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ - string(name: 'APM_URL_BASE', value: args.apm), - string(name: 'BEATS_URL_BASE', value: args.beats), - string(name: 'VERSION', value: args.version) + string(name: 'APM_URL_BASE', value: apm), + string(name: 'BEATS_URL_BASE', value: beats), + string(name: 'VERSION', value: version) ]) } else { - build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: args.version) ]) + build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: version) ]) } -} \ No newline at end of file +} diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml index 522abfa5e5c..ceb7a6955ad 100644 --- a/.ci/jobs/beats-tester.yml +++ b/.ci/jobs/beats-tester.yml @@ -1,8 +1,8 @@ --- - job: name: Beats/beats-tester - display-name: Beats Tester - description: Run the beats-tester + display-name: Beats Tester orchestrator + description: Orchestrate the beats-tester when packaging finished successfully view: Beats disabled: false project-type: multibranch diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index c8ecda572b0..f36087cbdd5 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -65,8 +65,29 @@ pipeline { options { skipDefaultCheckout() } steps { deleteDir() - gitCheckout(basedir: "${BASE_DIR}") + script { + if(isUpstreamTrigger()) { + try { + copyArtifacts(filter: 'packaging.properties', + flatten: true, + projectName: "Beats/beats/${env.JOB_BASE_NAME}", + selector: upstream(fallbackToLastSuccessful: true)) + def props = readProperties(file: 'packaging.properties') + gitCheckout(basedir: "${BASE_DIR}", branch: props.COMMIT) + } catch(err) { + // Fallback to the head of the branch as used to be. + gitCheckout(basedir: "${BASE_DIR}") + } + } else { + gitCheckout(basedir: "${BASE_DIR}") + } + } setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) + withMageEnv(){ + dir("${BASE_DIR}"){ + setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) + } + } stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") } } @@ -167,6 +188,17 @@ pipeline { } } } + post { + success { + writeFile(file: 'beats-tester.properties', + text: """\ + ## To be consumed by the beats-tester pipeline + COMMIT=${env.GIT_BASE_COMMIT} + BEATS_URL_BASE=https://storage.googleapis.com/${env.JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT} + VERSION=${env.BEAT_VERSION}-SNAPSHOT""".stripIndent()) // stripIdent() requires '''/ + archiveArtifacts artifacts: 'beats-tester.properties' + } + } } } } @@ -192,7 +224,7 @@ def pushCIDockerImages(){ } def tagAndPush(beatName){ - def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim() + def libbetaVer = env.BEAT_VERSION def aliasVersion = "" if("${env.SNAPSHOT}" == "true"){ aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version @@ -346,7 +378,7 @@ def publishPackages(baseDir){ uploadPackages("${bucketUri}/${beatsFolderName}", baseDir) // Copy those files to another location with the sha commit to test them - // aftewords. + // afterward. bucketUri = "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}" uploadPackages("${bucketUri}/${beatsFolderName}", baseDir) } diff --git a/Jenkinsfile b/Jenkinsfile index 2e579e01701..6bbfb01843b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,6 +76,7 @@ pipeline { withGithubNotify(context: "Lint") { withBeatsEnv(archive: false, id: "lint") { dumpVariables() + setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) cmd(label: "make check-python", script: "make check-python") cmd(label: "make check-go", script: "make check-go") cmd(label: "Check for changes", script: "make check-no-changes") @@ -122,6 +123,12 @@ pipeline { } } post { + success { + writeFile(file: 'packaging.properties', text: """## To be consumed by the packaging pipeline +COMMIT=${env.GIT_BASE_COMMIT} +VERSION=${env.VERSION}-SNAPSHOT""") + archiveArtifacts artifacts: 'packaging.properties' + } always { deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") diff --git a/Makefile b/Makefile index 94e8391c295..b55c590433e 100644 --- a/Makefile +++ b/Makefile @@ -191,6 +191,11 @@ python-env: test-apm: sh ./script/test_apm.sh +## get-version : Get the libbeat version +.PHONY: get-version +get-version: + @mage dumpVariables | grep 'beat_version' | cut -d"=" -f 2 | tr -d " " + ### Packaging targets #### ## snapshot : Builds a snapshot release.