Cherry-pick to 7.x: [packaging][beats-tester] use commit id binaries (#22027)

.ci/beats-tester.groovy

@@ -54,15 +54,13 @@ pipeline {
           options { skipDefaultCheckout() }
           when { branch 'master' }
           steps {
-            // TODO: to use the git commit that triggered the upstream build
             runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
           }
         }
         stage('Build *.x branch') {
           options { skipDefaultCheckout() }
           when { branch '*.x' }
           steps {
-            // TODO: to use the git commit that triggered the upstream build
             runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
           }
         }
@@ -79,7 +77,7 @@ pipeline {
           options { skipDefaultCheckout() }
           when {
             not {
-              allOf {
+              anyOf {
                 branch comparator: 'REGEXP', pattern: '(master|.*x)'
                 changeRequest()
               }
@@ -96,14 +94,28 @@ pipeline {
 }
 
 def runBeatsTesterJob(Map args = [:]) {
-  if (args.apm && args.beats) {
+  def apm = args.get('apm', '')
+  def beats = args.get('beats', '')
+  def version = args.version
+
+  if (isUpstreamTrigger()) {
+    copyArtifacts(filter: 'beats-tester.properties',
+                  flatten: true,
+                  projectName: "Beats/packaging/${env.JOB_BASE_NAME}",
+                  selector: upstream(fallbackToLastSuccessful: true))
+    def props = readProperties(file: 'beats-tester.properties')
+    apm = props.get('APM_URL_BASE', '')
+    beats = props.get('BEATS_URL_BASE', '')
+    version = props.get('VERSION', '8.0.0-SNAPSHOT')
+  }
+  if (apm?.trim() || beats?.trim()) {
     build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false,
           parameters: [
-            string(name: 'APM_URL_BASE', value: args.apm),
-            string(name: 'BEATS_URL_BASE', value: args.beats),
-            string(name: 'VERSION', value: args.version)
+            string(name: 'APM_URL_BASE', value: apm),
+            string(name: 'BEATS_URL_BASE', value: beats),
+            string(name: 'VERSION', value: version)
           ])
   } else {
-    build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: args.version) ])
+    build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: version) ])
   }
-}
+}

.ci/jobs/beats-tester.yml

@@ -1,8 +1,8 @@
 ---
 - job:
     name: Beats/beats-tester
-    display-name: Beats Tester
-    description: Run the beats-tester
+    display-name: Beats Tester orchestrator
+    description: Orchestrate the beats-tester when packaging finished successfully
     view: Beats
     disabled: false
     project-type: multibranch

.ci/packaging.groovy

@@ -55,9 +55,29 @@ pipeline {
         stage('Checkout') {
           options { skipDefaultCheckout() }
           steps {
-            deleteDir()
-            gitCheckout(basedir: "${BASE_DIR}")
+            script {
+              if(isUpstreamTrigger()) {
+                try {
+                  copyArtifacts(filter: 'packaging.properties',
+                                flatten: true,
+                                projectName: "Beats/beats/${env.JOB_BASE_NAME}",
+                                selector: upstream(fallbackToLastSuccessful: true))
+                  def props = readProperties(file: 'packaging.properties')
+                  gitCheckout(basedir: "${BASE_DIR}", branch: props.COMMIT)
+                } catch(err) {
+                  // Fallback to the head of the branch as used to be.
+                  gitCheckout(basedir: "${BASE_DIR}")
+                }
+              } else {
+                gitCheckout(basedir: "${BASE_DIR}")
+              }
+            }
             setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
+            withMageEnv(){
+              dir("${BASE_DIR}"){
+                setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
+              }
+            }
             stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
           }
         }
@@ -150,6 +170,17 @@ pipeline {
           }
         }
       }
+      post {
+        success {
+          writeFile(file: 'beats-tester.properties',
+                    text: """\
+                    ## To be consumed by the beats-tester pipeline
+                    COMMIT=${env.GIT_BASE_COMMIT}
+                    BEATS_URL_BASE=https://storage.googleapis.com/${env.JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}
+                    VERSION=${env.BEAT_VERSION}-SNAPSHOT""".stripIndent()) // stripIdent() requires '''/
+          archiveArtifacts artifacts: 'beats-tester.properties'
+        }
+      }
     }
   }
 }
@@ -185,7 +216,7 @@ def pushCIDockerImages(){
 }
 
 def tagAndPush(name){
-  def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+  def libbetaVer = env.BEAT_VERSION
   if("${env.SNAPSHOT}" == "true"){
     libbetaVer += "-SNAPSHOT"
   }
@@ -237,7 +268,7 @@ def publishPackages(baseDir){
   uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
 
   // Copy those files to another location with the sha commit to test them
-  // aftewords.
+  // afterward.
   bucketUri = "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}"
   uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
 }
@@ -275,4 +306,4 @@ def withBeatsEnv(Closure body) {
       }
     }
   }
-}
+}

Jenkinsfile

@@ -78,6 +78,7 @@ pipeline {
         withGithubNotify(context: "Lint") {
           withBeatsEnv(archive: false, id: "lint") {
             dumpVariables()
+            setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
             cmd(label: "make check-python", script: "make check-python")
             cmd(label: "make check-go", script: "make check-go")
             cmd(label: "Check for changes", script: "make check-no-changes")
@@ -124,6 +125,12 @@ pipeline {
     }
   }
   post {
+    success {
+      writeFile(file: 'packaging.properties', text: """## To be consumed by the packaging pipeline
+COMMIT=${env.GIT_BASE_COMMIT}
+VERSION=${env.VERSION}-SNAPSHOT""")
+      archiveArtifacts artifacts: 'packaging.properties'
+    }
     always {
       deleteDir()
       unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")

Makefile

@@ -191,6 +191,11 @@ python-env:
 test-apm:
 	sh ./script/test_apm.sh
 
+## get-version : Get the libbeat version
+.PHONY: get-version
+get-version:
+	@mage dumpVariables | grep 'beat_version' | cut -d"=" -f 2 | tr -d " "
+
 ### Packaging targets ####
 
 ## snapshot : Builds a snapshot release.