[7.x] [ci] backports (#4546)

* chore(ci): move CI scripts to the .ci directory (#4452)

* chore(ci): movee windows scripts to the CI location

* chore(ci): move more scripts

* chore(ci): move check-changelogs to CI folder

* chore(ci): move intake script to CI dir

* chore(ci): move bench script

* chore(ci): move sync script

* chore(ci): move package to CI dir

* chore(ci): move test-install to CI dir

* chore(ci): remove unused windows script

* ci: remove cron trigger from the check changelog job (#3957)

* fix: ci build and test of the golang-mage docker image (#3563)

* [CI][docker] prepare required context (#4136)

* packer: cache docker images (#3620)

* fix: check that docker is installed (#3889)

* Update Jenkinsfile

Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>

Co-authored-by: Manuel de la Peña <mdelapenya@gmail.com>
Co-authored-by: Ivan Fernandez Calvo <kuisathaverat@users.noreply.github.com>
Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>

.ci/check-changelogs.groovy

@@ -20,9 +20,6 @@ pipeline {
     rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
     quietPeriod(10)
   }
-  triggers {
-    cron 'H H(3-4) * * 1-5'
-  }
   stages {
     /**
      Checkout the code and stash it, to use it on other stages.
@@ -52,7 +49,7 @@ pipeline {
         deleteDir()
         unstash 'source'
         dir("${BASE_DIR}"){
-          sh(label: 'Run check changelogs', script: './script/jenkins/check-changelogs.sh')
+          sh(label: 'Run check changelogs', script: './.ci/scripts/check-changelogs.sh')
         }
       }
     }

.ci/docker/Makefile

@@ -41,6 +41,13 @@ convert-tests-results: ## convert TAP test results to JUnit
 	@APP=$*; docker run --rm -e APP=$${APP} -v "$(CURDIR)":/usr/src/app -w /usr/src/app node:${LTS_ALPINE} \
 					sh -c 'npm install tap-xunit -g && cat target/results.tap | tap-xunit --package="co.elastic.pipeline.$${APP}" > target/junit-$${APP}-results.xml'
 
+test-golang-mage: prepare-test ## Run the tests for the specific app
+	cp -f ../../go.* golang-mage
+	mkdir -p golang-mage/approvaltest && cp -f ../../approvaltest/go.* golang-mage/approvaltest
+	mkdir -p golang-mage/systemtest && cp -f ../../systemtest/go.* golang-mage/systemtest
+	@DOCKERFILE=golang-mage bats-core/bin/bats --tap tests | tee target/results.tap
+	@$(MAKE) -s convert-tests-results
+
 test-%: prepare-test ## Run the tests for the specific app
 	@DOCKERFILE=$* bats-core/bin/bats --tap tests | tee target/results.tap
 	@$(MAKE) -s convert-tests-results

.ci/packer_cache.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC1091
+source /usr/local/bin/bash_standard_lib.sh
+
+# shellcheck disable=SC1091
+source ./script/common.bash
+
+get_go_version
+
+DOCKER_IMAGES="
+docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm
+docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin
+docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main
+docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian7
+docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian8
+docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-mips
+docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-ppc
+docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-s390x
+golang:${GO_VERSION}
+"
+if [ -x "$(command -v docker)" ]; then
+  for image in ${DOCKER_IMAGES}
+  do
+  (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image, we continue"
+  done
+fi

.ci/scripts/build-darwin.sh

@@ -6,4 +6,4 @@ source ./script/common.bash
 
 jenkins_setup
 
-./script/jenkins/build.sh
+./.ci/scripts/build.sh

.ci/scripts/check-changelogs.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+docker run --rm \
+          -v "$PWD":/usr/src/myapp \
+          -w /usr/src/myapp python:3.6.8-alpine \
+          sh -c 'pip3 install requests; python3 script/check_changelogs.py'

.ci/scripts/package-docker-snapshot.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+#
+# Build the docker image for the apm-server, retag and push it to the given docker registry
+#
+# Arguments:
+# - NEW_TAG, this is the tag for the docker image to be pushed.
+# - NEW_IMAGE, this is the docker image name to be pushed.
+#
+set -euo pipefail
+
+NEW_TAG=${1:?Docker tag is not set}
+NEW_IMAGE=${2:?Docker image is not set}
+
+export PLATFORMS='linux/amd64'
+export TYPE='docker'
+export SNAPSHOT='true'
+export IMAGE="docker.elastic.co/apm/apm-server"
+
+echo 'INFO: Build docker images'
+make release
+
+echo 'INFO: Get the just built docker image'
+TAG=$(docker images ${IMAGE} --format "{{.Tag}}" | head -1)
+
+echo "INFO: Retag docker image (${IMAGE}:${TAG})"
+docker tag "${IMAGE}:${TAG}" "${NEW_IMAGE}:${NEW_TAG}"
+
+echo "INFO: Push docker image (${NEW_IMAGE}:${NEW_TAG})"
+docker push "${NEW_IMAGE}:${NEW_TAG}"

.ci/scripts/test-darwin.sh

@@ -17,4 +17,4 @@ source ./script/common.bash
 
 jenkins_setup
 
-script/jenkins/unit-test.sh
+./.ci/scripts/unit-test.sh

.gitignore

@@ -1,6 +1,9 @@
 /.ci/docker/bats-core
 /.ci/docker/tests/test_helper
 /.ci/docker/target
+/.ci/docker/golang-mage/go.*
+/.ci/docker/golang-mage/approvaltest/
+/.ci/docker/golang-mage/systemtest/
 **/*.idea
 /build
 /data

Jenkinsfile

@@ -10,6 +10,7 @@ pipeline {
     JOB_GCS_BUCKET = credentials('gcs-bucket')
     JOB_GCS_CREDENTIALS = 'apm-ci-gcs-plugin'
     CODECOV_SECRET = 'secret/apm-team/ci/apm-server-codecov'
+    ITS_PIPELINE = 'apm-integration-tests-selector-mbp/7.x'
     DIAGNOSTIC_INTERVAL = "${params.DIAGNOSTIC_INTERVAL}"
     ES_LOG_LEVEL = "${params.ES_LOG_LEVEL}"
     DOCKER_SECRET = 'secret/apm-team/ci/docker-registry/prod'
@@ -41,6 +42,7 @@ pipeline {
     booleanParam(name: 'bench_ci', defaultValue: true, description: 'Enable benchmarks')
     booleanParam(name: 'release_ci', defaultValue: true, description: 'Enable build the release packages')
     booleanParam(name: 'kibana_update_ci', defaultValue: true, description: 'Enable build the Check kibana Obj. Updated')
+    booleanParam(name: 'its_ci', defaultValue: true, description: 'Enable async ITs')
     string(name: 'DIAGNOSTIC_INTERVAL', defaultValue: "0", description: 'Elasticsearch detailed logging every X seconds')
     string(name: 'ES_LOG_LEVEL', defaultValue: "error", description: 'Elasticsearch error level')
   }
@@ -110,7 +112,7 @@ pipeline {
           deleteDir()
           unstash 'source'
           dir("${BASE_DIR}"){
-            sh(label: 'Run intake', script: './script/jenkins/intake.sh')
+            sh(label: 'Run intake', script: './.ci/scripts/intake.sh')
           }
         }
       }
@@ -138,7 +140,7 @@ pipeline {
                 dir(BASE_DIR){
                   retry(2) { // Retry in case there are any errors to avoid temporary glitches
                     sleep randomNumber(min: 5, max: 10)
-                    sh(label: 'Linux build', script: './script/jenkins/build.sh')
+                    sh(label: 'Linux build', script: './.ci/scripts/build.sh')
                   }
                 }
               }
@@ -168,8 +170,8 @@ pipeline {
               dir(BASE_DIR){
                 retry(2) { // Retry in case there are any errors to avoid temporary glitches
                   sleep randomNumber(min: 5, max: 10)
-                  powershell(label: 'Windows build', script: '.\\script\\jenkins\\windows-build.ps1')
-                  powershell(label: 'Run Window tests', script: '.\\script\\jenkins\\windows-test.ps1')
+                  powershell(label: 'Windows build', script: '.\\.ci\\scripts\\windows-build.ps1')
+                  powershell(label: 'Run Window tests', script: '.\\.ci\\scripts\\windows-test.ps1')
                 }
               }
             }
@@ -243,7 +245,7 @@ pipeline {
               deleteDir()
               unstash 'source'
               dir("${BASE_DIR}"){
-                sh(label: 'Run Unit tests', script: './script/jenkins/unit-test.sh')
+                sh(label: 'Run Unit tests', script: './.ci/scripts/unit-test.sh')
               }
             }
           }
@@ -285,7 +287,7 @@ pipeline {
               deleteDir()
               unstash 'source'
               dir("${BASE_DIR}"){
-                sh(label: 'Run Linux tests', script: './script/jenkins/linux-test.sh')
+                sh(label: 'Run Linux tests', script: './.ci/scripts/linux-test.sh')
               }
             }
           }
@@ -333,7 +335,7 @@ pipeline {
               unstash 'source'
               golang(){
                 dir("${BASE_DIR}"){
-                  sh(label: 'Run benchmarks', script: './script/jenkins/bench.sh')
+                  sh(label: 'Run benchmarks', script: './.ci/scripts/bench.sh')
                   sendBenchmarks(file: 'bench.out', index: "benchmark-server")
                 }
               }
@@ -364,78 +366,129 @@ pipeline {
               unstash 'source'
               dir("${BASE_DIR}"){
                 catchError(buildResult: 'SUCCESS', message: 'Sync Kibana is not updated', stageResult: 'UNSTABLE') {
-                  sh(label: 'Test Sync', script: './script/jenkins/sync.sh')
+                  sh(label: 'Test Sync', script: './.ci/scripts/sync.sh')
                 }
               }
             }
           }
         }
-      }
-    }
-    /**
-      build release packages.
-    */
-    stage('Package') {
-      agent { label 'linux && immutable' }
-      options { skipDefaultCheckout() }
-      environment {
-        PATH = "${env.PATH}:${env.WORKSPACE}/bin"
-        HOME = "${env.WORKSPACE}"
-        GOPATH = "${env.WORKSPACE}"
-        SNAPSHOT = "true"
-      }
-      when {
-        beforeAgent true
-        allOf {
-          expression { return params.release_ci }
-          expression { return env.ONLY_DOCS == "false" }
-          anyOf {
-            branch 'master'
-            branch pattern: '\\d+\\.\\d+', comparator: 'REGEXP'
-            tag pattern: 'v\\d+\\.\\d+\\.\\d+.*', comparator: 'REGEXP'
-            expression { return isPR() && env.BEATS_UPDATED != "false" }
-            expression { return env.GITHUB_COMMENT?.contains('package tests') || env.GITHUB_COMMENT?.contains('/package') }
-            expression { return params.Run_As_Master_Branch }
+        stage('Hey-Apm') {
+          agent { label 'linux && immutable' }
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            expression { return env.GITHUB_COMMENT?.contains('hey-apm tests') || env.GITHUB_COMMENT?.contains('/hey-apm')}
           }
-        }
-      }
-      stages {
-        stage('Package') {
           steps {
-            withGithubNotify(context: 'Package') {
+            withGithubNotify(context: 'Hey-Apm') {
               deleteDir()
               unstash 'source'
               golang(){
+                dockerLogin(secret: env.DOCKER_SECRET, registry: env.DOCKER_REGISTRY)
                 dir("${BASE_DIR}"){
-                  sh(label: 'Build packages', script: './script/jenkins/package.sh')
-                  sh(label: 'Test packages install', script: './script/jenkins/test-install-packages.sh')
-                  dockerLogin(secret: env.DOCKER_SECRET, registry: env.DOCKER_REGISTRY)
-                  sh(label: 'Package & Push', script: "./script/jenkins/package-docker-snapshot.sh ${env.GIT_BASE_COMMIT} ${env.DOCKER_IMAGE}")
+                  sh(label: 'Package & Push', script: "./.ci/scripts/package-docker-snapshot.sh ${env.GIT_BASE_COMMIT} ${env.DOCKER_IMAGE}")
                 }
               }
+              build(job: 'apm-server/apm-hey-test-benchmark', propagate: true, wait: true,
+                    parameters: [string(name: 'GO_VERSION', value: '1.12.1'),
+                                 string(name: 'STACK_VERSION', value: "${env.GIT_BASE_COMMIT}"),
+                                 string(name: 'APM_DOCKER_IMAGE', value: "${env.DOCKER_IMAGE}")])
             }
           }
         }
-        stage('Publish') {
+        stage('Package') {
+          agent { label 'linux && immutable' }
+          options { skipDefaultCheckout() }
           environment {
-            BUCKET_URI = """${isPR() ? "gs://${JOB_GCS_BUCKET}/pull-requests/pr-${env.CHANGE_ID}" : "gs://${JOB_GCS_BUCKET}/snapshots"}"""
+            PATH = "${env.PATH}:${env.WORKSPACE}/bin"
+            HOME = "${env.WORKSPACE}"
+            GOPATH = "${env.WORKSPACE}"
+            SNAPSHOT = "true"
           }
-          steps {
-            // Upload files to the default location
-            googleStorageUpload(bucket: "${BUCKET_URI}",
-              credentialsId: "${JOB_GCS_CREDENTIALS}",
-              pathPrefix: "${BASE_DIR}/build/distributions/",
-              pattern: "${BASE_DIR}/build/distributions/**/*",
-              sharedPublicly: true,
-              showInline: true)
+          when {
+            beforeAgent true
+            allOf {
+              expression { return params.release_ci }
+              expression { return env.ONLY_DOCS == "false" }
+              anyOf {
+                branch 'master'
+                branch pattern: '\\d+\\.\\d+', comparator: 'REGEXP'
+                tag pattern: 'v\\d+\\.\\d+\\.\\d+.*', comparator: 'REGEXP'
+                expression { return isPR() && env.BEATS_UPDATED != "false" }
+                expression { return env.GITHUB_COMMENT?.contains('package tests') || env.GITHUB_COMMENT?.contains('/package')}
+                expression { return params.Run_As_Master_Branch }
+              }
+            }
+          }
+          stages {
+            stage('Package') {
+              steps {
+                withGithubNotify(context: 'Package') {
+                  deleteDir()
+                  unstash 'source'
+                  golang(){
+                    dir("${BASE_DIR}"){
+                      sh(label: 'Build packages', script: './.ci/scripts/package.sh')
+                      sh(label: 'Test packages install', script: './.ci/scripts/test-install-packages.sh')
+                      dockerLogin(secret: env.DOCKER_SECRET, registry: env.DOCKER_REGISTRY)
+                      sh(label: 'Package & Push', script: "./.ci/scripts/package-docker-snapshot.sh ${env.GIT_BASE_COMMIT} ${env.DOCKER_IMAGE}")
+                    }
+                  }
+                }
+              }
+            }
+            stage('Publish') {
+              environment {
+                BUCKET_URI = """${isPR() ? "gs://${JOB_GCS_BUCKET}/pull-requests/pr-${env.CHANGE_ID}" : "gs://${JOB_GCS_BUCKET}/snapshots"}"""
+              }
+              steps {
+                // Upload files to the default location
+                googleStorageUpload(bucket: "${BUCKET_URI}",
+                  credentialsId: "${JOB_GCS_CREDENTIALS}",
+                  pathPrefix: "${BASE_DIR}/build/distributions/",
+                  pattern: "${BASE_DIR}/build/distributions/**/*",
+                  sharedPublicly: true,
+                  showInline: true)
 
-            // Copy those files to another location with the sha commit to test them afterward.
-            googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}",
-              credentialsId: "${JOB_GCS_CREDENTIALS}",
-              pathPrefix: "${BASE_DIR}/build/distributions/",
-              pattern: "${BASE_DIR}/build/distributions/**/*",
-              sharedPublicly: true,
-              showInline: true)
+                // Copy those files to another location with the sha commit to test them afterward.
+                googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}",
+                  credentialsId: "${JOB_GCS_CREDENTIALS}",
+                  pathPrefix: "${BASE_DIR}/build/distributions/",
+                  pattern: "${BASE_DIR}/build/distributions/**/*",
+                  sharedPublicly: true,
+                  showInline: true)
+              }
+            }
+          }
+        }
+        stage('APM Integration Tests') {
+          agent { label 'linux && immutable' }
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            allOf {
+              anyOf {
+                changeRequest()
+                expression { return !params.Run_As_Master_Branch }
+              }
+              expression { return params.its_ci }
+              expression { return env.ONLY_DOCS == "false" }
+            }
+          }
+          steps {
+            withGithubNotify(context: 'APM Integration Tests') {
+              script {
+                def buildObject = build(job: env.ITS_PIPELINE, propagate: false, wait: true,
+                      parameters: [string(name: 'INTEGRATION_TEST', value: 'All'),
+                                  string(name: 'BUILD_OPTS', value: "--apm-server-build https://github.com/elastic/${env.REPO}@${env.GIT_BASE_COMMIT}")])
+                copyArtifacts(projectName: env.ITS_PIPELINE, selector: specific(buildNumber: buildObject.number.toString()))
+              }
+            }
+          }
+          post {
+            always {
+              junit(testResults: "**/*-junit*.xml", allowEmptyResults: true, keepLongStdio: true)
+            }
           }
         }
       }