Merge remote-tracking branch 'upstream/master' into opentelemetry

* upstream/master: (56 commits)
  docs: update CHANGELOG.md
  [maven-release-plugin] prepare for next development iteration
  [maven-release-plugin] prepare release v1.1.209
  security: disable codecov bash updater
  chore(deps-dev): bump mockito-core from 3.8.0 to 3.9.0 (#1073)
  fix: use toJSON instead readJSON (#1071)
  test-infra: add curl validation on windows (#1069)
  CI: skip using ubuntu label in the beats test-infra (#1070)
  test: add Ubuntu 20.04 to the tests (#1068)
  docs: update CHANGELOG.md
  [maven-release-plugin] prepare for next development iteration
  [maven-release-plugin] prepare release v1.1.208
  Disable unique ephemeral worker by default (#1067)
  Fix and simplify buildStatus step (#1066)
  Add benchmark reporter image to build pipeline (#1065)
  docs: update CHANGELOG.md
  [maven-release-plugin] prepare for next development iteration
  [maven-release-plugin] prepare release v1.1.207
  feat: stage status cache (#1061)
  feat: step hasCommentAuthorWritePermissions (#1064)
  ...

.ci/Jenkinsfile

@@ -245,8 +245,7 @@ pipeline {
   }
   post {
     cleanup {
-      notifyBuildResult(prComment: true, slackComment: true, analyzeFlakey: true,
-                        flakyReportIdx: 'reporter-apm-shared-apm-pipeline-library-master')
+      notifyBuildResult(prComment: true, slackComment: true, analyzeFlakey: true, jobName: getFlakyJobName(withBranch: 'master'))
     }
   }
 }

.ci/buildDockerImages.groovy

@@ -59,8 +59,10 @@ pipeline {
     booleanParam(name: 'oracle_instant_client', defaultValue: "false", description: "")
     booleanParam(name: 'picklesdoc', defaultValue: "false", description: "Pickles Doc generator")
     booleanParam(name: 'python', defaultValue: "false", description: "")
+    booleanParam(name: 'rebuild_analyzer', defaultValue: "false", description: "Rebuild analyzer [https://github.com/elastic/observability-dev/tree/master/apps/automation/rebuild-analyzer]")
     booleanParam(name: 'ruby', defaultValue: 'false', description: '')
     booleanParam(name: 'rum', defaultValue: 'false', description: '')
+    booleanParam(name: 'slack_apm_report', defaultValue: 'false', description: "Slack APM Bridge [https://github.com/elastic/observability-dev/tree/master/tools/report-bridge]")
     booleanParam(name: 'testPlans', defaultValue: "false", description: "Test Plans app")
     booleanParam(name: 'weblogic', defaultValue: "false", description: "")
     booleanParam(name: 'load_orch', defaultValue: "false", description: "Load testing orchestrator [https://github.com/elastic/observability-dev/tree/master/apps/automation/bandstand]")
@@ -361,6 +363,25 @@ pipeline {
           folder: "apps/automation/build-analyzer")
       }
     }
+    stage('Rebuild analyzer'){
+      options {
+        skipDefaultCheckout()
+      }
+      when{
+        beforeAgent true
+        expression { return params.rebuild_analyzer}
+      }
+      steps {
+        deleteDir()
+        dockerLoginElasticRegistry()
+        buildDockerImage(
+          repo: 'https://github.com/elastic/observability-dev',
+          tag: 'rebuild-analyzer',
+          version: 'latest',
+          push: true,
+          folder: "apps/automation/rebuild-analyzer")
+      }
+    }
     stage('Build integrations test reporter'){
       options {
         skipDefaultCheckout()
@@ -380,6 +401,25 @@ pipeline {
           folder: "apps/automation/integrations/reporter")
       }
     }
+    stage('Build hey-apm reports Slack integration'){
+      options {
+        skipDefaultCheckout()
+      }
+      when{
+        beforeAgent true
+        expression { return params.slack_apm_report}
+      }
+      steps {
+        deleteDir()
+        dockerLoginElasticRegistry()
+        buildDockerImage(
+          repo: 'https://github.com/elastic/observability-dev',
+          tag: 'slack-bridge-hey-apm',
+          version: 'latest',
+          push: true,
+          folder: "tools/report-bridge")
+      }
+    }
     stage('Build Heartbeat'){
       options {
         skipDefaultCheckout()

.ci/validateWorkersBeatsCi.groovy

@@ -62,30 +62,29 @@ pipeline {
           axis {
             name 'PLATFORM'
             values (
-              'ubuntu && immutable',
-              'ubuntu-1404-i386',
-              'ubuntu-16',
+              'arm',  // ephemeral workers
+              'immutable && windows-2019',
+              'immutable && windows-2016',
+              'immutable && windows-2012-r2',
+              'immutable && windows-10',
+              'immutable && windows-8',
+              'immutable && windows-2008-r2',
+              'immutable && windows-7-32-bit',
+              'immutable && windows-7',
               'ubuntu-18',
               'ubuntu-20',
+              'worker-c07c6107jyw0',  // macOS workers https://beats-ci.elastic.co/label/macosx/
               'worker-c07jc1nzdwym',
               'worker-c07l34n6dwym',
               'worker-c07y20b6jyvy',
-              'worker-c07ll940dwyl',
               'worker-c07y20b9jyvy',
               'worker-c07y20b4jyvy',
               'worker-c07y20bcjyvy',
               'worker-c07mq25jdy3h',
               'worker-c07mq1u7dy3h',
-              'worker-395930',
-              'worker-0a434dec4bdcd060f',
-              'immutable && windows-2019',
-              'immutable && windows-2016',
-              'immutable && windows-2012-r2',
-              'immutable && windows-10',
-              'immutable && windows-8',
-              'immutable && windows-2008-r2',
-              'immutable && windows-7-32-bit',
-              'immutable && windows-7'
+              'worker-h2wdt2qxq6ny',
+              'worker-395930',  // metal workers https://beats-ci.elastic.co/label/metal/
+              'worker-1244230'
             )
           }
         }

.mergify.yml

@@ -11,7 +11,6 @@ pull_request_rules:
       merge:
         method: squash
         strict: smart+fasttrack
-        priority: high
   - name: backport patches to 7.x branch
     conditions:
       - base=master
@@ -30,4 +29,3 @@ pull_request_rules:
       merge:
         method: squash
         strict: smart+fasttrack
-        priority: low

CHANGELOG.md

@@ -1,9 +1,102 @@
 # Changelog
 
+## v1.1.209 (15/04/2021)
+
+#### 🚀 Enhancements
+
+-  test: add Ubuntu 20.04 to the tests [#1068](https://github.com/elastic/apm-pipeline-library/pull/1068)
+
+#### 🐛 Bug Fixes
+
+-  fix: use toJSON instead readJSON [#1071](https://github.com/elastic/apm-pipeline-library/pull/1071)
+
+#### 🙈 No user affected
+
+-  test-infra: add curl validation on windows [#1069](https://github.com/elastic/apm-pipeline-library/pull/1069)
+
+#### ⚙️ CI
+
+-  CI: skip using ubuntu label in the beats test-infra [#1070](https://github.com/elastic/apm-pipeline-library/pull/1070)
+
+## v1.1.208 (06/04/2021)
+
+#### 🚀 Enhancements
+
+-  Disable unique ephemeral worker by default [#1067](https://github.com/elastic/apm-pipeline-library/pull/1067)
+
+#### ⚙️ CI
+
+-  Add benchmark reporter image to build pipeline [#1065](https://github.com/elastic/apm-pipeline-library/pull/1065)
+
+## v1.1.207 (31/03/2021)
+
+#### 🚀 Enhancements
+
+-  feat: stage status cache [#1061](https://github.com/elastic/apm-pipeline-library/pull/1061)
+-  feat: step hasCommentAuthorWritePermissions [#1064](https://github.com/elastic/apm-pipeline-library/pull/1064)
+-  feat: allow to remove padding [#1059](https://github.com/elastic/apm-pipeline-library/pull/1059)
+
+#### 📚 Documentation
+
+-  docs: update metricbeat step docs [#1057](https://github.com/elastic/apm-pipeline-library/pull/1057)
+
+#### 🙈 No user affected
+
+-  ARM workers are now ephemeral [#1062](https://github.com/elastic/apm-pipeline-library/pull/1062)
+-  Update beats-ci workers for the test-infra validation [#1058](https://github.com/elastic/apm-pipeline-library/pull/1058)
+
+## v1.1.206 (29/03/2021)
+
+#### 🚀 Enhancements
+
+-  feat: metricbeat step [#1050](https://github.com/elastic/apm-pipeline-library/pull/1050)
+-  chore: refactor filebeat step [#1049](https://github.com/elastic/apm-pipeline-library/pull/1049)
+
+#### 🙈 No user affected
+
+-  Add GCE JCasC [#1053](https://github.com/elastic/apm-pipeline-library/pull/1053)
+-  chore: bump Elastic Stack versions [#1052](https://github.com/elastic/apm-pipeline-library/pull/1052)
+
+## v1.1.205 (23/03/2021)
+
+#### 🚀 Enhancements
+
+-  Add withAzureCredentials step [#1048](https://github.com/elastic/apm-pipeline-library/pull/1048)
+
+## v1.1.204 (22/03/2021)
+
+#### 🙈 No user affected
+
+-  Add rebuild analyzer docker generation [#1047](https://github.com/elastic/apm-pipeline-library/pull/1047)
+-  fix: set KIBANA_DOCKER_TAG env var properly [#1046](https://github.com/elastic/apm-pipeline-library/pull/1046)
+
+## v1.1.203 (22/03/2021)
+
+#### 🚀 Enhancements
+
+-  feat: add a new step to build and push a Docker image for Kibana [#1040](https://github.com/elastic/apm-pipeline-library/pull/1040)
+
+#### 🐛 Bug Fixes
+
+-  fix: use proper flag for recreate the local Jenkins services [#1043](https://github.com/elastic/apm-pipeline-library/pull/1043)
+-  Fix the getCause and getCauses for the AbortException [#1041](https://github.com/elastic/apm-pipeline-library/pull/1041)
+
+## v1.1.202 (17/03/2021)
+
+#### 🚀 Enhancements
+
+-  Add getFlakyJobName [#1038](https://github.com/elastic/apm-pipeline-library/pull/1038)
+-  Update to new schema for flaky tests [#1016](https://github.com/elastic/apm-pipeline-library/pull/1016)
+
+#### 📚 Documentation
+
+-  document Gren and GH labels for the automated release process [#1035](https://github.com/elastic/apm-pipeline-library/pull/1035)
+
 ## v1.1.201 (11/03/2021)
 
 #### 🚀 Enhancements
 
+-  chore: support filtering the upstream cause in the isUpstreamTrigger step [#1031](https://github.com/elastic/apm-pipeline-library/pull/1031)
 -  Slack notification with the total build time [#1033](https://github.com/elastic/apm-pipeline-library/pull/1033)
 
 ## v1.1.200 (10/03/2021)

docs/RELEASE.md

@@ -7,6 +7,7 @@ has a name like v[:number:].[:number:].[:number:] see [Semantic Versioning](http
 
 Follow the below steps:
 
+* Make sure your PRs contain the proper Github labels to group them under the proper changelog section, as defined in [Gren's configuration file, `groupBy` section](../.grenrc.js).
 * Navigate to the [APM Pipeline Library job](https://apm-ci.elastic.co/job/apm-shared/job/apm-pipeline-library-mbp/job/master/build?delay=0sec) 
 * Choose `Build with Parameters`
 * Select the `make_release` checkbox.

local/Makefile

@@ -73,24 +73,6 @@ start: master
 	@>&2 printf '\n'
 	@>&2 printf 'Jenkins master now running at http://localhost:18080\n'
 
-.PHONY: start-all
-start-all: # Start all the services
-	env \
-		VAULT_TOKEN=${VAULT_TOKEN} \
-		docker-compose \
-			--log-level ${LOG_LEVEL} \
-			--file docker-compose.yml \
-			up --detach --no-recreate
-
-.PHONY: recreate-all
-recreate-all: # Recreate all the services
-	env \
-		VAULT_TOKEN=${VAULT_TOKEN} \
-		docker-compose \
-			--log-level ${LOG_LEVEL} \
-			--file docker-compose.yml \
-			up --detach --recreate
-
 .PHONY: start-linux-worker
 start-linux-worker:  ## Start the local linux worker
 	VAGRANT_CWD=workers/linux/ vagrant up --provision

local/configs/google.yaml

@@ -13,33 +13,44 @@ jenkins:
   clouds:
   - computeEngine:
       cloudName: "elastic-observability"
+      credentialsId: "elastic-observability"
+      instanceCapStr: "300"
+      instanceId: "20d4387f-26d5-470f-8455-d8e4c15de6df"
+      projectId: "elastic-observability"
       configurations:
       - bootDiskAutoDelete: true
-        bootDiskSizeGb: 50
+        bootDiskSizeGb: 150
+        bootDiskSizeGbStr: "150"
         bootDiskSourceImageProject: "elastic-observability"
-        bootDiskType: "https://www.googleapis.com/compute/v1/projects/elastic-observability/zones/us-central1-f/diskTypes/pd-ssd"
-        description: "obs11 workers"
-        externalAddress: true
+        bootDiskType: "https://www.googleapis.com/compute/v1/projects/elastic-observability/zones/us-central1-a/diskTypes/pd-balanced"
+        description: "obs11 linux workers"
         javaExecPath: "java"
-        labels: "linux immutable obs11"
-        launchTimeoutSeconds: 600
-        machineType: "https://www.googleapis.com/compute/v1/projects/elastic-observability/zones/us-central1-f/machineTypes/g1-small"
+        labelSet:
+        - name: "immutable"
+        - name: "linux"
+        - name: "obs11"
+        - name: "ubuntu"
+        - name: "ubuntu-18"
+        - name: "x86_64"
+        labelString: "ubuntu ubuntu-18 immutable obs11 linux x86_64"
+        labels: "ubuntu ubuntu-18 immutable obs11 linux x86_64"
+        launchTimeoutSeconds: 300
+        launchTimeoutSecondsStr: "300"
+        machineType: "https://www.googleapis.com/compute/v1/projects/elastic-observability/zones/us-central1-a/machineTypes/n2-standard-2"
         mode: NORMAL
-        namePrefix: "obs11-linux"
+        namePrefix: "obs11-ubuntu-18-linux"
         networkConfiguration:
           autofilled:
             network: "https://www.googleapis.com/compute/v1/projects/elastic-observability/global/networks/default"
             subnetwork: "https://www.googleapis.com/compute/v1/projects/elastic-observability/regions/us-central1/subnetworks/default"
         numExecutors: 1
+        numExecutorsStr: "1"
         oneShot: true
         preemptible: true
         region: "https://www.googleapis.com/compute/v1/projects/elastic-observability/regions/us-central1"
-        retentionTimeMinutes: 10
-        runAsUser: "jenkins-gce"
+        retentionTimeMinutes: 6
+        retentionTimeMinutesStr: "6"
+        runAsUser: "jenkins"
         serviceAccountEmail: "jenkins-gce@elastic-observability.iam.gserviceaccount.com"
-        template: "https://www.googleapis.com/compute/v1/projects/elastic-observability/global/instanceTemplates/elastic-apm-ci-ubuntu-1804-lts-20210201222928"
-        zone: "https://www.googleapis.com/compute/v1/projects/elastic-observability/zones/us-central1-f"
-      credentialsId: "elastic-observability"
-      instanceCapStr: "300"
-      instanceId: "20d4387f-26d5-470f-8455-d8e4c15de6df"
-      projectId: "elastic-observability"
+        template: "https://www.googleapis.com/compute/v1/projects/elastic-observability/global/instanceTemplates/elastic-apm-ci-ubuntu-1804-lts-20210302222932"
+        zone: "https://www.googleapis.com/compute/v1/projects/elastic-observability/zones/us-central1-a"

local/configs/jenkins.yaml

@@ -108,11 +108,6 @@ unclassified:
   gitscm:
     globalConfigName: username
     globalConfigEmail: username@example.com
-
-  openTelemetry:
-    endpoint: "otel-collector-contrib:4317"
-    useTls: false
-
 jobs:
   - file: "/var/pipeline-library/src/test/resources/folders/it.dsl"
   - file: "/var/pipeline-library/src/test/resources/folders/beats.dsl"

local/configs/plugins.txt

@@ -4,6 +4,5 @@ google-compute-engine
 job-dsl
 metrics
 monitoring
-opentelemetry
 plot
 ssh-agent