Merge remote-tracking branch 'upstream/master' into feature/windows-2016

* upstream/master:
  [CI] Setup git config globally (elastic#21562)
  docs: update generate_fields_docs.py (elastic#21359)
  Add support for additional fields from V2 ALB logs (elastic#21540)
  Move Prometheus query & remote_write to GA (elastic#21507)
  feat: add a new step to run the e2e tests for certain parts of Beats (elastic#21100)
  [Elastic Agent] Add elastic agent ID and version to events from filebeat and metricbeat. (elastic#21543)
  Release cloudfoundry input and processor as GA (elastic#21525)
  [Packetbeat] New SIP protocol (elastic#21221)
  [Filebeat][New Module] Add support for Microsoft MTP / 365 Defender (elastic#21446)
  [Beats][pytest] Asserting if filebeat logs include errors (elastic#20999)
  junipersrx-module initial release (elastic#20017)
  Add a persistent cache for cloudfoundry metadata based on badger (elastic#20775)
  Add missing changelog entry for cisco umbrella (elastic#21550)
  [Elastic Agent] Add upgrade CLI to initiate upgrade of Agent locally (elastic#21425)
  Enable filestream input (elastic#21533)
  Add filestream input reader (elastic#21481)
  [CI] fix 'no matches found within 10000' (elastic#21466)
  Fix billing.go aws.GetStartTimeEndTime (elastic#21531)

.ci/packaging.groovy

@@ -5,12 +5,14 @@
 pipeline {
   agent none
   environment {
-    BASE_DIR = 'src/github.com/elastic/beats'
+    REPO = 'beats'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
     JOB_GCS_BUCKET = 'beats-ci-artifacts'
     JOB_GCS_BUCKET_STASH = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
+    GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests'
     SNAPSHOT = "true"
     PIPELINE_LOG_LEVEL = "INFO"
   }
@@ -119,6 +121,7 @@ pipeline {
                     release()
                     pushCIDockerImages()
                   }
+                  runE2ETestForPackages()
                 }
               }
               stage('Package Mac OS'){
@@ -209,6 +212,25 @@ def tagAndPush(name){
   }
 }
 
+def runE2ETestForPackages(){
+  def suite = ''
+
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to run e2e tests', stageResult: 'FAILURE') {
+    if ("${env.BEATS_FOLDER}" == "filebeat" || "${env.BEATS_FOLDER}" == "x-pack/filebeat") {
+      suite = 'helm,ingest-manager'
+    } else if ("${env.BEATS_FOLDER}" == "metricbeat" || "${env.BEATS_FOLDER}" == "x-pack/metricbeat") {
+      suite = ''
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
+      suite = 'ingest-manager'
+    } else {
+      echo("Skipping E2E tests for ${env.BEATS_FOLDER}.")
+      return
+    }
+
+    triggerE2ETests(suite)
+  }
+}
+
 def release(){
   withBeatsEnv(){
     dir("${env.BEATS_FOLDER}") {
@@ -218,6 +240,32 @@ def release(){
   }
 }
 
+def triggerE2ETests(String suite) {
+  echo("Triggering E2E tests for ${env.BEATS_FOLDER}. Test suite: ${suite}.")
+
+  def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
+  def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+  build(job: "${e2eTestsPipeline}",
+    parameters: [
+      booleanParam(name: 'forceSkipGitChecks', value: true),
+      booleanParam(name: 'forceSkipPresubmit', value: true),
+      booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+      booleanParam(name: 'USE_CI_SNAPSHOTS', value: true),
+      string(name: 'ELASTIC_AGENT_VERSION', value: "pr-${env.CHANGE_ID}"),
+      string(name: 'METRICBEAT_VERSION', value: "pr-${env.CHANGE_ID}"),
+      string(name: 'runTestsSuites', value: suite),
+      string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
+      string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+      string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
+    ],
+    propagate: false,
+    wait: false
+  )
+
+  def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME} for ${env.BEATS_FOLDER}"
+  githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
+}
+
 def withMacOSEnv(Closure body){
   withEnvMask( vars: [
       [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],

CHANGELOG.next.asciidoc

@@ -448,7 +448,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
 - Added experimental dataset `sophos/utm`. {pull}20820[20820]
 - Add Cloud Foundry tags in related events. {pull}21177[21177]
+- Cloud Foundry metadata is cached to disk. {pull}20775[20775]
 - Add option to select the type of index template to load: legacy, component, index. {pull}21212[21212]
+- Release `add_cloudfoundry_metadata` as GA. {pull}21525[21525]
 
 *Auditbeat*
 
@@ -605,6 +607,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add related.hosts ecs field to all modules {pull}21160[21160]
 - Keep cursor state between httpjson input restarts {pull}20751[20751]
 - Convert aws s3 to v2 input {pull}20005[20005]
+- Add support for additional fields from V2 ALB logs. {pull}21540[21540]
+- Release Cloud Foundry input as GA. {pull}21525[21525]
+- New Cisco Umbrella dataset {pull}21504[21504]
+- New juniper.srx dataset for Juniper SRX logs. {pull}20017[20017]
+- Adding support for Microsoft 365 Defender (Microsoft Threat Protection) {pull}21446[21446]
 
 *Heartbeat*
 
@@ -732,6 +739,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
 - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
 - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
+- Move Prometheus query & remote_write to GA. {pull}21507[21507]
 - Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486]
 - Map cloud data filed `cloud.account.id` to azure subscription.  {pull}21483[21483] {issue}21381[21381]
 
@@ -742,6 +750,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
   port. {pull}19209[19209]
 - Add ECS fields for x509 certs, event categorization, and related IP info. {pull}19167[19167]
 - Add 100-continue support {issue}15830[15830] {pull}19349[19349]
+- Add initial SIP protocol support {pull}21221[21221]
 
 
 *Functionbeat*

Jenkinsfile

@@ -74,7 +74,7 @@ pipeline {
       }
       steps {
         withGithubNotify(context: 'Lint') {
-          withBeatsEnv(archive: true) {
+          withBeatsEnv(archive: true, id: 'lint') {
             dumpVariables()
             cmd(label: 'make check', script: 'make check')
           }
@@ -265,8 +265,8 @@ def withBeatsEnv(Map args = [:], Closure body) {
         // See https://github.com/elastic/beats/issues/17787.
         sh(label: 'check git config', script: '''
           if [ -z "$(git config --get user.email)" ]; then
-            git config user.email "beatsmachine@users.noreply.github.com"
-            git config user.name "beatsmachine"
+            git config --global user.email "beatsmachine@users.noreply.github.com"
+            git config --global user.name "beatsmachine"
           fi''')
       }
       try {
@@ -345,8 +345,13 @@ def archiveTestOutput(Map args = [:]) {
     }
     cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py')
     dir('build') {
+      if (isUnix()) {
+        cmd(label: 'Delete folders that are causing exceptions (See JENKINS-58421)',
+            returnStatus: true,
+            script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;')
+      } else { log(level: 'INFO', text: 'Delete folders that are causing exceptions (See JENKINS-58421) is disabled for Windows.') }
       junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, stashedTestReports: stashedTestReports, id: args.id)
-      archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts)
+      tar(file: "test-build-artifacts-${args.id}.tgz", dir: '.', archive: true, allowMissing: true)
     }
     catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
       def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()