Merge remote-tracking branch 'upstream/master' into feature/2.0-skip-…

…ci-build-label-support * upstream/master: (60 commits) Skip publisher flaky tests (elastic#21657) backport: add 7.10 branch (elastic#21635) [CI: Packaging] fix: push ubi8 images too (elastic#21621) Docker build resiliance with a retry (elastic#21587) Fix flaky FSWatch/FSScanner tests (elastic#21625) chore: add versions 7.1x (elastic#21670) [Elastic Agent] Reload fleet.kibana.hosts from policy change (elastic#21599) Fix cyberark/corepas pipeline (elastic#21643) Add openstack ssl provider in add_cloud_metadata (elastic#21590) Add fips_enabled into all aws filesets (elastic#21626) [Filebeat S3] Change log.file.path to be nested object (elastic#21624) [CI] Change notification channel (elastic#21559) Add `add_observer_metadata` `geo.name` to Quickstart (elastic#21501) [E2E Tests] fix: set versions ony for PRs (elastic#21608) Fix badger build in 386 (elastic#21613) docs: Update timestamp.asciidoc (elastic#20395) Remove nil-zero metrics and linux-exclusive metrics from Metricbeat (elastic#21457) [Metricbeat] Use timestamp from CloudWatch for events (elastic#21498) [Filebeat][S3 Input] Add support for FIPS endpoints (elastic#21585) [Ingest Manager] Use new form of fleet API paths (elastic#21478) ...
v1v · Oct 8, 2020 · 36ed51a · 36ed51a
2 parents 2a3d98f + df03add
commit 36ed51a
Show file tree

Hide file tree

Showing 455 changed files with 28,768 additions and 4,770 deletions.
diff --git a/.backportrc.json b/.backportrc.json
@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [{ "name": "7.9"}, { "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
+  "branches": [ { "name": "7.x", "checked": true }, "7.10", "7.9", "7.8", "7.7"],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

diff --git a/.ci/jobs/apm-beats-update.yml b/.ci/jobs/apm-beats-update.yml
@@ -18,7 +18,7 @@
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
         discover-tags: true
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         disable-pr-notifications: true
         notification-context: 'apm-beats-update'
         repo: 'beats'
@@ -38,6 +38,9 @@
             - regex-name:
                 regex: '7\.[x789]'
                 case-sensitive: true
+            - regex-name:
+                regex: '7\.1\d'
+                case-sensitive: true
             - regex-name:
                 regex: '8\.\d+'
                 case-sensitive: true
@@ -48,7 +51,7 @@
             before: true
         prune: true
         shallow-clone: true
-        depth: 3
+        depth: 10
         do-not-fetch-tags: true
         submodule:
             disable: false

diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml
@@ -44,7 +44,7 @@
               before: true
           prune: true
           shallow-clone: true
-          depth: 3
+          depth: 10
           do-not-fetch-tags: true
           submodule:
               disable: false

diff --git a/.ci/jobs/beats-windows-mbp.yml b/.ci/jobs/beats-windows-mbp.yml
@@ -44,7 +44,7 @@
             before: true
           prune: true
           shallow-clone: true
-          depth: 4
+          depth: 10
           do-not-fetch-tags: true
           submodule:
             disable: false

diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml
@@ -17,7 +17,7 @@
         discover-pr-forks-strategy: 'merge-current'
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         discover-tags: true
         notification-context: "beats-ci"
         repo: 'beats'
@@ -38,6 +38,9 @@
             - regex-name:
                 regex: '7\.[x789]'
                 case-sensitive: true
+            - regex-name:
+                regex: '7\.1\d'
+                case-sensitive: true
             - regex-name:
                 regex: '8\.\d+'
                 case-sensitive: true
@@ -46,7 +49,7 @@
             before: true
         prune: true
         shallow-clone: true
-        depth: 3
+        depth: 10
         do-not-fetch-tags: true
         submodule:
             disable: false

diff --git a/.ci/jobs/golang-crossbuild-mbp.yml b/.ci/jobs/golang-crossbuild-mbp.yml
@@ -31,7 +31,7 @@
             before: true
           prune: true
           shallow-clone: true
-          depth: 4
+          depth: 10
           do-not-fetch-tags: true
           submodule:
             disable: false

diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml
@@ -14,7 +14,7 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+          head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
           disable-pr-notifications: true
           notification-context: 'beats-packaging'
           repo: 'beats'
@@ -34,6 +34,9 @@
               - regex-name:
                   regex: '7\.[x789]'
                   case-sensitive: true
+              - regex-name:
+                  regex: '7\.1\d'
+                  case-sensitive: true
               - regex-name:
                   regex: '8\.\d+'
                   case-sensitive: true
@@ -44,7 +47,7 @@
               before: true
           prune: true
           shallow-clone: true
-          depth: 3
+          depth: 10
           do-not-fetch-tags: true
           submodule:
               disable: false

diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -5,12 +5,14 @@
 pipeline {
   agent none
   environment {
-    BASE_DIR = 'src/github.com/elastic/beats'
+    REPO = 'beats'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
     JOB_GCS_BUCKET = 'beats-ci-artifacts'
     JOB_GCS_BUCKET_STASH = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
+    GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests'
     SNAPSHOT = "true"
     PIPELINE_LOG_LEVEL = "INFO"
   }
@@ -119,6 +121,7 @@ pipeline {
                     release()
                     pushCIDockerImages()
                   }
+                  runE2ETestForPackages()
                 }
               }
               stage('Package Mac OS'){
@@ -195,17 +198,49 @@ def tagAndPush(name){
     tagName = "pr-${env.CHANGE_ID}"
   }
 
-  def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
-  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${tagName}"
-  def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
   dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-  retry(3){
-    sh(label:'Change tag and push', script: """
-      docker tag ${oldName} ${newName}
-      docker push ${newName}
-      docker tag ${oldName} ${commitName}
-      docker push ${commitName}
-    """)
+
+  // supported image flavours
+  def variants = ["", "-oss", "-ubi8"]
+  variants.each { variant ->
+    def oldName = "${DOCKER_REGISTRY}/beats/${name}${variant}:${libbetaVer}"
+    def newName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${tagName}"
+    def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${env.GIT_BASE_COMMIT}"
+
+    def iterations = 0
+    retryWithSleep(retries: 3, seconds: 5, backoff: true)
+      iterations++
+      def status = sh(label:'Change tag and push', script: """
+        docker tag ${oldName} ${newName}
+        docker push ${newName}
+        docker tag ${oldName} ${commitName}
+        docker push ${commitName}
+      """, returnStatus: true) 
+      if ( status > 0 && iterations < 3) {
+        error('tag and push failed, retry')
+      } else if ( status > 0 ) {
+        log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
+      }
+    }
+  }
+}
+
+def runE2ETestForPackages(){
+  def suite = ''
+
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to run e2e tests', stageResult: 'FAILURE') {
+    if ("${env.BEATS_FOLDER}" == "filebeat" || "${env.BEATS_FOLDER}" == "x-pack/filebeat") {
+      suite = 'helm,ingest-manager'
+    } else if ("${env.BEATS_FOLDER}" == "metricbeat" || "${env.BEATS_FOLDER}" == "x-pack/metricbeat") {
+      suite = ''
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
+      suite = 'ingest-manager'
+    } else {
+      echo("Skipping E2E tests for ${env.BEATS_FOLDER}.")
+      return
+    }
+
+    triggerE2ETests(suite)
   }
 }
 
@@ -218,6 +253,38 @@ def release(){
   }
 }
 
+def triggerE2ETests(String suite) {
+  echo("Triggering E2E tests for ${env.BEATS_FOLDER}. Test suite: ${suite}.")
+
+  def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
+  def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+
+  def parameters = [
+    booleanParam(name: 'forceSkipGitChecks', value: true),
+    booleanParam(name: 'forceSkipPresubmit', value: true),
+    booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    string(name: 'runTestsSuites', value: suite),
+    string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
+    string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+    string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
+  ]
+  if (isPR()) {
+    def version = "pr-${env.CHANGE_ID}"
+    parameters.push(booleanParam(name: 'USE_CI_SNAPSHOTS', value: true))
+    parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
+    parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
+  }
+
+  build(job: "${e2eTestsPipeline}",
+    parameters: parameters,
+    propagate: false,
+    wait: false
+  )
+
+  def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME} for ${env.BEATS_FOLDER}"
+  githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
+}
+
 def withMacOSEnv(Closure body){
   withEnvMask( vars: [
       [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -93,6 +93,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Move service config under metrics and simplify metric types. {pull}18691[18691]
 - Fix ECS compliance of user.id field in system/users  metricset {pull}19019[19019]
 - Rename googlecloud stackdriver metricset to metrics. {pull}19718[19718]
+- Remove "invalid zero" metrics on Windows and Darwin, don't report linux-only memory and diskio metrics when running under agent. {pull}21457[21457]
 
 *Packetbeat*
 
@@ -349,6 +350,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601]
 - Fix k8s scheduler compatibility issue. {pull}19699[19699]
 - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898
+- Add support for azure light metricset app_stats. {pull}20639[20639]
 - Fix ec2 disk and network metrics to use Sum statistic method. {pull}20680[20680]
 - Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736]
 - The Kibana collector applies backoff when errored at getting usage stats {pull}20772[20772]
@@ -361,6 +363,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix remote_write flaky test. {pull}21173[21173]
 - Visualization title fixes in aws, azure and googlecloud compute dashboards. {pull}21098[21098]
 - Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
+- Use timestamp from CloudWatch API when creating events. {pull}21498[21498]
 
 *Packetbeat*
 
@@ -447,7 +450,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
 - Added experimental dataset `sophos/utm`. {pull}20820[20820]
 - Add Cloud Foundry tags in related events. {pull}21177[21177]
+- Cloud Foundry metadata is cached to disk. {pull}20775[20775]
 - Add option to select the type of index template to load: legacy, component, index. {pull}21212[21212]
+- Release `add_cloudfoundry_metadata` as GA. {pull}21525[21525]
+- Add support for OpenStack SSL metadata APIs in `add_cloud_metadata`. {pull}21590[21590]
 
 *Auditbeat*
 
@@ -603,6 +609,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Always attempt community_id processor on zeek module {pull}21155[21155]
 - Add related.hosts ecs field to all modules {pull}21160[21160]
 - Keep cursor state between httpjson input restarts {pull}20751[20751]
+- Convert aws s3 to v2 input {pull}20005[20005]
+- Add support for additional fields from V2 ALB logs. {pull}21540[21540]
+- Release Cloud Foundry input as GA. {pull}21525[21525]
+- New Cisco Umbrella dataset {pull}21504[21504]
+- New juniper.srx dataset for Juniper SRX logs. {pull}20017[20017]
+- Adding support for Microsoft 365 Defender (Microsoft Threat Protection) {pull}21446[21446]
+- Adding support for FIPS in s3 input {pull}21446[21446]
 
 *Heartbeat*
 
@@ -722,13 +735,17 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add billing data collection from Cost Explorer into aws billing metricset. {pull}20527[20527] {issue}20103[20103]
 - Migrate `compute_vm` metricset to a light one, map `cloud.instance.id` field. {pull}20889[20889]
 - Request prometheus endpoints to be gzipped by default {pull}20766[20766]
+- Add latency config parameter into aws module. {pull}20875[20875]
 - Release all kubernetes `state` metricsets as GA {pull}20901[20901]
 - Add billing metricset into googlecloud module. {pull}20812[20812] {issue}20738[20738]
 - Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985]
 - Sanitize `event.host`. {pull}21022[21022]
 - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
 - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
 - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
+- Move Prometheus query & remote_write to GA. {pull}21507[21507]
+- Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486]
+- Map cloud data filed `cloud.account.id` to azure subscription.  {pull}21483[21483] {issue}21381[21381]
 
 *Packetbeat*
 
@@ -737,6 +754,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
   port. {pull}19209[19209]
 - Add ECS fields for x509 certs, event categorization, and related IP info. {pull}19167[19167]
 - Add 100-continue support {issue}15830[15830] {pull}19349[19349]
+- Add initial SIP protocol support {pull}21221[21221]
 
 
 *Functionbeat*

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -24,7 +24,7 @@ pipeline {
     PIPELINE_LOG_LEVEL = 'INFO'
     PYTEST_ADDOPTS = "${params.PYTEST_ADDOPTS}"
     RUNBLD_DISABLE_NOTIFICATIONS = 'true'
-    SLACK_CHANNEL = "#beats-ci-builds"
+    SLACK_CHANNEL = "#beats-build"
     TERRAFORM_VERSION = "0.12.24"
     XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
   }
@@ -74,7 +74,7 @@ pipeline {
       }
       steps {
         withGithubNotify(context: 'Lint') {
-          withBeatsEnv(archive: true) {
+          withBeatsEnv(archive: true, id: 'lint') {
             dumpVariables()
             cmd(label: 'make check', script: 'make check')
           }
@@ -127,7 +127,7 @@ pipeline {
       runbld(stashedTestReports: stashedTestReports, project: env.REPO)
     }
     cleanup {
-      notifyBuildResult(prComment: true, slackComment: true)
+      notifyBuildResult(prComment: true, slackComment: true, slackNotify: (isBranch() || isTag()))
     }
   }
 }
@@ -270,8 +270,8 @@ def withBeatsEnv(Map args = [:], Closure body) {
         // See https://github.com/elastic/beats/issues/17787.
         sh(label: 'check git config', script: '''
           if [ -z "$(git config --get user.email)" ]; then
-            git config user.email "beatsmachine@users.noreply.github.com"
-            git config user.name "beatsmachine"
+            git config --global user.email "beatsmachine@users.noreply.github.com"
+            git config --global user.name "beatsmachine"
           fi''')
       }
       try {
@@ -350,8 +350,13 @@ def archiveTestOutput(Map args = [:]) {
     }
     cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py')
     dir('build') {
+      if (isUnix()) {
+        cmd(label: 'Delete folders that are causing exceptions (See JENKINS-58421)',
+            returnStatus: true,
+            script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;')
+      } else { log(level: 'INFO', text: 'Delete folders that are causing exceptions (See JENKINS-58421) is disabled for Windows.') }
       junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, stashedTestReports: stashedTestReports, id: args.id)
-      archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts)
+      tar(file: "test-build-artifacts-${args.id}.tgz", dir: '.', archive: true, allowMissing: true)
     }
     catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
       def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()