Merge branch 'master' into archive-system-tests

* master: (32 commits) [Metricbeat] Change Account ID to Project ID in `gcp.billing` module (elastic#26412) update libbeat fields.ecs.yml file and ecsVersion to 1.10.0 (elastic#26121) [Filebeat] Update AWS ELB ingest pipeline (elastic#26441) [FIlebeat] add strict_date_optional_time_nanos date format to PanOS module (elastic#26158) Fix the irregular and typo on prometheus module. (elastic#25726) [Filebeat] Parse additonal debug data fields for Okta module (elastic#25818) fix: update MSSQL Server linux image's Docker registry (elastic#26440) Update indexing.go godocs (elastic#26408) Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled (elastic#26411) Add support for copytruncate method when rotating input logs with an external tool in `filestream` input (elastic#23457) Allow fields with ip_range datatype (elastic#26444) Add Anomali ThreatStream support to threatintel module (elastic#26350) fix: use the right param type (elastic#26469) [Automation] Update elastic stack version to 8.0.0-7640093f for testing (elastic#26460) Set SM Filebeat modules as GA (elastic#26226) Fix rfc5464 date parsing in the syslog input (elastic#26419) Add linked account information into billing metricset (elastic#26285) [Filebeat] Update HA Proxy log grok patterns (elastic#25835) disable metricbeat logstash test_node_stats (elastic#26436) chore: pass BEAT_VERSION when running E2E tests (elastic#26291) ...
mdelapenya · Jun 28, 2021 · b6dbec3 · b6dbec3
2 parents ead2f67 + 076e0a6
commit b6dbec3
Show file tree

Hide file tree

Showing 167 changed files with 9,547 additions and 1,001 deletions.
diff --git a/.ci/bump-go-release-version.sh b/.ci/bump-go-release-version.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Given the Golang release version this script will bump the version.
+#
+# This script is executed by the automation we are putting in place
+# and it requires the git add/commit commands.
+#
+# Parameters:
+#	$1 -> the Golang release version to be bumped. Mandatory.
+#
+set -euo pipefail
+MSG="parameter missing."
+GO_RELEASE_VERSION=${1:?$MSG}
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+
+if [ "${OS}" == "darwin" ] ; then
+	SED="sed -i .bck"
+else
+	SED="sed -i"
+fi
+
+echo "Update go version ${GO_RELEASE_VERSION}"
+echo "${GO_RELEASE_VERSION}" > .go-version
+git add .go-version
+
+find . -maxdepth 3 -name Dockerfile -print0 |
+    while IFS= read -r -d '' line; do
+        ${SED} -E -e "s#(FROM golang):[0-9]+\.[0-9]+\.[0-9]+#\1:${GO_RELEASE_VERSION}#g" "$line"
+        ${SED} -E -e "s#(ARG GO_VERSION)=[0-9]+\.[0-9]+\.[0-9]+#\1=${GO_RELEASE_VERSION}#g" "$line"
+        git add "${line}"
+    done
+
+${SED} -E -e "s#(:go-version:) [0-9]+\.[0-9]+\.[0-9]+#\1 ${GO_RELEASE_VERSION}#g" libbeat/docs/version.asciidoc
+git add libbeat/docs/version.asciidoc
+
+git diff --staged --quiet || git commit -m "[Automation] Update go release version to ${GO_RELEASE_VERSION}"
+git --no-pager log -1
+
+echo "You can now push and create a Pull Request"
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -212,7 +212,6 @@ pipeline {
                   'x-pack/filebeat',
                   'x-pack/heartbeat',
                   'x-pack/metricbeat',
-                  'x-pack/osquerybeat',
                   'x-pack/packetbeat'
                 )
               }
@@ -426,11 +425,13 @@ def triggerE2ETests(String suite) {
 
   def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
   def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+  def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
 
   def parameters = [
     booleanParam(name: 'forceSkipGitChecks', value: true),
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    string(name: 'BEAT_VERSION', value: beatVersion),
     booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),

diff --git a/.gitignore b/.gitignore
@@ -51,5 +51,5 @@ x-pack/elastic-agent/pkg/agent/operation/tests/scripts/serviceable-1.0-darwin-x8
 *.terraform
 *.tfstate*
 
-# Files generated with the bump elastic stack version automation
-testing/environments/*.bck
+# Files generated with the bump version automations
+*.bck
diff --git a/.mergify.yml b/.mergify.yml
@@ -84,11 +84,25 @@ pull_request_rules:
       merge:
         method: squash
         strict: smart+fasttrack
-  - name: delete upstream branch after merging changes on testing/environments/snapshot*
+  - name: delete upstream branch after merging changes on testing/environments/snapshot* or it's closed
     conditions:
-      - merged
-      - label=automation
-      - head~=^update-stack-version
-      - files~=^testing/environments/snapshot.*\.yml$
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^update-stack-version
+        - files~=^testing/environments/snapshot.*\.yml$
+    actions:
+      delete_head_branch:
+  - name: delete upstream branch after merging changes on .go-version or it's closed
+    conditions:
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^update-go-version
+        - files~=^.go-version$
     actions:
       delete_head_branch:
diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc
@@ -208,3 +208,4 @@ The list below covers the major changes between 6.3.0 and 7.0.0-alpha2 only.
 - Allow/Merge fields.yml overrides {pull}9188[9188]
 - Filesets can now define multiple ingest pipelines, with the first one considered as the entry point pipeline. {pull}8914[8914]
 - Add `group_measurements_by_instance` option to windows perfmon metricset. {pull}8688[8688]
+- Bump ECS version to 1.10.0. {issue}25734[25734]
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -275,6 +275,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
 - Fix default config template values for paths on oracle module: {pull}26276[26276]
 - Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
+- Fix bug in `httpjson` that prevented `first_event` getting updated. {pull}26407[26407]
+- Fix bug in the Syslog input that misparsed rfc5424 days starting with 0. {pull}26419[26419]
+- Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411]
 
 *Filebeat*
 
@@ -383,6 +386,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
 - Add improvements to the azure activitylogs and platformlogs ingest pipelines. {pull}26148[26148]
 - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
+- Removed incorrect `http.request.referrer` field from `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 
 *Heartbeat*
 
@@ -491,6 +495,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
 - Major refactor of system/cpu and system/core metrics. {pull}25771[25771]
+- Fix GCP Project ID being ingested as `cloud.account.id` in `gcp.billing` module {issue}26357[26357] {pull}26412[26412]
 
 *Packetbeat*
 
@@ -818,11 +823,17 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make `filestream` input GA. {pull}26127[26127]
 - Add new `parser` to `filestream` input: `container`. {pull}26115[26115]
 - Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564]
+- Add possibility to include headers in resulting docs and preserve the original event in http_endpoint input {pull}26279[26279]
 - Add `preserve_original_event` option to `o365audit` input. {pull}26273[26273]
 - Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267]
 - Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
 - RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293]
-
+- Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835]
+- Update PanOS module's date processor formats to parse `strict_date_optional_time_nanos`. {issue}26033[26033] {pull}26158[26158]
+- Update Okta module to parse additional fields to `okta.debug_context.debug_data`. {issue}25689[25689] {pull}25818[25818]
+- Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350]
+- Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457]
+- Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 
 *Heartbeat*
 
@@ -957,6 +968,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
 - Migrate rds metricsets to use cloudwatch input. {pull}26077[26077]
 - Migrate sqs metricsets to use cloudwatch input. {pull}26117[26117]
+- Collect linked account information in AWS billing. {pull}26285[26285]
 - Add total CPU to vSphere virtual machine metrics. {pull}26167[26167]
 
 *Packetbeat*
@@ -1012,6 +1024,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Deprecate the MISP module. The Threat Intel module should be used instead. {issue}25240[25240]
 
 *Heartbeat*
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -520,6 +520,7 @@ def e2e(Map args = [:]) {
     def goVersionForE2E = readFile('.go-version').trim()
     withEnv(["GO_VERSION=${goVersionForE2E}",
               "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
+              "BEAT_VERSION=${env.VERSION}-SNAPSHOT",
               "LOG_LEVEL=TRACE"]) {
       def status = 0
       filebeat(output: dockerLogFile){

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -6133,11 +6133,11 @@ This Agreement is governed by the laws of the State of New York and the intellec
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/ecs
-Version: v1.8.0
+Version: v1.10.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.8.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.10.0/LICENSE.txt:
 
 
                                  Apache License

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl
@@ -292,6 +292,16 @@ filebeat.inputs:
   # original for harvesting but will report the symlink name as source.
   #prospector.scanner.symlinks: false
 
+  ### Log rotation
+
+  # When an external tool rotates the input files with copytruncate strategy
+  # use this section to help the input find the rotated files.
+  #rotation.external.strategy.copytruncate:
+  # Regex that matches the rotated files.
+  #  suffix_regex: \.\d$
+  # If the rotated filename suffix is a datetime, set it here. 
+  #  dateformat: -20060102
+
   ### State options
 
   # Files for the modification data is older then clean_inactive the state from the registry is removed