Merge branch '7.12' into mergify/bp/7.12/pr-25112

.ci/bump-stack-version.sh

@@ -35,14 +35,15 @@ done
 
 echo "Commit changes"
 if [ "$CREATE_BRANCH" = "true" ]; then
-	git checkout -b "update-stack-version-$(date "+%Y%m%d%H%M%S")"
+	base=$(git rev-parse --abbrev-ref HEAD | sed 's#/#-#g')
+	git checkout -b "update-stack-version-$(date "+%Y%m%d%H%M%S")-${base}"
 else
 	echo "Branch creation disabled."
 fi
 for FILE in ${FILES} ; do
 	git add $FILE
 done
-git diff --staged --quiet || git commit -m "bump stack version ${VERSION}"
+git diff --staged --quiet || git commit -m "[Automation] Update elastic stack version to ${VERSION} for testing"
 git --no-pager log -1
 
 echo "You can now push and create a Pull Request"

CHANGELOG.asciidoc

@@ -3,6 +3,61 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.12.1]]
+=== Beats version 7.12.1
+https://github.com/elastic/beats/compare/v7.12.0...v7.12.1[View commits]
+
+==== Breaking changes
+
+*Filebeat*
+
+- Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix templates being overwritten if there was an error when check for the template existance. {pull}24332[24332]
+- Fix Kubernetes autodiscovery provider to correctly handle pod states and avoid missing event data {pull}17223[17223]
+- Fix inode removal tracking code when files are replaced by files with the same name {pull}25002[25002]
+- Fix `mage GenerateCustomBeat` instructions for a new beat {pull}17679[17679]
+- Fix bug with annotations dedot config on k8s not used {pull}25111[25111]
+- Fix negative Kafka partition bug {pull}25048[25048]
+
+*Filebeat*
+
+- Properly update offset in case of unparasable line. {pull}22685[22685]
+- Fix Cisco ASA parser for message 722051. {pull}24410[24410]
+- Fix `google_workspace` pagination. {pull}24668[24668]
+- Fix netflow module ignoring detect_sequence_reset flag. {issue}24268[24268] {pull}24270[24270]
+- Fix Cisco ASA parser for message 302022. {issue}24405[24405] {pull}24697[24697]
+- Fix Cisco AMP `@metadata._id` calculation {issue}24717[24717] {pull}24718[24718]
+- Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694]
+- Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
+- Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799]
+- Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861]
+- Fix google_workspace and okta modules pagination when next page template is empty. {pull}24967[24967]
+- Fix gcp module field names to use gcp instead of googlecloud. {pull}25038[25038]
+
+*Heartbeat*
+
+- Fix panic when initialization of ICMP monitors fail twice. {pull}25073[25073]
+
+*Metricbeat*
+
+- Ignore unsupported derive types for filesystem metricset. {issue}22501[22501] {pull}24502[24502]
+
+
+==== Added
+
+*Filebeat*
+
+- Updating field mappings for Cisco AMP module, fixing certain fields. {pull}24661[24661]
+- Add support for upper case field names in Sophos XG module {pull}24693[24693]
+- Add `fail_on_template_error` option for httpjson input. {pull}24784[24784]
+
+
+
 [[release-notes-7.12.0]]
 === Beats version 7.12.0
 https://github.com/elastic/beats/compare/v7.11.2...v7.12.0[View commits]

CHANGELOG.next.asciidoc

@@ -40,13 +40,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Disable the option of running --machine-learning on its own. {pull}20241[20241]
 - Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808]
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
-- API address and shard ID are required settings in the Cloud Foundry input. {pull}21759[21759]
-- Remove `suricata.eve.timestamp` alias field. {issue}10535[10535] {pull}22095[22095]
-- Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571]
-- Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975]
-- Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041]
-- Rename `s3` input to `aws-s3` input. {pull}23469[23469]
-- Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295]
 
 *Heartbeat*
 
@@ -86,16 +79,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Affecting all Beats*
 
 - Fix events being dropped if they contain a floating point value of NaN or Inf. {pull}25051[25051]
-- Fix templates being overwritten if there was an error when check for the template existance. {pull}24332[24332]
-- Fix Kubernetes autodiscovery provider to correctly handle pod states and avoid missing event data {pull}17223[17223]
-- Fix `add_cloud_metadata` to better support modifying sub-fields with other processors. {pull}13808[13808]
-- TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
-- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
-- Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568]
 - Fix a race condition with the Kafka pipeline client, it is possible that `Close()` get called before `Connect()` . {issue}11945[11945]
 - Allow users to configure only `cluster_uuid` setting under `monitoring` namespace. {pull}14338[14338]
 - Update replicaset group to apps/v1 {pull}15854[15802]
-- Fix Kubernetes autodiscovery provider to correctly handle pod states and avoid missing event data {pull}17223[17223]
 - Fix missing output in dockerlogbeat {pull}15719[15719]
 - Do not load dashboards where not available. {pull}15802[15802]
 - Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038]
@@ -129,6 +115,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `mage GenerateCustomBeat` instructions for a new beat {pull}17679[17679]
 - Fix bug with annotations dedot config on k8s not used {pull}25111[25111]
 - Fix negative Kafka partition bug {pull}25048[25048]
+- Fix decode_xml processor config checks. {pull}25310[25310]
 
 *Auditbeat*
 
@@ -184,48 +171,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix event.kind for system/syslog pipeline {issue}20365[20365] {pull}20390[20390]
 - Fix event.type for zeek/ssl and duplicate event.category for zeek/connection {pull}20696[20696]
 - Add json body check for sqs message. {pull}21727[21727]
-- Properly update offset in case of unparasable line. {pull}22685[22685]
 - Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716]
-- Fix cisco umbrella module config by adding input variable. {pull}22892[22892]
-- Fix network.direction logic in zeek connection fileset. {pull}22967[22967]
-- Fix aws s3 overview dashboard. {pull}23045[23045]
-- Fix bad `network.direction` values in Fortinet/firewall fileset. {pull}23072[23072]
-- Fix Cisco ASA/FTD module's parsing of WebVPN log message 716002. {pull}22966[22966]
-- Add support for organization and custom prefix in AWS/CloudTrail fileset. {issue}23109[23109] {pull}23126[23126]
-- Simplify regex for organization custom prefix in AWS/CloudTrail fileset. {issue}23203[23203] {pull}23204[23204]
-- Fix syslog header parsing in infoblox module. {issue}23272[23272] {pull}23273[23273]
-- Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
-- Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534]
-- Change the `event.created` in Netflow events to be the time the event was created by Filebeat
-  to be consistent with ECS. {pull}23094[23094]
-- Fix Zoom module parameters for basic auth and url path. {pull}23779[23779]
-- Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777]
-- Use rfc6587 framing for fortinet firewall and clientendpoint filesets when transferring over tcp. {pull}23837[23837]
-- Fix httpjson input logging so it doesn't conflict with ECS. {pull}23972[23972]
-- Fix Logstash module handling of logstash.log.log_event.action field. {issue}20709[20709]
-- aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
-- Zoom module pipeline failed to ingest some chat_channel events. {pull}23904[23904]
-- Fix Netlow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110]
-- in httpjson input using encode_as "application/x-www-form-urlencoded" now sets Content-Type correctly {issue}24331[24331] {pull}24336[24336]
-- Fix default `scope` in `add_nomad_metadata`. {issue}24559[24559]
-- Fix Cisco ASA parser for message 722051. {pull}24410[24410]
-- Fix `google_workspace` pagination. {pull}24668[24668]
-- Fix netflow module ignoring detect_sequence_reset flag. {issue}24268[24268] {pull}24270[24270]
-- Fix Cisco ASA parser for message 302022. {issue}24405[24405] {pull}24697[24697]
-- Fix Cisco AMP `@metadata._id` calculation {issue}24717[24717] {pull}24718[24718]
-- Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694]
-- Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
-- Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799]
-- Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861]
-- Fix google_workspace and okta modules pagination when next page template is empty. {pull}24967[24967]
-- Fix gcp module field names to use gcp instead of googlecloud. {pull}25038[25038]
 
 *Heartbeat*
 
 - Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639]
 - Fixed scheduler shutdown issues which would in rare situations cause a panic due to semaphore misuse. {pull}16397[16397]
 - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549]
-- Fix panic when initialization of ICMP monitors fail twice. {pull}25073[25073]
 
 *Journalbeat*
 
@@ -280,7 +232,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix remote_write flaky test. {pull}21173[21173]
 - Remove io.time from windows {pull}22237[22237]
 - Fix `logstash` module when `xpack.enabled: true` is set from emitting redundant events. {pull}22808[22808]
-- Ignore unsupported derive types for filesystem metricset. {issue}22501[22501] {pull}24502[24502]
 
 *Packetbeat*
 
@@ -415,9 +366,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Keep cursor state between httpjson input restarts {pull}20751[20751]
 - New juniper.srx dataset for Juniper SRX logs. {pull}20017[20017]
 - Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291]
-- Updating field mappings for Cisco AMP module, fixing certain fields. {pull}24661[24661]
-- Add support for upper case field names in Sophos XG module {pull}24693[24693]
-- Add `fail_on_template_error` option for httpjson input. {pull}24784[24784]
 
 *Heartbeat*
 

auditbeat/docs/getting-started.asciidoc

@@ -40,9 +40,9 @@ include::{libbeat-dir}/tab-widgets/install-widget.asciidoc[]
 ==== Other installation options
 
 * <<setup-repositories,APT or YUM>>
-* https://www.elastic.co/downloads/beats/{beatname_lc}[Download page] 
+* https://www.elastic.co/downloads/beats/{beatname_lc}[Download page]
 * <<running-on-docker,Docker>>
-* <<running-on-kubernetes,Kubernetes>> 
+* <<running-on-kubernetes,Kubernetes>>
 
 [float]
 [[set-connection]]
@@ -56,11 +56,11 @@ include::{libbeat-dir}/shared/connecting-to-es.asciidoc[]
 
 {beatname_uc} uses <<auditbeat-modules,modules>> to collect audit information.
 
-By default, {beatname_uc} uses a configuration that's tailored to the operating 
+By default, {beatname_uc} uses a configuration that's tailored to the operating
 system where {beatname_uc} is running.
 
 To use a different configuration, change the module settings in
-+{beatname_lc}.yml+. 
++{beatname_lc}.yml+.
 
 The following example shows the `file_integrity` module configured to generate
 events whenever a file in one of the specified paths changes on disk:
@@ -99,7 +99,7 @@ include::{libbeat-dir}/tab-widgets/setup-widget.asciidoc[]
 +
 `-e` is optional and sends output to standard error instead of the configured log output.
 
-This step loads the recommended {ref}/indices-templates.html[index template] for writing to {es}
+This step loads the recommended {ref}/index-templates.html[index template] for writing to {es}
 and deploys the sample dashboards for visualizing the data in {kib}.
 
 [TIP]

filebeat/docs/getting-started.asciidoc

@@ -145,7 +145,7 @@ include::{libbeat-dir}/tab-widgets/setup-widget.asciidoc[]
 +
 `-e` is optional and sends output to standard error instead of the configured log output.
 
-This step loads the recommended {ref}/indices-templates.html[index template] for writing to {es}
+This step loads the recommended {ref}/index-templates.html[index template] for writing to {es}
 and deploys the sample dashboards for visualizing the data in {kib}.
 
 This step does not load the ingest pipelines used to parse log lines. By

heartbeat/docs/getting-started.asciidoc

@@ -100,14 +100,14 @@ include::{libbeat-dir}/shared/config-check.asciidoc[]
 [[configurelocation]]
 === Step 4: Configure the Heartbeat location
 
-Heartbeat can be deployed in multiple locations so that you can detect 
+Heartbeat can be deployed in multiple locations so that you can detect
 differences in availability and response times across those locations.
 Configure the Heartbeat location to allow {kib} to display location-specific
 information on Uptime maps and perform Uptime anomaly detection based
 on location.
 
-To configure the location of a Heartbeat instance, modify the 
-`add_observer_metadata` processor in +{beatname_lc}.yml+.  The following 
+To configure the location of a Heartbeat instance, modify the
+`add_observer_metadata` processor in +{beatname_lc}.yml+.  The following
 example specifies the `geo.name` of the `add_observer_metadata` processor as
 `us-east-1a`:
 
@@ -148,7 +148,7 @@ include::{libbeat-dir}/tab-widgets/setup-widget.asciidoc[]
 +
 `-e` is optional and sends output to standard error instead of the configured log output.
 
-This step loads the recommended {ref}/indices-templates.html[index template] for writing to {es}.
+This step loads the recommended {ref}/index-templates.html[index template] for writing to {es}.
 It does not install {beatname_uc} dashboards.  Heartbeat dashboards and
 installation steps are available in the
 https://github.com/elastic/uptime-contrib[uptime-contrib] GitHub repository.

journalbeat/docs/getting-started.asciidoc

@@ -37,7 +37,7 @@ include::{libbeat-dir}/tab-widgets/install-deb-rpm-linux-widget.asciidoc[]
 ==== Other installation options
 
 * <<setup-repositories,APT or YUM>>
-* https://www.elastic.co/downloads/beats/{beatname_lc}[Download page] 
+* https://www.elastic.co/downloads/beats/{beatname_lc}[Download page]
 * <<running-on-docker,Docker>>
 
 [float]
@@ -112,7 +112,7 @@ include::{libbeat-dir}/tab-widgets/setup-deb-rpm-linux-widget.asciidoc[]
 +
 `-e` is optional and sends output to standard error instead of the configured log output.
 
-This step loads the recommended {ref}/indices-templates.html[index template] for writing to {es}.
+This step loads the recommended {ref}/index-templates.html[index template] for writing to {es}.
 
 [TIP]
 =====

libbeat/docs/howto/load-index-templates.asciidoc

@@ -1,7 +1,7 @@
 [id="{beatname_lc}-template"]
 == Load the {es} index template
 
-{es} uses {ref}/indices-templates.html[index templates] to define:
+{es} uses {ref}/index-templates.html[index templates] to define:
 
 * Settings that control the behavior of your indices. The settings include the
 lifecycle policy used to manage indices as they grow and age.

libbeat/docs/release.asciidoc

@@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read
 <<breaking-changes>> for more detail about changes that affect
 upgrade.
 
+* <<release-notes-7.12.1>>
 * <<release-notes-7.12.0>>
 * <<release-notes-7.11.2>>
 * <<release-notes-7.11.1>>

libbeat/docs/shared-docker.asciidoc

@@ -5,13 +5,13 @@ Docker images for {beatname_uc} are available from the Elastic Docker
 registry. The base image is https://hub.docker.com/_/centos/[centos:7].
 
 A list of all published Docker images and tags is available at
-https://www.docker.elastic.co[www.docker.elastic.co]. 
+https://www.docker.elastic.co[www.docker.elastic.co].
 
-These images are free to use under the Elastic license. They contain open source 
-and free commercial features and access to paid commercial features.  
-{kibana-ref}/managing-licenses.html[Start a 30-day trial] to try out all of the 
-paid commercial features. See the 
-https://www.elastic.co/subscriptions[Subscriptions] page for information about 
+These images are free to use under the Elastic license. They contain open source
+and free commercial features and access to paid commercial features.
+{kibana-ref}/managing-licenses.html[Start a 30-day trial] to try out all of the
+paid commercial features. See the
+https://www.elastic.co/subscriptions[Subscriptions] page for information about
 Elastic license levels.
 
 ==== Pull the image
@@ -34,8 +34,8 @@ docker pull {dockerimage}
 ------------------------------------------------
 
 Alternatively, you can download other Docker images that contain only features
-available under the Apache 2.0 license. To download the images, go to 
-https://www.docker.elastic.co[www.docker.elastic.co]. 
+available under the Apache 2.0 license. To download the images, go to
+https://www.docker.elastic.co[www.docker.elastic.co].
 
 endif::[]
 
@@ -129,7 +129,7 @@ endif::apm-server[]
 ==== Configure {beatname_uc} on Docker
 
 The Docker image provides several methods for configuring {beatname_uc}. The
-conventional approach is to provide a configuration file via a volume mount, but 
+conventional approach is to provide a configuration file via a volume mount, but
 it's also possible to create a custom image with your
 configuration included.
 
@@ -244,6 +244,7 @@ ifeval::["{beatname_lc}"=="apm-server"]
 ["source", "sh", subs="attributes"]
 --------------------------------------------
 docker run -d \
+  -p 8200:8200 \
   --name={beatname_lc} \
   --user={beatname_lc} \
   --volume="$(pwd)/{beatname_lc}.docker.yml:/usr/share/{beatname_lc}/{beatname_lc}.yml:ro" \

libbeat/docs/template-config.asciidoc

@@ -7,7 +7,7 @@
 ++++
 
 The `setup.template` section of the +{beatname_lc}.yml+ config file specifies
-the {ref}/indices-templates.html[index template] to use for setting
+the {ref}/index-templates.html[index template] to use for setting
 mappings in Elasticsearch. If template loading is enabled (the default),
 {beatname_uc} loads the index template automatically after successfully
 connecting to Elasticsearch.