Merge branch 'master' into mongodb-json-format

.ci/packaging.groovy

@@ -208,7 +208,8 @@ pipeline {
                   'packetbeat',
                   'x-pack/auditbeat',
                   'x-pack/dockerlogbeat',
-                  'x-pack/elastic-agent',
+                  // See https://github.com/elastic/beats/issues/26239
+                  // 'x-pack/elastic-agent',
                   'x-pack/filebeat',
                   'x-pack/heartbeat',
                   'x-pack/metricbeat',

.ci/packer_cache.sh

@@ -45,6 +45,8 @@ function dockerPullImages() {
   docker.elastic.co/kibana/kibana:${SNAPSHOT}
   docker.elastic.co/logstash/logstash:${SNAPSHOT}
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm
+  docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armhf
+  docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armel
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main

.go-version

@@ -1 +1 @@
-1.16.4
+1.16.5

CHANGELOG-developer.next.asciidoc

@@ -116,3 +116,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.15.12. {pull}25629[25629]
 - Update Go version to 1.16.4. {issue}25346[25346] {pull}25671[25671]
 - Add sorting to array fields for generated data files (*-generated.json) {pull}25320[25320]
+- Update Go version to 1.16.5. {issue}26182[26182] {pull}26186[26186]

CHANGELOG.next.asciidoc

@@ -106,6 +106,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299]
 
 *Heartbeat*
+- Add support for screenshot blocks and use newer synthetics flags that only works in newer synthetics betas. {pull}25808[25808]
 
 *Journalbeat*
 
@@ -131,6 +132,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for the MemoryPressure, DiskPressure, OutOfDisk and PIDPressure status conditions in state_node. {pull}23905[23905]
 - Remove xpack enabled flag on ES, Logstash, Beats and Kibana {pull}24427[24427]
 - Adjust host fields to adopt new names from 1.9.0 ECS. {pull}24312[24312]
+- Add replicas.ready field to state_statefulset in Kubernetes module{pull}26088[26088]
 
 *Packetbeat*
 
@@ -238,6 +240,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix 'make setup' instructions for a new beat {pull}24944[24944]
 - Fix out of date FreeBSD vagrantbox. {pull}25652[25652]
 - Fix handling of `file_selectors` in aws-s3 input. {pull}25792[25792]
+- Fix ILM alias creation when write alias exists and initial index does not exist {pull}26143[26143]
+- Include date separator in the filename prefix of `dateRotator` to make sure nothing gets purged accidentally {pull}26176[26176]
+- In the script processor, the `decode_xml` and `decode_xml_wineventlog` processors are now available as `DecodeXML` and `DecodeXMLWineventlog` respectively.
 
 *Auditbeat*
 
@@ -259,6 +264,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693]
 - system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827]
 - Note incompatibility of system/socket on ARM. {pull}23381[23381]
+- auditd: Fix kernel deadlock when netlink congestion causes "no buffer space available" errors. {issue}26031[26031] {pull}26032[26032]
 
 *Filebeat*
 
@@ -275,6 +281,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523]
 - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
 - Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
+- Fix default config template values for paths on oracle module: {pull}26276[26276]
+- Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
 
 *Filebeat*
 
@@ -381,6 +389,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `checkpoint.action_reason` when its a string, not a Long. {issue}25575[25575] {pull}25609[25609]
 - Fix `fortinet.firewall.addr` when its a string, not an IP address. {issue}25585[25585] {pull}25608[25608]
 - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
+- Add improvements to the azure activitylogs and platformlogs ingest pipelines. {pull}26148[26148]
+- Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
+- o365: Avoid mapping exception for `Parameters` and `ExtendedProperties` fields of string type. {pull}26164[26164]
 
 *Heartbeat*
 
@@ -488,6 +499,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
 - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
+- Major refactor of system/cpu and system/core metrics. {pull}25771[25771]
 
 *Packetbeat*
 
@@ -585,6 +597,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012]
 - Add support for defining explicitly named dynamic templates without path/type match criteria {pull}25422[25422]
 - Improve ES output error insights. {pull}25825[25825]
+- Libbeat: report beat version to monitoring. {pull}26214[26214]
 
 *Auditbeat*
 
@@ -807,6 +820,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841]
 - Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686]
 - Support MongoDB 4.4 in filebeat's MongoDB module. {issue}20501[20501] {pull}24774[24774]
+- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368]
+- Move Filebeat azure module to GA. {pull}26114[26114] {pull}26168[26168]
+- http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764]
+- Make `filestream` input GA. {pull}26127[26127]
+- Add new `parser` to `filestream` input: `container`. {pull}26115[26115]
+- Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564]
+- Add `preserve_original_event` option to `o365audit` input. {pull}26273[26273]
+- Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267]
+- Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
 
 *Heartbeat*
 
@@ -939,6 +961,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add additional network metrics to docker/network {pull}25354[25354]
 - Migrate ec2 metricsets to use cloudwatch input. {pull}25924[25924]
 - Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
+- Migrate rds metricsets to use cloudwatch input. {pull}26077[26077]
+- Migrate sqs metricsets to use cloudwatch input. {pull}26117[26117]
+- Add total CPU to vSphere virtual machine metrics. {pull}26167[26167]
 
 *Packetbeat*
 

Vagrantfile

@@ -307,6 +307,7 @@ Vagrant.configure("2") do |config|
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: $freebsdShellUpdate, privileged: true
     c.vm.provision "shell", inline: gvmProvision(arch="amd64", os="freebsd"), privileged: false
+    c.vm.provision "shell", inline: "sudo mount -t linprocfs /dev/null /proc", privileged: false
   end
 
   # OpenBSD 6.0

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4
+FROM golang:1.16.5
 
 RUN \
     apt-get update \

auditbeat/module/auditd/audit_linux.go

@@ -51,6 +51,8 @@ const (
 
 	lostEventsUpdateInterval        = time.Second * 15
 	maxDefaultStreamBufferConsumers = 4
+
+	setPIDMaxRetries = 5
 )
 
 type backpressureStrategy uint8
@@ -137,10 +139,32 @@ func newAuditClient(c *Config, log *logp.Logger) (*libaudit.AuditClient, error)
 	return libaudit.NewAuditClient(nil)
 }
 
+func closeAuditClient(client *libaudit.AuditClient) error {
+	discard := func(bytes []byte) ([]syscall.NetlinkMessage, error) {
+		return nil, nil
+	}
+	// Drain the netlink channel in parallel to Close() to prevent a deadlock.
+	// This goroutine will terminate once receive from netlink errors (EBADF,
+	// EBADFD, or any other error). This happens because the fd is closed.
+	go func() {
+		for {
+			_, err := client.Netlink.Receive(true, discard)
+			switch err {
+			case nil, syscall.EINTR:
+			case syscall.EAGAIN:
+				time.Sleep(50 * time.Millisecond)
+			default:
+				return
+			}
+		}
+	}()
+	return client.Close()
+}
+
 // Run initializes the audit client and receives audit messages from the
 // kernel until the reporter's done channel is closed.
 func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
-	defer ms.client.Close()
+	defer closeAuditClient(ms.client)
 
 	if err := ms.addRules(reporter); err != nil {
 		reporter.Error(err)
@@ -164,7 +188,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
 		go func() {
 			defer func() { // Close the most recently allocated "client" instance.
 				if client != nil {
-					client.Close()
+					closeAuditClient(client)
 				}
 			}()
 			timer := time.NewTicker(lostEventsUpdateInterval)
@@ -178,7 +202,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
 						ms.updateKernelLostMetric(status.Lost)
 					} else {
 						ms.log.Error("get status request failed:", err)
-						if err = client.Close(); err != nil {
+						if err = closeAuditClient(client); err != nil {
 							ms.log.Errorw("Error closing audit monitoring client", "error", err)
 						}
 						client, err = libaudit.NewAuditClient(nil)
@@ -233,7 +257,7 @@ func (ms *MetricSet) addRules(reporter mb.PushReporterV2) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to create audit client for adding rules")
 	}
-	defer client.Close()
+	defer closeAuditClient(client)
 
 	// Don't attempt to change configuration if audit rules are locked (enabled == 2).
 	// Will result in EPERM.
@@ -350,10 +374,12 @@ func (ms *MetricSet) initClient() error {
 			return errors.Wrap(err, "failed to enable auditing in the kernel")
 		}
 	}
+
 	if err := ms.client.WaitForPendingACKs(); err != nil {
 		return errors.Wrap(err, "failed to wait for ACKs")
 	}
-	if err := ms.client.SetPID(libaudit.WaitForReply); err != nil {
+
+	if err := ms.setPID(setPIDMaxRetries); err != nil {
 		if errno, ok := err.(syscall.Errno); ok && errno == syscall.EEXIST && status.PID != 0 {
 			return fmt.Errorf("failed to set audit PID. An audit process is already running (PID %d)", status.PID)
 		}
@@ -362,6 +388,20 @@ func (ms *MetricSet) initClient() error {
 	return nil
 }
 
+func (ms *MetricSet) setPID(retries int) (err error) {
+	if err = ms.client.SetPID(libaudit.WaitForReply); err == nil || errors.Cause(err) != syscall.ENOBUFS || retries == 0 {
+		return err
+	}
+	// At this point the netlink channel is congested (ENOBUFS).
+	// Drain and close the client, then retry with a new client.
+	closeAuditClient(ms.client)
+	if ms.client, err = newAuditClient(&ms.config, ms.log); err != nil {
+		return errors.Wrapf(err, "failed to recover from ENOBUFS")
+	}
+	ms.log.Info("Recovering from ENOBUFS ...")
+	return ms.setPID(retries - 1)
+}
+
 func (ms *MetricSet) updateKernelLostMetric(lost uint32) {
 	if !ms.kernelLost.enabled {
 		return

dev-tools/mage/crossbuild.go

@@ -197,15 +197,22 @@ func crossBuildImage(platform string) (string, error) {
 	tagSuffix := "main"
 
 	switch {
-	case strings.HasPrefix(platform, "darwin"):
+	case platform == "darwin/amd64":
 		tagSuffix = "darwin-debian10"
-	case strings.HasPrefix(platform, "linux/armv7"):
-		tagSuffix = "armhf"
-	case strings.HasPrefix(platform, "linux/arm"):
+	case platform == "darwin/arm64":
+		tagSuffix = "darwin-arm64-debian10"
+	case platform == "linux/arm64":
 		tagSuffix = "arm"
+		// when it runs on a ARM64 host/worker.
 		if runtime.GOARCH == "arm64" {
 			tagSuffix = "base-arm-debian9"
 		}
+	case platform == "linux/armv5":
+		tagSuffix = "armel"
+	case platform == "linux/armv6":
+		tagSuffix = "armel"
+	case platform == "linux/armv7":
+		tagSuffix = "armhf"
 	case strings.HasPrefix(platform, "linux/mips"):
 		tagSuffix = "mips"
 	case strings.HasPrefix(platform, "linux/ppc"):

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4
+FROM golang:1.16.5
 
 RUN \
     apt-get update \

filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl

@@ -11,7 +11,7 @@ filebeat.inputs:
 #
 # Possible options are:
 # * log: Reads every line of the log file (default)
-# * filestream: Improved version of log input. Experimental.
+# * filestream: Improved version of log input
 # * stdin: Reads the standard in
 
 #------------------------------ Log input --------------------------------

filebeat/_meta/config/filebeat.inputs.yml.tmpl

@@ -50,7 +50,7 @@ filebeat.inputs:
   # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
   #multiline.match: after
 
-# filestream is an experimental input. It is going to replace log input in the future.
+# filestream is an input for collecting log messages from files. It is going to replace log input in the future.
 - type: filestream
 
   # Change to true to enable this input configuration.

filebeat/docs/fields.asciidoc

@@ -2675,30 +2675,13 @@ type: keyword
 
 --
 
-[float]
-=== properties
-
-Properties
-
-
-
-*`azure.activitylogs.properties.service_request_id`*::
+*`azure.activitylogs.properties`*::
 +
 --
-Service Request Id
-
-
-type: keyword
-
---
-
-*`azure.activitylogs.properties.status_code`*::
-+
---
-Status code
+Properties
 
 
-type: keyword
+type: flattened
 
 --
 
@@ -3198,13 +3181,13 @@ type: keyword
 
 --
 
-*`azure.platformlogs.properties.*`*::
+*`azure.platformlogs.properties`*::
 +
 --
-Properties
+Event inner properties
 
 
-type: object
+type: flattened
 
 --
 
@@ -125738,7 +125721,7 @@ S3 fields from s3 input.
 
 
 
-*`bucket_name`*::
+*`bucket.name`*::
 +
 --
 Name of the S3 bucket that this log retrieved from.
@@ -125748,7 +125731,17 @@ type: keyword
 
 --
 
-*`object_key`*::
+*`bucket.arn`*::
++
+--
+ARN of the S3 bucket that this log retrieved from.
+
+
+type: keyword
+
+--
+
+*`object.key`*::
 +
 --
 Name of the S3 object that this log retrieved from.
@@ -125758,6 +125751,15 @@ type: keyword
 
 --
 
+*`metadata`*::
++
+--
+AWS S3 object metadata values.
+
+type: flattened
+
+--
+
 [[exported-fields-santa]]
 == Google Santa fields