Merge remote-tracking branch 'upstream/master' into feature/beats-tes…

…ter-commit * upstream/master: (25 commits) [CI] set env variable for the params (elastic#22143) Fix zeek connection pipeline (elastic#22151) Fix Google Cloud Function configuration file issues (elastic#22156) Remove old TODO on kubernetes node update (elastic#22074) [CI] Enable winlogbeat (elastic#22142) [CI] support windows-10 (elastic#19804) Use default config when creating the input (elastic#22126) Change x509 mappings from file. to tls.server. (elastic#22097) Add fleet settings image (elastic#22065) Edit 7.9.3 changelog (elastic#22117) Edit 6.8.13 release notes (elastic#22120) Incorporate librpm fix feedback (elastic#22098) [libbeat] Add more disk queue unit tests and fix a size-check bug (elastic#22107) docs: move kerberos include (elastic#22109) Check context.Canceled and fix s3 input config (elastic#22036) Add max_number_of_messages into aws filebeat fileset vars (elastic#22057) Remove suricata.eve.timestamp alias (elastic#22095) [Ingest Manager] Use symlink path for reexecutions (elastic#21835) chore: use ubuntu 18 as linux agent (elastic#22084) docs: Prepare Changelog for 7.9.3 (elastic#22073) (elastic#22075) ...
v1v · Oct 26, 2020 · 1605aef · 1605aef
2 parents 43b878d + 5501ce8
commit 1605aef
Show file tree

Hide file tree

Showing 70 changed files with 1,124 additions and 168 deletions.
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -43,7 +43,7 @@ pipeline {
   }
   stages {
     stage('Filter build') {
-      agent { label 'ubuntu && immutable' }
+      agent { label 'ubuntu-18 && immutable' }
       when {
         beforeAgent true
         anyOf {
@@ -117,7 +117,7 @@ pipeline {
             }
             stages {
               stage('Package Linux'){
-                agent { label 'ubuntu && immutable' }
+                agent { label 'ubuntu-18 && immutable' }
                 options { skipDefaultCheckout() }
                 when {
                   beforeAgent true
@@ -181,7 +181,7 @@ pipeline {
           }
         }
         stage('Run E2E Tests for Packages'){
-          agent { label 'ubuntu && immutable' }
+          agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           steps {
             runE2ETests()

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -3,6 +3,31 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.9.3]]
+=== Beats version 7.9.3
+https://github.com/elastic/beats/compare/v7.9.2...v7.9.3[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- The `o365audit` input and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21258[21258]
+
+*Auditbeat*
+
+- system/socket: Fix a crash due to concurrent map read and write. {issue}21192[21192] {pull}21690[21690]
+
+*Filebeat*
+
+- Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382]
+
+*Metricbeat*
+
+- Fix `remote_write` flaky test. {pull}21173[21173]
+- Fix panic in Kubernetes autodiscovery caused by storing stateless keystores. {issue}21843[21843] {pull}21880[21880]
+- Remove redundant dockersock volume mount to avoid problems on Kubernetes deployments that do not use docker as the container runtime. {pull}22009[22009]
+
+
 [[release-notes-7.9.2]]
 === Beats version 7.9.2
 https://github.com/elastic/beats/compare/v7.9.1...v7.9.2[View commits]
@@ -2575,6 +2600,46 @@ https://github.com/elastic/beats/compare/v6.5.0...v7.0.0-alpha1[View commits]
 - Added support to calculate certificates' fingerprints (MD5, SHA-1, SHA-256). {issue}8180[8180]
 - Support new TLS version negotiation introduced in TLS 1.3. {issue}8647[8647].
 
+[[release-notes-6.8.13]]
+=== Beats version 6.8.13
+https://github.com/elastic/beats/compare/v6.8.12...v6.8.13[View commits]
+
+==== Added
+
+*Filebeat*
+
+- Add container image in Kubernetes metadata. {pull}13356[13356] {issue}12688[12688]
+
+[[release-notes-6.8.12]]
+=== Beats version 6.8.12
+https://github.com/elastic/beats/compare/v6.8.11...v6.8.12[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Fix Filebeat OOMs on very long lines {issue}19500[19500], {pull}19552[19552]
+
+[[release-notes-6.8.11]]
+=== Beats version 6.8.11
+https://github.com/elastic/beats/compare/v6.8.10...v6.8.11[View commits]
+
+==== Bugfixes
+
+*Metricbeat*
+
+- Fix bug incorrect parsing of float numbers as integers in Couchbase module {issue}18949[18949] {pull}19055[19055]
+
+[[release-notes-6.8.10]]
+=== Beats version 6.8.10
+https://github.com/elastic/beats/compare/v6.8.9...v6.8.10[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix `add_cloud_metadata` to better support modifying sub-fields with other processors. {pull}13808[13808]
+
 [[release-notes-6.8.9]]
 === Beats version 6.8.9
 https://github.com/elastic/beats/compare/v6.8.8...v6.8.9[View commits]

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -80,6 +80,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 - API address and shard ID are required settings in the Cloud Foundry input. {pull}21759[21759]
+- Remove `suricata.eve.timestamp` alias field. {issue}10535[10535] {pull}22095[22095]
 
 *Heartbeat*
 
@@ -286,6 +287,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix checkpoint module when logs contain time field. {pull}20567[20567]
 - Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382]
 - Fix syslog RFC 5424 parsing in the CheckPoint module. {pull}21854[21854]
+- Fix incorrect connection state mapping in zeek connection pipeline. {pull}22151[22151] {issue}22149[22149]
 
 *Heartbeat*
 
@@ -378,6 +380,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix azure storage event format. {pull}21845[21845]
 - Fix panic in kubernetes autodiscover related to keystores {issue}21843[21843] {pull}21880[21880]
 - [Kubernetes] Remove redundant dockersock volume mount {pull}22009[22009]
+- Revert change to report `process.memory.rss` as `process.memory.wss` on Windows. {pull}22055[22055]
+- Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
 
 *Packetbeat*
 
@@ -397,6 +401,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072]
 - Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070]
 - Fix catchall bucket config errors by adding more validation. {issue}17572[16282] {pull}20887[16287]
+- Fix Google Cloud Function configuration issue. {issue}20864[20864] {pull}22156[22156]
 
 ==== Added
 
@@ -820,3 +825,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 ==== Known Issue
 
 *Journalbeat*
+
+
+
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -12,6 +12,7 @@ pipeline {
   agent { label 'ubuntu-18 && immutable' }
   environment {
     AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
+    AWS_REGION = "${params.awsRegion}"
     REPO = 'beats'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
@@ -438,7 +439,7 @@ def withCloudTestEnv(Closure body) {
       error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'")
     }
     maskedVars.addAll([
-      [var: "AWS_REGION", password: params.awsRegion],
+      [var: "AWS_REGION", password: "${env.AWS_REGION}"],
       [var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
       [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
     ])

diff --git a/Makefile b/Makefile
@@ -12,7 +12,7 @@ GOLINT=golint
 GOLINT_REPO=golang.org/x/lint/golint
 REVIEWDOG=reviewdog
 REVIEWDOG_OPTIONS?=-diff "git diff master"
-REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog
+REVIEWDOG_REPO=github.com/reviewdog/reviewdog/cmd/reviewdog
 XPACK_SUFFIX=x-pack/
 
 # PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support

diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
@@ -69,3 +69,14 @@ stages:
                 - "windows-2012"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+    windows-10:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-10"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test auditbeat for windows-10"
+            labels:
+                - "windows-10"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
diff --git a/auditbeat/docs/configuring-howto.asciidoc b/auditbeat/docs/configuring-howto.asciidoc
@@ -42,6 +42,10 @@ include::./reload-configuration.asciidoc[]
 
 include::{libbeat-dir}/outputconfig.asciidoc[]
 
+ifndef::no_kerberos[]
+include::{libbeat-dir}/shared-kerberos-config.asciidoc[]
+endif::[]
+
 include::{libbeat-dir}/shared-ssl-config.asciidoc[]
 
 include::{libbeat-dir}/shared-ilm.asciidoc[]

diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go
@@ -26,6 +26,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/magefile/mage/sh"
 	"github.com/pkg/errors"
@@ -71,7 +72,13 @@ func (b *dockerBuilder) Build() error {
 
 	tag, err := b.dockerBuild()
 	if err != nil {
-		return errors.Wrap(err, "failed to build docker")
+		fmt.Println(">> Building docker images again (after 10 seconds)")
+		// This sleep is to avoid hitting the docker build issues when resources are not available.
+		time.Sleep(10)
+		tag, err = b.dockerBuild()
+		if err != nil {
+			return errors.Wrap(err, "failed to build docker")
+		}
 	}
 
 	if err := b.dockerSave(tag); err != nil {

diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
@@ -57,3 +57,16 @@ stages:
                 - "windows-2016"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+    windows-10:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-10"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test filebeat for windows-10"
+            labels:
+                - "windows-10"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
diff --git a/filebeat/docs/configuring-howto.asciidoc b/filebeat/docs/configuring-howto.asciidoc
@@ -44,6 +44,10 @@ include::./reload-configuration.asciidoc[]
 
 include::{libbeat-dir}/outputconfig.asciidoc[]
 
+ifndef::no_kerberos[]
+include::{libbeat-dir}/shared-kerberos-config.asciidoc[]
+endif::[]
+
 include::{libbeat-dir}/shared-ssl-config.asciidoc[]
 
 include::../../libbeat/docs/shared-ilm.asciidoc[]

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -143952,15 +143952,6 @@ type: keyword
 
 --
 
-*`suricata.eve.timestamp`*::
-+
---
-type: alias
-
-alias to: @timestamp
-
---
-
 *`suricata.eve.in_iface`*::
 +
 --

diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml
@@ -67,4 +67,15 @@ stages:
                 - "windows-2012"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+    windows-10:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-10"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test heartbeat for windows-10"
+            labels:
+                - "windows-10"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
 
diff --git a/heartbeat/docs/configuring-howto.asciidoc b/heartbeat/docs/configuring-howto.asciidoc
@@ -38,6 +38,10 @@ include::{libbeat-dir}/shared-path-config.asciidoc[]
 
 include::{libbeat-dir}/outputconfig.asciidoc[]
 
+ifndef::no_kerberos[]
+include::{libbeat-dir}/shared-kerberos-config.asciidoc[]
+endif::[]
+
 include::{libbeat-dir}/shared-ssl-config.asciidoc[]
 
 include::{libbeat-dir}/shared-ilm.asciidoc[]

diff --git a/heartbeat/docs/monitors/monitor-http.asciidoc b/heartbeat/docs/monitors/monitor-http.asciidoc
@@ -161,7 +161,7 @@ Under `check.response`, specify these options:
 
 *`status`*:: A list of expected status codes. 4xx and 5xx codes are considered `down` by default. Other codes are considered `up`.
 *`headers`*:: The required response headers.
-*`body`*:: A list of regular expressions to match the the body output. Only a single expression needs to match. HTTP response
+*`body`*:: A list of regular expressions to match the body output. Only a single expression needs to match. HTTP response
 bodies of up to 100MiB are supported.
 
 Example configuration:

diff --git a/journalbeat/docs/configuring-howto.asciidoc b/journalbeat/docs/configuring-howto.asciidoc
@@ -34,6 +34,10 @@ include::{libbeat-dir}/shared-path-config.asciidoc[]
 
 include::{libbeat-dir}/outputconfig.asciidoc[]
 
+ifndef::no_kerberos[]
+include::{libbeat-dir}/shared-kerberos-config.asciidoc[]
+endif::[]
+
 include::{libbeat-dir}/shared-ssl-config.asciidoc[]
 
 include::{libbeat-dir}/shared-ilm.asciidoc[]

diff --git a/libbeat/autodiscover/providers/kubernetes/node.go b/libbeat/autodiscover/providers/kubernetes/node.go
@@ -105,7 +105,6 @@ func (n *node) OnUpdate(obj interface{}) {
 		time.AfterFunc(n.config.CleanupTimeout, func() { n.emit(node, "stop") })
 	} else {
 		n.logger.Debugf("Watcher Node update: %+v", obj)
-		// TODO: figure out how to avoid stop starting when node status is periodically being updated by kubelet
 		n.emit(node, "stop")
 		n.emit(node, "start")
 	}

diff --git a/libbeat/cfgfile/cfgfile.go b/libbeat/cfgfile/cfgfile.go
@@ -205,6 +205,10 @@ func LoadList(file string) ([]*common.Config, error) {
 	return c, nil
 }
 
+func SetConfigPath(path string) {
+	*configPath = path
+}
+
 // GetPathConfig returns ${path.config}. If ${path.config} is not set, ${path.home} is returned.
 func GetPathConfig() string {
 	if *configPath != "" {

diff --git a/libbeat/docs/outputs-list.asciidoc b/libbeat/docs/outputs-list.asciidoc
@@ -83,9 +83,5 @@ ifdef::requires_xpack[]
 endif::[]
 include::{libbeat-outputs-dir}/codec/docs/codec.asciidoc[]
 endif::[]
-ifndef::no_kerberos[]
-include::{libbeat-dir}/shared-kerberos-config.asciidoc[]
-endif::[]
-
 
 //# end::outputs-include[]
diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc
@@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read
 <<breaking-changes>> for more detail about changes that affect
 upgrade.
 
+* <<release-notes-7.9.3>>
 * <<release-notes-7.9.2>>
 * <<release-notes-7.9.1>>
 * <<release-notes-7.9.0>>
@@ -39,6 +40,10 @@ upgrade.
 * <<release-notes-7.0.0-beta1>>
 * <<release-notes-7.0.0-alpha2>>
 * <<release-notes-7.0.0-alpha1>>
+* <<release-notes-6.8.13>>
+* <<release-notes-6.8.12>>
+* <<release-notes-6.8.11>>
+* <<release-notes-6.8.10>>
 * <<release-notes-6.8.9>>
 * <<release-notes-6.8.8>>
 * <<release-notes-6.8.7>>