Merge remote-tracking branch 'upstream/master' into feature/windows-2016

* upstream/master: (27 commits)
  [Ingest Manager] Split index restrictions into type,dataset, namespace parts (elastic#21406)
  Update Filebeat module expected logs files (elastic#21454)
  Edit SQL module docs and fix broken doc structure (elastic#21233)
  [Ingest Manager] Send snapshot flag together with metadata (elastic#21285)
  Revert "[JJBB] Set shallow cloning to 10 (elastic#21409)" (elastic#21447)
  [JJBB] Use reference repo for fast checkouts (elastic#21410)
  Add initial skeleton of filestream input (elastic#21427)
  Initial spec file for apm-server (elastic#21225)
  [Ingest Manager] Upgrade Action: make source URI optional (elastic#21372)
  Add field limit check for AWS Cloudtrail flattened fields (elastic#21388)
  [Winlogbeat] Move winlogbeat javascript processor to libbeat (elastic#21402)
  ci: pipeline to generate the changelog (elastic#21426)
  [JJBB] Set shallow cloning to 10 (elastic#21409)
  docs: add link to release notes for 7.9.2 (elastic#21405) (elastic#21419)
  docs: Prepare Changelog for 7.9.2 (elastic#21229) (elastic#21403)
  fix: mark flaky tests (elastic#21300)
  fix: use a fixed version of setuptools (elastic#21393)
  Move Kubernetes events metricset to its own block in reference config (elastic#21407)
  [libbeat] Enable WriteAheadLimit in the disk queue (elastic#21391)
  docs: fix apt/yum formatting (elastic#21362)
  ...

.ci/jobs/beats-release-changelog.yml

@@ -0,0 +1,20 @@
+---
+- job:
+    name: Beats/Release/beats-release-changelog
+    display-name: 'Prepare the Changelog for a Release'
+    description: 'Automate the steps to prepare the Changelog for a Release'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: release_scripts/pipeline-release-changelog.groovy
+        scm:
+        - git:
+            url: git@github.com:elastic/ingest-dev.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+            branches:
+                - master

.ci/jobs/beats.yml

@@ -53,6 +53,7 @@
             recursive: true
             parent-credentials: true
             timeout: 100
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
         timeout: '15'
         use-author: true
         wipe-workspace: true

CHANGELOG.asciidoc

@@ -3,6 +3,47 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.9.2]]
+=== Beats version 7.9.2
+https://github.com/elastic/beats/compare/v7.9.1...v7.9.2[View commits]
+
+==== Breaking changes
+
+*Affecting all Beats*
+
+- Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
+- Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197]
+
+*Filebeat*
+
+- Provide backwards compatibility for the `set` processor when Elasticsearch is less than 7.9.0. {pull}20908[20908]
+- Fix an error updating file size being logged when EOF is reached. {pull}21048[21048]
+- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943]
+
+*Metricbeat*
+
+- The Kibana collector applies backoff when errored at getting usage stats {pull}20772[20772]
+- The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938]
+- Fix panic index out of range error when getting AWS account name. {pull}21101[21101] {issue}21095[21095]
+- Handle missing counters in the application_pool metricset. {pull}21071[21071]
+
+*Functionbeat*
+
+- Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072]
+- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070]
+
+==== Added
+
+*Affecting all Beats*
+
+- Add container ECS fields in kubernetes metadata. {pull}20984[20984]
+
 [[release-notes-7.9.1]]
 === Beats version 7.9.1
 https://github.com/elastic/beats/compare/v7.9.0...v7.9.1[View commits]

CHANGELOG.next.asciidoc

@@ -23,6 +23,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
 - Remove redundant `cloudfoundry.*.timestamp` fields. This value is set in `@timestamp`. {pull}21175[21175]
 - Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs https://github.com/elastic/beats/pull/21179
+- Update to Golang 1.12.1. {pull}11330[11330]
+- Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
 
 *Auditbeat*
 
@@ -74,6 +76,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808]
 - Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
+- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 
 *Heartbeat*
 
@@ -272,6 +275,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Handle multiple upstreams in ingress-controller. {pull}21215[21215]
 - Provide backwards compatibility for the `append` processor when Elasticsearch is less than 7.10.0. {pull}21159[21159]
 - Fix checkpoint module when logs contain time field. {pull}20567[20567]
+- Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382]
 
 *Heartbeat*
 
@@ -356,6 +360,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix timestamp handling in remote_write. {pull}21166[21166]
 - Fix remote_write flaky test. {pull}21173[21173]
 - Visualization title fixes in aws, azure and googlecloud compute dashboards. {pull}21098[21098]
+- Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
 
 *Packetbeat*
 
@@ -597,6 +602,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add type and sub_type to panw panos fileset {pull}20912[20912]
 - Always attempt community_id processor on zeek module {pull}21155[21155]
 - Add related.hosts ecs field to all modules {pull}21160[21160]
+- Keep cursor state between httpjson input restarts {pull}20751[20751]
 
 *Heartbeat*
 

NOTICE.txt

@@ -10907,6 +10907,36 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/blakerouse/service
+Version: v1.1.1-0.20200924160513-057808572ffa
+Licence type (autodetected): Zlib
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/blakerouse/service@v1.1.1-0.20200924160513-057808572ffa/LICENSE:
+
+Copyright (c) 2015 Daniel Theophanes
+
+This software is provided 'as-is', without any express or implied
+warranty. In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+   1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+
+   2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+
+   3. This notice may not be removed or altered from any source
+   distribution.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/lib/pq
 Version: v1.1.2-0.20190507191818-2ff3cb3adc01
@@ -11843,6 +11873,37 @@ Contents of probable licence file $GOMODCACHE/github.com/oklog/ulid@v1.3.1/LICEN
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/otiai10/copy
+Version: v1.2.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/otiai10/copy@v1.2.0/LICENSE:
+
+The MIT License (MIT)
+
+Copyright (c) 2018 otiai10
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/pierrre/gotestcover
 Version: v0.0.0-20160517101806-924dca7d15f0
@@ -34009,6 +34070,54 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runtime-
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/otiai10/curr
+Version: v1.0.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/otiai10/curr@v1.0.0/LICENSE:
+
+The MIT License (MIT)
+
+Copyright (c) 2020 Hiromu Ochiai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
+--------------------------------------------------------------------------------
+Dependency : github.com/otiai10/mint
+Version: v1.3.1
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/otiai10/mint@v1.3.1/LICENSE:
+
+Copyright 2017 otiai10 (Hiromu OCHIAI)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/oxtoacart/bpool
 Version: v0.0.0-20150712133111-4e1c5567d7c2

auditbeat/module/file_integrity/eventreader_test.go

@@ -41,6 +41,7 @@ func init() {
 const ErrorSharingViolation syscall.Errno = 32
 
 func TestEventReader(t *testing.T) {
+	t.Skip("Flaky test: about 1/10 of bulds fails https://github.com/elastic/beats/issues/21302")
 	// Make dir to monitor.
 	dir, err := ioutil.TempDir("", "audit")
 	if err != nil {
@@ -240,6 +241,7 @@ func TestEventReader(t *testing.T) {
 }
 
 func TestRaces(t *testing.T) {
+	t.Skip("Flaky test: about 1/20 of bulds fails https://github.com/elastic/beats/issues/21303")
 	const (
 		fileMode os.FileMode = 0640
 		N                    = 100

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -36,8 +36,6 @@ data:
                     - state_cronjob
                     - state_resourcequota
                     - state_statefulset
-                    # Uncomment this to get k8s events:
-                    #- event
                 - module: kubernetes
                   metricsets:
                     - apiserver
@@ -46,6 +44,10 @@ data:
                   ssl.certificate_authorities:
                     - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                   period: 30s
+                # Uncomment this to get k8s events:
+                #- module: kubernetes
+                #  metricsets:
+                #    - event
         # To enable hints based autodiscover uncomment this:
         #- type: kubernetes
         #  node: ${NODE_NAME}

deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml

@@ -36,8 +36,6 @@ data:
                     - state_cronjob
                     - state_resourcequota
                     - state_statefulset
-                    # Uncomment this to get k8s events:
-                    #- event
                 - module: kubernetes
                   metricsets:
                     - apiserver
@@ -46,6 +44,10 @@ data:
                   ssl.certificate_authorities:
                     - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                   period: 30s
+                # Uncomment this to get k8s events:
+                #- module: kubernetes
+                #  metricsets:
+                #    - event
         # To enable hints based autodiscover uncomment this:
         #- type: kubernetes
         #  node: ${NODE_NAME}

dev-tools/dependencies-report

@@ -41,3 +41,14 @@ go list -m -json all $@ | go run go.elastic.co/go-licence-detector \
 		-noticeTemplate "$SRCPATH/notice/dependencies.csv.tmpl" \
 		-noticeOut "$outfile" \
 		-depsOut ""
+
+
+# Fill-in required values for UBI images
+# Check headers in $SRCPATH/notice/dependencies.csv.tmpl:
+# name,url,version,revision,license
+ubi8url='https://catalog.redhat.com/software/containers/ubi8/ubi-minimal/5c359a62bed8bd75a2c3fba8'
+ubi8source='https://oss-dependencies.elastic.co/redhat/ubi/ubi-minimal-8-source.tar.gz'
+ubilicense='Custom;https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf,https://oss-dependencies.elastic.co/redhat/ubi/ubi-minimal-8-source.tar.gz'
+cat <<EOF >> $outfile
+Red Hat Universal Base Image,$ubi8url,8,,$ubilicense,$ubi8source
+EOF

dev-tools/notice/dependencies.csv.tmpl

@@ -1,7 +1,7 @@
 {{- define "depInfo" -}}
 {{- range $i, $dep := . }}
-{{ $dep.Name }},{{ $dep.URL }},{{ $dep.Version | canonicalVersion }},{{ $dep.Version | revision }},{{ $dep.LicenceType }}
+{{ $dep.Name }},{{ $dep.URL }},{{ $dep.Version | canonicalVersion }},{{ $dep.Version | revision }},{{ $dep.LicenceType }},{{ $dep.URL }}
 {{- end -}}
 {{- end -}}
 
-name,url,version,revision,license{{ template "depInfo" .Direct }}{{ template "depInfo" .Indirect }}
+name,url,version,revision,license,sourceURL{{ template "depInfo" .Direct }}{{ template "depInfo" .Indirect }}

dev-tools/notice/rules.json

@@ -9,7 +9,8 @@
     "ISC",
     "MIT",
     "MPL-2.0",
-    "Public Domain"
+    "Public Domain",
+    "Zlib"
   ],
   "maybelist": [
     "EPL-1.0",

dev-tools/packaging/templates/linux/elastic-agent.sh.tmpl

@@ -6,6 +6,5 @@
 exec /usr/share/{{.BeatName}}/bin/{{.BeatName}} \
   --path.home /var/lib/{{.BeatName}} \
   --path.config /etc/{{.BeatName}} \
-  --path.data /var/lib/{{.BeatName}}/data \
   --path.logs /var/log/{{.BeatName}} \
   "$@"