Merge remote-tracking branch 'upstream/master' into feature/retry-win…

…dows-7

* upstream/master: (41 commits)
  Fix version parser regex for packaging (elastic#22581)
  Fix local_dynamic documentation and add providers inline doc. (elastic#22657)
  fix: use proper param name for e2e tests (elastic#22836)
  [Heartbeat] Fix exit on disabled monitor (elastic#22829)
  Update Golang to 1.14.12 (elastic#22790)
  docs: fix setup.template.overwrite typos (elastic#22804)
  Add docs section for ECS EC2 monitoring (elastic#22784)
  Fixing logic to keep list of unique cluster UUIDs (elastic#22808)
  Skip somewhat flaky UDP system test on Windows (elastic#22810)
  Fix polling node when it is not ready and monitor by hostname (elastic#22666)
  Skip Filebeat test_shutdown on windows 7 (elastic#22797)
  Make monitoring Namespace thread-safe (elastic#22640)
  Drop pkt_dstaddr and pkt_srcaddr when equals to "-" (elastic#22721)
  Add support for reading from UNIX datagram sockets (elastic#22699)
  Fix export dashboard command from Elastic Cloud (elastic#22746)
  Skip flaky winlogbeat test on Windows-7 (elastic#22754)
  Missing `>` (elastic#22763) (elastic#22766)
  Fix k8s watcher issue when node access to list nodes and ns (elastic#22714)
  [Metricbeat/Kibana/stats] Enforce `exclude_usage=true` (elastic#22732)
  Avoid sending non-numeric floats in cloud foundry integrations (elastic#22634)
  ...

.ci/packaging.groovy

@@ -135,9 +135,11 @@ pipeline {
                     'linux/386',
                     'linux/arm64',
                     'linux/armv7',
-                    'linux/ppc64le',
-                    'linux/mips64',
-                    'linux/s390x',
+                    // The platforms above are disabled temporarly as crossbuild images are
+                    // not available. See: https://github.com/elastic/golang-crossbuild/issues/71
+                    //'linux/ppc64le',
+                    //'linux/mips64',
+                    //'linux/s390x',
                     'windows/amd64',
                     'windows/386',
                     (params.macos ? '' : 'darwin/amd64'),
@@ -344,7 +346,7 @@ def triggerE2ETests(String suite) {
   ]
   if (isPR()) {
     def version = "pr-${env.CHANGE_ID}"
-    parameters.push(booleanParam(name: 'USE_CI_SNAPSHOTS', value: true))
+    parameters.push(booleanParam(name: 'ELASTIC_AGENT_USE_CI_SNAPSHOTS', value: true))
     parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
     parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
   }

.go-version

@@ -1 +1 @@
-1.14.7
+1.14.12

CHANGELOG-developer.next.asciidoc

@@ -57,6 +57,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 
 - Stop using `mage:import` in community beats. This was ignoring the vendorized beats directory for some mage targets, using the code available in GOPATH, this causes inconsistencies and compilation problems if the version of the code in the GOPATH is different to the vendored one. Use of `mage:import` will continue to be unsupported in custom beats till beats is migrated to go modules, or mage supports vendored dependencies. {issue}13998[13998] {pull}14162[14162]
 - Metricbeat module builders call host parser only once when instantiating light modules. {pull}20149[20149]
+- Fix export dashboard command when running against Elastic Cloud hosted Kibana. {pull}22746[22746]
 
 ==== Added
 
@@ -102,4 +103,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.14.7. {pull}20508[20508]
 - Add packaging for docker image based on UBI minimal 8. {pull}20576[20576]
 - Make the mage binary used by the build process in the docker container to be statically compiled. {pull}20827[20827]
-- Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267]
+- Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267]

CHANGELOG.next.asciidoc

@@ -215,6 +215,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed documentation for commands in beats dev guide {pull}22194[22194]
 - Fix parsing of expired licences. {issue}21112[21112] {pull}22180[22180]
 - Fix duplicated pod events in kubernetes autodiscover for pods with init or ephemeral containers. {pull}22438[22438]
+- Fix FileVersion contained in Windows exe files. {pull}22581[22581]
+- Fix index template loading when the new index format is selected. {issue}22482[22482] {pull}22682[22682]
+
 
 *Auditbeat*
 
@@ -232,6 +235,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/socket: Fix kprobe grouping to allow running more than one instance. {pull}20325[20325]
 - system/socket: Fixed a crash due to concurrent map read and write. {issue}21192[21192] {pull}21690[21690]
 - file_integrity: stop monitoring excluded paths {issue}21278[21278] {pull}21282[21282]
+- auditd: Fix an error condition causing a lot of `audit_send_reply` kernel threads being created. {pull}22673[22673]
+- system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693]
 
 *Filebeat*
 
@@ -332,6 +337,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361]
 - Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696]
 - Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377]
+- Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716]
 
 *Heartbeat*
 
@@ -341,6 +347,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Heartbeat*
 
 - The `service_name` monitor option is being replaced with `service.name` which is more correct. We will support the old option till 8.0. {pull}20330[20330]
+- Fix exit on monitors with `enabled: false` {pull}22829[22829]
 
 *Journalbeat*
 
@@ -435,6 +442,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change Session ID type from int to string {pull}22359[22359]
 - Fix filesystem types on Windows in filesystem metricset. {pull}22531[22531]
 - Fix failiures caused by custom beat names with more than 15 characters {pull}22550[22550]
+- Stop generating NaN values from Cloud Foundry module to avoid errors in outputs. {pull}22634[22634]
+- Update NATS dashboards to leverage connection and route metricsets {pull}22646[22646]
+- Fix `logstash` module when `xpack.enabled: true` is set from emitting redundant events. {pull}22808[22808]
 
 *Packetbeat*
 
@@ -531,6 +541,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added Kafka version 2.2 to the list of supported versions. {pull}22328[22328]
 - Add support for ephemeral containers in kubernetes autodiscover and `add_kubernetes_metadata`. {pull}22389[22389] {pull}22439[22439]
 - Added support for wildcard fields and keyword fallback in beats setup commands. {pull}22521[22521]
+- Fix polling node when it is not ready and monitor by hostname {pull}22666[22666]
 
 *Auditbeat*
 
@@ -716,12 +727,17 @@ from being added to events by default. {pull}18159[18159]
 - Add SSL option to checkpoint module {pull}19560[19560]
 - Add max_number_of_messages config into s3 input. {pull}21993[21993]
 - Update Okta documentation for new stateful restarts. {pull}22091[22091]
+- Rename googlecloud module to gcp module. {pull}22214[22214]
 - Rename awscloudwatch input to aws-cloudwatch. {pull}22228[22228]
 - Rename google-pubsub input to gcp-pubsub. {pull}22213[22213]
 - Copy tag names from MISP data into events. {pull}21664[21664]
 - Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291]
 - Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696]
+- Add platform logs in the azure filebeat module. {pull}22371[22371]
 - Added `event.ingested` field to data from the Netflow module. {pull}22412[22412]
+- Improve panw ECS url fields mapping. {pull}22481[22481]
+- Improve Nats filebeat dashboard. {pull}22726[22726]
+- Add support for UNIX datagram sockets in `unix` input. {issues}18632[18632] {pull}22699[22699]
 
 *Heartbeat*
 
@@ -841,8 +857,11 @@ same journal. {pull}18467[18467]
 - Map cloud data filed `cloud.account.id` to azure subscription.  {pull}21483[21483] {issue}21381[21381]
 - Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703]
 - Duplicate system.process.cmdline field with process.command_line ECS field name. {pull}22325[22325]
+- Add awsfargate module task_stats metricset to monitor AWS ECS Fargate. {pull}22034[22034]
 - Add connection and route metricsets for nats metricbeat module to collect metrics per connection/route. {pull}22445[22445]
 - Add unit file states to system/service {pull}22557[22557]
+- Add io.ops in fields exported by system.diskio. {pull}22066[22066]
+- `kibana` module: `stats` metricset no-longer collects usage-related data. {pull}22732[22732]
 
 *Packetbeat*
 

Jenkinsfile

@@ -35,7 +35,7 @@ pipeline {
   }
   options {
     timeout(time: 3, unit: 'HOURS')
-    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    buildDiscarder(logRotator(numToKeepStr: '60', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
     timestamps()
     ansiColor('xterm')
     disableResume()
@@ -216,7 +216,7 @@ def generateStages(Map args = [:]) {
 }
 
 def cloud(Map args = [:]) {
-  node(args.label) {
+  withNode(args.label) {
     startCloudTestEnv(name: args.directory, dirs: args.dirs)
   }
   withCloudTestEnv() {
@@ -231,7 +231,7 @@ def cloud(Map args = [:]) {
 def k8sTest(Map args = [:]) {
   def versions = args.versions
   versions.each{ v ->
-    node(args.label) {
+    withNode(args.label) {
       stage("${args.context} ${v}"){
         withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){
           withGithubNotify(context: "${args.context} ${v}") {
@@ -292,7 +292,7 @@ def runCommand(Map args = [:]) {
   def directory = args.get('directory', '')
   def withModule = args.get('withModule', false)
   def isMage = args.get('isMage', false)
-  node(args.label) {
+  withNode(args.label) {
     withGithubNotify(context: "${context}") {
       withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
         dumpVariables()
@@ -306,6 +306,16 @@ def runCommand(Map args = [:]) {
   }
 }
 
+/**
+* This method wraps the node call with some latency to avoid the known issue with the scalabitity in gobld.
+*/
+def withNode(String label, Closure body) {
+  sleep randomNumber(min: 10, max: 200)
+  node(label) {
+    body()
+  }
+}
+
 /**
 * This method wraps all the environment setup and pre-requirements to run any commands.
 */

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14.7
+FROM golang:1.14.12
 
 RUN \
     apt-get update \

auditbeat/module/auditd/audit_linux.go

@@ -163,7 +163,11 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
 			ms.log.Errorw("Failure creating audit monitoring client", "error", err)
 		}
 		go func() {
-			defer client.Close()
+			defer func() { // Close the most recently allocated "client" instance.
+				if client != nil {
+					client.Close()
+				}
+			}()
 			timer := time.NewTicker(lostEventsUpdateInterval)
 			defer timer.Stop()
 			for {
@@ -175,6 +179,15 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
 						ms.updateKernelLostMetric(status.Lost)
 					} else {
 						ms.log.Error("get status request failed:", err)
+						if err = client.Close(); err != nil {
+							ms.log.Errorw("Error closing audit monitoring client", "error", err)
+						}
+						client, err = libaudit.NewAuditClient(nil)
+						if err != nil {
+							ms.log.Errorw("Failure creating audit monitoring client", "error", err)
+							reporter.Error(err)
+							return
+						}
 					}
 				}
 			}

dev-tools/mage/common.go

@@ -782,7 +782,7 @@ func binaryExtension(goos string) string {
 	return ""
 }
 
-var parseVersionRegex = regexp.MustCompile(`(?m)^[^\d]*(?P<major>\d)+\.(?P<minor>\d)+(?:\.(?P<patch>\d)+.*)?$`)
+var parseVersionRegex = regexp.MustCompile(`(?m)^[^\d]*(?P<major>\d+)\.(?P<minor>\d+)(?:\.(?P<patch>\d+).*)?$`)
 
 // ParseVersion extracts the major, minor, and optional patch number from a
 // version string.

dev-tools/mage/common_test.go

@@ -33,6 +33,8 @@ func TestParseVersion(t *testing.T) {
 		{"1.2.3-SNAPSHOT", 1, 2, 3},
 		{"1.2.3rc1", 1, 2, 3},
 		{"1.2", 1, 2, 0},
+		{"7.10.0", 7, 10, 0},
+		{"10.01.22", 10, 1, 22},
 	}
 
 	for _, tc := range tests {

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14.7
+FROM golang:1.14.12
 
 RUN \
     apt-get update \