Merge remote-tracking branch 'upstream/master' into feature/packaging…

…-arm

* upstream/master:
  [Metricbeat][Kubernetes] Extend state_node with more conditions (elastic#23905)
  [CI] googleStorageUploadExt step (elastic#24048)
  Check fields are documented for aws metricsets (elastic#23887)
  Update go-concert to 0.1.0 (elastic#23770)
  [Libbeat][New Processor] XML Decode (elastic#23678)
  Fix: bad substitution of API key (elastic#24036)
  [Filebeat] Add Pensando DFW Module (elastic#21063)
  [Filebeat] Check if processor is supported by ES version (elastic#23763)
  Syslog system tests: be more forgiving (elastic#24021)

.ci/packaging.groovy

@@ -17,6 +17,7 @@ pipeline {
     JOB_GCS_BUCKET = 'beats-ci-artifacts'
     JOB_GCS_BUCKET_STASH = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
+    JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials'
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
     GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests'
@@ -448,14 +449,11 @@ def publishPackages(baseDir){
   uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
 }
 
-def uploadPackages(bucketUri, baseDir){
-  googleStorageUpload(bucket: bucketUri,
-    credentialsId: "${JOB_GCS_CREDENTIALS}",
-    pathPrefix: "${baseDir}/build/distributions/",
-    pattern: "${baseDir}/build/distributions/**/*",
-    sharedPublicly: true,
-    showInline: true
-  )
+def uploadPackages(bucketUri, beatsFolder){
+  googleStorageUploadExt(bucket: bucketUri,
+    credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+    pattern: "${beatsFolder}/build/distributions/**/*",
+    sharedPublicly: true)
 }
 
 /**
@@ -510,4 +508,4 @@ def fixPermissions() {
       }
     }
   }
-}
+}

CHANGELOG-developer.next.asciidoc

@@ -104,6 +104,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.14.7. {pull}20508[20508]
 - Add packaging for docker image based on UBI minimal 8. {pull}20576[20576]
 - Make the mage binary used by the build process in the docker container to be statically compiled. {pull}20827[20827]
+- Add Pensando distributed firewall module. {pull}21063[21063]
 - Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267]
 - Add support for customized monitoring API. {pull}22605[22605]
 - Update Go version to 1.15.7. {pull}22495[22495]

CHANGELOG.next.asciidoc

@@ -126,6 +126,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - kubernetes.container.cpu.limit.cores and kubernetes.container.cpu.requests.cores are now floats. {issue}11975[11975]
 - Change types of numeric metrics from Kubelet summary api to double so as to cover big numbers. {pull}23335[23335]
 - Add container.image.name and containe.name ECS fields for state_container. {pull}23802[23802]
+- Add support for the MemoryPressure, DiskPressure, OutOfDisk and PIDPressure status conditions in state_node. {pull}[23905]
 
 *Packetbeat*
 
@@ -497,6 +498,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286]
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
 - Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647]
+- Unskip s3_request integration test. {pull}23887[23887]
 - Add system.hostfs configuration option for system module. {pull}23831[23831]
 
 *Packetbeat*
@@ -598,6 +600,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595]
 - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564]
 - Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629]
+- Added new decode_xml processor to libbeat that is available to all beat types. {pull}23678[23678]
 - Add deployment name in pod's meta. {pull}23610[23610]
 - Add `selector` information in kubernetes services' metadata. {pull}23730[23730]
 
@@ -825,6 +828,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added `application/x-www-form-urlencoded` as encode option for httpjson input {pull}23521[23521]
 - Move aws-s3 input to GA. {pull}23631[23631]
 - Populate `source.mac` and `destination.mac` for Suricata EVE events. {issue}23706[23706] {pull}23721[23721]
+- Added feature to modules to adapt Ingest Node pipelines for compatibility with older Elasticsearch versions by
+  removing unsupported processors. {pull}23763[23763]
 - Added RFC6587 framing option for tcp and unix inputs {issue}23663[23663] {pull}23724[23724]
 - Added string splitting for httpjson input {pull}24022[24022]
 
@@ -956,6 +961,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Apache: convert status.total_kbytes to status.total_bytes in fleet mode. {pull}23022[23022]
 - Release MSSQL as GA {pull}23146[23146]
 - Enrich events of `state_service` metricset with kubernetes services' metadata. {pull}23730[23730]
+- Check fields are documented in aws metricsets. {pull}23887[23887]
 
 *Packetbeat*
 

Jenkinsfile

@@ -14,6 +14,7 @@ pipeline {
     DOCKER_REGISTRY = 'docker.elastic.co'
     JOB_GCS_BUCKET = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
+    JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials'
     OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
     PIPELINE_LOG_LEVEL = 'INFO'
     PYTEST_ADDOPTS = "${params.PYTEST_ADDOPTS}"
@@ -324,13 +325,10 @@ def publishPackages(beatsFolder){
 * @param beatsFolder the beats folder.
 */
 def uploadPackages(bucketUri, beatsFolder){
-  googleStorageUpload(bucket: bucketUri,
-    credentialsId: "${JOB_GCS_CREDENTIALS}",
-    pathPrefix: "${beatsFolder}/build/distributions/",
+  googleStorageUploadExt(bucket: bucketUri,
+    credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
     pattern: "${beatsFolder}/build/distributions/**/*",
-    sharedPublicly: true,
-    showInline: true
-  )
+    sharedPublicly: true)
 }
 
 /**
@@ -706,11 +704,10 @@ def archiveTestOutput(Map args = [:]) {
 */
 def tarAndUploadArtifacts(Map args = [:]) {
   tar(file: args.file, dir: args.location, archive: false, allowMissing: true)
-  googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
-                      credentialsId: "${JOB_GCS_CREDENTIALS}",
-                      pattern: "${args.file}",
-                      sharedPublicly: true,
-                      showInline: true)
+  googleStorageUploadExt(bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
+                         credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+                         pattern: "${args.file}",
+                         sharedPublicly: true)
 }
 
 /**

NOTICE.txt

@@ -6336,11 +6336,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-concert
-Version: v0.0.4
+Version: v0.1.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-concert@v0.0.4/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-concert@v0.1.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl

@@ -40,7 +40,7 @@ function enroll(){
       local apikeyId=$(echo $enrollResp | jq -r '.list[] | select((.name | startswith("Default ")) and (.active == true)) | .id')
       echo $apikeyId
 
-      if [[ -z "${$apikeyId}" ]]; then
+      if [[ -z "${apikeyId}" ]]; then
         echo "Default agent policy was not found. Please consider using own enrollment token (FLEET_ENROLLMENT_TOKEN)."
         exit 1
       fi

filebeat/docs/fields.asciidoc

@@ -69,6 +69,7 @@ grouped in the following categories:
 * <<exported-fields-oracle>>
 * <<exported-fields-osquery>>
 * <<exported-fields-panw>>
+* <<exported-fields-pensando>>
 * <<exported-fields-postgresql>>
 * <<exported-fields-process>>
 * <<exported-fields-proofpoint>>
@@ -105827,6 +105828,147 @@ Specifies the sub type of the log
 
 --
 
+[[exported-fields-pensando]]
+== Pensando fields
+
+pensando Module
+
+
+
+[float]
+=== pensando
+
+Fields from Pensando logs.
+
+
+
+[float]
+=== dfw
+
+Fields for Pensando DFW
+
+
+
+*`pensando.dfw.action`*::
++
+--
+Action on the flow. 
+
+
+type: keyword
+
+--
+
+*`pensando.dfw.app_id`*::
++
+--
+Application ID 
+
+
+type: integer
+
+--
+
+*`pensando.dfw.destination_address`*::
++
+--
+Address of destination. 
+
+
+type: keyword
+
+--
+
+*`pensando.dfw.destination_port`*::
++
+--
+Port of destination. 
+
+
+type: integer
+
+--
+
+*`pensando.dfw.direction`*::
++
+--
+Direction of the flow 
+
+
+type: keyword
+
+--
+
+*`pensando.dfw.protocol`*::
++
+--
+Protocol of the flow 
+
+
+type: keyword
+
+--
+
+*`pensando.dfw.rule_id`*::
++
+--
+Rule ID that was matched. 
+
+
+type: keyword
+
+--
+
+*`pensando.dfw.session_id`*::
++
+--
+Session ID of the flow 
+
+
+type: integer
+
+--
+
+*`pensando.dfw.session_state`*::
++
+--
+Session state of the flow. 
+
+
+type: keyword
+
+--
+
+*`pensando.dfw.source_address`*::
++
+--
+Source address of the flow. 
+
+
+type: keyword
+
+--
+
+*`pensando.dfw.source_port`*::
++
+--
+Source port of the flow. 
+
+
+type: integer
+
+--
+
+*`pensando.dfw.timestamp`*::
++
+--
+Timestamp of the log. 
+
+
+type: date
+
+--
+
 [[exported-fields-postgresql]]
 == PostgreSQL fields
 

filebeat/docs/modules/pensando.asciidoc

@@ -0,0 +1,69 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-pensando]]
+:modulename: pensando
+:has-dashboards: true
+
+== pensando module
+
+The +{modulename}+ module parses distributed firewall logs created by the
+http://pensando.io/[Pensando] distributed services card (DSC).
+
+
+include::../include/what-happens.asciidoc[]
+
+include::../include/gs-link.asciidoc[]
+
+[float]
+=== Compatibility
+
+The Pensando module has been tested with 1.12.0-E-54 and later.
+
+include::../include/configuring-intro.asciidoc[]
+The following example shows how to set parameters in the +modules.d/{modulename}.yml+
+file to listen for firewall logs sent from the Pensando DSC(s) on port 5514 (default is 9001):
+
+["source","yaml",subs="attributes"]
+-----
+- module: pensando
+  access:
+    enabled: true
+    var.syslog_host: 0.0.0.0
+    var.syslog_port: [9001]
+-----
+:fileset_ex: dfw
+
+include::../include/config-option-intro.asciidoc[]
+
+TODO: document the variables from each fileset. If you're describing a variable
+that's common to other modules, you can reuse shared descriptions by including
+the relevant file. For example:
+
+[float]
+==== `dfw` log fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+[float]
+=== Example dashboard
+
+This module comes with a sample dashboard. For example:
+
+[role="screenshot"]
+image::./images/filebeat-pensando-dfw.png[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-pensando,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -50,6 +50,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-oracle>>
   * <<filebeat-module-osquery>>
   * <<filebeat-module-panw>>
+  * <<filebeat-module-pensando>>
   * <<filebeat-module-postgresql>>
   * <<filebeat-module-proofpoint>>
   * <<filebeat-module-rabbitmq>>
@@ -121,6 +122,7 @@ include::modules/okta.asciidoc[]
 include::modules/oracle.asciidoc[]
 include::modules/osquery.asciidoc[]
 include::modules/panw.asciidoc[]
+include::modules/pensando.asciidoc[]
 include::modules/postgresql.asciidoc[]
 include::modules/proofpoint.asciidoc[]
 include::modules/rabbitmq.asciidoc[]