Cherry-pick #17456 to 7.x: [Filebeat] Add access_key_id, secret_access_key and session_token into aws module config

CHANGELOG.next.asciidoc

@@ -298,6 +298,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS categorization field mappings in mongodb module. {issue}16170[16170] {pull}17371[17371]
 - Improve ECS categorization field mappings for mssql module. {issue}16171[16171] {pull}17376[17376]
 - Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275]
+- Added access_key_id, secret_access_key and session_token into aws module config. {pull}17456[17456]
 
 *Heartbeat*
 

filebeat/docs/modules/aws.asciidoc

@@ -5,6 +5,8 @@ This file is generated! See scripts/docs_collector.py
 [[filebeat-module-aws]]
 [role="xpack"]
 
+:libbeat-xpack-dir: ../../../x-pack/libbeat
+
 :modulename: aws
 :has-dashboards: true
 
@@ -23,6 +25,11 @@ from network interfaces in AWS VPC. ELB access logs captures detailed informatio
 about requests sent to the load balancer. CloudTrail logs contain events
 that represent actions taken by a user, role or AWS service.
 
+The `aws` module requires AWS credentials configuration in order to make AWS API calls.
+Users can either use `access_key_id`, `secret_access_key` and/or
+`session_token`, or use shared AWS credentials file.
+Please see <<aws-credentials-options,AWS credentials options>> for more details.
+
 include::../include/gs-link.asciidoc[]
 
 [float]
@@ -38,6 +45,9 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
@@ -47,6 +57,9 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
@@ -56,6 +69,9 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
@@ -65,6 +81,9 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
@@ -74,6 +93,9 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
@@ -83,23 +105,17 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
-
 ----
 
 *`var.queue_url`*::
 
-AWS SQS queue url.
-
-*`var.shared_credential_file`*::
-
-Filename of AWS credential file.
-
-*`var.credential_profile_name`*::
-
-AWS credential profile name.
+(Required) AWS SQS queue url.
 
 *`var.visibility_timeout`*::
 
@@ -114,6 +130,23 @@ Maximum duration before AWS API request will be interrupted. Default to be 120 s
 
 Custom endpoint used to access AWS APIs.
 
+*`var.shared_credential_file`*::
+
+Filename of AWS credential file.
+
+*`var.credential_profile_name`*::
+
+AWS credential profile name.
+
+*`var.access_key_id`*::
+First part of access key.
+
+*`var.access_key_id`*::
+Second part of access key.
+
+*`var.access_key_id`*::
+Required when using temporary security credentials.
+
 [float]
 === cloudtrail fileset
 
@@ -188,6 +221,9 @@ This fileset comes with a predefined dashboard:
 [role="screenshot"]
 image::./images/filebeat-aws-vpcflow-overview.png[]
 
+[id="aws-credentials-options"]
+include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[]
+
 
 [float]
 === Fields

x-pack/filebeat/filebeat.reference.yml

@@ -111,6 +111,11 @@ filebeat.modules:
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -137,6 +142,42 @@ filebeat.modules:
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
+    # The duration that the received messages are hidden from ReceiveMessage request
+    # Default to be 300s
+    #var.visibility_timeout: 300s
+
+    # Maximum duration before AWS API request will be interrupted
+    # Default to be 120s
+    #var.api_timeout: 120s
+
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
+  ec2:
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    #var.credential_profile_name: fb-aws
+
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -163,6 +204,11 @@ filebeat.modules:
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -189,6 +235,11 @@ filebeat.modules:
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -215,6 +266,11 @@ filebeat.modules:
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s

x-pack/filebeat/module/aws/_meta/config.yml

@@ -14,6 +14,11 @@
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -40,6 +45,42 @@
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
+    # The duration that the received messages are hidden from ReceiveMessage request
+    # Default to be 300s
+    #var.visibility_timeout: 300s
+
+    # Maximum duration before AWS API request will be interrupted
+    # Default to be 120s
+    #var.api_timeout: 120s
+
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
+  ec2:
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    #var.credential_profile_name: fb-aws
+
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -66,6 +107,11 @@
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -92,6 +138,11 @@
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -118,6 +169,11 @@
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s