Add support for IAM role arn in aws config (#17658) (#17726)

* add support for role arn in aws config (cherry picked from commit fa03922)
elastic · Apr 20, 2020 · 62e2e60 · 62e2e60
1 parent 3a5caa5
commit 62e2e60
Show file tree

Hide file tree

Showing 22 changed files with 180 additions and 9 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -253,6 +253,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update supported versions of `redis` output. {pull}17198[17198]
 - Update documentation for system.process.memory fields to include clarification on Windows os's. {pull}17268[17268]
 - Add `urldecode` processor to for decoding URL-encoded fields. {pull}17505[17505]
+- Add support for AWS IAM `role_arn` in credentials config. {pull}17658[17658] {issue}12464[12464]
 
 *Auditbeat*
 

diff --git a/filebeat/docs/aws-credentials-examples.asciidoc b/filebeat/docs/aws-credentials-examples.asciidoc
@@ -22,6 +22,16 @@ filebeat.inputs:
   session_token: '${AWS_SESSION_TOKEN:""}'
 ----
 
+* Use IAM role ARN
++
+[source,yaml]
+----
+filebeat.inputs:
+- type: s3
+  queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue
+  role_arn: arn:aws:iam::123456789012:role/test-mb
+----
+
 * Use shared AWS credentials file
 +
 [source,yaml]

diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc
@@ -27,7 +27,8 @@ that represent actions taken by a user, role or AWS service.
 
 The `aws` module requires AWS credentials configuration in order to make AWS API calls.
 Users can either use `access_key_id`, `secret_access_key` and/or
-`session_token`, or use shared AWS credentials file.
+`session_token`, or use `role_arn` AWS IAM role, or use shared AWS credentials file.
+
 Please see <<aws-credentials-options,AWS credentials options>> for more details.
 
 include::../include/gs-link.asciidoc[]
@@ -51,6 +52,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 
   cloudwatch:
     enabled: false
@@ -63,6 +65,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 
   ec2:
     enabled: false
@@ -75,6 +78,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 
   elb:
     enabled: false
@@ -87,6 +91,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 
   s3access:
     enabled: false
@@ -99,6 +104,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 
   vpcflow:
     enabled: false
@@ -111,6 +117,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 ----
 
 *`var.queue_url`*::
@@ -147,6 +154,9 @@ Second part of access key.
 *`var.session_token`*::
 Required when using temporary security credentials.
 
+*`var.role_arn`*::
+AWS IAM Role to assume.
+
 [float]
 === cloudtrail fileset
 

diff --git a/metricbeat/docs/aws-credentials-examples.asciidoc b/metricbeat/docs/aws-credentials-examples.asciidoc
@@ -26,6 +26,18 @@ metricbeat.modules:
   session_token: '${AWS_SESSION_TOKEN:""}'
 ----
 
+* Use IAM role ARN
++
+[source,yaml]
+----
+metricbeat.modules:
+- module: aws
+  period: 300s
+  metricsets:
+    - ec2
+  role_arn: arn:aws:iam::123456789012:role/test-mb
+----
+
 * Use shared AWS credentials file
 +
 [source,yaml]

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -127,6 +127,9 @@ filebeat.modules:
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
   cloudwatch:
     enabled: false
 
@@ -158,6 +161,9 @@ filebeat.modules:
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
   ec2:
     enabled: false
 
@@ -189,6 +195,9 @@ filebeat.modules:
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
   elb:
     enabled: false
 
@@ -220,6 +229,9 @@ filebeat.modules:
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
   s3access:
     enabled: false
 
@@ -251,6 +263,9 @@ filebeat.modules:
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
   vpcflow:
     enabled: false
 
@@ -282,6 +297,9 @@ filebeat.modules:
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
 #-------------------------------- Azure Module --------------------------------
 - module: azure
   # All logs

diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml
@@ -30,6 +30,9 @@
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
   cloudwatch:
     enabled: false
 
@@ -61,6 +64,9 @@
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
   ec2:
     enabled: false
 
@@ -92,6 +98,9 @@
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
   elb:
     enabled: false
 
@@ -123,6 +132,9 @@
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
   s3access:
     enabled: false
 
@@ -154,6 +166,9 @@
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
 
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+
   vpcflow:
     enabled: false
 
@@ -184,3 +199,6 @@
 
     # Custom endpoint used to access AWS APIs
     #var.endpoint: amazonaws.com
+
+    # AWS IAM Role to assume
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc
@@ -22,7 +22,8 @@ that represent actions taken by a user, role or AWS service.
 
 The `aws` module requires AWS credentials configuration in order to make AWS API calls.
 Users can either use `access_key_id`, `secret_access_key` and/or
-`session_token`, or use shared AWS credentials file.
+`session_token`, or use `role_arn` AWS IAM role, or use shared AWS credentials file.
+
 Please see <<aws-credentials-options,AWS credentials options>> for more details.
 
 include::../include/gs-link.asciidoc[]
@@ -46,6 +47,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 
   cloudwatch:
     enabled: false
@@ -58,6 +60,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 
   ec2:
     enabled: false
@@ -70,6 +73,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 
   elb:
     enabled: false
@@ -82,6 +86,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 
   s3access:
     enabled: false
@@ -94,6 +99,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 
   vpcflow:
     enabled: false
@@ -106,6 +112,7 @@ Example config:
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
+    #var.role_arn: arn:aws:iam::123456789012:role/test-mb
 ----
 
 *`var.queue_url`*::
@@ -142,6 +149,9 @@ Second part of access key.
 *`var.session_token`*::
 Required when using temporary security credentials.
 
+*`var.role_arn`*::
+AWS IAM Role to assume.
+
 [float]
 === cloudtrail fileset
 

diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml
@@ -33,3 +33,7 @@ secret_access_key: {{ .secret_access_key }}
 {{ if .session_token }}
 session_token: {{ .session_token }}
 {{ end }}
+
+{{ if .role_arn }}
+role_arn: {{ .role_arn }}
+{{ end }}
diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml
@@ -12,6 +12,7 @@ var:
   - name: access_key_id
   - name: secret_access_key
   - name: session_token
+  - name: role_arn
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml
@@ -32,3 +32,7 @@ secret_access_key: {{ .secret_access_key }}
 {{ if .session_token }}
 session_token: {{ .session_token }}
 {{ end }}
+
+{{ if .role_arn }}
+role_arn: {{ .role_arn }}
+{{ end }}
diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml
@@ -12,6 +12,7 @@ var:
   - name: access_key_id
   - name: secret_access_key
   - name: session_token
+  - name: role_arn
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/module/aws/ec2/config/s3.yml b/x-pack/filebeat/module/aws/ec2/config/s3.yml
@@ -32,3 +32,7 @@ secret_access_key: {{ .secret_access_key }}
 {{ if .session_token }}
 session_token: {{ .session_token }}
 {{ end }}
+
+{{ if .role_arn }}
+role_arn: {{ .role_arn }}
+{{ end }}
diff --git a/x-pack/filebeat/module/aws/ec2/manifest.yml b/x-pack/filebeat/module/aws/ec2/manifest.yml
@@ -12,6 +12,7 @@ var:
   - name: access_key_id
   - name: secret_access_key
   - name: session_token
+  - name: role_arn
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/module/aws/elb/config/s3.yml b/x-pack/filebeat/module/aws/elb/config/s3.yml
@@ -32,3 +32,7 @@ secret_access_key: {{ .secret_access_key }}
 {{ if .session_token }}
 session_token: {{ .session_token }}
 {{ end }}
+
+{{ if .role_arn }}
+role_arn: {{ .role_arn }}
+{{ end }}
diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml
@@ -12,6 +12,7 @@ var:
   - name: access_key_id
   - name: secret_access_key
   - name: session_token
+  - name: role_arn
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml

diff --git a/x-pack/filebeat/module/aws/s3access/config/s3.yml b/x-pack/filebeat/module/aws/s3access/config/s3.yml
@@ -32,3 +32,7 @@ secret_access_key: {{ .secret_access_key }}
 {{ if .session_token }}
 session_token: {{ .session_token }}
 {{ end }}
+
+{{ if .role_arn }}
+role_arn: {{ .role_arn }}
+{{ end }}
diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml
@@ -12,6 +12,7 @@ var:
   - name: access_key_id
   - name: secret_access_key
   - name: session_token
+  - name: role_arn
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml
@@ -35,6 +35,10 @@ secret_access_key: {{ .secret_access_key }}
 session_token: {{ .session_token }}
 {{ end }}
 
+{{ if .role_arn }}
+role_arn: {{ .role_arn }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 
 type: log

diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml
@@ -12,6 +12,7 @@ var:
   - name: access_key_id
   - name: secret_access_key
   - name: session_token
+  - name: role_arn
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/input.yml