elastic · andrewkroh · Dec 2, 2020 · Nov 19, 2020
diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log
@@ -0,0 +1 @@
+{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-25T18:45:11Z"}}},"eventTime":"2014-03-25T21:08:14Z","eventSource":"iam.amazonaws.com","eventName":"AddUserToGroup","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"AWSConsole","requestParameters":{"userName":"Bob","groupName":"admin"},"responseElements":null}
diff --git a/...aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-config.json b/...aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-config.json
@@ -0,0 +1,5 @@
+{
+    "dynamic_fields": {
+        "event.ingested": ".*"
+    }
+}
diff --git a/...s/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json b/...s/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json
@@ -0,0 +1,69 @@
+{
+    "expected": [
+        {
+            "cloud": {
+                "region": "us-east-2",
+                "account": {
+                    "id": "123456789012"
+                }
+            },
+            "@timestamp": "2014-03-25T21:08:14.000Z",
+            "related": {
+                "user": [
+                    "Bob"
+                ]
+            },
+            "source": {
+                "address": "127.0.0.1",
+                "ip": "127.0.0.1"
+            },
+            "event": {
+                "ingested": "2020-11-19T22:16:17.114840Z",
+                "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-25T18:45:11Z\"}}},\"eventTime\":\"2014-03-25T21:08:14Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AddUserToGroup\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"AWSConsole\",\"requestParameters\":{\"userName\":\"Bob\",\"groupName\":\"admin\"},\"responseElements\":null}",
+                "provider": "iam.amazonaws.com",
+                "kind": "event",
+                "action": "AddUserToGroup",
+                "type": [
+                    "group",
+                    "change"
+                ],
+                "category": [
+                    "iam"
+                ],
+                "outcome": "success"
+            },
+            "aws": {
+                "cloudtrail": {
+                    "event_version": "1.0",
+                    "flattened": {
+                        "request_parameters": {
+                            "userName": "Bob",
+                            "groupName": "admin"
+                        }
+                    },
+                    "user_identity": {
+                        "access_key_id": "EXAMPLE_KEY_ID",
+                        "session_context": {
+                            "mfa_authenticated": "false",
+                            "creation_date": "2014-03-25T18:45:11.000Z"
+                        },
+                        "type": "IAMUser",
+                        "arn": "arn:aws:iam::123456789012:user/Alice"
+                    },
+                    "request_parameters": "{groupName=admin, userName=Bob}"
+                }
+            },
+            "user": {
+                "name": "Alice",
+                "id": "EX_PRINCIPAL_ID"
+            },
+            "user_agent": {
+                "name": "Other",
+                "device": {
+                    "name": "Other"
+                },
+                "original": "AWSConsole"
+            }
+        }
+    ]
+}
diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log
@@ -0,0 +1 @@
+{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIN5ATK5U7KEXAMPLE:JohnRole1","arn":"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1","accountId":"111111111111","accessKeyId":"AKIAI44QH8DHBEXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2019-10-02T21:50:54Z"},"sessionIssuer":{"type":"Role","principalId":"AROAIN5ATK5U7KEXAMPLE","arn":"arn:aws:iam::111111111111:role/JohnRole1","accountId":"111111111111","userName":"JohnDoe"}}},"eventTime":"2019-10-02T22:12:29Z","eventSource":"sts.amazonaws.com","eventName":"AssumeRole","awsRegion":"us-east-2","sourceIPAddress":"123.145.67.89","userAgent":"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239","requestParameters":{"incomingTransitiveTags":{"Department":"Engineering"},"tags":[{"value":"johndoe@example.com","key":"Email"},{"value":"12345","key":"CostCenter"}],"roleArn":"arn:aws:iam::111111111111:role/JohnRole2","roleSessionName":"Role2WithTags","transitiveTagKeys":["Email","CostCenter"],"durationSeconds":3600},"responseElements":{"credentials":{"accessKeyId":"ASIAWHOJDLGPOEXAMPLE","expiration":"Oct 2, 2019 11:12:29 PM","sessionToken":"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN"},"assumedRoleUser":{"assumedRoleId":"AROAIFR7WHDTSOYQYHFUE:Role2WithTags","arn":"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags"}},"requestID":"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE","eventID":"1917948f-3042-46ec-98e2-62865EXAMPLE","resources":[{"ARN":"arn:aws:iam::111122223333:role/JohnRole2","accountId":"111111111111","type":"AWS::IAM::Role"}],"eventType":"AwsApiCall","recipientAccountId":"111111111111"}
diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-config.json
@@ -0,0 +1,5 @@
+{
+    "dynamic_fields": {
+        "event.ingested": ".*"
+    }
+}
diff --git a/...ges/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json b/...ges/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json
@@ -0,0 +1,125 @@
+{
+    "expected": [
+        {
+            "cloud": {
+                "region": "us-east-2",
+                "account": {
+                    "id": "111111111111"
+                }
+            },
+            "@timestamp": "2019-10-02T22:12:29.000Z",
+            "source": {
+                "geo": {
+                    "continent_name": "Asia",
+                    "region_iso_code": "CN-CQ",
+                    "country_name": "China",
+                    "region_name": "Chongqing",
+                    "location": {
+                        "lon": 106.5531,
+                        "lat": 29.5569
+                    },
+                    "country_iso_code": "CN"
+                },
+                "as": {
+                    "number": 4837,
+                    "organization": {
+                        "name": "CHINA UNICOM China169 Backbone"
+                    }
+                },
+                "address": "123.145.67.89",
+                "ip": "123.145.67.89"
+            },
+            "event": {
+                "ingested": "2020-11-19T22:16:17.142969600Z",
+                "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE:JohnRole1\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1\",\"accountId\":\"111111111111\",\"accessKeyId\":\"AKIAI44QH8DHBEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-10-02T21:50:54Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE\",\"arn\":\"arn:aws:iam::111111111111:role/JohnRole1\",\"accountId\":\"111111111111\",\"userName\":\"JohnDoe\"}}},\"eventTime\":\"2019-10-02T22:12:29Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"123.145.67.89\",\"userAgent\":\"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239\",\"requestParameters\":{\"incomingTransitiveTags\":{\"Department\":\"Engineering\"},\"tags\":[{\"value\":\"johndoe@example.com\",\"key\":\"Email\"},{\"value\":\"12345\",\"key\":\"CostCenter\"}],\"roleArn\":\"arn:aws:iam::111111111111:role/JohnRole2\",\"roleSessionName\":\"Role2WithTags\",\"transitiveTagKeys\":[\"Email\",\"CostCenter\"],\"durationSeconds\":3600},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAWHOJDLGPOEXAMPLE\",\"expiration\":\"Oct 2, 2019 11:12:29 PM\",\"sessionToken\":\"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAIFR7WHDTSOYQYHFUE:Role2WithTags\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags\"}},\"requestID\":\"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE\",\"eventID\":\"1917948f-3042-46ec-98e2-62865EXAMPLE\",\"resources\":[{\"ARN\":\"arn:aws:iam::111122223333:role/JohnRole2\",\"accountId\":\"111111111111\",\"type\":\"AWS::IAM::Role\"}],\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"111111111111\"}",
+                "provider": "sts.amazonaws.com",
+                "kind": "event",
+                "action": "AssumeRole",
+                "id": "1917948f-3042-46ec-98e2-62865EXAMPLE",
+                "type": [
+                    "info"
+                ],
+                "category": [
+                    "authentication"
+                ],
+                "outcome": "success"
+            },
+            "aws": {
+                "cloudtrail": {
+                    "event_version": "1.05",
+                    "flattened": {
+                        "request_parameters": {
+                            "incomingTransitiveTags": {
+                                "Department": "Engineering"
+                            },
+                            "transitiveTagKeys": [
+                                "Email",
+                                "CostCenter"
+                            ],
+                            "durationSeconds": 3600,
+                            "roleArn": "arn:aws:iam::111111111111:role/JohnRole2",
+                            "roleSessionName": "Role2WithTags",
+                            "tags": [
+                                {
+                                    "value": "johndoe@example.com",
+                                    "key": "Email"
+                                },
+                                {
+                                    "value": "12345",
+                                    "key": "CostCenter"
+                                }
+                            ]
+                        },
+                        "response_elements": {
+                            "assumedRoleUser": {
+                                "assumedRoleId": "AROAIFR7WHDTSOYQYHFUE:Role2WithTags",
+                                "arn": "arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags"
+                            },
+                            "credentials": {
+                                "accessKeyId": "ASIAWHOJDLGPOEXAMPLE",
+                                "sessionToken": "AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN",
+                                "expiration": "Oct 2, 2019 11:12:29 PM"
+                            }
+                        }
+                    },
+                    "event_type": "AwsApiCall",
+                    "user_identity": {
+                        "access_key_id": "AKIAI44QH8DHBEXAMPLE",
+                        "session_context": {
+                            "mfa_authenticated": "false",
+                            "session_issuer": {
+                                "account_id": "111111111111",
+                                "type": "Role",
+                                "arn": "arn:aws:iam::111111111111:role/JohnRole1",
+                                "principal_id": "AROAIN5ATK5U7KEXAMPLE"
+                            },
+                            "creation_date": "2019-10-02T21:50:54.000Z"
+                        },
+                        "type": "AssumedRole",
+                        "arn": "arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1"
+                    },
+                    "recipient_account_id": "111111111111",
+                    "request_parameters": "{incomingTransitiveTags={Department=Engineering}, transitiveTagKeys=[Email, CostCenter], durationSeconds=3600, roleArn=arn:aws:iam::111111111111:role/JohnRole2, roleSessionName=Role2WithTags, tags=[{value=johndoe@example.com, key=Email}, {value=12345, key=CostCenter}]}",
+                    "response_elements": "{assumedRoleUser={assumedRoleId=AROAIFR7WHDTSOYQYHFUE:Role2WithTags, arn=arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags}, credentials={accessKeyId=ASIAWHOJDLGPOEXAMPLE, sessionToken=AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN, expiration=Oct 2, 2019 11:12:29 PM}}"
+                }
+            },
+            "user": {
+                "name": "JohnDoe",
+                "id": "AROAIN5ATK5U7KEXAMPLE:JohnRole1"
+            },
+            "user_agent": {
+                "name": "aws-cli",
+                "original": "aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239",
+                "os": {
+                    "name": "Linux",
+                    "version": "4.9.184",
+                    "full": "Linux 4.9.184"
+                },
+                "device": {
+                    "name": "Spider"
+                },
+                "version": "1.16.248"
+            }
+        }
+    ]
+}
diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log
@@ -0,0 +1,2 @@
+{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice"},"eventTime":"2020-01-09T00:09:33Z","eventSource":"iam.amazonaws.com","eventName":"ChangePassword","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","errorCode":"AccessDeniedException","errorMessage":"An unknown error occurred","requestParameters":null,"responseElements":null,"requestID":"EXAMPLE-5204-4fed-9c60-9c6EXAMPLE","eventID":"EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"}
+{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice"},"eventTime":"2020-01-09T00:03:36Z","eventSource":"iam.amazonaws.com","eventName":"ChangePassword","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","requestParameters":null,"responseElements":null,"requestID":"EXAMPLE-5c16-4eda-9724-EXAMPLE","eventID":"EXAMPLE-35a7-4c25-9fc7-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"}
diff --git a/...s/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-config.json b/...s/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-config.json
@@ -0,0 +1,5 @@
+{
+    "dynamic_fields": {
+        "event.ingested": ".*"
+    }
+}